diff --git a/.dockerignore b/.dockerignore index d18057c9b..24924a4bf 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,10 @@ .github +.git +**/.git +__pycache__ +**/__pycache__ +*.pyc +**/*.pyc .venv .vscode .data diff --git a/.github/workflows/build-dev-image.yaml b/.github/workflows/build-dev-image.yaml index 623a9bf99..ad7bc310b 100644 --- a/.github/workflows/build-dev-image.yaml +++ b/.github/workflows/build-dev-image.yaml @@ -40,6 +40,8 @@ jobs: with: context: . push: true + build-args: | + LANGBOT_BUILD_REVISION=${{ github.sha }} tags: | rockchin/langbot:${{ steps.image.outputs.branch_tag }} rockchin/langbot:${{ steps.image.outputs.sha_tag }} diff --git a/.github/workflows/build-docker-image.yml b/.github/workflows/build-docker-image.yml index 4575c4fba..215783735 100644 --- a/.github/workflows/build-docker-image.yml +++ b/.github/workflows/build-docker-image.yml @@ -41,7 +41,14 @@ jobs: run: docker buildx create --name mybuilder --use - name: Build for Release # only relase, exlude pre-release if: ${{ github.event.release.prerelease == false }} - run: docker buildx build --platform linux/arm64,linux/amd64 -t rockchin/langbot:${{ steps.check_version.outputs.version }} -t rockchin/langbot:latest . --push + run: docker buildx build --build-arg LANGBOT_BUILD_REVISION=${{ github.sha }} --platform linux/arm64,linux/amd64 -t rockchin/langbot:${{ steps.check_version.outputs.version }} -t rockchin/langbot:latest . --push - name: Build for Pre-release # no update for latest tag if: ${{ github.event.release.prerelease == true }} - run: docker buildx build --platform linux/arm64,linux/amd64 -t rockchin/langbot:${{ steps.check_version.outputs.version }} . --push \ No newline at end of file + env: + RELEASE_VERSION: ${{ steps.check_version.outputs.version }} + run: | + tags=(-t "rockchin/langbot:$RELEASE_VERSION") + if [[ "$RELEASE_VERSION" =~ ^v[0-9]+\.[0-9]+\.[0-9]+-beta\.[0-9]+$ ]]; then + tags+=(-t rockchin/langbot:beta) + fi + docker buildx build --build-arg LANGBOT_BUILD_REVISION=${{ github.sha }} --platform linux/arm64,linux/amd64 "${tags[@]}" . --push \ No newline at end of file diff --git a/.github/workflows/build-release-artifacts.yaml b/.github/workflows/build-release-artifacts.yaml index ed36fae92..5f92a3c64 100644 --- a/.github/workflows/build-release-artifacts.yaml +++ b/.github/workflows/build-release-artifacts.yaml @@ -31,6 +31,9 @@ jobs: echo ::set-output name=version::${GITHUB_REF} fi + - name: Stamp build revision + run: python3 scripts/stamp_build_revision.py + - name: Make Temp Directory run: | mkdir -p /tmp/langbot_build_web diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 50366f093..57f12cfd7 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -37,8 +37,8 @@ jobs: from pathlib import Path tag = os.environ['RELEASE_TAG'] - if not re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', tag): - raise SystemExit('source_ref must be an existing release tag: vX.Y.Z') + if not re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+(?:-(?:alpha|beta|rc)\.[0-9]+)?', tag): + raise SystemExit('source_ref must be an existing release tag: vX.Y.Z or vX.Y.Z-beta.N') def revision(ref): return subprocess.check_output(['git', 'rev-parse', '--verify', ref], text=True).strip() if revision('HEAD') != revision(f'refs/tags/{tag}^{{}}'): @@ -70,6 +70,7 @@ jobs: - name: Build package run: | + python3 scripts/stamp_build_revision.py uv build - name: Publish to PyPI diff --git a/AGENTS.md b/AGENTS.md index b6c6dc401..760cf06b6 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -83,7 +83,7 @@ Config keys to verify in `data/config.yaml` / `src/langbot/templates/config.yaml ## Change Rules - HTTP API changes that should be agent-accessible must update the matching MCP tool in `src/langbot/pkg/api/mcp/server.py` and the relevant skill under `skills/` in the same pass. -- New schema changes use Alembic under `src/langbot/pkg/persistence/alembic/versions/`; do not add legacy `dbmXXX` migrations. +- New schema changes use Alembic under `src/langbot/pkg/persistence/alembic/versions/`. LangBot 4.x does not support upgrading 3.x databases. - New platform behavior belongs in platform adapters only for platform translation; pipeline/business logic belongs in `pkg/pipeline/` or services. - User-facing strings must support i18n (`en_US`, `zh_Hans`; include `ja_JP` where the repo already does). - Code comments and docstrings must be English. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 552a104dc..688d8f5c1 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -52,12 +52,13 @@ LangBot/ │ │ ├── api/ # HTTP API + MCP server mount │ │ ├── platform/ # IM adapters and runtime bot manager │ │ ├── pipeline/ # Message routing and pipeline stages -│ │ ├── provider/ # LLM runners, model manager, tools +│ │ ├── provider/ # Model providers and Host-owned tools +│ │ ├── agent/ # Agent/Runner orchestration and run state │ │ ├── plugin/ # LangBot-side Plugin Runtime connector/handler │ │ ├── box/ # LangBot-side Box service/connector │ │ ├── skill/ # Skill metadata/activation integration │ │ ├── rag/ , vector/ # Knowledge-base and vector DB integration -│ │ ├── persistence/ # SQLAlchemy/SQLModel, Alembic, legacy migrations +│ │ ├── persistence/ # SQLAlchemy/SQLModel and Alembic migrations │ │ ├── storage/ # Local/S3 file storage abstraction │ │ └── config/, entity/, utils/, telemetry/, survey/ │ ├── libs/ # Vendored third-party platform SDKs @@ -80,7 +81,7 @@ Platform adapter → Controller → RuntimePipeline → PipelineStage chain - → RequestRunner / ToolManager / PluginRuntimeConnector / BoxService + → Runner orchestrator / ToolManager / PluginRuntimeConnector / BoxService → response via adapter ``` @@ -107,7 +108,7 @@ Inbound platform messages enter through adapter-specific SDK callbacks. The comm 3. `MessageAggregator` batches/normalizes messages before adding a `Query` to `QueryPool`. 4. `Controller` in `pkg/pipeline/controller.py` selects queries subject to global pipeline concurrency and per-session concurrency. 5. `RuntimePipeline` in `pkg/pipeline/pipelinemgr.py` runs configured pipeline stages using a responsibility-chain style executor that supports generator stages. -6. The chat stage emits plugin events, calls a configured `RequestRunner`, handles streaming/non-streaming responses, records telemetry, and appends conversation history. +6. The chat stage emits plugin events and projects the current query into the Runner Host orchestrator. The selected plugin Runner returns streaming or final results while the Host owns authorization, tools, telemetry, and conversation history. 7. Output stages send text, cards, chunks, files, or error notices back through the original platform adapter. Pipeline components are registered by decorators and package import side effects. When adding a new stage, loader, runner, or adapter, check the corresponding preregistration mechanism instead of inventing a second registry. @@ -117,6 +118,7 @@ Pipeline components are registered by decorators and package import side effects Platform code lives under `pkg/platform/`. - `botmgr.py` owns runtime bots, routing rules, event logging, webhook pushing, and adapter lifecycle. +- Bots store exclusive Agent/Pipeline routes in `event_bindings` and independent plugin subscriptions in `plugin_processors` (`processor_uuid`, `enabled`). Subscriptions resolve event patterns from the installed Runner and fan out alongside the primary route. Configuration, state, debug and run logs belong to the reusable processor instance. - `sources/` contains adapter implementations. Each adapter subclasses `langbot_plugin.api.definition.abstract.platform.adapter.AbstractMessagePlatformAdapter` from the SDK. - Platform entities such as `MessageChain`, `Image`, `At`, `Voice`, and events come from `langbot-plugin-sdk`, not from this repo. @@ -136,12 +138,12 @@ Important pieces: Pipelines are configuration-driven. Prefer adding a stage or extending an existing stage family over hard-coding behavior in platform adapters. -## Provider, RAG, and Tools +## Agents, Providers, RAG, and Tools -Provider code lives under `pkg/provider/`. +Agent orchestration lives under `pkg/agent/`; model providers and tools live under `pkg/provider/`. - `modelmgr/` manages configured model providers and requesters. -- `runners/` implements request runners such as the local agent runner and external workflow integrations. +- `pkg/agent/runner/` discovers plugin Runner components, resolves bindings, constructs run-scoped context/resources, and records execution state. - `tools/toolmgr.py` aggregates tools from native tools, plugin tools, external MCP servers, and skill-authoring tools. - `tools/loaders/mcp.py` is the MCP client side: external MCP servers that LangBot connects to for agent tools. - RAG lives across `pkg/rag/`, `pkg/vector/`, model services, and plugin KnowledgeEngine actions. @@ -170,6 +172,23 @@ The Plugin Runtime supports stdio and WebSocket control transports. Direct local ## Box Runtime and Skills +Runner plugins own sandbox policy: enablement, reuse-key interpolation, acquisition, +binding, and explicit file import/export. The Host exposes authenticated resource +APIs (`get_box_status`, `list_boxes`, `acquire_box`) and run-bound operations +(`bind_box`, `import_box_attachments`, `export_box_files`, `reply_files`). Box Runtime +owns atomic capacity enforcement and container reuse/lifecycle. Reusing an existing +Box is allowed when no additional capacity remains. + +A run binds one Box before native sandbox tools or file transfer. The Host does not +choose a conversation scope or stage attachments before starting the Runner. Input +references retain the original attachment metadata; import yields paths specific to +the run. Output export reads only that run's outbox and returns opaque file handles. +`message.completed.file_ids` attaches explicitly exported files to Pipeline output; +Agents send them explicitly with `ctx.reply_files()`, subject to event reply permission. +The Pipeline wrapper never scans a Box. Binding ends with the run; the reusable Box +remains subject to Runtime idle expiry. Persistent workspace files survive expiry, +but processes and container-local state do not. + Box is the sandbox subsystem used by native agent tools, stdio MCP servers, skill authoring, and managed processes. In this repo: @@ -213,8 +232,9 @@ Persistence is centered on `pkg/persistence/mgr.py`. - SQLite is the default database; PostgreSQL is supported. - Models live under `pkg/entity/persistence/`. -- Fresh schemas are created from metadata, then legacy migrations run up to the frozen 3.x baseline, then Alembic migrations run to head. -- New schema changes should use Alembic under `pkg/persistence/alembic/versions/`; do not extend the frozen legacy migration chain. +- Timezone-less SQL `DateTime` columns store UTC-naive values on both backends. Normalize aware values with `pkg/persistence/datetime_utils.py::as_naive_utc` at the bind boundary, including deadlines, leases, event times, and query/retention cutoffs; do not merely strip an offset. Existing naive rows represent UTC. Restore UTC awareness for application comparisons and epoch serialization. Runner ledger/event/transcript stores follow this existing schema contract without a data migration or changes to journal/tenant authorization. +- Fresh schemas are created from current metadata, then Alembic migrations run to head. LangBot 4.x does not upgrade 3.x databases. +- New schema changes use Alembic under `pkg/persistence/alembic/versions/`; there is no legacy migration chain in 4.x. Configuration starts from `src/langbot/templates/config.yaml` and is generated into `data/config.yaml` on first run. Most long-lived managers read from `ap.instance_config.data`. @@ -246,7 +266,7 @@ When one of these changes, update the others if the behavior or contract changed - New LLM tool source: extend `pkg/provider/tools/loaders/` and `ToolManager` intentionally. - New plugin component/API/protocol: change `langbot-plugin-sdk` first or in lockstep, then update LangBot bridge code. - New Box capability: change both `pkg/box/` and `langbot-plugin-sdk/src/langbot_plugin/box/`, plus config and tests. -- New database schema: add an Alembic migration, not a legacy `dbmXXX` migration. +- New database schema: add an Alembic migration. ## Design Biases diff --git a/Dockerfile b/Dockerfile index 1d1351c72..3768a5802 100644 --- a/Dockerfile +++ b/Dockerfile @@ -31,6 +31,9 @@ WORKDIR /app COPY . . +ARG LANGBOT_BUILD_REVISION +RUN if [ -n "$LANGBOT_BUILD_REVISION" ]; then python3 scripts/stamp_build_revision.py --revision "$LANGBOT_BUILD_REVISION"; fi + COPY --from=node /app/web/dist ./web/dist # nsjail binary built in the dedicated stage above. Self-contained sandbox diff --git a/docs/agent-runner-pluginization/AGENT_CONTEXT_PROTOCOL.md b/docs/agent-runner-pluginization/AGENT_CONTEXT_PROTOCOL.md new file mode 100644 index 000000000..049a86738 --- /dev/null +++ b/docs/agent-runner-pluginization/AGENT_CONTEXT_PROTOCOL.md @@ -0,0 +1,149 @@ +# Agent-owned Context 协议设计 + +本文档描述插件化 Runner 场景下的上下文边界**设计理由**。结论先行：LangBot 不应成为最终 agentic context manager；它提供 context substrate，Runner 或其背后的 runtime 自己决定如何管理历史、压缩、召回和 KV cache。 + +> 涉及的数据结构（`RunnerContext`、`ContextAccess`、`RunnerAPIProxy` 等）唯一定义在 [PROTOCOL_V1.md](./PROTOCOL_V1.md)。本文只讲语义和约束，不重抄 schema。 + +## 1. 设计原则 + +### 1.1 Agent 拥有上下文策略 + +不同 runner 背后的 runtime 差异很大： + +- 官方 local-agent 可能依赖 LangBot 的模型、工具、知识库和存储。 +- Claude Code SDK / Codex 类 runtime 有自己的 session、transcript、tool loop 和上下文压缩。 +- Pi Agent SDK 或外部 agent 平台可能只需要当前事件和一个外部 conversation key。 + +因此 LangBot 不应强行决定最终传给模型的历史窗口。Host 只提供：当前事件的完整结构化信息、稳定身份和会话引用、可授权读取的 history / event / state API、sandbox/workspace 文件能力、可投影给外部 harness 的 scoped context / SDK-owned MCP bridge / resource handles、payload hard cap 和权限 guardrail。 + +### 1.2 Host 不定义通用历史窗口 + +历史窗口策略不是 Runner 协议或 Query entry adapter 的核心概念。Host 只提供 history pull API、cursor、hard cap 和权限边界；runner 自己决定是否读取、读取多少、如何截断和压缩。 + +正确的问题不是"LangBot 每轮裁几轮历史给 agent"，而是： + +- 这类 runner 是否自管 context？ +- 事件到来时 host 应 inline 哪些最小信息？ +- agent 需要更多上下文时通过什么 API 拉取？ +- host 如何保证安全、可审计和可分页？ + +### 1.3 Host 保存事实源，Agent 管理 working context + +三类数据要分开： + +- `EventLog`: Host 保存原始事件、工具调用、投递结果、错误和系统事件。 +- `Transcript`: Host 从 EventLog 投影出的对话视图，用于 UI、审计和按需历史读取。 +- `Working context`: Agent 本轮实际送进模型或 runtime 的上下文，由 Runner 决定。 + +LangBot 不提供 host-side inline history window。简单 runner 如果需要历史窗口，应在 runner 内部通过 Host history API 拉取并裁剪。 + +## 2. Event 到来时传什么 + +默认 `RunnerContext`（PROTOCOL_V1 §5.2）应尽量小且稳定。默认规则： + +- Host MUST NOT inline full history by default. +- Host SHOULD inline only current event / input and context handles. +- Runner owns working-context assembly. +- Runner MAY use Host history / event / state / storage API and sandbox/workspace file tools when authorized. +- Official runners MUST consume Host infrastructure through the same public API as third-party runners. + +### 2.1 必须 inline 的内容 + +当前 event 的类型/id/时间/source；当前输入文本和结构化内容；附件/文件/图片的 metadata、path 或 URL；actor / subject / conversation / thread / bot / workspace；delivery 能力；已授权资源列表；context cursors 和可用 API 能力；Agent/runner config。这些是 agent 决定下一步所需的最低信息。 + +### 2.2 默认不 inline 的内容 + +完整历史消息、大文件全文、大工具结果、全量知识库内容、平台原始 payload 大对象、每轮重新生成的大段 summary。这些会破坏跨进程序列化成本、泄露范围、KV cache 稳定性，也会迫使 host 替 agent 做 context 策略。 + +### 2.3 不提供 Host Inline History Window + +`RunnerContext` 不包含 `bootstrap` 字段。Host 不下发历史窗口，也不通过 Pipeline 配置决定窗口大小。runner 若需要类似 `recent_tail` 的策略，应在自己的 manifest/config schema 中声明参数，并在 runner 内部通过 history API 读取、裁剪和压缩。Host 只负责权限、分页、hard cap 和事实源。 + +## 3. ContextAccess 的作用 + +`ContextAccess`（PROTOCOL_V1 §5.8）是 host 交给 agent 的上下文读取入口描述，告诉 agent：当前事件位于哪条 conversation / thread、若需要更多历史从哪个 cursor 开始拉、host inline 了什么没 inline 什么、当前 run 有哪些 context API 权限。 + +## 4. Agent 如何获取更多上下文 + +所有 API 都走 `RunnerAPIProxy`（PROTOCOL_V1 §8），由 host 用 `run_id` 校验。 + +外部 harness 不能直接访问 LangBot 资源。无论是 history、event、state、model、tool、knowledge base，还是 LangBot skills，都必须通过 SDK runtime 转发到 Host API，并由 Host 按 active `run_id`、runner identity、binding resource policy 和 caller plugin identity 校验。当前运行文件进入授权 sandbox/workspace 后，再由 runner 用 read/write/exec 类工具按需访问。harness 自己的 native tools 只属于 harness 执行环境，不能绕过 SDK runtime 访问 LangBot 内部资源。 + +### 4.1 History + +```python +await api.history_page(conversation_id=ctx.context.conversation_id, + before_cursor=ctx.context.latest_cursor, + limit=50, direction="backward", include_attachments=False) +``` + +返回 `HistoryPage`（schema 见 PROTOCOL_V1 §8）。 + +约束：`limit` 有 host hard cap；默认只能读当前 conversation / thread；跨会话读取需 binding policy / run authorization snapshot 授权；可返回 attachment ref，不默认返回大文件内容。 + +### 4.2 Search + +```python +await api.history_search(query="用户之前提到的数据库连接信息", + filters={"conversation_id": ..., "event_types": ["message.received"]}, + top_k=10) +``` + +Search 可先用数据库全文索引，后续接 embedding recall。它是 host 检索能力，不等于 agent 的长期记忆策略。 + +### 4.3 Event / State + +- Event API（`events.get` / `events.page`）用于读取非消息事件、工具事件、系统事件。Agent 不应把所有事件都当成 user/assistant message。 +- State API（`state.get` / `set`）是可选寄宿能力。自管 runtime 可以完全不用；依附 LangBot 的官方 runner 可以使用，例如 `external.session_id`、`summary.checkpoint`。 + +### 4.4 大文件与工具协作 + +大文件、多模态输入和工具产物不要内联进 prompt 或 tool result：message/content 里只放小文本和必要摘要；当前事件附件由 Host staged 到授权 sandbox/workspace，并在 input attachment 中给出轻量 metadata/path。工具之间传递大结果时传 sandbox path 或 attachment ref，不传完整 blob。Host 只保证当前 run 授权范围，默认不允许插件直接读任意本地路径；临时文件由 sandbox 生命周期和清理机制管理。 + +### 4.5 External harness context projection + +外部 harness 的总体边界以 [HOST_SDK_INFRASTRUCTURE.md](./HOST_SDK_INFRASTRUCTURE.md) §4.8 为准。本节只描述 context projection 的推荐形态。 + +Claude Code、Codex、Kimi Code 这类 runtime 通常已有自己的 session、工具 loop、MCP 加载、上下文压缩和工作目录。LangBot 不应把它们改造成"host prompt assembler"，而应提供可审计的事件和资源投影。推荐 projection 形态： + +- `agent-context.json`：结构化 JSON，包含 `run_id`、`event`、`actor`、`subject`、`input`、`delivery`、`resources`、`context`、`state`、`runtime`。 +- `LANGBOT_CONTEXT.md`：人类可读摘要。 +- `resources`：只包含本次 run 授权后的资源句柄和能力摘要，不暴露 Host 内部私有对象、secret 或资源内容。 +- `skills`：LangBot skills 不是直接投影给 harness native tool loop 的文件能力，而是**一组被授权的 tool**。发现走 `list_skills`（或 `langbot_list_assets` 增加 skills 一类），激活/注册走 `activate` / `register_skill`，包内操作走 native exec/read/write，统一通过 `ctx.resources.tools`、`RunnerAPIProxy` 或 SDK-owned MCP bridge 暴露。Host 不向 prompt 注入 skill 索引（无 progressive-disclosure 注入）；harness 通过调用发现工具主动查询 skill 清单。`agent-context.json` 的 `skills` 字段仅作发现工具的数据来源与可选 `suggested_skill_prompt` 的输入。 +- `MCP config`：只投影 per-run、scoped 的 SDK-owned bridge 或外部 MCP 连接配置；LangBot 资源访问必须回到 SDK runtime / Host API，不允许 harness 通过自带 MCP/native tool 直接读 Host 内部资源。 +- `state pointers`：外部 session id、working directory、checkpoint 等小型 JSON 状态通过 Host state API 保存。 + +当前官方外部 harness 路径由 ACP / Claude Code / Codex 等 runner 插件承担（现状见 OFFICIAL_RUNNER_PLUGINS §7）。这类 projection 是"把 LangBot 事实源和授权资源句柄交给 harness"，不是"把 LangBot 资源本体或内部权限交给 harness"，也不是"由 LangBot 决定最终模型上下文"。 + +## 5. Runner 上下文边界 + +Host 只给当前事件、当前输入和 context handles。Runner 是否能拉取历史、事件、state 或 storage、是否能访问 sandbox/workspace 文件，以运行时 `ctx.context.available_apis` 和工具授权为准；runner 自己决定是否拉取历史、是否搜索、何时摘要、如何构造最终 prompt。 + +## 6. KV cache 友好的上下文管理 + +支持 Claude Code SDK、Codex、Pi Agent SDK 等 runtime 时，必须避免每轮由 LangBot 重组大块 prompt： + +- 稳定 session key：`workspace/bot/binding/runner/conversation/thread`。 +- 每轮只传 delta：当前 event、attachment refs/path、少量 runtime metadata。 +- 历史 append-only：不要每轮改写同一段 history 文本。 +- Summary checkpoint 稳定：只有压缩发生时产生新 checkpoint。 +- 大文件和工具结果写入 sandbox/workspace。 +- Tool/context API schema 稳定，数据通过 API 拉取而非塞入 prompt。 +- 对自管 runtime，优先让它复用自身 session/cache，而不是强制 LangBot 每轮重放 transcript。 +- 模型窗口元信息应作为 resource/runtime metadata 暴露给 runner，由 runner 决定预算和压缩策略。 + +稳定 session key 的用途是隔离外部 runtime 的 resume/cache/state，不是改变 PROTOCOL_V1 §13 定义的 Agent 复用和 dispatch 边界。只有当某个外部 harness 的同一 native session 不支持并发 turn 时，runner 或 future runtime control plane 才应按 external session key 做 turn-level 串行化。 + +对长期运行的 external harness / daemon，推荐运行形态是 reader 与 writer 分离：一个 session reader 独占读取 stdout/SSE/native event stream，并把 native event 转成 `RunnerResult` 或 task progress；用户输入只作为 turn write 进入该 session。当前一次性 CLI subprocess runner 可以继续在单次 `run(ctx)` 内同步收集 stdout，但后续改成长连接时不应让多个 request 同时读取同一 native stream。 + +## 7. Host guardrail + +Agent 自管 context 不代表无限制访问。LangBot 仍必须控制：每次 run 的 active `run_id`、runner identity、当前 binding 的 resource policy、conversation / actor / subject scope、page size / sandbox file read size / API rate limit、跨会话读取权限、数据脱敏和敏感变量过滤、审计日志。Host 不负责"最佳上下文策略"，但负责"不越权、不爆内存、不不可审计"。 + +外部 harness 的 native tools、shell、MCP 或 skill 机制不构成 LangBot 资源授权边界。只要访问的是 LangBot 持有的资源，就必须经 SDK runtime 转发并接受 Host 校验；完整边界见 HOST_SDK §4.8。 + +## 8. 官方 runner 与业务编排边界 + +官方 runner 插件可以把状态寄宿在 LangBot，但必须和第三方 runner 一样通过公开 Host API 消费。LangBot core 不内置官方 agent 的业务流程（prompt 组装、tool loop、RAG 编排、summary/compaction、"local-agent 专用"状态字段）。 + +官方 local-agent 应作为"依附 LangBot 基础设施的复杂 runner 参考实现"：transcript/history 通过 `api.history_page()` / `api.history_search()` 读取，summary/checkpoint/外部 session id/用户偏好通过 `api.state_get()` / `api.state_set()` 或 storage 方法保存，图片/文件/工具大结果通过 sandbox/workspace read/write 工具访问，模型/工具/知识库通过 `api.invoke_llm()` / `api.call_tool()` / `api.retrieve_knowledge()` 调用。这样 LangBot 保持为通用 agent host，不变成内置 agent 框架。具体迁移要求见 [OFFICIAL_RUNNER_PLUGINS.md](./OFFICIAL_RUNNER_PLUGINS.md)。 diff --git a/docs/agent-runner-pluginization/AGENT_RUNNER_QA_GUIDE.md b/docs/agent-runner-pluginization/AGENT_RUNNER_QA_GUIDE.md new file mode 100644 index 000000000..f0e6e090c --- /dev/null +++ b/docs/agent-runner-pluginization/AGENT_RUNNER_QA_GUIDE.md @@ -0,0 +1,259 @@ +# Runner QA 指南 + +本文档是 agent-runner 插件化下一轮测试的唯一 QA 入口。它合并并取代旧的 Phase 1 验收矩阵与 2026-05-18 / 2026-05-29 两份本地 QA 报告。 + +目标不是保留完整历史流水账，而是指导测试 agent 用最小但高价值的路径判断当前分支是否仍然健康。 + +## 1. 测试边界 + +当前主线验证的是 Runner Protocol v1： + +```text +event -> binding -> runner.run(ctx) -> result stream +``` + +本指南验证： + +- Host 能通过当前 Query entry adapter 进入 event-first `run(event, binding)` 主链路。 +- Runner 来自插件 registry，而不是旧内置 runner 分支。 +- `local-agent` 能消费 Host 模型、工具、知识库、history、state、sandbox 文件等基础设施。 +- 外部 harness runner（ACP / Claude Code / Codex 等直接 runner 插件）能消费 event-first context，并把外部 session 指针写回 host-owned state。 +- 错误、权限裁剪、无输出、timeout 等路径不会破坏主聊天流程。 + +本指南不验证： + +- 完整外部 harness daemon 管控和分布式业务队列；已实现的 run ledger / heartbeat / claim 原语仍应执行定向回归。 +- 尚未实现的通用事件订阅和定时自动化；已集成的 Bot 路由、独立 Agent 和 Pipeline 必须纳入当前产品验收。 +- 发布级 path isolation、secret filtering、MCP allowlist、资源配额和 workspace cleanup。 +- 所有外部服务 runner 的真实凭据联调。 + +这些属于后续能力或发布门槛，分别见 [RUNTIME_CONTROL_PLANE_V2.md](./RUNTIME_CONTROL_PLANE_V2.md) 与 [SECURITY_HARDENING.md](./SECURITY_HARDENING.md)。 + +## 2. 状态定义 + +测试报告只使用以下状态： + +| 状态 | 含义 | +| --- | --- | +| PASS | 按步骤执行，用户可见行为和日志证据都满足通过条件。 | +| FAIL | 环境可用，但行为不满足通过条件。 | +| BLOCKED | 凭据、CLI、外部服务、测试数据或本地配置缺失导致无法执行。必须写清阻塞原因。 | +| N/A | 当前 runner 或平台明确不支持该能力。必须引用 manifest、文档或配置说明。 | + +不能使用“看起来正常”“大概通过”“基本没问题”等模糊状态。 + +## 3. 执行顺序 + +2026-09-05 更新：先记录 Core/SDK/Runner 提交、发行版元数据与实际 import 路径，区分 editable 工作区和正式包安装。当前定向测试及前端未通过断言见 [STATUS.md](./STATUS.md)。本指南的步骤不是已执行记录。 + +本轮发布验收至少覆盖： + +- 空白实例从市场安装 Runner，创建 Agent 与 Pipeline，并绑定到 Bot。 +- 同一 Bot 的消息事件走 Pipeline，非消息事件走独立 Agent；dry-run 与保存后的路由结果一致。 +- `event_*` 使用冻结目标；未授权 `platform_*`、额外参数和失效机器人调用被拒绝；SDK/Python 与 MCP gateway 均走 Host 工具授权。 +- `interaction.requested` 回调恢复原处理器，重复/过期/跨作用域提交被拒绝。 +- 分别记录合成事件、mock provider、真实平台和真实 provider 的结果，不能相互替代。 + +推荐按以下顺序执行，前一层失败时不要继续扩大测试面： + +1. Host / SDK / runner 单测。 +2. WebUI 登录与 Pipeline Debug Chat 基础 smoke。 +3. `local-agent` 高价值场景。 +4. 外部 code-agent harness smoke。 +5. 权限和错误路径补充检查。 +6. 汇总 PASS / FAIL / BLOCKED，并给出下一步建议。 + +用户可见流程必须通过 WebUI 或真实消息平台验证。API / curl 只能作为诊断证据，不能单独让 UI case PASS。 + +## 4. 必跑基线 + +### 4.1 单测基线 + +在 LangBot 仓库运行： + +```bash +uv run --frozen pytest tests/unit_tests/agent +``` + +如果本次改动只触及默认配置或 API service，也至少补跑相关目标测试，例如： + +```bash +uv run pytest tests/unit_tests/api/test_pipeline_service_defaults.py +``` + +通过条件： + +- agent 单测全 PASS，或失败项已确认与本次 agent-runner 路径无关。 +- 若失败来自 `context_builder`、`orchestrator`、`session_registry`、`resource_builder`、`plugin/handler.py` 的 run action 权限路径，不应进入 UI smoke。 + +### 4.2 环境基线 + +用 `langbot-skills` 做环境检查： + +```bash +cd "$LANGBOT_SKILLS_REPO" +bin/lbs env doctor +bin/lbs case list +``` + +`LANGBOT_SKILLS_REPO` 指向当前工作区里的 `langbot-skills` 仓库。优先使用已有 case，而不是临时发明测试路径。 + +推荐首批 case： + +- `webui-login-state` +- `pipeline-debug-chat` +- `local-agent-basic-debug-chat` +- `local-agent-rag-debug-chat`（改动涉及 RAG / knowledge） +- `local-agent-plugin-tool-call-debug-chat`（改动涉及 tool / resource policy） + +## 5. WebUI 主链路 Smoke + +### 5.1 Runner registry + +步骤： + +1. 打开 WebUI Pipeline 配置页。 +2. 查看 AI runner 下拉列表。 +3. 选择 `plugin:langbot-team/LocalAgent/default`。 +4. 保存并刷新页面。 + +通过条件： + +- runner 选项来自插件 registry。 +- 保存后配置仍为 `ai.runner.id` + `ai.runner_config[id]`。 +- `runner_config` 表示 Agent/runner config，不表示插件实例状态。 +- 不读取或回写旧 `ai.runner.runner` 字段。 +- 不出现旧内置 runner stage 名（例如裸 `local-agent`）作为当前选中项或配置 surface。 +- 插件没有循环重启或 metadata 加载失败。 + +### 5.2 主聊天路径 + +步骤： + +1. 使用绑定 `plugin:langbot-team/LocalAgent/default` 的 Pipeline。 +2. 在 Debug Chat 发送确定性普通文本。 +3. 查看 WebUI 回复和后端日志。 + +通过条件： + +- 用户可见回复正常。 +- 后端日志显示走 `AgentRunOrchestrator` / `RUN_RUNNER`。 +- 不走旧内置 local-agent 主执行分支。 +- conversation transcript 写入用户消息和助手消息。 + +## 6. `local-agent` 高价值测试 + +只保留最能覆盖架构边界的场景。 + +| ID | 场景 | 操作 | 通过条件 | +| --- | --- | --- | --- | +| LA-01 | 绑定 prompt | 配置 system prompt 后发送文本。 | runner 使用 `ctx.config.prompt`，不读取 `ctx.adapter.extra["prompt"]`；回复体现绑定 prompt。 | +| LA-02 | history API | 连续两轮对话，第二轮引用第一轮 marker。 | runner 通过 Host history API 或自管上下文读取历史，不依赖 inline history window。 | +| LA-03 | 流式 / 非流式 | 分别用支持流式和关闭流式的路径发送文本。 | 流式 UI 不重复、不空白；非流式只输出最终消息。 | +| LA-04 | 工具调用 | 绑定测试工具，发送会触发工具的 prompt。 | `ctx.resources.tools` 只包含授权工具；工具调用 started/completed；最终回复包含工具结果。 | +| LA-05 | RAG | 绑定测试知识库，发送命中文档的 prompt。 | `ctx.resources.knowledge_bases` 包含所选知识库；runner 通过授权 API 检索；回复使用检索内容。 | +| LA-06 | 多模态 | 发送图片输入。 | `ctx.input.contents` 保留图片；支持视觉模型时正常处理，不支持时受控失败。 | +| LA-07 | fallback / 错误 | 模拟 primary 模型失败或 runner 抛错。 | fallback 或 `run.failed` 行为受控；后续请求不受影响。 | +| LA-08 | 无输出保护 | 测试 runner 完成但不产出消息。 | 不产生空白成功回复；按受控失败或明确缺陷处理。 | +| LA-09 | steering / 运行中追加消息 | 使用支持 steering 的 runner，第一条消息触发长 run；run 未结束时在同 conversation 追加第二条消息。 | 第二条消息被 active run claim，不启动并发 run；runner 通过 `steering_pull` 看到追加输入；EventLog 有 `queued` -> `steering.injected`，若未消费则有 `steering.dropped` 终态；后续普通消息仍可处理。 | + +Rerank、remove-think、文件输入等场景只在本次改动直接涉及时补测，不作为每轮必跑项。 + +## 7. Code-agent Harness Smoke + +这些测试用于验证 ACP、Claude Code、Codex 这类自管 runtime 能走同一条 Host 协议路径。若目标 harness 没有 CLI/daemon、登录态、代理配置或远端 workspace，标记 BLOCKED，不要伪造 PASS。 + +Smoke 前应优先保留一层轻量单测或 fixture 测试：session 创建/复用、消息发送、结果解析、`run_id` 注入和 LangBot MCP gateway 必须有稳定测试覆盖。WebUI smoke 证明真实链路可用，但不能替代转换层和错误映射测试。 + +### 7.1 外部 harness runner + +步骤： + +1. 确认目标 harness（例如 ACP daemon、Claude Code 或 Codex）在对应机器上可执行且已登录。 +2. 绑定目标 runner，例如 `plugin:langbot-team/ACPRunner/default`、`plugin:langbot-team/ClaudeCodeAgent/default` 或 `plugin:langbot-team/CodexAgent/default`。 +3. 配置 runner 必要字段，例如 remote target、workspace、provider、startup timeout、reuse session 等。 +4. 在 Debug Chat 执行一次确定性真实 smoke。 +5. 检查 LangBot MCP gateway、`run_id` 回填和 host-owned state。 + +通过条件： + +- WebUI 可见回复包含预期 sentinel。 +- 发送给 harness 的消息包含当前 LangBot `run_id` 和可访问资源摘要。 +- Harness 通过 gateway 调用 `langbot_history_page`、`langbot_retrieve_knowledge` 或 `langbot_call_tool` 时必须携带正确 `run_id`；错误 run id 被拒绝。 +- `external.session_id` 写入 host-owned state。 +- 外部 harness 错误、timeout、empty output 都转成受控 `run.failed`。 +- resume 到同一 external session 时，全局锁边界符合 PROTOCOL_V1 §13。 + +### 7.2 API 型外部 runner + +Dify、n8n、Coze、DashScope、Langflow、Tbox 等外部服务 runner 不作为每轮必跑项。只有在本次改动触及对应 runner 或凭据已经可用时执行 smoke。 + +通过条件： + +- runner 可选，配置可保存。 +- 请求成功，或外部服务错误被清晰返回。 +- 外部服务凭据缺失时标记 BLOCKED，并记录缺失项。 + +## 8. 权限与隔离补充 + +以下优先用单测 / targeted fixture 覆盖，不要求每次通过 UI 人工构造恶意 runner。 + +| 场景 | 推荐证据 | +| --- | --- | +| 未授权模型调用被拒绝 | `plugin/handler.py` run action 权限测试或目标单测。 | +| 未授权工具调用被拒绝 | `ctx.resources.tools` 与 host action 拒绝日志。 | +| 未授权知识库检索被拒绝 | `ctx.resources.knowledge_bases` 与 host action 拒绝日志。 | +| run_id 结束后复用被拒绝 | session registry 注销测试。 | +| 插件身份不匹配被拒绝 | `caller_plugin_identity` mismatch 测试。 | +| 绑定插件身份的 run_id 省略 caller identity 被拒绝 | `_validate_run_authorization(..., caller_plugin_identity=None)` 返回错误。 | +| 未注册 Runtime 连接伪造插件身份被剥离 | SDK runtime forwarding 测试：请求自带 `caller_plugin_identity` 时，未注册连接转发前必须 `pop`，已注册连接必须覆盖为真实插件身份。 | +| storage/state scope 越权被拒绝 | state/storage proxy 单测。 | +| steering claim 异常不杀 consumer loop | controller 单测：无效 runner / registry 异常只让当前消息回到普通 session 槽位路径，消息消费循环继续。 | +| steering queue 未消费有终态 | session registry / orchestrator 单测：队列有上限；run unregister 时未 pull 项写 `steering.dropped` 审计。 | + +如果这些单测失败，不能用 WebUI 正常回复替代。 + +## 9. 证据要求 + +每轮测试报告至少记录： + +- LangBot commit、SDK commit、相关 runner 插件 commit。 +- Pipeline UUID/name、runner id、关键 runner config 摘要。 +- WebUI 截图或 Playwright 操作记录。 +- 后端日志中对应 query id / run id 的关键行。 +- `langbot-skills` case/report 路径。 +- 外部 harness runner 的 context 文件、session id、working directory、CLI 错误摘要。 +- FAIL/BLOCKED 的复现步骤和归属仓库建议。 + +报告结论必须回答： + +- 是否建议继续进入下一阶段测试。 +- 是否存在主聊天路径阻塞。 +- 是否只是凭据 / 外部服务 / 本机 CLI 缺失导致 BLOCKED。 +- 是否需要进入 [SECURITY_HARDENING.md](./SECURITY_HARDENING.md) 的发布级验收。 + +## 10. 历史高价值记录 + +历史高价值记录与当前 runner 验收状态见 [STATUS.md](./STATUS.md)。本指南只保留可重复执行的测试步骤和证据要求。 + +### Event debug execution trace + +The Agent workbench uses `POST /api/v1/agents/{uuid}/debug/stream` (NDJSON, `runtime.operate`). Verify incremental text and provider-returned reasoning, tool call arguments/results in execution order, automatic scrolling, and preservation of partial output on errors. Thinking is only shown when returned by the runner. Disconnecting cancels the debug task. The existing `/debug` endpoint and MCP `debug_agent` return final text and a bounded `execution_events` snapshot; they are not live transports. + +Platform tools now use Host-owned mock adapters in synthetic debug runs. The model really invokes the authorized tool, validates its parameters, and receives a result; the Host does not call a live platform adapter. Event tools retain their frozen targets and platform tools retain explicit targets. Native, plugin and MCP tools continue to execute normally. A successful mock platform result is completion of that action in the debug run; runners must communicate that context to the model to avoid repeated attempts at real delivery. + +The optional `mock` payload field (also editable under **Mock 场景（JSON）**) supports: + +```json +{ + "errors": {"event_reply": "Simulated permission denied"}, + "results": {"event_get_actor": {"id": "user-42", "nickname": "Fixture User"}}, + "unsupported_apis": ["delete_message"] +} +``` + +`errors` and `results` cannot both override the same tool. Unknown tools/APIs and malformed options are rejected before model execution. Default read fixtures follow SDK `User`, `UserGroup`, `UserGroupMember` and `MessageReceivedEvent` structures; list reads return a synthetic fixture list (override with `[]` to test emptiness). `unsupported_apis` participates in the normal capability intersection, so unsupported tools are not offered to the model. Event payloads should include the intended group/member/request IDs; the built-in presets supply examples. + +Verify a welcome event shows `event_reply`, the expected text and frozen target, and **模拟执行成功 · Mock**. Plain text alone must not count as a successful platform action. A simulated failure must show **模拟执行失败 · Mock**. Also verify **停止调试** retains the partial trace and allows the next run, and that entering a new draft while streaming does not erase it on completion. + +Full event/Mock regression evidence: [2026-09-05 follow-up](./EVENT_DEBUG_FULL_QA_2026-09-05.md). diff --git a/docs/agent-runner-pluginization/EVENT_BASED_AGENT.md b/docs/agent-runner-pluginization/EVENT_BASED_AGENT.md new file mode 100644 index 000000000..269ada788 --- /dev/null +++ b/docs/agent-runner-pluginization/EVENT_BASED_AGENT.md @@ -0,0 +1,96 @@ +# Event Based Agent 接入设计 + +> 更新：2026-09-05。EBA 平台事件、Bot 路由和独立 Agent 已集成到 `dev/4.11.x`。通用事件订阅、通知与定时自动化仍是后续扩展。 +> +> 数据结构唯一定义在 [PROTOCOL_V1.md](./PROTOCOL_V1.md)（runner 可见）与 [HOST_SDK_INFRASTRUCTURE.md](./HOST_SDK_INFRASTRUCTURE.md)（Host 内部模型）；本文只讲 EBA 语义，不重抄 schema。 +> 与当前 runner 外化分支、后续 Agent Platform / Runtime Control Plane 的边界见 [EXTENSION_SCOPE_MATRIX.md](./EXTENSION_SCOPE_MATRIX.md)。 + +本文描述当前事件如何进入 LangBot、如何在平级的 Pipeline / Agent 之间路由，以及 Agent 如何复用插件化 Runner。路由逻辑由 `pkg/platform/botmgr.py::RuntimeBot` 承担；文中的 EventRouter 表示职责，不代表独立进程或同名类。 + +## 1. 设计目标 + +- 消息、撤回、入群、好友申请、定时任务、API 调用都能抽象为 host event。 +- EventRouter 可以根据 event type、bot、workspace、conversation、actor、subject 选择一个 Pipeline 或 Agent 处理器。 +- Pipeline 目标执行完整消息 Stage 链；Agent 目标通过统一 orchestrator 调用 Runner。 +- 非消息事件不伪造成用户文本消息。 +- 平台动作通过已授权的语义工具执行；结构化交互通过 `action.requested` 中的 `interaction.requested` 白名单执行。 + +## 2. 事件不是消息 + +`message.received` 只是事件的一种。协议不应假设：一定有用户文本、一定有 conversation history、一定要返回一条聊天消息、actor 一定等于 sender、subject 一定等于当前消息。 + +| event_type | actor | subject | input | +| --- | --- | --- | --- | +| `message.received` | 发消息的人 | 当前消息 | 文本、图片、文件等 | +| `message.deleted` | 撤回操作者，未知时为系统 | 被删除消息 | 通常为空 | +| `group.member_joined` | 新成员或邀请人 | 群/成员关系 | 通常为空 | +| `friend.request_received` | 申请人 | 好友申请 | 验证消息或申请理由 | +| `schedule.triggered` | 系统 | 定时任务 | 任务 payload | +| `api.invoked` | API caller | API request | request payload | + +## 3. 稳定事件名 + +当前平台事件名示例（定时任务与 API 事件示例仅表示未来入口；实际能力以 SDK 实体和适配器声明为准）： + +- `message.received` +- `message.deleted` +- `group.member_joined` +- `friend.request_received` + +平台原始事件名只能进入 `ctx.event.source_event_type` / `raw_ref`，不能成为 `ctx.event.event_type` 的公共契约。 + +## 4. Event Envelope 与 Binding + +- 入口事件用 `AgentEventEnvelope`（HOST_SDK §4.1）承载；顶层字段使用 LangBot 稳定协议名，平台原始事件名和原始 payload 放 `metadata` / `raw_ref`。 +- EBA 持久路由通过 `event_pattern`、`filters`、`target_type` 和 `target_uuid` 选择处理器。只有 `target_type=agent`，或 Pipeline AI Stage 需要调用 runner 时，才进一步解析 `AgentBinding`（HOST_SDK §4.2）。 + +EBA 每个事件只选择一个有效处理器；Runner 调用的基数、Agent 复用和 fan-out 边界以 PROTOCOL_V1 §13 为准。 + +路由 scope 示例：workspace 全局、bot 级、platform channel 级、conversation / group / thread 级、user / actor 级。Pipeline 是 `message.*` 场景的一等处理器，适合需要预处理、AI、后处理、扩展和输出控制的消息链路；Agent 是 runner 驱动的一等处理器，可处理其声明支持的消息与非消息事件。二者都不会被转换成对方。 + +Event Source 可包括：`platform_adapter`（飞书、QQ、微信、Telegram 等）、`webui`、`http_api`、`scheduler`、`system`。EventRouter 不应写死平台 adapter 的类名。 + +## 5. EventRouter 调用链 + +```text +Platform Adapter canonical event + -> RuntimeBot record adapter event + -> Plugin EventListener observer broadcast + -> RuntimeBot match saved event_bindings and resolve one Processor target + -> target_type=pipeline: MessageAggregator -> QueryPool -> Pipeline stages + -> target_type=agent: resolve AgentBinding -> AgentRunOrchestrator + -> RunnerContextBuilder -> PluginRuntimeConnector.run_runner() + -> RunnerResult stream + -> Host result delivery / authorized platform tool +``` + +约束：Pipeline 和 Agent 是 EventRouter 的平级目标；Pipeline 仅接受消息事件，Agent 受其事件能力声明约束。任何 Runner 调用都必须复用现有 orchestrator，不能为 EBA 单独实现另一套 plugin runner 协议；非消息事件不能绕过 resource authorization；delivery 和 platform action 走统一权限模型；外部 harness runner 也通过同一套 envelope/binding/context/result 协议接入。observer / fan-out / parallel arbitration 的额外语义仍按 PROTOCOL_V1 §13 处理。 + +## 6. 平台动作执行 + +平台动作走统一工具入口，详见 [PLATFORM_ACTION_TOOLS.md](./PLATFORM_ACTION_TOOLS.md)： + +- `event_*` 的目标由 Host 从当前事件冻结；Agent 只填写动作参数。 +- `platform_*` 允许填写目标，必须在 `allowed_platform_tools` 中显式选择。 +- 最终资源与 Runner 权限、适配器能力及事件目标求交，执行时再校验运行身份和当前机器人。 +- SDK/Python `call_tool` 和 scoped MCP gateway 使用同一 Host 授权；原始 `call_platform_api` 不作为 Agent 工具开放。 +- `action.requested` 只执行白名单 `interaction.requested`，用于持久化交互和回调恢复；其它 action 仍是 telemetry，不能用于执行好友审核等平台动作。 + +事件可能没有默认 reply target；Host 不为缺少目标的事件猜测投递对象。Runner 只能使用当前授权允许的工具和投递能力（`DeliveryContext` 见 PROTOCOL_V1 §5.7）。 +当前 Host 会把 adapter 声明的通用 API 投影到 +`DeliveryContext.platform_capabilities.supported_apis`，并据此设置 +`supports_edit` / `supports_reaction`。该投影只供 runner 选择输出形态，不构成 +平台动作授权；合成测试 adapter 会移除副作用能力并抑制实际出站调用。 + +## 7. 与 Context 协议的关系 + +EBA 事件进入 Runner 时仍遵循 [AGENT_CONTEXT_PROTOCOL.md](./AGENT_CONTEXT_PROTOCOL.md)：inline 当前事件、大 payload 用 raw/staged file ref、不默认 inline 完整 history、agent 按需通过 API 拉取、Host 保留 EventLog 和权限 guardrail。非消息事件可以被投影进 Transcript，但不能强制伪装为 user message；Runner 根据 event type 自己决定是否纳入模型上下文。 + +## 8. 当前集成状态 + +当前分支已完成 EventRouter、Pipeline / Agent 平级处理器路由、Bot +`event_bindings` 持久化与 WebUI、AgentBinding 投影、路由 dry-run、合成测试事件、 +运行状态和真实 OneBot 非消息事件到 Agent 的闭环。Pipeline 消息链和独立 Agent +均复用同一个 Runner orchestrator / context / result 协议。 + +平台动作授权和结构化交互已实现，但真实平台/provider 验收不等同于单测通过。SDK 的 `platform_tools` 分类发现于 2026-09-05 检视时仍是未提交工作区改动。剩余发布工作和历史验证边界见 [STATUS.md](./STATUS.md)。通用订阅、Scheduler、Workflow 和多 Agent 串并联仍未作为产品交付。 diff --git a/docs/agent-runner-pluginization/EVENT_DEBUG_E2E_2026-09-05.md b/docs/agent-runner-pluginization/EVENT_DEBUG_E2E_2026-09-05.md new file mode 100644 index 000000000..6e9f5b5d5 --- /dev/null +++ b/docs/agent-runner-pluginization/EVENT_DEBUG_E2E_2026-09-05.md @@ -0,0 +1,74 @@ +# Event debug end-to-end regression — 2026-09-05 + +Environment: Windows, Vite :3000, Core :5300, standalone Plugin Runtime :5400/:5401, standalone Box :5410, Docker backend. Tests use browser controls in Edge and an isolated unbound Agent `E2E Debug 0905` (`49e52ae0-6a99-46ce-975e-70164b2786ec`). Local plugin maintenance uses the authenticated localhost management API because Edge file upload is disabled. + +## Coverage inventory + +- Message input: empty input, plain text, multi-turn context, draft edits during streaming. +- Structured events: group join/leave/ban, friend request/add, feedback, bot state, message edit/delete/reaction, platform and custom events. +- Payload validation: invalid JSON, non-object JSON, Unicode and nested fields. +- Execution trace: intermediate/final text, returned thinking, tool arguments/results, tool failure, no-output failure, completion without duplication. +- Lifecycle: save-and-run, provider failure and recovery, long output/autoscroll, switch Agent during run, reconnect. +- Visual checks: scroll containment, long tool JSON, error card, narrow viewport. + +## Findings and fixes + +1. Synthetic debug Query omitted public Workspace fields. Native tools failed during skill mount lookup. Project trusted ExecutionContext fields onto synthetic queries; real Docker exec returned `E2E_TOOL_OK` after repair. +2. LocalAgent context assembly omitted structured event data. Add a bounded user-role event facts message before current input, preserving ordinary empty-data message behavior. Browser join/leave/friend/feedback returned unique JSON probes after installing the patched local plugin. +3. A model failure before visible output left an empty Agent card. Hide output entries with no visible execution steps. +4. A completed message request cleared a newly edited draft. Clear only the submitted input value; the browser retained `NEXT_DRAFT_SHOULD_REMAIN`. +5. Tool transport completion was shown as success even with `result.ok=false`. Treat explicit failed tool results as failure; real exec exit 7/stderr is preserved. +6. Native file tools resolved a Core host path before selecting remote Box execution. Choose Box first for remote/no-host-root deployments, validate virtual path boundaries, and cover all five file tools. +7. This local legacy config had no `box.local.host_root`, so Docker had no writable `/workspace` mount. Add an explicit local development root and keep the read-only root filesystem enabled. + +8. LocalAgent repeats previous turns in cumulative chunks after tools. Strip the already displayed prefix for those chunks; real thought text now appears once across write/read. +9. Remote grep generated Python `include = null` when no filter was supplied. Serialize it as a Python literal and test optional/quoted values. + +## External configuration observations + +`gpt-4.1-mini` under LangBot Models returned no available channel; NewAPI rockchin returned invalid token. These are reported by the trace; tests continued with working `claude-opus-4-8` and `deepseek-v4-flash` models. The test Agent now uses the latter. No credentials were changed. + +## Results + +### Browser outcomes + +All 15 preset event types and one named custom event were executed through the UI. These exercise synthetic debug event dispatch, not delivery from real messaging platforms. + +| Events | Observed outcome | +| --- | --- | +| `message.received` | Plain text marker, multi-turn tool execution and recovery succeeded. | +| `message.edited`, `message.deleted`, `message.reaction` | Returned `EDIT_10`, `DELETE_11`, `REACTION_12`. | +| `group.member_joined`, `group.member_left`, `group.member_banned` | Returned Unicode member data/`JOIN_42`, `LEFT_7`, `BANNED_20`. | +| `friend.request_received`, `friend.added` | Returned `FRIEND_8`, `ADDED_19`. | +| `bot.muted`, `bot.unmuted`, `bot.invited_to_group`, `bot.removed_from_group` | Returned `MUTED_13`, `UNMUTED_16`, `INVITED_17`, `REMOVED_18`. | +| `feedback.received` | Returned `FEEDBACK_9` and numeric rating from JSON. | +| `platform.specific`, `custom.e2e_probe` | Read nested JSON; returned `PLATFORM_14` and custom event type/`CUSTOM_15`. | + +- Empty message, malformed JSON and non-object JSON did not start execution. Unicode, nested arrays/objects and an empty event object were exercised. Empty custom event name validation was not conclusively verified. +- Real Docker `exec`, `write`, `read`, `edit`, `glob`, and `grep` all succeeded after fixes. Files were limited to `/workspace/e2e-debug-0905`. Final grep without `include` returned `probe.txt`, line 1, `E2E_EDIT_OK`, total 1. +- Nonzero exec exit preserved exit code 7/stderr and displayed failure. `sleep 3` with `timeout_sec=1` returned `timed_out`, `ok=false`, about 1037 ms, and displayed failure with the actual timeout message. +- Provider-returned thinking, intermediate text, tool arguments/results and final text appeared in order. A repeated prior thought appeared once after the cumulative-prefix fix. Models that do not return thinking are not expected to display it. +- Two browser tabs ran independent debug sessions without mixed transcripts. While `exec sleep 30` showed running, switching Agents cleared the old transcript/running state. Returning and sending a new request produced `AFTER_CANCEL_OK`. This verifies UI cancellation/recovery; termination timing of the already launched container command was not independently measured. +- Core restart and standalone Plugin Runtime reconnection recovered successfully. The development services remain running. +- At the 1280 x 800 test viewport, the final transcript measured height 315, scroll height 5203, distance from bottom 0, and client/scroll widths both 333: automatic scrolling and horizontal containment passed. Screenshot inspected; viewport override reset afterward. + +### Automated verification + +- Core: 130 passed, 24 skipped across orchestrator, execution context, debug controller/service, native tools and skill tools. Skips are POSIX secure-host-filesystem cases unavailable on Windows; they still require Linux verification. Remote Box routing is covered on both capability branches. +- LocalAgent full suite: 185 passed with `PYTHONUTF8=1`. +- Frontend trace reducer: 5 passed; TypeScript `tsc --noEmit` passed. +- Ruff checks on changed Python implementation/tests and Git whitespace checks passed. Existing Pydantic deprecation warnings remain. + +The isolated unbound Agent `E2E Debug 0905` is retained for reproduction. No real platform messages were sent. This run does not claim coverage of every provider, model fallback policy, Linux host file operations, or live platform adapter ingress. + +## Follow-up: mock platform actions + +The original run did not verify platform action tools: the synthetic envelope had no supported platform APIs or reply target. This was a coverage gap. Debug now supplies mock adapter capabilities and a synthetic reply target while retaining tool selection, event compatibility and parameter validation. Event targets remain frozen by the Host. Only platform operations are simulated; native tools still execute normally. + +Verified in the browser on the user's `localagent test` Agent (`0dc5d7d3-07b2-4c4c-bb2b-a4f8e40e3a76`) without changing its welcome system prompt: `group.member_joined` caused a real `event_reply` tool call with `text: Hello，Debug User`. Its mock result contained `api: send_message`, `target_type: group`, `target_id: debug-group`, `mock: true`, and `delivery: simulated`. The UI labels this as mock execution rather than text output or real delivery, and shows the tool count after completion. + +Mock platform responses are deterministic fixtures, not evidence of real adapter support or delivery. Read operations return synthetic information; list operations return empty fixture lists. Service/platform-tool regressions: 44 passed, including mock reply, explicit-target send, identity/group lookup, request rejection, validation and assertions that the real bot manager is never accessed. TypeScript and Ruff checks passed. + +The same browser run subsequently exposed repeated `event_reply` calls after successful mock results, and one invalid `event_get_actor` call with an unexpected `_call` parameter. The invalid call was rejected and shown as failure. The run was cancelled by switching Agents. Therefore this verifies the mock call/result path, but does not establish that this configured model completes the welcome workflow exactly once. At this checkpoint the cause had not been determined. + +For the subsequent investigation, updated SDK-shaped Mock fixtures (including non-empty default lists), cancellation fix, complete event matrix, Linux verification and remaining provider/stream-pressure limits, see [the full follow-up report](./EVENT_DEBUG_FULL_QA_2026-09-05.md). Its results supersede the checkpoint counts and fixture description above. diff --git a/docs/agent-runner-pluginization/EVENT_DEBUG_FULL_QA_2026-09-05.md b/docs/agent-runner-pluginization/EVENT_DEBUG_FULL_QA_2026-09-05.md new file mode 100644 index 000000000..1f85e05ea --- /dev/null +++ b/docs/agent-runner-pluginization/EVENT_DEBUG_FULL_QA_2026-09-05.md @@ -0,0 +1,99 @@ +# Event debug full QA and release verification — 2026-09-05 + +## Scope and inventory + +This follow-up investigates repeated welcome actions, verifies the debug surface with standalone Box/Plugin Runtime, and checks the changes before committing/pushing Core, SDK, and LocalAgent. Real messaging-platform effects must remain isolated from debug runs. + +| Area | Required checks | Evidence | +| --- | --- | --- | +| Root cause | Follow-up messages/tool IDs preserved; direct model replay before/after mock guidance; original welcome prompt | Sanitized local traces, browser tool count | +| Events | All 15 built-in event types plus custom; nested/Unicode data and identity/target mapping | Browser results and parameterized service tests | +| Platform actions | Every catalog tool; frozen and explicit targets; permission/event/API intersection | Parameterized host boundary tests, browser reply/query/moderation/request cases | +| Mock scenarios | Success, error, query fixture, unsupported API; malformed options rejected | Browser and service tests | +| Input | Empty message/name, invalid/non-object JSON, unsupported event, actor/subject validation | Browser and request tests | +| Trace | Thinking/text/tool ordering, final snapshot deduplication, status/error correctness, no-call summary | Browser and reducer tests | +| Lifecycle | Cancel retains partial record; rerun; tabs/Agent switching; provider failure/fallback; reconnect | Browser and transport/runner tests | +| Native tools | Exec/file operations, nonzero exit, timeout and path escape | Browser and native-tool tests | +| Layout | Long output/parameters, automatic bottom scroll, narrow view, KB card spacing | DOM geometry and screenshots | +| Release | Python/SDK suites, frontend build/lint, whitespace checks, change review, commit and push | Commands, counts and remote SHAs | + +## Investigation + +The real follow-up request contained the assistant tool call and successful tool result with matching IDs. No loss occurred at the LocalAgent → SDK → Host model boundary. Direct replay to the configured provider, bypassing the runner, reproduced `event_reply` after success. Adding explicit mock completion semantics stopped further calls in two direct replays. The previous LocalAgent system context omitted the debug/mock semantics although the tool result said no real platform operation occurred. + +The configured upstream `claude-opus-4-8` also returned an unsolicited CLI identity statement in direct replay. This originates in the provider response, not the debug renderer. It is separate from preserving tool results; it must not be presented as normal LangBot-generated status. + +## Final results + +### Browser event matrix + +Executed through the real Edge WebUI, against Core `:5300`, Vite `:3000`, standalone Plugin Runtime `:5400/:5401`, and Docker-backed standalone Box `:5410`. The unbound `E2E Debug 0905` Agent uses `deepseek-v4-flash`; the user's original Agent and welcome prompt remain unchanged and use `claude-opus-4-8`. + +| Event | Observed result in this pass | +| --- | --- | +| `bot.invited_to_group` | Actor/group queries and `event_respond_group_invite(approve=true)` succeeded; request ID frozen to `debug-group-request`. | +| `bot.muted`, `bot.removed_from_group` | One `event_get_group` call each, correct group target. | +| `bot.unmuted` | One actor query; correct user target. | +| `feedback.received` | One mock reply, `FEEDBACK_MOCK_OK`, person target. | +| `friend.added` | One mock reply, `FRIEND_ADDED_OK`, person target. | +| `friend.request_received` | Acceptance and separate explicit rejection exercised; rejection preserved `approve=false`, Unicode remark and frozen request ID. | +| `group.member_banned` | One member lookup, SDK-compatible nested `user` and `group_id`. | +| `group.member_left` | One group lookup, SDK-compatible `id`/`name`. | +| `group.member_joined` | Welcome reply, configured failure, query fixture, unsupported API, and mute/unmute/kick scenarios. Original-model repeat caveat below. | +| `message.deleted` | One actor lookup, SDK-compatible `id`/`nickname`. | +| `message.edited` | One simulated deletion; frozen group, chat and message IDs. | +| `message.reaction` | One reply, `REACTION_OK`, and correct `👍` event data. | +| `message.received` | Real six-tool file/exec chain, plain-text recovery, cancellation, draft retention, nonzero exit and timeout. | +| `platform.specific` | Preserved nested arrays, booleans, null and `测试🙂`; no tools called. | +| `custom.event`, `custom.e2e` | Empty object and named custom event with `CUSTOM_中文🙂`; no tools called. | + +The first invitation/default friend-request runs submitted the preset text; they are recorded as default-behavior cases, not as explicit one-call/rejection tests. The rejection was separately rerun with the actual submitted instruction verified. Browser automation reads the controlled input after filling before treating a scenario as submitted. + +### Mock and execution behavior + +- Platform Mock runs the actual model/tool selection, authorization, parameter validation and frozen-target resolution; only the adapter boundary is simulated. Tests assert that mock execution never accesses the real bot manager. Native tools retain real Box behavior. +- The platform catalog contains 24 tools. Parameterized tests exercise success and configured failure for all 24, plus permission/event/API filtering, explicit/frozen targets, SDK fixture shapes and invalid options. This is full catalog contract coverage, not 24 separate browser clicks or proof of live adapter support. +- `errors.event_reply = "E2E permission denied"` produced one failed call and the **模拟执行失败 · Mock** status. The model reported the actual error without retrying in this scenario. +- `results.event_get_actor` returned the configured `fixture-user-77` / `测试用户🙂`; the model used these values instead of the original event identity. Default query results now serialize SDK platform models; default lists contain one synthetic entry and can be overridden with `[]`. +- Disabling `send_message` removed `event_reply` from the model's available tools. It returned `REPLY_UNAVAILABLE`, with the explicit no-tool-call summary. Clearing Mock options restored action availability; the three moderation calls succeeded with the correct group/member IDs. +- Mock `[]`, unknown tool names, whitespace-only message/custom name, malformed event JSON and event JSON `[]` were rejected. Browser transcript counts did not increase for client-side validation failures. Backend tests additionally cover falsey/non-object actor, subject and data values, conflicting outcomes and invalid API names. +- Thinking returned by the provider, intermediate text, tool parameters, results and final text remain distinct and ordered. Missing thinking is not fabricated. Cumulative final snapshots and prior tool-turn prefixes are deduplicated without hiding actual repeated tool calls. + +### Lifecycle, native tools and layout + +- A browser cancellation regression exposed a stuck **停止调试** button: the `finally` block skipped resetting state for aborted requests. It now clears the matching controller and resets running state even after user cancellation. Retest retained partial thinking/tool arguments, marked the unfinished call **未返回结果**, showed the cancellation notice and restored **运行测试**. +- Two subsequent recovery requests encountered the bounded-stream error below. A later request in the same tab/session returned `42` successfully; further real tool runs also completed. Cancellation and recovery are therefore verified, but immediate model success after cancellation is not guaranteed. +- Editing `NEXT_DRAFT_FINAL_0905` while a `sleep 5` run was still awaiting its final response preserved the draft after completion. +- Real `write → read → edit → glob → grep → exec` succeeded. `final.txt` changed from `FINAL_WRITE` to `FINAL_EDIT`; grep without `include` found one match; exec returned `FINAL_EXEC` with exit code 0. Writes stayed under `/workspace/e2e-debug-0905`. +- `printf E2E_ERROR >&2; exit 7` preserved stderr and exit code 7 and displayed failure. `sleep 3` with `timeout_sec=1` returned `timed_out` in 1025 ms and displayed failure. Neither was retried. +- Separate tabs retained separate transcripts during concurrent runs. Agent switching reset the debug surface. File workspace sharing remains intentional; transcript isolation does not mean separate Box filesystems. +- The final transcript measured 674 px high with 16,630 px scroll height, distance from bottom 0, and equal client/scroll widths of 468 px. Visual inspection confirmed contained parameters/results and accessible input controls. The earlier pass also verified 1280 × 800; the final attempted viewport override did not change this tab, so that attempt is not counted as an additional narrow-screen result. Temporary overrides were reset. +- Knowledge-base card geometry: retrieval card bottom 1413 px, danger card top 1437 px, giving the expected **24 px** gap. Both adjacent card gaps measured 24 px. +- Core health check returned `ok`; Plugin Runtime was connected and all development service ports remained listening. Previous restart/reconnect evidence is in the earlier report. + +### Automated checks + +| Suite | Result | +| --- | --- | +| Core agent unit directory plus debug service/controller, native/skill tools and model conversion | 659 passed, 24 Windows-only skips | +| Final changed platform/debug service/controller checks | 133 passed | +| Linux Docker: native tools, skill tools and all platform tools | 150 passed, 0 skipped; includes the 24 POSIX cases skipped on Windows | +| SDK API suite plus runtime I/O handler | 411 passed | +| LocalAgent full suite (`PYTHONUTF8=1`) | 187 passed | +| Frontend unit suite | 69 passed | +| Skills CLI suite | 122 passed | +| Frontend TypeScript/Vite build and changed-file ESLint | Passed; existing large-bundle advisory remains | +| Changed Python Ruff; Git whitespace checks | Passed | +| Skills index generation, validate and index consistency | Passed | + +Counts overlap where a final focused/Linux run repeats an earlier suite; do not add them as unique tests. Existing Pydantic deprecation warnings remain. This is the relevant subsystem regression set, not the entire repository/integration matrix. Provider fallback has deterministic LocalAgent tests; live fallback after a partially emitted response is deliberately unsupported and was not claimed as a successful browser fallback. + +The full regression also found Windows portability problems in the skills tooling: LF-only frontmatter parsing, `/cases/` detection and native path separators in generated references. These are fixed and covered by the CLI suite. Two frontend source-contract expectations were stale after the existing three-tab workbench/layout changes; they were updated to the actual product structure. + +### Remaining limits and reproduction + +1. **Original provider is still nondeterministic.** With the latest LocalAgent guidance, one rerun made exactly one successful `event_reply`; another made repeated successful replies before stopping. The trace faithfully records each call. Earlier invalid `_call` arguments were rejected. Direct provider replay proved that repeated calls can originate upstream even with a valid matching tool result. Prompt guidance reduces ambiguity but does not establish exactly-once actions. No silent deduplication or fabricated success was added. +2. **Bounded stream pressure can fail long responses.** Two recovery runs emitted thinking and then `Streaming action consumer is too slow; response buffer full`; later recovery and six-tool runs succeeded. The error originates in the SDK's existing 128-frame response queue and is preserved in the UI. It is not a missing-tool-result or 30-second HTTP timeout. Full end-to-end flow control under sustained overload remains open; queue limits were not removed to conceal the failure. +3. Mock results verify debug behavior, not real platform credentials, permissions, delivery, or every adapter/event Cartesian combination. Already launched external tool termination timing was not independently measured. A development-branch push does not mean these remaining production release gates passed. + +For reproduction, use the retained test Agent, the Mock examples in [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md), and the original welcome Agent. Local diagnostic traces and fixture artifacts are excluded from Git; they contain model responses and local runtime state. This report contains outcomes rather than raw provider reasoning or secrets. diff --git a/docs/agent-runner-pluginization/EXTENSION_SCOPE_MATRIX.md b/docs/agent-runner-pluginization/EXTENSION_SCOPE_MATRIX.md new file mode 100644 index 000000000..d3451627e --- /dev/null +++ b/docs/agent-runner-pluginization/EXTENSION_SCOPE_MATRIX.md @@ -0,0 +1,41 @@ +# Runner 与产品扩展边界 + +更新：2026-09-05，适用于 `dev/4.11.x`。EBA、独立 Agent、Bot 事件绑定和处理器 UI 已与 Runner 插件化合并。当前状态和测试证据以 [STATUS.md](./STATUS.md) 为准；runner 可见 schema 与调度基数以 [PROTOCOL_V1.md](./PROTOCOL_V1.md) 为准。 + +## 当前职责 + +| 层 | 已实现职责 | 边界 | +| --- | --- | --- | +| LangBot 产品层 | 独立 Agent CRUD、Pipeline、处理器工作台、Bot 事件绑定、Runner 安装与调试 | Agent 与 Pipeline 各自持久化；聚合列表不转换实体 | +| 平台层 | 适配器事件转换、能力声明、observer 广播、路由匹配、平台 API 与回复 | 路由逻辑在 RuntimeBot；不为每个入口重建 runner 协议 | +| Host Agent 底座 | envelope/binding 投影、统一编排、资源授权、run session、EventLog/Transcript/State、run/result ledger | SDK 不持有 Host 私有 Query 或数据库 | +| SDK / Plugin Runtime | typed contract、Runner 组件和脚手架、proxy、MCP bridge、结果流转发、installation worker 管理 | 具体 Agent 执行策略由 Runner 插件承担 | +| Box Runtime | 沙盒会话、文件、托管进程、Skill、资源限制与作用域 | 不等于外部 harness 的通用托管承诺；存储统计不等于硬配额 | + +## 已有能力与后续扩展 + +| 能力 | 当前状态 | 后续工作与接入点 | +| --- | --- | --- | +| Agent / binding | Agent 表、API、配置 UI、Bot event_bindings 已存在；AgentBinding 是运行投影 | 新产品模型复用现有投影，不把 Pipeline 持久化成 Agent | +| 事件路由 | observer 广播后按 pattern/filter/priority 选择一个 Pipeline、Agent 或 discard | 通用订阅、通知和其他事件源仍需单独设计 | +| 平台动作 | event_* 冻结目标，platform_* 显式授权；通过 Host 工具调用 | 新动作先定义语义、schema 和授权，不开放任意原始 action | +| 结构化交互 | interaction.requested 白名单、持久回调关联、TTL/作用域/幂等、原处理器恢复 | 补真实 provider/platform 验收；其它 action.requested 仍仅 telemetry | +| Run / runtime | 持久 AgentRun/AgentRunEvent、取消/结果/终态、heartbeat/claim/reconcile 原语 | 业务队列、任务生产、唤醒、跨 Host 执行和运维产品面 | +| Plugin worker | 独立安装进程、依赖环境、supervisor、退避及重启协调器 | 最终部署故障注入、出站网络策略及硬存储配额 | +| External harness | 通过 Runner 消费协议、按 run 访问 Host 资源 | 通用 daemon supervisor、登录态诊断、分布式调度；不要与 Plugin worker 混淆 | +| History / state / storage | Host 事实源、按需读取、state/checkpoint、sandbox 文件能力 | EventLog/Transcript 的定时 retention 接入和完整文件生命周期 | +| Scheduler / Automation | 仅保留可扩展的事件入口 | 用户定时任务必须走事件、授权和运行记录链路，不直调插件绕过 Host | +| Workflow / 多 Agent | 尚无完整产品实现 | 先定义串并联、失败恢复、投递与状态冲突语义 | +| Solution | 尚无导出/导入实现 | 处理器、路由模板、依赖、变量与文档；不导出凭据或已安装 UUID | +| 长期 memory 产品 | 提供 history/state/storage 基础 | 由 Runner 或后续产品定义召回策略，不把全量 memory 默认塞入 context | +| Cloud | 作用域与运行时隔离底座已合入；OSS 为单 Workspace 多成员 | 生产激活独立通过网络、硬配额、事务代次切换和部署验收 | + +平台动作详见 [PLATFORM_ACTION_TOOLS.md](./PLATFORM_ACTION_TOOLS.md)，控制面规划详见 [RUNTIME_CONTROL_PLANE_V2.md](./RUNTIME_CONTROL_PLANE_V2.md)，Cloud 门禁见 [剩余验证清单](../multi-tenant/cloud-v2-pending-verification.md)。 + +## 扩展规则 + +- 新入口构造 Host event 和有效 binding，继续调用统一 orchestrator；Pipeline AI Stage 使用 QueryEntryAdapter。 +- Host 保持 run/result、授权、事件、状态和历史的事实源；插件负责自己的执行策略及 provider 私有 continuation。 +- 新增业务表、调度或 UI 不要求修改 runner 可见协议；需要协议扩展时先更新 canonical spec，再同步 SDK、Runtime、模板与测试。 +- fan-out、并行仲裁和自动重试必须明确副作用、幂等、状态冲突和审计语义，不能通过多个隐式回复者实现。 +- 外部 harness 自带 shell、文件系统和网络权限由部署环境负责；manifest permissions 约束的是 LangBot 持有的资源。 diff --git a/docs/agent-runner-pluginization/HOST_SDK_INFRASTRUCTURE.md b/docs/agent-runner-pluginization/HOST_SDK_INFRASTRUCTURE.md new file mode 100644 index 000000000..daa303932 --- /dev/null +++ b/docs/agent-runner-pluginization/HOST_SDK_INFRASTRUCTURE.md @@ -0,0 +1,266 @@ +# LangBot Host 与 SDK 基础设施设计 + +本文档描述 LangBot 作为 agent host 的内部能力与分层架构，以及 Host 内部模型。 + +- SDK ↔ Host 的协议数据结构（`RunnerContext`、`RunnerManifest`、`RunnerResult`、`RunnerAPIProxy` 等）的**唯一定义在** [PROTOCOL_V1.md](./PROTOCOL_V1.md)；本文只引用，不重抄。 +- 测试执行入口和 smoke 记录见 [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md)；安全发布门槛见 [SECURITY_HARDENING.md](./SECURITY_HARDENING.md)。 +- 本文定义的 Host 内部模型（`AgentEventEnvelope`、`AgentBinding`、`RunnerDescriptor`）不属于 SDK 协议字段。 + +## 1. 目标 + +LangBot 要转为 agent host，而不是内置 runner 容器： + +- 接收 IM、WebUI、API 和当前 RuntimeBot 事件路由产生的事件。 +- 接收 EBA 选中的 Agent 处理器，并根据事件、bot、workspace、scope 解析 Runner binding。 +- 发现、校验和调用插件提供的 Runner。 +- 为每次 run 提供受限资源、状态、存储、上下文引用和生命周期控制。 +- 接收 Runner 返回的事件流，投递到 IM、WebUI 或其他 output surface。 + +## 2. 非目标 + +- 不定义 Pipeline 的 Stage 编排语义；Pipeline 是 EBA 的同级处理器，其 AI Stage 只在需要 runner 时接入本 Host 边界。 +- 不要求所有 Runner 依赖 LangBot 的上下文管理。 +- 不要求官方 local-agent 的旧行为反向塑造 host 协议。 +- 不在 host 中实现通用 agentic prompt assembler。 +- 不强制 runner 使用 LangBot state / storage；只提供可选、受控的寄宿能力。 +- 不在 runner 底座重复实现平台路由；当前分支的 RuntimeBot 已承担 observer 广播和 Bot event_bindings 匹配，runner 层消费其 envelope/binding 投影。 + +## 3. 分层架构 + +```text +IM platform event + | + v +RuntimeBot event normalization / observer broadcast + | + v +RuntimeBot event_bindings -> one Processor target + |-- target_type=pipeline -> Pipeline Stage chain + | + `-- target_type=agent -> AgentBindingResolver + | + v + AgentRunOrchestrator + |-- RunnerRegistry + |-- AgentResourceBuilder + |-- AgentContextBuilder + |-- AgentRunSessionRegistry + |-- PersistentStateStore / EventLogStore / TranscriptStore + |-- Sandbox / workspace file tools + v + Plugin Runtime / Runner + | + v + RunnerResult stream + | + v +Delivery / Renderer / Platform API +``` + +Pipeline 与 Agent 是事件路由的平级目标。Agent 直接解析 `AgentBinding`；Pipeline 执行完整 Stage 链，仅在 AI Stage 调用 runner 时通过 QueryEntryAdapter 构造一次性运行投影，不会持久化成 Agent。WebUI/API 调试按服务入口构造调试事件或 Query，再使用同一编排器；不要求绕行真实 Bot 回调。调度基数、Agent 复用和 fan-out 边界以 [PROTOCOL_V1.md](./PROTOCOL_V1.md) §13 为准。 + +## 4. LangBot 侧能力 + +### 4.1 事件入口与路由（已集成） + +> 2026-09-05：平台转换、Bot event_bindings、独立 Agent 和配置 UI 已集成。源码入口为 `pkg/platform/botmgr.py`、`pkg/api/http/service/agent.py` 和 `pkg/agent/runner/`。通用订阅与定时自动化仍是后续能力。 + +各入口将消息或非消息输入投影为 `AgentEventEnvelope`。以下为概念字段摘要，精确字段及默认值以 `pkg/agent/runner/host_models.py` 为准： + +```python +class AgentEventEnvelope(BaseModel): + event_id: str + event_type: str + event_time: int | None + source: str + bot_id: str | None + workspace_id: str | None + conversation_id: str | None + thread_id: str | None + actor: ActorRef | None + subject: SubjectRef | None + input: AgentInput # 见 PROTOCOL_V1 §5.6 + delivery: DeliveryContext # 见 PROTOCOL_V1 §5.7 + raw_ref: RawEventRef | None + metadata: dict[str, Any] = {} +``` + +`AgentEventEnvelope` 是 Host 内部入口模型；投影给 runner 的是 `ctx.event`（PROTOCOL_V1 §5.4）。原始平台 payload 存为 raw event 或 staged file reference，不扩散到 runner 协议顶层。 + +**当前 adapter source**：`QueryEntryAdapter.query_to_event(query)` 从 Query 生成 `AgentEventEnvelope`。 + +### 4.2 AgentConfig 与 AgentBinding + +`AgentConfig` 是 Host 内部的一次 Runner 调用配置投影（不暴露给 SDK）。独立 Agent 从自己的持久配置生成它；Pipeline 只在 AI Stage 调用 runner 时，由 Query entry adapter 从该 Stage 的当前配置生成它。两种来源随后都由 BindingResolver 结合事件和 scope 解析为 `AgentBinding`。Pipeline 本身不是 `AgentConfig`，该调用投影也不会创建或更新持久 Agent。 + +```python +class AgentConfig(BaseModel): + agent_id: str | None = None + runner_id: str + runner_config: dict[str, Any] = {} + resource_policy: ResourcePolicy = ResourcePolicy() + state_policy: StatePolicy = StatePolicy() + delivery_policy: DeliveryPolicy = DeliveryPolicy() + event_types: list[str] = ["message.received"] + enabled: bool = True + metadata: dict[str, Any] = {} +``` + +`AgentBinding` 是"什么事件调用哪个 Runner、带什么 Agent 配置"的 Host 内部运行投影（不暴露给 SDK）。它是 EventRouter / 当前 QueryEntryAdapter 在一次运行前解析出的有效绑定。 + +```python +class AgentBinding(BaseModel): + binding_id: str + enabled: bool + scope: BindingScope + event_types: list[str] + filters: list[EventFilter] = [] # EBA 阶段使用，见 EVENT_BASED_AGENT + runner_id: str + runner_config: dict[str, Any] + resource_policy: ResourcePolicy + state_policy: StatePolicy + delivery_policy: DeliveryPolicy +``` + +BindingResolver 的基数、fan-out 和冲突处理约束见 PROTOCOL_V1 §13；本节只定义 Host 内部投影形态。 + +**当前 adapter source**：`QueryEntryAdapter.config_to_agent_config(query, runner_id)` +只在 Pipeline AI Stage 调用 runner 时，把 current config 临时投影为运行期 `AgentConfig`，再由 +`AgentBindingResolver.resolve_one(event, [agent_config])` 解析出唯一 +`AgentBinding`。该调用从 Pipeline 取得 AI runner config +→ runner_config、extension preference → resource_policy、output settings → +delivery_policy，但 Pipeline 仍执行并拥有完整 Stage/config 语义。该适配不会把 Pipeline 持久化为 Agent；独立 Agent 由用户自行新增和绑定。 + +### 4.3 RunnerRegistry + +Registry 收集 runner descriptor（来自插件 runtime、开发期本地插件）： + +```python +class RunnerDescriptor(BaseModel): + id: str + source: Literal["plugin"] + label: I18nObject + description: I18nObject | None = None + plugin_author: str + plugin_name: str + runner_name: str + capabilities: RunnerCapabilities # 见 PROTOCOL_V1 §4.3 + permissions: RunnerPermissions # 见 PROTOCOL_V1 §4.4 + config_schema: list[DynamicFormItemSchema] + plugin_version: str | None = None + raw_manifest: dict[str, Any] = {} +``` + +职责：调用 `plugin_connector.list_runners()` 拉取 runner、校验 typed `RunnerManifest`、输出 descriptor、缓存 discovery 结果并提供 `refresh()`。单个插件 manifest 失败只记 warning，不影响其它 runner。`plugin:author/name/runner` 是稳定 id 格式；插件实例边界见 PROTOCOL_V1 §13。 + +Host 内置 runner / adapter 不能作为 `RunnerDescriptor.source` 绕过插件 +runtime、`run_id`、`ctx.resources` 和 `RunnerAPIProxy` 权限链。若需要 +开发期调试 adapter，应放在 Host 内部测试入口，不进入可选 runner 列表。 + +刷新触发点：插件安装/卸载/升级/重启后；Pipeline metadata 请求时发现缓存为空；可选 TTL（优先保证正确性）。 + +### 4.4 AgentRunOrchestrator + +Orchestrator 是唯一运行入口： + +```text +run(event, binding) + -> resolve runner descriptor + -> build resources + -> build context + -> register run session + -> call plugin runtime + -> normalize result stream + -> update state + -> unregister run session +``` + +它负责：`run_id` 生成和生命周期、timeout/deadline/cancellation、插件异常隔离、result schema 校验和大小限制、`state.updated` 处理、delivery backpressure 和 telemetry。 + +典型 run 时序： + +```text +QueryEntryAdapter / EventRouter + -> AgentRunOrchestrator.run(event, binding) + -> RunnerRegistry.resolve(runner_id) + -> AgentResourceBuilder.freeze_snapshot(binding, event) + -> AgentRunSessionRegistry.register(run_id, runner_id, snapshot) + -> AgentContextBuilder.build(event, binding, snapshot) + -> PluginRuntimeConnector.run_runner(ctx) + -> RunnerAPIProxy action + -> validate active run session + caller identity + snapshot + -> Host API / Store + <- RunnerResult stream + -> apply state.updated to PersistentStateStore + -> write message.completed to Transcript + -> keep current-run files and large tool outputs in sandbox/workspace + -> render delivery or raise RunnerExecutionError + -> AgentRunSessionRegistry.unregister(run_id) +``` + +`run_from_query()` 保留为 Query entry adapter 入口，但内部转换成 event + binding 后走统一 `run()`。约束：`ChatMessageHandler` 不解析 `plugin:*`、不实例化 wrapper、不知道 runner 组件细节；`PipelineService` 从 registry 读取 metadata，不直接访问插件 runtime；跨请求持久化状态必须走授权 storage / 外部服务。 + +### 4.5 Resource Authorization + +LangBot 在每次 run 前生成 `ctx.resources`（PROTOCOL_V1 §6），来自 manifest permissions 与 binding policy 的交集： + +1. `descriptor.permissions` 声明 runner 需要的 LangBot 资源访问上限。 +2. binding / resource policy 允许的资源范围。 +3. Agent/runner config 中选择的模型、知识库、文件等资源。 +4. 当前 event / actor / bot / workspace 的实际权限。 +5. `ctx.context.available_apis` 暴露的 pull API 能力。 + +这次裁剪结果必须冻结为 run-scoped authorization snapshot，并由 +`AgentRunSessionRegistry` 按 `run_id` 保存。`ctx.resources` 是投影给 runner +看的同一份授权结果；运行期每个 proxy action 只依据该 snapshot 校验 active +run session、caller plugin identity、resource id、scope、payload size、rate +limit 和 deadline。Handler 不应重新执行授权裁剪，否则 build-time 与 runtime +授权逻辑会漂移。 + +SDK 侧本地校验只用于开发体验，host 侧 run authorization snapshot 才是安全边界。`spec.capabilities` 只帮助 Host 判断 runner 是否需要 tool / knowledge 等资源投影，不能替代 permissions 或 binding policy。skill 不由独立 capability 决定是否投影——它通过统一 tool 授权（`resource_policy.allowed_tool_names`）消费，`skill_authoring` 仅作为「一键授权这组 skill tool + sandbox」的便捷开关。 + +资源裁剪应通用，不写死 local-agent。selector 与资源的映射示例：`model-fallback-selector` → primary/fallback LLM、`llm-model-selector` → LLM、`rerank-model-selector` → rerank 模型、`knowledge-base-multi-selector` → 知识库；新增 selector 时在 resource builder 中统一扩展。 + +构造 `ctx.resources.tools` 时，Host 尽可能一次提供完整 schema（`ToolResource.parameters`），减少逐个 detail 查询。Runner 仍需兼容 `parameters=None` 并按需调用 `get_tool_detail`。平台动作以 `tool_type=platform` 使用同一资源面，授权与冻结目标见 [PLATFORM_ACTION_TOOLS.md](./PLATFORM_ACTION_TOOLS.md)。 + +执行/文件/skill/MCP 等能力的接入方向：先由 Host / sandbox 封装成普通 scoped tool，再通过 `ctx.resources.tools` 和 SDK runtime 转发进入 runner；runner 不应识别或硬编码执行环境 provider。外部 harness 的 native tools 不能直接访问 LangBot 资源。skill 的整个生命周期都走统一 tool：发现走 `list_skills` / `langbot_list_assets`，激活/注册走 `activate` / `register_skill`，包内操作走 native exec/read/write——runner 不需要独立的 skill 渲染或门控。 + +### 4.6 State / Storage + +LangBot 可提供 host-owned state 让 runner 寄宿状态（conversation / actor / subject / runner / binding / workspace state），但**不是强制**。Host 只需提供：授权开关、scope key、get/set/list/delete API（见 PROTOCOL_V1 §8）、持久化 backend、审计和清理策略。外部 agent runtime 可维护自己的 session 和 memory。进程内 state store 只能作为过渡实现，不能作为正式生产语义。 + +部分 host-owned state 由 Host 自身直接写：例如 `activate` tool 在 Host 侧执行时，把已激活 skill 写入 conversation scope 的 `host.activated_skills`。host 直接写与 runner `state.updated` 写到同一 key 时按 **last-write-wins** 合并，runner 可覆盖。 + +### 4.7 EventLog / Transcript / Sandbox Files（事实源） + +- `EventLog`: durable append-only，保存原始事件、系统事件、工具调用、投递结果、错误。 +- `Transcript`: 从 EventLog 投影出的对话视图，用于 UI、审计和按需历史读取。 +- `Sandbox / workspace files`: 当前 run 的上传文件、平台附件、工具大结果和临时产物。Host 负责 staging 与授权边界，runner 通过 read/write/exec 类工具按需访问。 + +三类数据与 working context 的边界、读取约束见 [AGENT_CONTEXT_PROTOCOL.md](./AGENT_CONTEXT_PROTOCOL.md)。Runner 可读取这些能力，但不被迫使用 LangBot 作为唯一记忆系统。 + +### 4.8 External harness resource projection + +Claude Code、Codex、Kimi Code 等外部 harness runner 可能不直接调用 LangBot 的 model/tool loop，而是把 LangBot 事件和授权资源句柄投影到自己的 harness 执行。Host 侧仍保持统一边界：Host 负责构造 event-first context、资源授权、state/storage、EventLog/Transcript、sandbox/workspace 文件边界和审计；Host 或 binding policy 决定哪些 MCP bridge、skill-backed tool、sandbox path、history/state 句柄可投影给 runner；runner plugin 把 scoped projection 转成目标 harness 可消费形式；所有 LangBot 资源访问必须经 SDK runtime / `RunnerAPIProxy` / SDK-owned MCP bridge 转发并接受 Host 校验；外部 harness 负责自己的 native session、tool loop、压缩、权限模式和 resume，但不能用 native tools 绕过 Host 授权。 + +投影的具体形态（context 文件、resource handles、LangBot MCP gateway、state pointers）见 AGENT_CONTEXT_PROTOCOL §4.5；当前 code-agent harness runner 形态见 OFFICIAL_RUNNER_PLUGINS §7。发布级隔离要求见 SECURITY_HARDENING。 + +## 5. SDK 侧协议 + +SDK 组件入口如下；所有数据结构定义见 PROTOCOL_V1。 + +```python +class Runner(BaseComponent): + __kind__ = "Runner" + + @classmethod + def get_config_schema(cls) -> list[dict]: ... + + async def run(self, ctx: RunnerContext) -> AsyncGenerator[RunnerResult, None]: ... + # ctx: PROTOCOL_V1 §5.2 ; RunnerResult: PROTOCOL_V1 §7 +``` + +- Manifest / capabilities / effective access：PROTOCOL_V1 §4。Capabilities 来自组件 manifest 的 `spec.capabilities`，不是 SDK 基类 classmethod。 +- `RunnerContext`：PROTOCOL_V1 §5.2。`messages` / `bootstrap` 不是协议字段。 +- `RunnerResult`：PROTOCOL_V1 §7。 +- `RunnerAPIProxy`：PROTOCOL_V1 §8，是 runner 访问 host 能力的唯一入口，所有请求带 `run_id`。 diff --git a/docs/agent-runner-pluginization/OFFICIAL_RUNNER_PLUGINS.md b/docs/agent-runner-pluginization/OFFICIAL_RUNNER_PLUGINS.md new file mode 100644 index 000000000..b067929b2 --- /dev/null +++ b/docs/agent-runner-pluginization/OFFICIAL_RUNNER_PLUGINS.md @@ -0,0 +1,138 @@ +# 官方 Runner 插件迁移计划 + +本文档描述内置 `RequestRunner` 迁出 LangBot 后，官方 runner 插件如何组织、迁移和验收。它是 [HOST_SDK_INFRASTRUCTURE.md](./HOST_SDK_INFRASTRUCTURE.md) 和 [AGENT_CONTEXT_PROTOCOL.md](./AGENT_CONTEXT_PROTOCOL.md) 的下游落地计划，不是 LangBot 宿主协议的设计前提。QA 入口和 smoke 记录见 [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md)。 + +官方 `local-agent` 可以外移，也可以重写。设计重点不是保留旧内置 runner 的内部结构，而是验证一个依附 LangBot host 基础设施的官方 agent 能否完整工作。同时，LangBot host 协议必须服务 Claude Code SDK、Codex、Pi Agent SDK、外部 Agent 平台等自管 context/runtime 的 runner，不能被官方插件的实现细节绑死。 + +## 1. 仓库组织 + +官方 runner 插件与 LangBot 主仓库、SDK 仓库以不同节奏迭代：LangBot 主仓库只维护宿主协议和调度，SDK 仓库维护 Runner 组件和 runtime 协议，官方 runner 插件承载业务 runner 的具体实现和第三方平台适配。 + +当前推荐"官方插件可独立发布，必要时共享 SDK helper"。开发期采用本地多目录布局： + +```text +langbot-app/ + langbot-local-agent/ # plugin:langbot-team/LocalAgent/default + manifest.yaml + components/runner/default.{yaml,py} + langbot-agent-runner/ # 外部服务 runner 仓库 + acp-agent-runner/ claude-code-agent/ codex-agent/ dify-agent/ n8n-agent/ ... +``` + +后续可聚合进 monorepo，也可继续独立发布——这个选择不影响协议设计。重复逻辑优先沉淀到 SDK 或明确的共享 helper 包，不要把宿主私有结构泄漏给插件。旧 `src/langbot/pkg/provider/runners/*` 只作为历史行为对齐基准；当前未发布分支不提供旧内置 runner 的运行时 fallback。 + +## 2. 插件命名和 runner id + +| 旧 runner | 官方插件 | runner id | +| --- | --- | --- | +| `local-agent` | `langbot-team/LocalAgent` | `plugin:langbot-team/LocalAgent/default` | +| `dify-service-api` | `langbot-team/DifyAgent` | `plugin:langbot-team/DifyAgent/default` | +| `n8n-service-api` | `langbot-team/N8nAgent` | `plugin:langbot-team/N8nAgent/default` | +| `coze-api` | `langbot-team/CozeAgent` | `plugin:langbot-team/CozeAgent/default` | +| - | `langbot-team/ACPRunner` | `plugin:langbot-team/ACPRunner/default` | +| - | `langbot-team/ClaudeCodeAgent` | `plugin:langbot-team/ClaudeCodeAgent/default` | +| - | `langbot-team/CodexAgent` | `plugin:langbot-team/CodexAgent/default` | +| `dashscope-app-api` | `langbot-team/DashScopeAgent` | `plugin:langbot-team/DashScopeAgent/default` | +| `deerflow-api` | `langbot-team/DeerFlowAgent` | `plugin:langbot-team/DeerFlowAgent/default` | +| `langflow-api` | `langbot-team/LangflowAgent` | `plugin:langbot-team/LangflowAgent/default` | +| `tbox-app-api` | `langbot-team/TboxAgent` | `plugin:langbot-team/TboxAgent/default` | +| `weknora-api` | `langbot-team/WeKnoraAgent` | `plugin:langbot-team/WeKnoraAgent/default` | + +每个插件可后续提供多个 runner，但迁移目标的默认 runner 统一叫 `default`。 + +## 3. 迁移批次 + +- **Batch 1（打通协议）**：`local-agent`（能力最完整基准）、`acp-agent-runner` / `claude-code-agent` / `codex-agent`（外部 code-agent harness 路径）、`dify-agent`（传统 service API runner）。 +- **Batch 2（外部 workflow）**：`n8n-agent`、`langflow-agent`（webhook/workflow 输入输出、timeout、外部 conversation id）。 +- **Batch 3（平台 Agent API）**：`coze-agent`、`dashscope-agent`、`tbox-agent`、`deerflow-agent`、`weknora-agent`（平台特有响应格式、引用资料、文件/图片输入、外部 thread/session 状态）。 + +## 4. 每个官方插件的组件要求 + +每个插件至少包含一个 `Runner` 组件，manifest 示例： + +```yaml +apiVersion: langbot/v1 +kind: Runner +metadata: + name: default + label: { en_US: Dify Agent, zh_Hans: Dify Agent } + description: + en_US: Run a Dify application as a LangBot Runner. + zh_Hans: 将 Dify 应用作为 LangBot Runner 运行。 +spec: + config: [] + capabilities: # 字段语义见 PROTOCOL_V1 §4.3 + streaming: true +execution: + python: { path: ./main.py, attr: DefaultRunner } +``` + +## 5. local-agent 插件方向 + +`local-agent` 是官方插件中能力最完整的消费者，但不是宿主协议的设计中心。它需要证明：一个主要依附 LangBot host 能力的 agent runner 可以通过公开协议完成模型、工具、知识库、状态、history、sandbox 文件访问、上下文压缩和消息投递。 + +迁移或重写需覆盖旧内置 runner 的用户可见能力：model primary/fallback 选择、prompt、knowledge-bases、rerank-model、rerank-top-k、function calling、streaming、multimodal input、conversation history、monitoring metadata。 + +责任边界与 Host API 消费方式见 AGENT_CONTEXT_PROTOCOL §8。关键约束： + +- 从 `ctx.config` 读取静态绑定 `prompt`，**不**读取 `ctx.adapter.extra["prompt"]`；不消费 Query entry adapter 生成的历史窗口。 +- 通过 `RunnerAPIProxy.history` 拉取 transcript，而不是依赖 host 每轮强塞历史窗口。 +- `ctx.input.contents` 保留图片/文件等多模态内容；RAG 只替换/插入文本部分，不丢图片/文件。 +- 不能绕过 `ctx.resources` 调用未授权模型、工具或知识库。 +- manifest 声明功能能力、LangBot 资源 permissions 和配置表单；实际授权来自 manifest permissions 与 binding resource policy、runner config、`ctx.context.available_apis` 和 Host run session snapshot 的交集。 + +### 5.1 Native Execution / Skills 后续接入 + +本阶段不把 sandbox/skills 做成 Runner 协议字段。后续 sandbox/skills 分支合并后，命令执行、文件操作、skill、MCP managed process 应先由 Host / sandbox 封装成 scoped tools，再通过 `ctx.resources.tools` 和 SDK runtime 转发暴露给 runner。这让 local-agent 只消费授权后的 Host 基础设施，而不是直接持有宿主机执行能力。 + +## 6. 外部 runner 插件要求 + +外部平台 runner 迁移遵循：旧配置字段尽量保持同名便于 migration 复制；输出统一转换为 `RunnerResult`；外部 API timeout 从 runner config 读取；平台 conversation id 存 plugin storage 或 context runtime state，不依赖 LangBot 内置 conversation uuid 私有结构；流式按平台能力声明，没有流式就只发 `message.completed`。 + +### 6.1 Code-agent harness runner + +Claude Code、Codex、Kimi Code 这类 runner 不一定通过 LangBot 的模型/工具 loop 执行，可以依赖自己的 harness，但仍必须遵守统一 Host 边界。总体边界见 [HOST_SDK_INFRASTRUCTURE.md](./HOST_SDK_INFRASTRUCTURE.md) §4.8；context projection 形态见 [AGENT_CONTEXT_PROTOCOL.md](./AGENT_CONTEXT_PROTOCOL.md) §4.5；发布级要求见 [SECURITY_HARDENING.md](./SECURITY_HARDENING.md)。 + +本文件只补充官方 runner 的实现要求：输入来自 `ctx.event` / `ctx.input`，不依赖 Pipeline 私有 `Query`；外部 session id / workspace / checkpoint 写入 Host state 或 plugin storage；插件实例边界见 PROTOCOL_V1 §13；CLI / subprocess runner 必须处理 timeout、取消、空输出、非零退出和 stderr 映射。 + +实现结构应把 provider-native output 解析与 LangBot result stream 组装分开：Claude stream-json、Codex JSONL、Kimi / OpenCode 事件等只在 runner adapter 内解析，输出统一归一为 `RunnerResult`（`message.completed` / `message.delta`、`state.updated`、`run.completed` / `run.failed`）。文件和工具大结果留在当前 run 的 sandbox/workspace，通过消息 metadata、attachment ref 或 path 指向。未知 native event 不应导致 run 崩溃；应记录诊断 metadata 或 warning。新增 harness 时优先补 native fixture -> `RunnerResult` 的转换测试，再接 WebUI smoke。 + +并发约束应按外部 session 粒度表达，而不是按 Agent / runner id / 插件实例表达；Agent 复用和全局锁边界见 PROTOCOL_V1 §13。若 runner 使用 `external.session_id` / `thread_id` resume 到同一 native session，且该 harness 不支持并发 turn，runner 应按稳定 external session key 串行写入；一次性 subprocess runner 可以只在单次 `run(ctx)` 内处理，长连接/daemon runner 则应采用 reader 独占 native stream、turn writer 串行写入的结构。 + +### 6.2 LangBot MCP gateway + +外部 harness 不能直接持有进程内的 `plugin_runtime_handler`，也不能用自己的 native tools 直接访问 LangBot 资源。外部 harness runner 应通过稳定 HTTP MCP gateway 或 SDK-owned bridge 把 harness 的工具请求转回 SDK runtime / Host API： + +- Gateway 由 runner 插件启动，暴露稳定的 `langbot_history_page`、`langbot_retrieve_knowledge`、`langbot_call_tool` 等最小工具面。 +- Harness 每次调用必须携带当前 LangBot `run_id`；Host 仍按 run session、caller identity 和授权快照校验。 +- Gateway 只转发 LangBot 资产访问，不承担外部 harness 的文件、进程或 native tool 权限边界。 + +第一批工具保持很小：history page、knowledge retrieve、authorized tool call。新增工具必须先有 Host action 权限与 run-scoped authorization，再由 gateway 投影。 + +## 7. Code-agent harness runner 当前形态 + +外部 code-agent harness 由直接 runner 插件承接，例如 `acp-agent-runner`、`claude-code-agent`、`codex-agent`，每个 runner 负责把目标 harness 的 native session、workspace、MCP bridge 和输出事件转换为统一 `RunnerResult`。本地 smoke 验收入口与记录见 [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md)。 + +当前形态： + +- Runner ID 示例：`plugin:langbot-team/ACPRunner/default`、`plugin:langbot-team/ClaudeCodeAgent/default`、`plugin:langbot-team/CodexAgent/default`。 +- Runner 可通过 ACP、远端 daemon、本机 subprocess 或外部 HTTP API 调用 harness；harness 的安装、登录态、workspace 和 provider-native 权限由该运行环境负责。 +- Runner 会把当前 LangBot `run_id`、可访问资源摘要和 gateway 使用规则注入本次消息；harness 通过 gateway 回填 `run_id` 后访问 LangBot 资产。 +- 外部 session id / workspace / checkpoint 写回 Host state 或 plugin storage，后续轮次可复用目标 harness 会话。 + +### 7.1 当前限制 + +这不是发布级安全边界实现；LangBot 只约束 LangBot 持有资产的访问，外部 harness 的文件、进程、workspace、provider-native MCP 和模型凭据由对应 runner 的运行环境承担。当前 `run_id` 可由系统提示词、ACP metadata 或 runner 自有 session metadata 传递给 harness 并由 gateway 校验。runtime 管控面方向见 [RUNTIME_CONTROL_PLANE_V2.md](./RUNTIME_CONTROL_PLANE_V2.md)。 + +## 8. 发布和安装策略 + +最终 LangBot 安装/升级时需保证官方 runner 插件可用，可选方案：首次启动检测缺失并提示安装，或由用户从 marketplace 安装。当前分支未发布，因此不保留历史 Pipeline Agent 配置兼容、旧内置 runner fallback，也不把旧 Pipeline 内的 Agent 配置迁移成独立 Agent。4.x 只读取 `ai.runner.id` 与 `ai.runner_config[id]`；升级后由用户选择或安装需要的 Runner。 + +## 9. 验收标准 + +- 每个目标 runner 都有对应官方 Runner 插件和稳定 runner id；当前配置只使用 `ai.runner.id` + `ai.runner_config[id]`。 +- LangBot 主聊天路径不再通过 `RequestRunner` 执行业务 runner。 +- 官方插件测试覆盖非流式、流式、错误、timeout、配置缺失。 +- `local-agent` 能完成模型 fallback、tool calling、知识库检索、多模态输入、静态绑定 prompt 消费、history API 拉取、rerank。 +- 外部 code-agent harness runner 能消费 event-first context、投影 scoped resources、保存 external session state，并通过 WebUI Debug Chat smoke。 +- `local-agent` 覆盖旧内置 runner 的用户可见核心能力；代码结构和运行路径不需要相同。 diff --git a/docs/agent-runner-pluginization/PLATFORM_ACTION_TOOLS.md b/docs/agent-runner-pluginization/PLATFORM_ACTION_TOOLS.md new file mode 100644 index 000000000..4c50eb836 --- /dev/null +++ b/docs/agent-runner-pluginization/PLATFORM_ACTION_TOOLS.md @@ -0,0 +1,75 @@ +# Agent 工具权限 + +Agent 配置页展示同一次运行中可能投射给 Runner 的完整工具目录： + +- 事件级工具由 Agent 选择的事件范围自动启用。 +- `allowed_platform_tools` 管理需要 Agent 自行指定目标的平台级动作。 +- `allowed_tools` 管理沙盒内置工具、MCP 工具、插件工具和技能工具。 + +Host 会按当前 Workspace 实时解析工具来源。未安装的插件、未连接的 MCP、不可用的 Box +沙盒以及名称存在歧义的工具不会进入可选目录。旧 Agent 若尚未保存 `allowed_tools`，继续 +沿用运行器原有的工具策略；一旦在配置页保存，就转为明确的顶层白名单。 + +Agent 不直接持有平台适配器，也不能调用任意原始平台接口。每次运行时，Host 根据当前 +事件自动加入兼容的事件级工具，并加入 `allowed_platform_tools` 中选择的平台级工具， +再与 Runner 权限、当前适配器声明的 API、当前事件能够安全绑定的目标取交集， +得到 `ctx.resources.tools` 中 `tool_type=platform` 的最终工具集合。 + +## 两类工具 + +- 事件级工具以 `event_` 开头。用户、群组、消息或请求标识由 Host 从当前事件冻结， + Agent 只能填写回复文本、审核结果、禁言时长等动作参数。 +- 平台级工具以 `platform_` 开头。Agent 可以填写目标用户、群组或消息标识，因此权限 + 更宽，配置页将其与事件级工具分开展示。 + +当前事件级工具包括：回复当前会话、删除当前消息、查询事件发起者或相关群组/成员、 +禁言/解除禁言/移出相关成员、同意或拒绝好友请求、同意或拒绝入群邀请。 + +当前平台级工具包括：发送/查询/删除消息，查询群组、群列表、群成员，修改群名称， +禁言/解除禁言/移出成员、退出群组，以及查询用户和好友列表。 + +`call_platform_api` 不在 Agent 工具目录中。平台私有透传接口必须先在 Host 中定义为 +具有固定名称、JSON Schema、风险级别和授权规则的语义工具，不能让 Agent 自行传入 +原始 action 名称。 + +## 运行时投射 + +```text +current event type ── compatible event tools +Agent.allowed_platform_tools ── selected platform tools + │ + ├─ Runner capability tool_calling is enabled + ├─ Runner manifest permissions.tools contains call + ├─ current adapter.get_supported_apis() + └─ current event type and frozen target are compatible + │ + ▼ +ctx.resources.tools[tool_type=platform] + │ + ├─ Local Agent: RunnerAPIProxy.call_tool + └─ External Runner: langbot_list_assets / langbot_get_tool_detail / + langbot_call_tool (MCP Asset Gateway) + │ + ▼ +Host revalidates run_id, runner plugin identity, operation and frozen source + │ + ▼ +current bot adapter semantic API +``` + +本地和外部 Runner 因此使用同一个工具名、参数 Schema 和 Host 授权快照。外部 +平台不会获得适配器对象或长期凭据；MCP 网关中的 run token 和 Host 中的 run session +都只对应当前运行。 + +## 失败语义 + +- Agent 未选择当前事件：不会触发运行，也不会生成事件级工具。 +- 平台级工具未在 Agent 配置中选择：不进入运行资源。 +- Runner 未启用 `tool_calling` 或没有 `tools.call` 权限：所有平台动作均不可用。 +- 当前适配器不声明对应 API：该工具记入 `platform_capabilities.unavailable_tools`，不投射。 +- 事件类型不匹配或缺少可冻结目标：事件级工具不投射。 +- 调用期间机器人下线或适配器能力变化：Host 拒绝执行并返回具体错误。 +- 参数包含 Schema 之外的字段：Host 拒绝执行。 + +这些规则保证配置白名单不是唯一防线；真正的执行授权始终由单次运行快照和执行时检查 +共同决定。 diff --git a/docs/agent-runner-pluginization/PROTOCOL_V1.md b/docs/agent-runner-pluginization/PROTOCOL_V1.md new file mode 100644 index 000000000..cc6973a8c --- /dev/null +++ b/docs/agent-runner-pluginization/PROTOCOL_V1.md @@ -0,0 +1,820 @@ +# LangBot Runner Protocol v1 + +本文档是 LangBot Host 与插件 SDK / Runtime / Runner 之间协议合同的**唯一规范来源（single source of truth）**。 + +- 本文件描述当前 Protocol v1 稳定合同，不混入验收流水。当前实现状态见 [STATUS.md](./STATUS.md)，测试执行入口见 [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md)，安全发布门槛见 [SECURITY_HARDENING.md](./SECURITY_HARDENING.md)。 +- 本文件之外的任何文档**不得重新定义这里的数据结构**，只能引用，例如"见 PROTOCOL_V1 §4.2"。 +- Host 内部模型（`AgentEventEnvelope`、`AgentBinding`、Descriptor、各 Store）不属于 SDK 协议，定义在 [HOST_SDK_INFRASTRUCTURE.md](./HOST_SDK_INFRASTRUCTURE.md)。 + +## 1. 协议目标 + +Protocol v1 只解决四件事： + +- LangBot 如何发现插件提供的 Runner。 +- LangBot 如何把一次事件调用封装成 `RunnerContext`。 +- Runner 如何以事件流形式返回运行结果。 +- Runner 如何通过受限 API 访问 LangBot host 能力。 + +Protocol v1 **不定义**： + +- LangBot 内部如何持久化 `AgentBinding`（见 HOST_SDK）。 +- Runner 内部如何组装 prompt、压缩历史、管理 memory（见 [AGENT_CONTEXT_PROTOCOL.md](./AGENT_CONTEXT_PROTOCOL.md)）。 +- 官方 runner 的具体实现（见 [OFFICIAL_RUNNER_PLUGINS.md](./OFFICIAL_RUNNER_PLUGINS.md)）。 +- Pipeline 的长期配置模型。 +- 发布级安全 hardening 的完整实现（见 [SECURITY_HARDENING.md](./SECURITY_HARDENING.md)）。 + +## 2. 参与方 + +| 名称 | 职责 | +| --- | --- | +| LangBot Host | 事件入口、绑定解析、权限、资源、存储、生命周期、结果投递。 | +| Plugin Runtime | 加载插件，响应 Host 的 runner discovery 和 run 调用。 | +| Runner | 插件提供的 agent 执行组件。 | +| RunnerAPIProxy | Runner 访问 Host 能力的受限 API。 | +| AgentBinding | Host 内部的事件到 runner 绑定配置，不直接暴露给 SDK（见 HOST_SDK §4.2）。 | + +产品层同时保留 Pipeline 与独立 `Agent`：现有 Pipeline 不迁移为 Agent； +用户可以新建 Agent 并绑定到 bot / IM channel，一个 Agent 可以被多个 bot / channel 复用。Host 内部的 +`AgentBinding` 是一次事件运行前解析出的有效绑定，只影响 Host 构造出的 +`ctx.config`、`ctx.resources`、`ctx.context` 和 `ctx.delivery`。SDK 不需要知道 +Agent / binding 的持久化形态。 + +外部 harness runner（Claude Code、Codex、Kimi Code 等）也是 `Runner`：它们消费 event-first `RunnerContext`、返回 `RunnerResult`，并通过 Host 授权的 state/storage API 保存跨轮次指针；当前运行文件和工具大结果进入 sandbox/workspace。它们内部可以继续使用自己的 session、tool loop、MCP、上下文压缩和权限模型。 + +## 3. 协议演进 + +当前 Runner 合同不暴露显式 `protocol_version` 字段。协议演进先按字段级兼容规则处理： + +- 新增可选字段保持向后兼容。 +- 删除字段或改变既有字段语义，需要在 SDK 发布前完成；发布后应走新的显式兼容方案。 +- 结果流演进：Host **必须忽略未知 result type 并记录 warning**（除非该 type 明确要求强校验）。SDK envelope 接收入站未知 `type` 字符串，runner 侧可按原字符串转发或忽略；新增 result type 不提升大版本。 +- SDK 入站 context 类实体偏宽松，用于兼容 Host 附加的非核心字段；Host 返回 DTO（history/event/steering/run/runtime/stats/error 等）忽略未知字段，保证 Host 增加可选返回字段时旧 SDK 不会解析失败；manifest、runner result payload 等由插件/runner 提交给 Host 的输入合同仍偏严格。安全边界仍在 Host，SDK 校验只提升开发体验。 + +## 4. Discovery 协议 + +### 4.1 LIST_RUNNERS + +Host 调用 Plugin Runtime 获取当前插件暴露的 runner 列表，请求无额外 payload。返回： + +```python +class ListRunnersResponse(BaseModel): + runners: list[RunnerDiscovery] + +class RunnerDiscovery(BaseModel): + plugin_author: str + plugin_name: str + runner_name: str + manifest: RunnerManifest +``` + +`manifest` 是 SDK typed `RunnerManifest`，由 Runtime 从插件组件 manifest 解析并校验后返回。`plugin_author` / `plugin_name` / `runner_name` 保留为 transport 寻址字段；Host 以它们生成稳定 runner id，并把 `manifest.id` 校验为 `plugin:author/name/runner`。单个 runner manifest 解析失败时 Runtime/Host 记录 warning 并跳过该 runner，不影响同一插件或其它插件的 runner discovery。 + +### 4.2 RunnerManifest + +这里的 manifest 指 Runtime 返回给 Host 的 typed runner manifest： + +```python +class RunnerManifest(BaseModel): + id: str + name: str + label: I18nObject + description: I18nObject | None = None + capabilities: RunnerCapabilities = RunnerCapabilities() + permissions: RunnerPermissions = RunnerPermissions() + config_schema: list[DynamicFormItemSchema] = [] + metadata: dict[str, Any] = {} +``` + +- runner id 由 Host 生成，格式 `plugin:author/name/runner`。 +- `name` 是插件内 runner 名称，例如 `default`。 +- `config_schema` 只描述绑定配置表单，不代表插件实例状态。 +- `capabilities` 是 Host 用于 UI 和资源投影的 typed bool model；它不是权限授予。 +- `permissions` 是 runner 申请的 LangBot 资源访问上限；实际授权仍必须与 binding policy 求交。 +- `metadata` 只放展示、诊断、非稳定扩展信息。 + +### 4.3 Capabilities + +```python +class RunnerCapabilities(BaseModel): + streaming: bool = False + tool_calling: bool = False + knowledge_retrieval: bool = False + multimodal_input: bool = False + skill_authoring: bool = False + interrupt: bool = False + steering: bool = False + interactions: bool = False + + model_config = ConfigDict(extra="forbid") +``` + +- `streaming`: runner 可以返回 `message.delta`。 +- `tool_calling`: runner 可能调用 Host tool API。 +- `knowledge_retrieval`: runner 可能调用 Host knowledge API。 +- `multimodal_input`: runner 可以处理非纯文本 input / attachment。 +- `skill_authoring`:（降级为便捷开关，非访问硬前提）声明该 runner 期望使用 LangBot skill 工具链。skill 本身通过**统一 tool 授权**获得——发现走 `list_skills` / `langbot_list_assets`，激活/注册走 `activate` / `register_skill`，操作走 native exec/read/write，全部计入 `resource_policy.allowed_tool_names`。该 capability 仅作为「一键授权这组 skill tool + sandbox」的便捷开关，不再单独决定 skill 是否可用。 +- `interrupt`: runner 支持取消或中断。 +- `steering`: runner 支持在 turn 边界通过 Host pull API 消费同 conversation 在途追加消息。 +- `interactions`: runner 可以请求 Host 展示结构化交互，并处理后续 `interaction.submitted` 事件。 + +Capabilities 字段全部是 `bool`，未知 key 禁止进入 typed manifest。早期草案里的上下文/会话类 capability 已删除；对应语义由 event-first context 和 runner-owned context 原则表达。 + +### 4.4 Permissions 与 Effective Access + +```python +class RunnerPermissions(BaseModel): + models: list[Literal["invoke", "stream", "rerank"]] = [] + tools: list[Literal["detail", "call"]] = [] + knowledge_bases: list[Literal["list", "retrieve"]] = [] + history: list[Literal["page", "search"]] = [] + events: list[Literal["get", "page"]] = [] + storage: list[Literal["plugin", "workspace"]] = [] + files: list[Literal["config", "knowledge"]] = [] + interactions: list[Literal["request"]] = [] + + model_config = ConfigDict(extra="forbid") +``` + +通用交互投递是当前唯一允许执行的 `action.requested` 白名单动作。Runner 必须同时声明 +`capabilities.interactions=true` 和 `permissions.interactions=["request"]`，Host 还必须将其与 +当前 binding delivery policy、run authorization snapshot 和 adapter delivery capability 求交。 +其它 `action.requested` 动作仍只记录 telemetry，不得作为任意平台动作执行器。平台语义动作已经通过 `ctx.resources.tools` 中 `tool_type="platform"` 的工具提供：Runner 需具备 `tool_calling` capability 和 `permissions.tools` 的 `call` 操作，Host 再与 Agent 工具策略、适配器能力和当前事件目标求交。`event_*` 的目标由 Host 冻结，`platform_*` 需显式选择；调用仍走统一 `call_tool`。这不新增 manifest permission 字段，规则见 [PLATFORM_ACTION_TOOLS.md](./PLATFORM_ACTION_TOOLS.md)。 + +Runner 实际可用 LangBot 资源来自 Host 在 run 前冻结的授权快照： + +```text +effective_access = manifest.permissions ∩ binding.resource_policy ∩ current scope/config +``` + +具体落地： + +1. `AgentResourceBuilder` 先用 manifest permissions 与 binding resource policy / runner config 求交，生成 `ctx.resources`。 +2. `AgentContextBuilder` 用 manifest permissions 与 binding state/storage policy 求交，生成 `ctx.context.available_apis`。 +3. `AgentRunSessionRegistry` 冻结 run-scoped resources 与 available APIs。 +4. Runtime handler / `RunnerAPIProxy` 按 active `run_id`、runner identity、caller plugin identity、resource id、scope、payload size、rate limit 和 deadline 校验每次调用。 + +反承诺：manifest permissions **只约束 LangBot 持有的资源访问**。它不承诺限制外部 harness 的 native shell、文件系统、CLI、MCP、网络或本机权限；这些能力由 operator/runtime/sandbox 另行约束，见 HOST_SDK §4.8 与 SECURITY_HARDENING。 + +默认原则： + +- Host 不得默认 inline 全量历史。 +- Host 只 inline 当前 event / input 和 context handles。 +- Runner 拥有 working context assembly。 +- Runner 可在授权后通过 Host history / event / state API 拉取更多上下文，并通过授权 sandbox/workspace 工具访问当前运行文件。 +- 历史窗口策略不属于 Protocol v1 字段，也不属于 Host 通用语义。 + +context 边界的设计理由见 [AGENT_CONTEXT_PROTOCOL.md](./AGENT_CONTEXT_PROTOCOL.md)。 + +## 5. Run 协议 + +### 5.1 RUN_RUNNER + +Host 调用 Runtime： + +```python +class AgentRunRequest(BaseModel): + runner_id: str + runner_name: str + context: RunnerContext +``` + +Runtime 返回 `RunnerResult` 异步流。底层 transport 可继续用 `plugin_author` / `plugin_name` / `runner_name` 定位组件，但协议语义以 `runner_id` 和 `context` 为准。 + +### 5.2 RunnerContext + +这是 SDK 看到的**唯一权威 context 定义**。 + +```python +class RunnerContext(BaseModel): + run_id: str + trigger: AgentTrigger + event: AgentEventContext + conversation: ConversationContext | None = None + actor: ActorContext | None = None + subject: SubjectContext | None = None + input: AgentInput + delivery: DeliveryContext + resources: AgentResources + context: ContextAccess + state: AgentRunState + runtime: AgentRuntimeContext + config: dict[str, Any] = {} + adapter: AdapterContext | None = None + metadata: dict[str, Any] = {} +``` + +核心约束： + +- `event` 是必选字段，Protocol v1 是 event-first。 +- `input` 表示当前事件的主输入，不等于历史消息。 +- `bootstrap` / `messages` **不是协议字段**；Host 不内联历史窗口。 +- `adapter` 只放入口 adapter 的非核心元数据，runner 不应依赖它做长期能力。 +- `config` 是 Agent/runner config，不是插件实例状态。 + +### 5.3 AgentTrigger + +```python +class AgentTrigger(BaseModel): + type: str + source: Literal["platform", "webui", "api", "scheduler", "system", "host_adapter"] + timestamp: int | None = None +``` + +`trigger.type` 应与 `event.event_type` 一致或更粗粒度。例如入口适配器触发消息时： + +```json +{ "type": "message.received", "source": "host_adapter" } +``` + +### 5.4 AgentEventContext + +```python +class AgentEventContext(BaseModel): + event_id: str + event_type: str + event_time: int | None = None + source: str + source_event_type: str | None = None + raw_ref: RawEventRef | None = None + data: dict[str, Any] = {} +``` + +- `event_type` 使用 LangBot 稳定协议名，例如 `message.received`。稳定事件名清单见 [EVENT_BASED_AGENT.md](./EVENT_BASED_AGENT.md)。 +- 平台原始事件名放入 `source_event_type`。 +- 大型原始 payload 必须放入 `raw_ref` 或 staged file，不应直接塞入 `data`。 + +### 5.5 Conversation / Actor / Subject + +```python +class ConversationContext(BaseModel): + conversation_id: str | None = None + thread_id: str | None = None + launcher_type: str | None = None + launcher_id: str | None = None + sender_id: str | None = None + bot_id: str | None = None + workspace_id: str | None = None + session_id: str | None = None + +class ActorContext(BaseModel): + actor_type: str + actor_id: str | None = None + actor_name: str | None = None + metadata: dict[str, Any] = {} + +class SubjectContext(BaseModel): + subject_type: str + subject_id: str | None = None + data: dict[str, Any] = {} +``` + +示例： + +- 消息事件：actor 是发消息的人，subject 是当前消息。 +- 入群事件：actor 是新成员或邀请人，subject 是群/成员关系。 +- 定时事件：actor 可以是 system，subject 是 schedule。 + +### 5.6 AgentInput + +```python +class AgentInput(BaseModel): + text: str | None = None + contents: list[ContentElement] = [] + attachments: list[InputAttachment] = [] + interaction: InteractionSubmission | None = None +``` + +- 文本、多模态、附件都属于当前 event input。 +- 大文件、图片、音频、工具大结果应进入授权 sandbox/workspace，input attachment 只携带轻量 metadata/path/url/content。 +- 平台原始消息链不属于 SDK `AgentInput`；需要诊断时放在 Host 内部 envelope 或 `ctx.adapter.extra` 的一次性兼容字段中，不作为长期 runner 合同。 +- `interaction` 只在 `event.event_type == "interaction.submitted"` 时出现，承载经过 Host 校验和归一化的用户提交；平台原始 callback payload 不进入该字段。 + +### 5.7 DeliveryContext + +```python +class DeliveryContext(BaseModel): + surface: str + reply_target: dict[str, Any] | None = None + supports_streaming: bool = False + supports_edit: bool = False + supports_reaction: bool = False + max_message_size: int | None = None + interactions: InteractionDeliveryCapabilities | None = None + platform_capabilities: dict[str, Any] = {} +``` + +Runner 可参考 delivery 能力决定返回 `message.delta`、`message.completed` 或交互请求。 +`interactions is not None` 表示当前 surface 可以投递结构化交互；其中的 field/action 能力 +决定 Host 可原生渲染的范围。Runner 仍必须提供 `fallback_text`，供不支持富交互或投递失败时降级。 +平台事件进入独立 Agent 时，Host 会从当前 adapter 的 `get_supported_apis()` 投影 +`supports_edit`、`supports_reaction`，并把去重后的 API 名称写入 +`platform_capabilities.supported_apis`。这些字段只描述当前投递表面的能力，不授予 +任意平台动作权限；交互请求按 §5.8 和 §7.3 的白名单约束执行。合成路由测试使用的 +adapter 会过滤所有已知副作用 API，因此测试事件不会向 runner 宣告真实出站能力。 + +### 5.8 Structured Interaction + +```python +InteractionFieldType = Literal[ + "text", "textarea", "select", "multiselect", "number", "boolean", "file" +] + +class InteractionOption(BaseModel): + value: str + label: str + description: str | None = None + +class InteractionField(BaseModel): + id: str + label: str + type: InteractionFieldType + required: bool = False + options: list[InteractionOption] = [] + placeholder: str | None = None + default: JSONValue | None = None + +class InteractionAction(BaseModel): + id: str + label: str + style: Literal["default", "primary", "danger"] = "default" + +class InteractionRequest(BaseModel): + interaction_id: str + kind: Literal["form", "confirmation", "choice"] = "form" + title: str + description: str | None = None + fields: list[InteractionField] = [] + actions: list[InteractionAction] = [] + expires_at: int | None = None + fallback_text: str + +class InteractionSubmission(BaseModel): + interaction_id: str + action_id: str | None = None + values: dict[str, JSONValue] = {} + submitted_at: int | None = None + +class InteractionDeliveryCapabilities(BaseModel): + field_types: list[InteractionFieldType] = [] + action_styles: list[Literal["default", "primary", "danger"]] = [] + supports_updates: bool = False + max_fields: int | None = None +``` + +Runner 使用 `RunnerResult.interaction_requested()` 生成 +`action.requested(action="interaction.requested")`。Host 只能把请求投递到当前 run 冻结的 +delivery target，Runner 不得通过 `target` 改写 bot、conversation 或用户。Host 为请求保存 +`interaction_id -> processor/binding/conversation/expiry` 关联；平台 callback 必须先经过签名、 +目标、过期和幂等校验，再生成 `interaction.submitted` 事件。 + +Provider 私有 continuation token、workflow id、凭据或原始 callback payload 不属于该协议。 +Runner 应把这些值保存到受授权的 Host state 或 plugin storage，并只用 `interaction_id` 关联提交。 +Host 必须使用服务器接收时间判断 callback 是否过期，并覆盖 `InteractionSubmission.submitted_at`； +平台事件时间只可作为原始事件诊断信息，不能控制 TTL 或进入 Runner 的可信提交字段。 + +### 5.9 ContextAccess + +```python +class ContextAccess(BaseModel): + conversation_id: str | None = None + thread_id: str | None = None + latest_cursor: str | None = None + event_seq: int | None = None + transcript_seq: int | None = None + has_history_before: bool = False + inline_policy: InlineContextPolicy + available_apis: ContextAPICapabilities + +class InlineContextPolicy(BaseModel): + mode: Literal["none", "current_event", "recent_tail", "summary_tail"] + delivered_count: int = 0 + source_total_count: int | None = None + messages_complete: bool = False + reason: str | None = None + +class ContextAPICapabilities(BaseModel): + prompt_get: bool = False + history_page: bool = False + history_search: bool = False + event_get: bool = False + event_page: bool = False + state: bool = False + storage: bool = False + steering_pull: bool = False +``` + +`ContextAccess` 告诉 runner：Host inline 了什么、没 inline 什么、需要更多上下文时走哪些 API。它是 runner 按需读取上下文的入口说明，不是 Host 的业务上下文编排策略。 + +### 5.9 AgentRuntimeContext + +```python +class AgentRuntimeContext(BaseModel): + langbot_version: str | None = None + trace_id: str | None = None + deadline_at: float | None = None + metadata: dict[str, Any] = {} +``` + +### 5.10 AgentRunState + +```python +class AgentRunState(BaseModel): + conversation: dict[str, Any] = {} + actor: dict[str, Any] = {} + subject: dict[str, Any] = {} + runner: dict[str, Any] = {} +``` + +State 是可选 host-owned snapshot。Runner 也可以完全自管状态。 + +## 6. Resources + +```python +class ToolResource(BaseModel): + tool_name: str + tool_type: str | None = None + description: str | None = None + parameters: dict[str, Any] | None = None # 完整 JSON schema，由 Host 一次塞齐 + operations: list[Literal["detail", "call"]] = [] + +class SkillResource(BaseModel): + skill_name: str + display_name: str | None = None + description: str | None = None + +class AgentResources(BaseModel): + models: list[ModelResource] = [] + tools: list[ToolResource] = [] + knowledge_bases: list[KnowledgeBaseResource] = [] + skills: list[SkillResource] = [] + storage: StorageResource = StorageResource() + platform_capabilities: dict[str, Any] = {} +``` + +`tools` 携带每个授权工具的完整 schema（`parameters`），由 Host 在构造 `ctx.resources` 时一次塞齐，runner 不需再逐个调用 `get_tool_detail` 拉取，减少 N 次往返。 + +`skills` 是本次 run 中 pipeline-visible 的 skill facts（`skill_name`、`display_name`、`description`）。**skill 通过统一 tool 形式消费，不是独立资源类别**：发现走 `list_skills` tool（或 `langbot_list_assets` 增加 skills 一类），激活走 `activate`，操作走 native exec/read/write。Host **不**把 skill 索引注入 system prompt，也不做 progressive-disclosure 注入；LLM 通过调用发现工具主动查询 skill 清单。Host **可选**在 ctx 提供预渲染的 `suggested_skill_prompt`（首轮延迟优化，runner 可忽略 / override），但它不是访问前提。`skills` 字段本身仅作为发现工具的数据来源与该可选预渲染的输入。 + +资源列表是本次 run 的授权结果。History / Event / State / Storage 访问通过 `ctx.context.available_apis` 和 Host 侧 run session 校验控制，不作为可枚举 resource list 暴露。Runner 只能通过 `RunnerAPIProxy` 访问这些能力。当前事件的文件和工具大结果优先进入授权 sandbox/workspace，由 runner 通过 read/write/exec 类工具按需读取。 + +## 7. Result Stream + +### 7.1 RunnerResult envelope + +```python +JSONValue = str | int | float | bool | None | list["JSONValue"] | dict[str, "JSONValue"] + +ResultType = Literal[ + "message.delta", + "message.completed", + "tool.call.started", + "tool.call.completed", + "state.updated", + "action.requested", + "run.completed", + "run.failed", +] + +class RunnerResult(BaseModel): + run_id: str + type: RunnerResultType | str + data: dict[str, Any] = {} + usage: LLMTokenUsage | None = None + sequence: int | None = None + timestamp: int | None = None +``` + +SDK 当前实现是单一 envelope：`type` 枚举 + `data` dict。Payload 由 SDK typed model 构造并 dump，但 wire 不改成 discriminated union；这样新旧版本偏斜时 Host 仍可按 §3 忽略未知 `type`。 + +`usage` 是 runner 可选上报的 token 使用量，沿用 SDK `LLMTokenUsage`： + +```python +class LLMTokenUsage(BaseModel): + prompt_tokens: int | None = None + completion_tokens: int | None = None + total_tokens: int | None = None + # provider-specific detail/cached/reasoning counters are preserved as extra fields +``` + +约束： + +- 运行时能观测到 provider/runtime usage 时，SHOULD 在 terminal `run.completed.usage` 上报本次 run 的最终聚合 token usage。 +- `run.failed.usage` MAY 上报失败前已经产生的部分 usage。 +- 不能观测 usage 的 runner 合法地省略该字段；缺失表示 unknown，Host 不得按 0 处理。 +- ACP 等外部协议不保证统一 usage；ACP runner 只能在具体 provider/native event 提供 usage 时填充本字段。 +- cost 不作为 runner result 的权威字段。Host 后续应基于 usage、model identity、时间和自身价格表计算账单成本；provider 原始 cost 如需保留，可放在 `usage` extra 字段中作为非权威 telemetry。 + +Host 边界分级校验： + +- `message.delta`、`message.completed`、`state.updated`、`action.requested`、`run.completed`、`run.failed` 属于会影响投递或 Host 副作用的严格 payload；校验失败时丢弃该 result 并记录 warning。 +- `tool.call.started`、`tool.call.completed` 当前只作为 telemetry，payload 宽松兼容。 +- 未知 `type` 忽略并记录 warning。 + +### 7.2 稳定 result payloads + +| type | `data` payload | +| --- | --- | +| `message.delta` | `{ "chunk": MessageChunk }` | +| `message.completed` | `{ "message": Message }` | +| `tool.call.started` | `{ "tool_call_id": str, "tool_name": str, "parameters": dict }` | +| `tool.call.completed` | `{ "tool_call_id": str, "tool_name": str, "result": dict \| None, "error": str \| None }` | +| `state.updated` | `{ "scope": "conversation" \| "actor" \| "subject" \| "runner", "key": str, "value": JSONValue }` | +| `action.requested` | `{ "action": str, "target": dict \| None, "payload": dict \| None }` | +| `run.completed` | `{ "finish_reason": str, "message"?: Message }` | +| `run.failed` | `{ "code": str, "error": str, "retryable": bool }` | + +Runner 生成的大文件、工具输出和临时产物不通过 result event 回传；应写入当前 run 的授权 sandbox/workspace，再用消息文本、metadata 或 attachment reference 指向它们。 + +### 7.3 稳定 result types + +| type | 说明 | 当前消费 | +| --- | --- | --- | +| `message.delta` | 流式消息片段。 | ✅ | +| `message.completed` | 完整消息。 | ✅ | +| `tool.call.started` | 工具调用开始的可观测事件。 | telemetry | +| `tool.call.completed` | 工具调用完成的可观测事件。 | telemetry | +| `state.updated` | runner 请求更新 host-owned state。 | ✅ | +| `action.requested` | runner 请求 Host 执行受控动作。 | `interaction.requested` 可执行；其它动作仅 telemetry | +| `run.completed` | run 正常结束。 | ✅ | +| `run.failed` | run 失败。 | ✅ | + +`action.requested` 仍是严格白名单协议表面。当前 Host 只执行 +`action="interaction.requested"`，并要求 payload 可校验为 `InteractionRequest`、Runner 声明 +interaction capability/permission、当前 binding 允许交互且 delivery surface 支持交互。 +Host 忽略 Runner 提供的外部 target，只使用 run authorization snapshot 中冻结的 delivery target。 +其它 action 只记 telemetry，不执行；通用 platform action executor 仍不属于本协议。 + +Host 必须校验 `state.updated` 的 scope、key、value 大小和 JSON 可序列化性。Host 还必须限制 +交互字段/动作数量、字符串长度、payload 总大小、过期时间和 interaction id 唯一性，并对 callback +应用默认 30 分钟、最长 24 小时的有效期；request 与 submission payload 均不得超过 256 KiB。 +执行目标绑定、签名、幂等和过期校验。 + +除 runner 经 `state.updated` 写之外，Host 自身也可直接写部分 host-owned state。例如 `activate` tool 在 Host 侧执行时，直接把已激活 skill 写入 conversation scope 的 `host.activated_skills` 快照。当 host 直接写与 runner `state.updated` 写到同一 key 时，按 **last-write-wins** 合并——runner 可以覆盖 host 写的快照。 + +### 7.4 Stream delivery semantics + +- Host 按 Runtime stream 顺序消费 result。当前 v1 不定义跨连接 replay，也不承诺 at-least-once；从 Host 视角，收到的 result 最多应用一次。 +- `sequence` 是单个 `run_id` 内的结果序号。in-process / stdio 这类天然有序的在线 stream 可以省略；任何会缓冲、重放、跨进程队列或 runtime-managed task 的 transport 必须提供从 1 开始严格递增的 `sequence`。 +- Host 看到已提供 `sequence` 的 result 时，应按 `(run_id, sequence)` 做重复检测，并在缺号或乱序时记录 warning；除非 transport 明确声明 replay 语义，Host 不应自行等待缺失序号重排用户可见输出。 +- `run.failed.data.retryable` 只表示整次 run 理论上可由上层重试；Protocol v1 不自动重试 run，也不自动重试 proxy action。 +- History / Event / Transcript cursor 是 opaque token。runner 不得解析 cursor，也不得假设 cursor 在不同 API、conversation、thread 或 retention window 之间可比较；当前实现即使返回数字字符串，也只是实现细节。 + +### 7.5 示例 + +```json +{ "type": "message.delta", "data": { "chunk": { "role": "assistant", "content": "hel" } } } +{ "type": "message.completed", "data": { "message": { "role": "assistant", "content": "hello" } } } +{ "type": "state.updated", "data": { "scope": "conversation", "key": "external.session_id", "value": "abc" } } +{ "type": "action.requested", "data": { "action": "interaction.requested", "payload": { "interaction_id": "form_1", "kind": "choice", "title": "Approve?", "actions": [{"id": "approve", "label": "Approve", "style": "primary"}], "fallback_text": "Reply approve or reject." } } } +``` + +## 8. RunnerAPIProxy + +所有 proxy action 必须携带 `run_id`。Host 必须校验：active run session 存在、caller plugin identity 匹配、resource 在本次 `ctx.resources` 中授权、scope 不越界、payload size / rate limit / deadline 合法。 + +```python +# Model +await api.invoke_llm(llm_model_uuid, messages, funcs=None, extra_args=None) +await api.invoke_llm_with_usage(llm_model_uuid, messages, funcs=None, extra_args=None) +async for chunk in api.invoke_llm_stream(llm_model_uuid, messages, funcs=None, extra_args=None): + ... +async for event in api.invoke_llm_stream_events(llm_model_uuid, messages, funcs=None, extra_args=None): + ... +await api.invoke_rerank(rerank_model_id, query, documents, top_k=None) + +# Tool +await api.get_tool_detail(tool_name) +await api.call_tool(tool_name, parameters) + +# Knowledge +await api.retrieve_knowledge(kb_id, query_text, top_k=5, filters=None) + +# History（返回 Transcript projection，不返回原始平台 payload） +await api.get_prompt() +await api.history_page(conversation_id=None, before_cursor=None, after_cursor=None, + limit=50, direction="backward", include_attachments=False) +await api.history_search(query, filters=None, top_k=10) + +# Event（返回稳定 event envelope 或受限 raw ref，不默认返回大 payload） +await api.event_get(event_id) +await api.event_page(conversation_id=None, event_types=None, before_cursor=None, limit=50) +await api.steering_pull(mode="all", limit=None) + +# State / Storage +await api.state_get(scope, key); await api.state_set(scope, key, value); await api.state_delete(scope, key) +await api.state_list(scope, prefix=None, limit=100) +await api.get_plugin_storage(key); await api.set_plugin_storage(key, value); await api.delete_plugin_storage(key) +await api.get_plugin_storage_keys() +await api.get_workspace_storage(key); await api.set_workspace_storage(key, value); await api.delete_workspace_storage(key) +await api.get_workspace_storage_keys() + +# Host info +await api.get_langbot_version() +``` + +`invoke_llm()` / `invoke_llm_stream()` 的第一个参数在 SDK 中命名为 +`llm_model_uuid`，wire payload 字段也是 `llm_model_uuid`。该值对 runner +仍是 opaque identifier，不应解析其内部格式。 + +`invoke_llm()` 和 `invoke_llm_stream()` 保持兼容：前者返回 `Message`，后者只 +yield `MessageChunk`。需要 provider 真实 token 计量的 runner 应使用 +`invoke_llm_with_usage()` 或 `invoke_llm_stream_events()`。Host response 可在 +原有 `{message: ...}` / `{chunk: ...}` 外额外携带可选 `usage` 字段；streaming +场景允许在所有 chunk 之后追加一个 usage-only event。`usage` 至少保留 +OpenAI-compatible 的 `prompt_tokens`、`completion_tokens`、`total_tokens`， +若 provider 返回 `prompt_tokens_details` / `completion_tokens_details` 或 +cache token counters，Host / SDK 不应丢弃这些字段。没有 usage 的 provider +必须继续返回成功响应，SDK 将 usage 置为 `None`。 + +`get_prompt()` 返回当前 query-backed run 的 Host effective prompt messages： +`list[Message]` 的 JSON 形式。该能力只在 `ctx.context.available_apis.prompt_get` +为 true 时可用；没有 query 缓存、prompt 已过期或非 query entry run 时 Host +可以返回错误或空列表。Runner 应在不可用时回退到自己的 config/prompt 策略。 + +`steering_pull(mode="all")` 是推荐默认：Host 按 claim 顺序返回全部 pending steering 输入并清空对应队列。`mode="one-at-a-time"` 仅用于 runner 主动节流，每次返回一条。Host 不合并多条用户消息；runner 负责在 turn 边界决定模型侧格式。 + +Steering 审计使用 EventLog 而不是 Transcript schema 扩展：被 active run 吸收的原始 `message.received` 事件保留原事件类型，并在 `metadata.steering` 标记 `status="queued"`、`trigger_behavior="absorbed_into_active_run"`、`claimed_by_run_id`、`claimed_runner_id`、`claimed_at`。Runner 成功 pull 后，Host 追加 `steering.injected` EventLog 记录，`metadata.steering.status="injected"` 并引用 `source_event_id`。若 run 结束时仍有已 claim 但未 pull 的 steering 输入，Host 追加 `steering.dropped` EventLog 记录，`metadata.steering.status="dropped"` 并引用 `source_event_id`；这不是用户消息事实的删除，只是 dispatch 终态。Transcript 继续只表示会话事实，不承担 dispatch 行为标记。 + +`state` 与 `storage` 的建议边界：`state` 放小型 JSON（conversation / actor / subject / runner），`storage` 放 blob 或较大数据（插件私有数据、workspace 数据、checkpoint）。 + +Compaction checkpoint 的推荐 state 约定： + +- scope: `conversation` +- key: `runner.compaction.checkpoint` +- value: + +```json +{ + "schema_version": "langbot.local_agent.compaction_checkpoint.v1", + "summary": "...", + "covers_until": "transcript-cursor-or-seq", + "tokens_before": 12345, + "created_at": 1710000000, + "conversation_id": "conv-..." +} +``` + +`covers_until` 是摘要覆盖到的 transcript 游标锚点。Runner 读取 checkpoint 后应只拉取该游标之后的 transcript；若 checkpoint 缺失、schema 不匹配、conversation 不匹配或游标不可用，应回退到无 checkpoint 的尾部历史拉取行为。 + +Proxy 返回数据结构也属于本协议： + +```python +class TranscriptItem(BaseModel): + transcript_id: str + event_id: str + conversation_id: str | None = None + thread_id: str | None = None + role: str + item_type: str = "message" + content: str | None = None + content_json: dict[str, Any] | None = None + attachment_refs: list[dict[str, Any]] = [] + seq: int | None = None + cursor: str | None = None + created_at: int | None = None + metadata: dict[str, Any] = {} + +class HistoryPage(BaseModel): + items: list[TranscriptItem] = [] + next_cursor: str | None = None + prev_cursor: str | None = None + has_more: bool = False + total_count: int | None = None + +class HistorySearchResult(BaseModel): + items: list[TranscriptItem] = [] + total_count: int | None = None + query: str + +class AgentEventRecord(BaseModel): + event_id: str + event_type: str + event_time: int | None = None + source: str + bot_id: str | None = None + workspace_id: str | None = None + conversation_id: str | None = None + thread_id: str | None = None + actor_type: str | None = None + actor_id: str | None = None + actor_name: str | None = None + subject_type: str | None = None + subject_id: str | None = None + input_summary: str | None = None + input_ref: str | None = None + raw_ref: str | None = None + seq: int | None = None + cursor: str | None = None + created_at: int | None = None + metadata: dict[str, Any] = {} + +class EventPage(BaseModel): + items: list[AgentEventRecord] = [] + next_cursor: str | None = None + prev_cursor: str | None = None + has_more: bool = False + total_count: int | None = None + +class SteeringInputItem(BaseModel): + claimed_run_id: str + runner_id: str + claimed_at: int | None = None + event: AgentEventContext + input: AgentInput + conversation: ConversationContext | None = None + actor: ActorContext | None = None + subject: SubjectContext | None = None + metadata: dict[str, Any] = {} + +class SteeringPullResult(BaseModel): + items: list[SteeringInputItem] = [] +``` + +## 9. 错误模型 + +```python +class AgentAPIError(BaseModel): + code: str + message: str + retryable: bool = False + details: dict[str, Any] = {} +``` + +| code | 说明 | +| --- | --- | +| `unauthorized` | 未授权访问资源或 scope。 | +| `not_found` | 资源不存在或对当前 runner 不可见。 | +| `deadline_exceeded` | 超过 run deadline。 | +| `payload_too_large` | 请求或响应过大。 | +| `rate_limited` | Host 限流。 | +| `invalid_argument` | 参数错误。 | +| `runtime_error` | Host 或下游能力错误。 | + +SDK runner-facing proxy 在 Host 返回结构化错误或畸形响应时抛出 `AgentAPIException`，其中 `error` 字段为 `AgentAPIError`。Legacy transport 只返回字符串错误时，SDK 使用 `host.action_error` 包装，避免 runner 继续依赖裸 `KeyError` 或字符串匹配。 + +Runner 失败使用 `run.failed`： + +```json +{ "type": "run.failed", "data": { "code": "runner.error", "error": "failed to call external agent", "retryable": false } } +``` + +## 10. Timeout 与 Cancellation + +- Host 在 `ctx.runtime.deadline_at` 下发总 deadline；SDK proxy 必须用该 deadline 限制单次 action timeout。 +- Host 可以取消 active run；Runtime 应尽力中断 runner。 +- Protocol v1 的 run 绑定当前 Host 进程和当前 runtime channel，不保证跨 Host 重启恢复。Host 重启、runtime channel 断开或 run session 丢失时，Runtime / external harness connector 必须 fail-fast 并尽力取消仍在执行的 runner，不得继续使用旧 `run_id` 调用 Host API。 +- Runner 支持中断时应返回或触发 `run.failed`，code 为 `cancelled`。 +- Host 必须 unregister active run session。 + +## 11. Security 与 Guardrail（协议层） + +Protocol v1 的安全边界在 Host： + +- Runner 不能直接访问未授权 model/tool/kb/history/storage/sandbox。 +- SDK 本地校验只提升开发体验，不能替代 Host 校验。 +- 所有 resource id 对 runner 来说都是 opaque。 +- 默认只能访问当前 conversation / thread 的 history；跨会话、workspace 级访问必须额外授权。 +- 大 payload 不应塞进 result event；当前 run 的文件和工具大结果应进入授权 sandbox/workspace，由 read/write/exec 类工具按需访问。 +- Host 必须记录 run_id、runner_id、action、resource、scope、result。 + +Host 不负责业务编排：不拼接全量历史、不替 runner 做 prompt assembly、不内置 agent memory / tool loop / 上下文压缩策略。这些由官方或第三方 Runner 插件实现。 + +外部 harness runner 的边界统一见 HOST_SDK §4.8。简言之：harness native permission mode、allowed/disallowed tools、shell/MCP 权限只是额外执行约束，不能替代 Host 对 LangBot 资源的授权。 + +> 发布级路径隔离、MCP allowlist、secret redaction、配额、workspace 清理等**不属于** v1 协议闭环，是生产默认启用前的 release gate，见 [SECURITY_HARDENING.md](./SECURITY_HARDENING.md)。 + +## 12. Pipeline AI Stage Adapter 边界 + +Pipeline 与 Agent 是 EBA 中平级的处理器：Pipeline 处理消息事件并执行完整 +Stage 链，Agent 处理其声明支持的消息或非消息事件。本协议只约束 Runner +调用，因此 Pipeline 仅在 AI Stage 调用 runner 时进入 Query entry adapter； +该适配不会把 Pipeline 变成 Agent，也不会创建或更新持久 Agent。adapter 负责： + +- 从 `Query` 构造 `AgentEventContext` 和临时 `AgentBinding`（见 HOST_SDK §4.2）。 +- 从当前 Agent/runner config 构造 `ctx.config`。 +- 将 Query-only 字段放入 `ctx.adapter`，例如 filtered params 放 `ctx.adapter.extra["params"]`。 + +约束： + +- adapter **不**定义历史窗口、prompt 组装或 agentic context 策略。 +- `ctx.adapter.extra` 只允许承载一次性、JSON-safe、入口相关的非核心元数据，例如 `params`；不得承载 `prompt`、history window、RAG 结果、tool schema 或授权资源。 +- 静态绑定 prompt 属于 `ctx.config.prompt`。preprocessing / hook 后的动态有效指令不通过 `ctx.adapter.extra` 主动推送；后续如需要保留这类能力，应通过 Host prompt/instruction pull API 暴露（占位见 HOST_SDK §4.8）。 +- 新 runner 不应长期依赖 `adapter`，应只依赖 event-first context 和 Host API。 + +## 13. 已确认约束 + +- EBA 路由层是 `one event -> one Processor target (Pipeline | Agent)`；同一 bot / channel 可以让不同事件绑定不同类型的处理器。 +- 进入 Runner Protocol 后，调用基数是 `one AgentBinding -> one run_id -> one runner`。这既适用于独立 Agent，也适用于 Pipeline AI Stage 的单次 runner 调用。 +- 一个 Agent 可以被多个 bot / channel 复用。如果 Agent 分支出现多个匹配 binding，BindingResolver 必须按明确规则选出一个或拒绝配置，不应默认 fan-out。 +- observer agent、多 runner fan-out、并行裁决、result 合并等能力需要单独设计 delivery、state、platform action 和 audit 语义，不属于当前 v1 契约。 +- `RunnerDescriptor.source` 只允许 `plugin`；Host 内置 adapter 不能作为 runner source 绕过插件/runtime/proxy 权限链。 +- `ctx.resources` 与 proxy action 校验必须来自同一个 run authorization snapshot；runtime handler 不应重新执行资源裁剪。 +- v1 不要求 Agent、Runner 插件实例或 runner id 全局串行。多个 bot / channel 可复用同一个 Agent；并发隔离依赖 `run_id`、binding、conversation / thread scope 和 Host authorization snapshot。 +- 外部 harness runner 当前是 MVP / dev path，证明协议可接入，不代表发布级安全边界或 Docker 生产可用性完成。 + +## 14. 开放问题 + +- `AgentBinding` 是否需要进入 SDK 文档作为只读诊断信息，还是完全 Host 内部。 +- State 与 Storage 的边界是否需要更强类型。 +- platform action 的审批模型如何表达。 +- Host 侧 scoped MCP / workspace projection 是否需要从 runner config 上移为一等 resource projection API。（skill 一项已收敛：skill 全 tool 化，作为被授权 tool 暴露，不再是独立 projection。） diff --git a/docs/agent-runner-pluginization/README.md b/docs/agent-runner-pluginization/README.md new file mode 100644 index 000000000..8d6708211 --- /dev/null +++ b/docs/agent-runner-pluginization/README.md @@ -0,0 +1,156 @@ +# Runner 插件化文档入口 + +本文档是 agent-runner 插件化工作的路由页。具体设计拆到独立文档中维护，避免把 LangBot 宿主架构、SDK 协议、上下文管理、EBA 接入边界和官方 runner 迁移混在同一份 README 里。 + +## 背景与问题 + +旧 runner 路径主要围绕 Pipeline / Query 和 `pkg/provider/runners` 内置实现展开，扩展外部 agent runtime 时容易把 runner 选择、上下文裁剪、资源授权和消息投递绑在同一条聊天链路里。这个分支要把 LangBot 收敛成 Agent Host：Host 负责事件、绑定、授权、事实源和结果投递；Runner 作为插件或外部 harness 消费统一协议并自主管理 prompt / history / memory。 + +## 文档维护原则（单一事实源） + +- **协议数据结构（schema）唯一定义在 [PROTOCOL_V1.md](./PROTOCOL_V1.md)。** 其他文档不得重抄 schema，只能引用，例如"见 PROTOCOL_V1 §4.2"。 +- 当前实现状态、spec 差距与 runner 验收状态归 [STATUS.md](./STATUS.md)；测试执行入口归 [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md)，安全发布门槛归 [SECURITY_HARDENING.md](./SECURITY_HARDENING.md)。 +- Host 内部模型（`AgentEventEnvelope`、`AgentBinding`、Descriptor、各 Store）定义在 [HOST_SDK_INFRASTRUCTURE.md](./HOST_SDK_INFRASTRUCTURE.md)，不属于 SDK 协议。 +- 其余专题文档只讲"为什么/边界/怎么用"，避免重复叙述。 + +## 本分支目标 + +**本分支目标：Runner 外化 / 插件化基础设施** + +本分支只做 LangBot 作为 Agent Host 的基础能力建设，让现有 Pipeline 与用户新建的独立 `Agent` 都能调用插件化 Runner；不负责把两者做持久化迁移： + +- LangBot 与 SDK 的稳定协议合同（Protocol v1） +- Host-side `AgentEventEnvelope` / `AgentBinding` 模型 +- `run(event, binding)` event-first 入口 +- `QueryEntryAdapter`：Query → AgentEventEnvelope + AgentBinding +- EventLog / Transcript / PersistentStateStore +- History / Event / State pull APIs +- Sandbox/workspace read/write/exec 文件能力，用于当前 run 的上传文件、工具大结果和临时产物 +- SDK runtime forwarding pull APIs + `caller_plugin_identity` 验证路径 + +## 当前已集成与后续扩展 + +截至 2026-09-05，`dev/4.11.x` 已合并 EBA 与 Runner 插件化。下面按当前代码划分实现边界： + +- **已实现的平台事件路由**：`RuntimeBot` 负责事件转换后的 observer 广播、`event_bindings` 匹配和 Pipeline / Agent / discard 单目标分派；EventRouter 是逻辑职责，不是另一个独立服务。 +- **已实现的持久化与 UI**：独立 Agent、Bot 事件绑定、处理器工作台、Runner 市场安装、事件范围与工具权限、路由诊断和调试入口。 +- **后续的通用事件订阅与通知**：不把当前适配器回调和 Bot 路由理解成通用订阅产品。 +- **后续的 Scheduler / Background event source**：用户可配置的定时自动化和后台任务入口。 +- **完整 Agent Platform / daemon control plane**：Host-owned `AgentRun` / `AgentRunEvent`、run control primitives、最小 runtime heartbeat/claim lease 已作为 v2 foundation 落地；业务队列、Platform UI、daemon supervisor、runtime wakeup channel 和分布式 runtime 管控仍不属于 Protocol v1 主线。 + +平台事件、Agent 配置及 Pipeline AI Stage 已共同使用 host-side envelope/binding models 和 `run(event, binding)`。后续入口应复用这条链路。当前实现与发布验收分别见 [STATUS.md](./STATUS.md)，平台动作授权见 [PLATFORM_ACTION_TOOLS.md](./PLATFORM_ACTION_TOOLS.md)。 + +本分支与外部 EBA / Agent Platform / Runtime Control Plane 的扩展边界见 [EXTENSION_SCOPE_MATRIX.md](./EXTENSION_SCOPE_MATRIX.md)。 + +## 目标产品模型 + +产品层同时保留 Pipeline 与独立 `Agent`。现有 Pipeline 保持原实体和执行链，也可继续被 bot 绑定；新建 Agent 则携带 runner id、runner config、resource/state/delivery policy 等配置，并可被多个 bot / IM channel 复用。统一产品列表可以聚合展示两类处理器，但不会把 Pipeline 或其内嵌 runner 配置自动迁移成 Agent；用户需要 Agent 时自行新增并绑定。 + +调度基数、Agent 复用、插件实例无状态、Pipeline adapter 和 fan-out 边界的规范来源是 [PROTOCOL_V1.md](./PROTOCOL_V1.md) §13；README 不复写这些约束。 + +## Pipeline 与 Runner 的关系 + +**Pipeline 与 Agent 是 EBA 中平级的处理器；`QueryEntryAdapter` 只适配 Pipeline 内部的 Runner 调用。** + +EBA 先根据 `target_type` 选择 Pipeline 或 Agent。Pipeline 目标执行完整 Stage 链；当 Pipeline 的 AI Stage 调用 runner 时，`run_from_query()` 经 `QueryEntryAdapter` 把 `Query` 转换为 `AgentEventEnvelope` + `AgentBinding`，再委托到统一的 `run(event, binding, ...)`。Agent 目标则直接从自己的持久配置构造 binding。两条路径可以复用同一套 Runner Host capabilities，但 Pipeline 本身不会被投影或持久化为 Agent。 + +下一轮测试路径、状态定义和 smoke 记录见 [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md)。 + +## 术语表 + +| 术语 | 含义 | +| --- | --- | +| Protocol v1 | Host 调用 Runner 的 runner 可见合同：discovery、`RunnerContext`、result stream、Host pull API 和错误模型。 | +| Processor | EBA 的处理器上位概念；当前平级类型为 Pipeline 与 Agent。 | +| Agent | 目标产品层配置对象，保存 runner id、runner config 和资源/状态/投递策略；不等于插件实例。 | +| AgentConfig | Host 内部的单次 Runner 调用配置投影，可由 Pipeline AI Stage 或持久 Agent 生成；投影本身不会创建 Agent。 | +| AgentBinding / binding | Host 在一次事件运行前解析出的有效绑定，决定调用哪个 runner 以及带什么策略。 | +| envelope | Host 内部事件封装，即 `AgentEventEnvelope`；runner 看到的是由它投影出的 `ctx.event`。 | +| descriptor / manifest | runner discovery 的能力和配置描述；manifest 来自插件，descriptor 是 Host 校验后的注册表视图。 | +| EBA | Event Based Agent；平台事件路由已集成，定时任务等通用事件源仍是后续扩展。 | +| harness runner | ACP、Claude Code、Codex 等已有自身 session / tool loop / MCP / 压缩机制的外部 runtime adapter。 | +| projection | Host 把内部事实源、授权资源或配置裁剪成 runner / harness 可消费视图的过程。 | +| Runtime Control Plane | v2 Host 能力层，当前已落地 Host-owned run/result ledger、run control primitives、最小 runtime heartbeat/claim lease；完整 daemon worker 管控、task wakeup 和 Agent Platform 产品形态不是 Protocol v1 主线。 | + +## 设计文档 + +| 文档 | 关注点 | +| --- | --- | +| [PROTOCOL_V1.md](./PROTOCOL_V1.md) | **🔒 唯一 schema 事实源**。LangBot Host 与 SDK / Runtime / Runner 的协议合同：版本协商、discovery、run context、result stream、proxy actions、错误和 adapter 边界。 | +| [HOST_SDK_INFRASTRUCTURE.md](./HOST_SDK_INFRASTRUCTURE.md) | LangBot 宿主能力与分层架构、Host 内部模型（`AgentEventEnvelope` / `AgentBinding` / Descriptor / 各 Store）、runner 发现、绑定、资源授权、状态、存储、生命周期和调用链。 | +| [AGENT_CONTEXT_PROTOCOL.md](./AGENT_CONTEXT_PROTOCOL.md) | Agent-owned context 方向：事件到来时 LangBot 传什么，agent 如何按需拉取更多历史 / state、如何访问 sandbox/workspace 文件，以及如何支持 KV cache 友好的上下文管理。 | +| [EXTENSION_SCOPE_MATRIX.md](./EXTENSION_SCOPE_MATRIX.md) | Runner 外化与外部 EBA / Agent Platform / Runtime Control Plane 的扩展边界矩阵，说明哪些是本分支底座、哪些由外部分支接入。 | +| [EVENT_BASED_AGENT.md](./EVENT_BASED_AGENT.md) | 已集成的事件路由、处理器分派、平台工具和结构化交互边界。 | +| [eba-productization-release.md](./eba-productization-release.md) | EBA 适配器与 Runner 插件化合并后的产品化 / 发布计划，说明非技术用户快速上手差距、Bot 与处理器边界、未来 Solution 分发标的，以及多 namespace SaaS 支持要求。 | +| [RUNTIME_CONTROL_PLANE_V2.md](./RUNTIME_CONTROL_PLANE_V2.md) | Agent Platform v2 / runtime 管控面决策：`AgentRun` / `AgentRunEvent` / run control 已作为 Host 事实源落地，最小 runtime heartbeat/claim lease 已落地；完整 runtime registry / daemon 管控仍是后续可选阶段。 | +| [OFFICIAL_RUNNER_PLUGINS.md](./OFFICIAL_RUNNER_PLUGINS.md) | 官方 runner 插件迁移，包括 local-agent 和外部 runner。它是下游落地计划，不是 LangBot 基础能力设计的前置约束。 | +| [RUN_STEERING_AND_CHECKPOINT.md](./RUN_STEERING_AND_CHECKPOINT.md) | 运行中消息注入（steering / follow-up）与压缩摘要持久化（compaction checkpoint）的设计与落地状态记录；schema 仍以 PROTOCOL_V1 为准。 | +| [STATUS.md](./STATUS.md) | 当前实现状态、spec 与实现已知差距、runner 验收状态和历史高价值记录。 | +| [PLATFORM_ACTION_TOOLS.md](./PLATFORM_ACTION_TOOLS.md) | 当前事件工具、平台动作和普通工具的配置、投射及执行授权。 | +| [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md) | Runner QA 指南：保留最高价值测试路径，指导 agent 开展下一轮 WebUI / runner smoke 验证。 | +| [SECURITY_HARDENING.md](./SECURITY_HARDENING.md) | 安全发布级 hardening 的后续发布门槛：路径隔离、权限边界、secret、资源配额、MCP / skill 投影和审计。 | + +## 工作拆分 + +### 1. LangBot + SDK 基础设施 + +目标是把 LangBot 从内置 runner 执行器变成 agent host： + +- LangBot 与 SDK 的稳定协议合同 +- runner manifest / descriptor / registry +- Agent / binding 配置解析 +- run orchestration 和生命周期管理 +- resource authorization 与 `run_id` 级权限校验 +- host-owned state / storage / event log / transcript 能力 +- sandbox/workspace 文件 staging 与 read/write/exec 能力 +- SDK `Runner`、`RunnerContext`、`RunnerResult`、`RunnerAPIProxy` + +协议合同详见 [PROTOCOL_V1.md](./PROTOCOL_V1.md)。 + +详见 [HOST_SDK_INFRASTRUCTURE.md](./HOST_SDK_INFRASTRUCTURE.md)。 + +### 2. Agent-owned context + +LangBot 不应成为最终 agentic context manager。它应提供事实源、默认上下文引用和按需读取 API；agent 或其背后的 runtime 负责历史剪裁、摘要、召回和 KV cache 策略。 + +Host 不定义通用历史窗口字段或策略；runner 通过 Host pull API 按需拉取历史并自行管理 working context。 + +详见 [AGENT_CONTEXT_PROTOCOL.md](./AGENT_CONTEXT_PROTOCOL.md)。 + +### 3. Event Based Agent（已集成） + +消息只是事件的一种。当前平台适配器中的 `message.received`、`message.deleted`、`group.member_joined`、`friend.request_received` 等事件通过统一事件 envelope 进入处理器路由，具体支持范围以适配器能力声明为准。 + +EBA dispatch 的基数和 fan-out 边界仍以 PROTOCOL_V1 §13 为准；新增事件源复用以下入口点。 + +**当前共同使用的执行底座：** +- event-first envelope (`AgentEventEnvelope`) +- AgentBinding model +- `run(event, binding)` 入口 +- QueryEntryAdapter（当前 AgentEventEnvelope / AgentBinding 的 Query entry adapter source） + +详见 [EVENT_BASED_AGENT.md](./EVENT_BASED_AGENT.md)。 + +### 4. 官方 runner 插件 + +官方 `local-agent` 和外部 runner 迁移是下游工作。它们需要依附 LangBot 提供的宿主能力，但不应反过来决定宿主协议。 + +`local-agent` 可以外移，也可以重写。验收重点是它能完整消费 LangBot 的模型、工具、知识库、存储、事件、history API 和 result stream，而不是保留旧内置 runner 的内部结构。 + +详见 [OFFICIAL_RUNNER_PLUGINS.md](./OFFICIAL_RUNNER_PLUGINS.md)。 + +### 5. Runtime Control Plane v2（Foundation Partial） + +当前 Runner v1 主线仍以 `event -> binding -> runner.run(ctx) -> result stream` 为 runner 可见合同。Host 侧已经新增持久 `AgentRun` / `AgentRunEvent`、result persistence、cancel/finalize/query 等通用 run control primitives，并提供受权限保护的最小 runtime register/heartbeat/list、claim/renew/release 和 reconcile 原语。 + +在这些 Host 能力之上，可以构建独立 agent 管控面插件；插件负责 UI、策略和编排体验，runtime/task 的事实源仍由 Host 持有。完整 daemon supervisor、任务唤醒/长轮询/WebSocket、跨 Host 分布式锁、provider 登录态诊断和产品化业务队列仍是后续工作。 + +详见 [RUNTIME_CONTROL_PLANE_V2.md](./RUNTIME_CONTROL_PLANE_V2.md)。 + +## 约束事实源 + +本分支已确认约束不在 README 重写： + +- Runner 可见协议、result stream 和调度边界见 [PROTOCOL_V1.md](./PROTOCOL_V1.md)。 +- Host 内部 `AgentConfig` / `AgentBinding` 投影见 [HOST_SDK_INFRASTRUCTURE.md](./HOST_SDK_INFRASTRUCTURE.md)。 +- 外部 EBA / Agent Platform / Runtime Control Plane 接入边界见 [EXTENSION_SCOPE_MATRIX.md](./EXTENSION_SCOPE_MATRIX.md)。 diff --git a/docs/agent-runner-pluginization/RUNTIME_CONTROL_PLANE_V2.md b/docs/agent-runner-pluginization/RUNTIME_CONTROL_PLANE_V2.md new file mode 100644 index 000000000..6f43ec877 --- /dev/null +++ b/docs/agent-runner-pluginization/RUNTIME_CONTROL_PLANE_V2.md @@ -0,0 +1,544 @@ +# Agent Platform / Runtime Control Plane Decision Note + +本文档记录 Runner 插件化之后，LangBot 如何继续演进成 Agent Platform 基础设施层。这里讨论的是 Host capability layer，不是 `Runner Protocol v2`，也不是把某个具体 Agent Platform 产品写进 LangBot core。 + +> 本文是当前决策版。协议数据结构仍以 [PROTOCOL_V1.md](./PROTOCOL_V1.md) 为准；测试执行入口见 [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md)；扩展边界见 [EXTENSION_SCOPE_MATRIX.md](./EXTENSION_SCOPE_MATRIX.md)。 +> +> 实现状态说明：本文描述的是 Runtime Control Plane v2 的目标能力和分阶段落地建议。当前 Runner 插件化主线已经具备 event-first context、run-scoped authorization、EventLog / Transcript / State / sandbox 文件等 Host capability，并已落地持久 `AgentRun` / `AgentRunEvent` ledger、run control actions、最小 runtime heartbeat/claim lease 和 admin reconcile 原语。完整 Agent Platform 产品形态、daemon supervisor、runtime wakeup channel 和分布式 runtime 管控仍未完成。当前实现状态以 [STATUS.md](./STATUS.md) 为准。 + +## 1. 当前决策 + +LangBot 后续定位应更像 **Agent Host / infrastructure provider / transfer layer**，而不是把某个完整 Agent Platform 产品固化进 core。 + +结论： + +- **Agent Platform 产品形态做成插件**。插件负责 agent 管理、策略、业务队列、UI、编排、多 agent 协作和产品体验。 +- **Agent Platform 所需的基础事实源做进 Host**。当前 Host 已保存 event、state、transcript、sandbox 文件边界、active run 权限快照、持久 run/result ledger、审计关联和通用控制状态。 +- **最小 runtime registry / heartbeat / claim lease 已作为 Host 原语落地，但不等于完整 daemon worker 管控**。远程 harness / daemon 的进程托管、wakeup channel、provider 登录态诊断和分布式调度仍可以先由 Runner 插件和 SDK remote layer 自己维护。 +- **不把业务调度写进 Host**。Host 提供通用 run/result/control primitives，Platform 插件决定哪些事件触发哪些 agent、如何排队、如何分配、是否 fan-out。 + +推荐分层： + +```text +LangBot Host + Current base: EventLog / runtime AgentBinding / State / Transcript / sandbox files / active run authorization + Current v2 foundation: Run / RunEvent / audit / result persistence / control primitives / minimal runtime heartbeat and claim lease + Current product: persisted Agent / Bot event_bindings / processor UI / event routing + Planned: external harness daemon supervisor / wakeup channel / distributed runtime operations + +Agent Platform plugin + Agent management UI / project-task model / event routing policy + Business queue / multi-agent orchestration / runtime selection policy + +Runner plugin / external harness runtime + Connects ACP / remote daemon / local subprocess / HTTP API + Executes and converts provider-native events to RunnerResult +``` + +## 2. Platform 与非 Platform 的区别 + +当前 LangBot 已经具备 Agent Host 的核心特征： + +- 抹平不同 Runner。 +- 从 IM / Pipeline 入口触发 runner。 +- 有 event-first context 方向。 +- 有 Host-owned EventLog / Transcript / State 和 sandbox/workspace 文件边界。 +- 有 runner config 下发和 active run-scoped authorization。 +- 有 `run_id` 串联 event、transcript、state、sandbox 文件和内存授权上下文。 + +这还不是完整 Agent Platform。完整 Platform 至少还需要： + +- 可管理的 agent 资产：agent profile、binding、resource policy、runner config、可用状态。 +- 可观察的执行生命周期：run status、result stream、失败原因、文件引用、审计、回放。 +- 可运营的控制面：取消、重试、排队、并发、超时、恢复、诊断。 +- 可产品化的调度体验：事件订阅、路由策略、任务板、多 agent 协作、项目/工作区视图。 + +因此，区别不只是“有没有调度”，而是是否具备： + +```text +managed agent assets + observable run lifecycle + operational run control +``` + +Host 负责这些能力的通用事实源和安全边界；Platform 插件负责把它们组装成具体产品。 + +### 2.1 当前实现边界 + +当前代码中的 `run_id` 已经连接 active run 授权、持久 run ledger 和多个 Host 事实源： + +- `EventLog` 保存输入事件和审计入口，并记录 `run_id` / `runner_id`。 +- `Transcript` 保存对话历史投影，并用 `run_id` 关联 assistant 输出。 +- Sandbox/workspace 保存当前运行输入文件和 runner 产物，并用 `run_id` 做访问边界的一部分。 +- `PersistentStateStore` 保存 runner state，但不等同于 run lifecycle。 +- `AgentRunSessionRegistry` 保存 active run 的内存态授权快照，用于 proxy action 校验；进程结束或 run 结束后不作为可回放事实源。 +- `AgentRun` 保存 run lifecycle、scope、authorization snapshot、queue/claim 状态、cancel intent、usage/cost 和 metadata。 +- `AgentRunEvent` 保存 runner/result/admin event stream，按 `run_id + sequence` 做可回放分页。 +- `AgentRuntime` 保存最小 runtime registry / heartbeat 事实，用于 runtime list、stale mark 和 claim lease reconcile。 + +因此本文后续提到的 `AgentRun` / `AgentRunEvent`、`run_append_result`、`run_finalize`、`run_cancel`、`runtime_register`、`runtime_heartbeat`、`run_claim` 等基础原语已经存在。2026-09-05 核对：独立 Agent 持久模型、Bot `event_bindings` 和处理器 UI 也已实现；`AgentBinding` 仍是单次运行投影，不是独立配置表。仍未完成的是独立 platform `run_create` action、业务队列产品形态、外部 harness daemon supervisor、runtime wakeup channel、跨 Host 分布式锁和 provider/runtime 诊断面。 + +SDK Plugin Runtime 已有 installation worker 的 supervisor、重启退避和 Runtime 重启协调器；这与这里规划的外部 Agent harness 进程托管不同。当前模型和状态以 [STATUS.md](./STATUS.md) 为准，本文后续阶段是能力拆分，不能作为尚未实现功能的清单。 + +## 3. 基础概念 + +### 3.1 Event + +Event 表示“发生了什么”： + +```text +message.received +github.issue.opened +scheduler.tick +user.approved +system.webhook.received +``` + +EBA 负责把外部输入标准化成 event。Event 本身不是 queue，也不等同于一次 agent 执行。当前 `EventLog` 记录的是输入事件和审计事实；未来 `AgentRunEvent` 记录的是某次 run 的输出事件流，二者不能混用。 + +### 3.2 Run + +Run 表示“某个 agent / binding / runner 针对某个 event 的一次执行”。 + +Run 应由 Host 持久化，成为执行状态、结果、权限和审计的事实源： + +```text +run_id +event_id +agent_id / binding_id +runner_id +status +created_at / started_at / finished_at +error / failure_reason +delivery target +metadata +``` + +当前 `AgentRunSessionRegistry` 只保存 active run 的内存态授权信息，不足以支撑 Platform 的回放、审计、取消、重试和异步执行。 + +### 3.3 RunEvent / RunResult + +RunEvent 是一次 run 过程中产生的结果事件流，对应 runner 返回的 `RunnerResult`。它不同于 EBA/EventLog 的输入事件： + +```text +message.delta +message.completed +tool.call.started +tool.call.completed +state.updated +action.requested +run.completed +run.failed +``` + +Host 应保存这些输出事件，按 `run_id + sequence` 可回放。Transcript、State 可以由这些 result event 触发写入现有 store，并保留能回溯到 `AgentRunEvent` 的关联。文件和工具大结果留在当前 run 的 sandbox/workspace 中，不作为 result event blob 回传。 + +### 3.4 Queue + +Queue 不是 EBA 的替代品。 + +EBA 负责产生 event；queue 负责处理“这个 event 对应的执行 work item 何时执行、谁来执行、如何取消/重试/恢复”。 + +队列可以分两层： + +- **业务队列**：由 Platform 插件管理，例如项目任务、优先级、agent team、workflow、人工审批。 +- **执行队列 / run queue**：可选 Host 原语，例如 queued / running / completed / failed / cancelled、claim lease、dispatch timeout、orphan recovery。 + +第一阶段不要求 Host 内置完整执行队列。Platform 插件可以先管理业务队列；在 Phase 1 / Phase 2 能力落地前，插件仍只能通过现有 `AgentRunOrchestrator.run(...)` 同步执行路径和现有 Host stores 获得有限的 run 关联能力。 + +### 3.5 Runtime / Daemon + +Runtime / daemon 表示执行位置或执行能力，例如某台机器上的 Claude Code / Codex CLI。 + +当前决策： + +- Host 不在第一阶段维护完整 runtime registry。 +- Runner 插件可以通过 SDK remote layer 与 daemon 保持连接、心跳和执行通道。 +- 外部 harness / agent 不应直接访问 LangBot Host 或数据库。访问 LangBot 资源必须通过 daemon / Runner plugin / SDK runtime / `RunnerAPIProxy` / scoped MCP bridge，并接受 run-scoped authorization 校验。 +- 如果后续多个插件都需要共享 runtime 状态，再把薄的 `RuntimeLease` / registry 下沉为 Host 通用能力。 + +## 4. Host 应新增的最小能力 + +第一阶段最重要的不是 daemon registry，而是让 Host 成为 run/result 的事实源。 + +### 4.1 AgentRun Store + +新增持久 `AgentRun`： + +```text +id / run_id +event_id +agent_id +binding_id +runner_id +conversation_id / thread_id +workspace_id / bot_id +status +status_reason +created_at / started_at / finished_at / updated_at +deadline_at +cancel_requested_at +usage_json +cost_json +metadata_json +``` + +建议 status 至少包含： + +```text +created +running +completed +failed +cancelled +timeout +``` + +如果后续加执行队列，再引入： + +```text +queued +claimed +dispatching +``` + +### 4.2 AgentRunEvent Store + +新增持久 `AgentRunEvent`： + +```text +id +run_id +sequence +type +data_json +usage_json +created_at +source +metadata_json +``` + +约束： + +- 同一 `run_id` 内 `sequence` 单调递增。 +- append 必须幂等，支持远程 daemon / plugin 重试。 +- 未知 result type 可保存但 Host 只对已知类型执行副作用。 +- 大 payload 仍应进入 sandbox/workspace，不直接塞入 result event。 +- `usage_json` 保存 `RunnerResult.usage` 原样结构；缺失表示 unknown，不等于 0。 + +### 4.3 Run Control API + +Host 提供通用控制原语： + +```text +run.create +run.get +run.list +run.events.page +run.cancel +run.append_result +run.finalize +``` + +语义： + +- `run.create` 创建 Host-owned run 和授权快照。 +- `run.append_result` 只允许受信 SDK/runtime 路径调用，必须绑定 run 创建时固化的授权快照，写入 `AgentRunEvent` 并触发 transcript/state/delivery 副作用。 +- `run.finalize` 关闭 run，更新 terminal status。 +- `run.cancel` 设置取消意图；同步 runner 通过 context/deadline 感知，远程 runner 通过插件/daemon 通道感知。 + +第一阶段可以只暴露给插件 runtime action，不一定先做公开 HTTP API。 + +### 4.4 Result Persistence In Orchestrator + +当前 `AgentRunOrchestrator.run()` 已经处理： + +```text +event -> binding -> context -> runner invocation -> result normalization +``` + +需要补齐： + +- run 开始时创建 `AgentRun`。 +- 每个 `RunnerResult` 进入 `AgentRunEvent`。 +- `run.completed` / 正常 generator 结束时标记 completed。 +- `run.failed` / exception / timeout 标记 failed 或 timeout。 +- terminal result 携带 usage 时，写入 `AgentRunEvent.usage_json` 并汇总到 `AgentRun.usage_json`。 +- `state.updated`、transcript 写入继续走现有 journal，但应与 `AgentRunEvent` 有可追踪关系。 + +### 4.5 Usage / Cost Accounting + +SDK 侧 `RunnerResult` 已提供可选 `usage` 字段，用于把不同 runner / external harness / provider-native event 的 token usage 归一到同一个 run result envelope。 + +语义： + +- `run.completed.usage` SHOULD 表示本次 run 的最终聚合 token usage。 +- `run.failed.usage` MAY 表示失败前已知的部分 token usage。 +- 没有 usage 表示 upstream runtime 没有报告或 adapter 暂未接入；Host 不得按 0 计费或按 0 判断上下文消耗。 +- Host 应把 event-level usage 原样写入 `AgentRunEvent.usage_json`，并在 terminal event 或 finalize 阶段汇总到 `AgentRun.usage_json`。 +- cost 应由 Host 根据 usage、runner/model identity、发生时间和价格表计算，写入 `AgentRun.cost_json`；runner/provider 上报的 cost 只能作为非权威 telemetry 保留在 metadata 或 usage extra 中。 + +这层约束先解决协议位置和持久化位置；具体 ACP、remote daemon、local subprocess runner 如何从 native event 中抽取 usage，可在各插件后续适配。 + +### 4.6 Authorization Snapshot + +异步或远程执行时，run 创建时必须固化授权快照： + +- runner identity +- binding identity +- caller plugin identity +- resource policy +- allowed tools/models/files/knowledge bases/storage scopes +- state scopes +- conversation/thread/workspace scope + +后续 append result、state API、history API 和 sandbox/workspace 文件访问都以这个 snapshot 校验，不重新扩大权限。 + +## 5. SDK 侧应新增的最小能力 + +SDK 不需要马上定义完整 daemon registry，但需要让插件和 runner 使用 Host run/result 能力。 + +### 5.1 Entities + +新增或补齐： + +```text +AgentRun +AgentRunStatus +AgentRunEvent +RunEventPage +RunCreateRequest / RunCreateResult +RunAppendResultRequest +``` + +这些是 Host control primitives，不替代 `RunnerContext` / `RunnerResult`。 + +### 5.2 Proxy Methods + +在 SDK proxy 中提供： + +```python +create_run(...) +get_run(run_id) +list_runs(...) +page_run_events(run_id, cursor=None, limit=...) +cancel_run(run_id) +append_run_result(run_id, result, sequence=None) +finalize_run(run_id, status, error=None) +``` + +访问边界： + +- 普通 Runner 在同步 `run(ctx)` 内不一定需要直接调用这些 API；Host orchestrator 可自动记录。 +- Platform 插件可以创建/查询/取消 run。 +- Runner 插件或 daemon bridge 可以 append/finalize 自己负责的 run。 +- 外部 harness 仍不能直接调用 Host；必须经 SDK runtime / proxy / bridge。 + +### 5.3 Plugin-Daemon Heartbeat + +远程 daemon 的初始心跳可以是 SDK / Runner plugin 私有能力： + +```text +daemon <-> Runner plugin / SDK remote layer <-> LangBot plugin runtime <-> Host +``` + +Host 第一阶段只需要知道： + +- 相关插件是否在线。 +- run 是否有 progress/result。 +- run 是否超时或取消。 + +如果后续需要跨插件共享 daemon 可用性，再把 heartbeat/registry 下沉为 Host 能力。 + +## 6. Platform 插件应负责什么 + +Agent Platform 插件可以负责： + +- 管理哪些 agent 可用。 +- 维护产品层 agent profile、项目、任务板、workflow、team。 +- 订阅 EBA event，决定哪些 event 触发哪些 agent。 +- 维护业务 queue：优先级、重试策略、人工审批、分配规则。 +- 选择 runner / runtime / daemon。 +- 在 Run Control API 落地后，调用 Host run API 创建、取消、查询执行。 +- 展示 run status、result stream、文件引用、失败原因和审计。 + +Platform 插件不应负责： + +- 在 Host Run Ledger 落地后，私有保存通用 run/result 事实源。 +- 绕过 Host 直接写 transcript/state 或越权访问 sandbox/workspace 文件。 +- 让外部 harness 直接访问 LangBot DB 或 Host 内部资源。 +- 把某个业务队列语义强塞进 Runner Protocol v1。 + +## 7. 与 EBA 的关系 + +EBA 做好后，事件流可以进入两种路径。 + +直接执行路径： + +```text +EventGateway + -> EventRouter resolves AgentBinding + -> AgentRunOrchestrator.run(event, binding) + -> Host records AgentRun / AgentRunEvent (after Run Ledger lands) + -> delivery +``` + +Platform 插件编排路径： + +```text +EventGateway + -> Platform plugin receives/subscribes event + -> plugin applies policy / business queue + -> plugin creates Host run (after Run Control API lands) + -> runner/plugin/daemon executes + -> Host records result and state + -> plugin displays / Host delivers +``` + +这两条路径最终应共享 Host run/result/state 事实源和 sandbox/workspace 文件边界。当前阶段可共享的是 event/transcript/state、sandbox 文件和同步执行链路；持久 run/result ledger 需要 Runtime Control Plane v2 Phase 1 补齐。区别在于是否有 Platform 插件参与产品化调度和业务队列。 + +## 8. 与 Runner Protocol v1 的关系 + +本设计不改变 v1 的 runner 可见合同： + +```text +RunnerContext -> Runner.run(ctx) -> RunnerResult stream +``` + +必须保持： + +- `RunnerContext` 不塞入 daemon/worker/pod 细节。 +- `RunnerResult` 仍是 runner 输出的统一事件流。 +- 普通 runner 不需要知道 task queue / runtime registry。 +- 远程 harness 可以自管 session、tool loop、MCP、上下文压缩，但访问 LangBot 资源必须通过 SDK proxy / bridge。 +- Runtime-managed execution 是 placement / transport 选择，不是普通 runner 协议的强制概念。 + +## 9. 分阶段实施建议 + +### Phase 1: Run Ledger（Foundation Implemented） + +目标：Host 成为执行状态和结果事实源。 + +范围： + +- `AgentRun` 表。 +- `AgentRunEvent` 表。 +- Orchestrator 自动创建/更新 run。 +- Journal 持久化每个 `RunnerResult`。 +- Run 查询和事件分页 API。 +- SDK entities + proxy 方法。 + +复杂度：中等。 + +预计改动： + +```text +Host: 12-20 个文件 +SDK: 4-8 个文件 +Tests: 8-15 个文件 +``` + +### Phase 2: Platform Plugin Queue On Host Run Primitives（Control Primitives Partially Implemented; Product Queue Pending） + +目标：Platform 插件管理业务 queue，Host 提供 run/result/cancel 原语。 + +范围： + +- `run.create` +- `run.cancel` +- `run.append_result` +- `run.finalize` +- result append 的 sequence/idempotency。 +- 受权限保护的远程 append/finalize。 +- Platform 插件可基于 Host run 构建任务板和调度体验。 + +复杂度：中等偏高。 + +预计改动： + +```text +Host: 20-35 个文件 +SDK: 8-14 个文件 +Tests: 15-25 个文件 +``` + +### Phase 3: Optional Host Execution Queue / Claim Lease（Claim Lease Primitive Implemented; Full Queue Pending） + +目标：当多个插件重复实现 claim/cancel/retry/recovery 时，再下沉执行队列到 Host。 + +范围： + +- `queued/running/completed/failed/cancelled` 状态机扩展。 +- `claim_run` / `lease_until`。 +- dispatch timeout。 +- retry / orphan recovery。 +- cancel propagation。 +- 并发 claim 防重。 + +复杂度：高。 + +预计改动： + +```text +Host: 35-55 个文件 +SDK: 12-20 个文件 +Tests: 25-40 个文件 +``` + +### Phase 4: Optional Runtime Registry（Minimal Registry Implemented; Full Daemon Control Pending） + +目标：当 Host 需要统一管理多个 daemon / worker 时，再引入 runtime registry。 + +范围： + +- runtime register / heartbeat / deregister。 +- capability report：provider、version、login status、workspace access、slot。 +- runtime online/offline。 +- runtime scoped auth。 +- runtime audit。 +- runtime gone recovery。 +- task wakeup / long polling / websocket。 +- 多 Host 实例下的 relay / distributed lock。 + +复杂度：很高。 + +预计改动： + +```text +Host: 55-80+ 个文件 +SDK: 18-30 个文件 +Tests: 40+ 个文件 +``` + +不建议现在直接进入此阶段。 + +## 10. 设计原则 + +- 先把 run/result 事实源做进 Host，再谈完整 runtime control plane。 +- Agent Platform 产品做插件；Host 做基础设施。 +- Host 不写业务调度策略，但要保存通用状态、结果、权限和审计。 +- EBA event 不是 queue；queue 是执行生命周期问题。 +- 业务 queue 可以先在 Platform 插件里；执行 queue 只有在复用需求明确后再下沉 Host。 +- Daemon registry 不应污染 Runner Protocol v1。 +- 外部 harness 不直接访问 LangBot Host 或 DB。 +- 所有 LangBot 资源访问必须走 SDK runtime / `RunnerAPIProxy` / scoped MCP bridge。 +- Docker / remote / local subprocess 只是 runtime placement，不是 runner 协议差异。 + +## 11. 非目标 + +当前阶段不做： + +- 完整 Multica 式 runtime registry。 +- Host 内置项目管理、任务板、agent team、workflow 产品逻辑。 +- 把 daemon heartbeat / worker liveness 放进 `RunnerContext`。 +- 把业务 queue 定义为 Runner Protocol 字段。 +- 让 Platform 插件私有保存 run/result 事实源。 +- 让外部 agent/harness 直连 Host 内部资源。 + +## 12. 待定问题 + +- 已确认：Agent 与 Pipeline 都持久存在，并由 EBA 处理器 binding 指向其中之一；Pipeline 仅在 AI Stage 调用 runner 时投影一次性 `AgentBinding`，不会充当持久 Agent 的替代物。 +- Platform 插件创建 run 时，是否传完整 `AgentBinding` snapshot，还是引用 Host-owned binding id。 +- `AgentRunEvent` 与现有 `EventLog` / `Transcript` 的查询关系：直接 join，还是通过专门 view 聚合。 +- `run.append_result` 的认证粒度：runner plugin identity、run token、scoped capability token，或 SDK runtime 内部 channel。 +- 取消语义：同步 runner、external harness runtime/session 如何统一感知 cancel。 +- 何时把插件私有 daemon heartbeat 提升为 Host `RuntimeLease`。 +- 若未来 Host 做 claim lease，Platform 插件业务 queue 与 Host execution queue 如何避免双队列混乱。 diff --git a/docs/agent-runner-pluginization/RUN_STEERING_AND_CHECKPOINT.md b/docs/agent-runner-pluginization/RUN_STEERING_AND_CHECKPOINT.md new file mode 100644 index 000000000..e87e691b7 --- /dev/null +++ b/docs/agent-runner-pluginization/RUN_STEERING_AND_CHECKPOINT.md @@ -0,0 +1,154 @@ +# Run Steering 与 Compaction Checkpoint（Design Note） + +本文档记录两项 Host/runner 协作能力：**运行中消息注入（steering / follow-up）**和 +**压缩摘要持久化（compaction checkpoint）**。两者来自官方 local-agent 对照 +Pi agent harness（`pi-mono/packages/agent`，下称 pi-agent-core）的差距分析： +local-agent 已移植 Pi 的事件生命周期、并行工具语义、hook 扩展点和压缩预算模型， +这两项需要 Host 协议、授权与 runner turn 边界协同才能闭环。 + +> 本文是设计备忘，不是 schema 事实源。涉及的数据结构最终落到 +> [PROTOCOL_V1.md](./PROTOCOL_V1.md)；上下文边界语义以 +> [AGENT_CONTEXT_PROTOCOL.md](./AGENT_CONTEXT_PROTOCOL.md) 为准； +> run 持久化与控制原语以 [RUNTIME_CONTROL_PLANE_V2.md](./RUNTIME_CONTROL_PLANE_V2.md) 为准。 + +## 1. Run Steering / Follow-up（运行中消息注入） + +### 1.1 问题 + +IM 场景下用户在 agent 运行中追加消息非常常见（补充信息、纠正方向、"算了别查了"）。 +EBA 先按事件选择一个 Pipeline 或 Agent 处理器；进入 Runner 后，当前调用链是 `one AgentBinding -> one run_id -> one runner` +（PROTOCOL_V1 §13）：同会话的新消息要么等待当前 run 结束后触发新 run， +要么并发触发独立 run。两种行为都无法把新消息送进**正在执行的 tool loop**， +用户体验是"agent 自顾自跑完过期任务，然后才看到新消息"。 + +cancel（PROTOCOL_V1 §10）不解决这个问题：cancel 丢弃已完成的工作； +steering 是在保留当前进度的前提下改变后续方向。 + +### 1.2 Pi 的参考语义 + +pi-agent-core 区分两个队列，注入时机都在 turn 边界，不打断进行中的模型流或工具执行： + +- **steering**：运行中插入。当前 assistant 消息的全部 tool call 完成后、 + 下一次模型调用前，注入排队的用户消息；模型在下一 turn 看到它们。 +- **follow-up**：排队后续工作。仅当没有 pending tool call 且没有 steering 消息、 + run 即将自然结束时检查；若有排队消息则注入并继续下一 turn，而不是结束 run。 + +两个队列各自支持 `one-at-a-time`（每次注入一条）和 `all`（一次注入全部）模式。 + +### 1.3 设计方向 + +职责划分遵循既有原则：Host 拥有事件路由和会话事实源，runner 拥有 turn 边界。 + +- **Host 侧**：BindingResolver / dispatch 层识别"同 conversation 存在 active run + 且 runner 声明支持 steering"的新消息事件，将其写入 run-scoped steering queue， + 并标记该事件已被在途 run 认领（不再触发新 run，避免破坏 §13 的基数约束）。 + 事件仍照常进 EventLog / Transcript（事实源不变，改变的只是触发行为）。 +- **Runner 侧**：在 turn 边界（tool batch 完成后、下一次模型调用前，以及 run + 即将自然结束前）通过 run-scoped pull API 拉取 pending steering 输入， + 注入 working context。local-agent 的 `AgentLoopHooks.prepare_next_turn` / + `should_stop_after_turn` 已预留了对应的注入点。 +- **能力协商**：runner manifest 声明 `steering` capability（参照 PROTOCOL_V1 §4.3）； + 未声明的 runner 保持现状（新消息按现有规则另起 run）。 +- **回执**：被 steering 消费的事件通过 EventLog 审计。原始 `message.received` + 记录在 `metadata.steering` 标记 queued/absorbed 与 `claimed_by_run_id`； + runner 成功 pull 后，Host 追加 `steering.injected` 记录并引用源事件。 + run 结束时仍未被 pull 的已 claim 输入，Host 追加 `steering.dropped` 记录作为 + dispatch 终态；原始 Transcript 事实不删除。 + Transcript 继续只表示会话事实，不扩展 dispatch 行为字段。 + +已落地的协议面（最终定义归 PROTOCOL_V1）： + +1. `ContextAccess.available_apis` 增加 steering pull 能力位。 +2. `RunnerAPIProxy` 增加 steering 拉取 action：默认 `mode=all`，Host 保序返回全部 + pending 输入；`one-at-a-time` 仅作为 runner 主动节流选项。 +3. dispatch 层的"认领"规则：`message.received` 可被同 conversation 的 active run + 吸收，原事件写 EventLog / Transcript，dispatch 行为写入 EventLog metadata。 +4. Host 对单 run steering queue 设置内存上限，队列满时不再 claim 新消息，消息回到 + 正常 dispatch 路径，避免 active run 无限吞入同会话输入。 + +### 1.4 边界 + +- 不引入 Host 替 runner 做 prompt 拼接：Host 只递队列，注入位置和格式由 runner 决定。 +- 不与 observer / fan-out 混淆：steering 仍是单 run 内的输入补充，不产生第二个 runner。 +- 远程 / 外部 harness runner（claude-code、codex 等）若其底层 session 自带 + steering 能力，adapter 可以直接转发；协议面保持一致。 + +## 2. Compaction Checkpoint 持久化 + +### 2.1 问题 + +local-agent 当前是无状态 runner：每次 run 重新拉取 transcript 尾部 +（默认 50 条）、重新估算 token、重新生成压缩摘要。后果： + +- 长会话中每 run 重复压缩计算，摘要每次重新生成，不同 run 之间措辞漂移， + 对 provider KV cache 不友好（AGENT_CONTEXT_PROTOCOL §"Summary checkpoint 稳定" + 已写明期望：只有压缩发生时才产生新 checkpoint）。 +- 历史一旦超过 fetch limit，更早的内容永久不可见——没有 checkpoint 记录 + "已压缩到哪里、压缩出了什么"。 + +pi-agent-core 把 compaction 条目持久化进 session tree：摘要带 +`tokensBefore` 和覆盖范围，后续 turn 直接复用，只在再次越过阈值时增量压缩。 + +### 2.2 现状盘点 + +协议面和主消费路径已具备： + +- State / Storage API 已定义（PROTOCOL_V1 §8 "State / Storage"）， + 且 AGENT_CONTEXT_PROTOCOL 已点名 `summary.checkpoint` 是 state 的预期用法。 +- Host 会根据 binding state policy 暴露 `ContextAccess.available_apis.state`。 +- local-agent 会在 state API 可用时读取/写入 `runner.compaction.checkpoint`； + 缺失、schema 不匹配、conversation 不匹配或游标失败时回退尾部历史拉取。 +- LLM 生成摘要**不依赖**本项 Host 能力——runner 用已授权的 `invoke_llm` + 即可生成；checkpoint 只解决"存下来、下次复用"。 + +### 2.3 设计方向 + +- **存放位置**：state，scope=`conversation`（小 JSON，符合 PROTOCOL_V1 §8 + 对 state/storage 的边界建议）。若未来摘要膨胀，超出部分放 storage 并在 + state 中留引用。 +- **key 约定**：`runner.compaction.checkpoint`（runner 命名空间内）。 +- **内容约定**（schema 落 PROTOCOL_V1 或 runner 文档，此处只列语义）： + - `schema_version` + - `summary`：压缩摘要文本（LLM 生成或确定性生成） + - `covers_until`：已被摘要覆盖的 transcript 游标（seq / message id）， + 是增量压缩和"从哪继续拉历史"的锚点 + - `tokens_before` / `created_at`：诊断与失效判断 +- **消费流程**：run 开始时读 checkpoint → 只拉取 `covers_until` 之后的 + transcript → 压缩触发时基于旧摘要增量生成新摘要、写回新 checkpoint。 + checkpoint 缺失或解析失败时回退到现行为（全量拉尾部），保证向后兼容。 +- **失效规则**：`covers_until` 在 Host transcript 中不存在（会话被清理 / 重置） + 即作废；runner 不得信任跨 conversation 的 checkpoint。 +- **授权**：Host 对声明需要 state 的 runner binding 开启 + `available_apis.state`；校验沿用现有 run-scoped state 校验 + （scope、key、value 大小、JSON 可序列化，见 PROTOCOL_V1 §7.2 对 + `state.updated` 的要求）。 + +### 2.4 相关但独立的工作 + +- **tokenizer / usage metadata 透传**：runner 目前用 chars/4 启发式估 token， + 对 CJK 偏低 3-4 倍，压缩触发系统性偏晚。Host 应在模型响应或 + `ctx.runtime.metadata` 透传 provider usage（prompt/completion tokens）与 + model context window（LiteLLM model-info 工作）。该项不阻塞 checkpoint + 落地，但决定压缩触发的准确性。 + +## 3. 实施拆分 + +| 项 | 归属 | 依赖 | +| --- | --- | --- | +| steering queue、事件认领、基础审计 | LangBot Host（dispatch / binding 层） | 已落地，含队列上限与未消费 dropped 终态 | +| steering pull API + capability 位 | PROTOCOL_V1 + SDK proxy | 已落地 | +| turn 边界拉取与注入 | langbot-local-agent | 已落地 | +| local-agent 对 state API 的 checkpoint 读写 | langbot-local-agent | 已落地 | +| checkpoint key / 内容 / 失效约定 | PROTOCOL_V1 + local-agent README | 已落地 | +| LLM 压缩摘要生成 | langbot-local-agent | 已落地（`invoke_llm`，失败回退确定性摘要） | +| usage / context-window metadata 透传 | LangBot Host（model 层） | LiteLLM model-info | + +剩余工作应优先补 usage / context-window metadata。streaming delivery 衔接依赖 +`ctx.delivery` 编辑/追加语义，不建议在协议能力缺失时硬编码。 + +## 4. 开放问题 + +- streaming delivery 下 steering 注入后，前序 turn 已流出的内容与新 turn + 输出在 IM 消息编辑面的衔接（涉及 `ctx.delivery` 能力，待 delivery 演进定）。 +- checkpoint 是否需要 Host 侧主动失效通知（如会话清空时删除对应 state key）。 + 当前实现靠 runner 读取时校验并回退，功能不阻塞。 diff --git a/docs/agent-runner-pluginization/SECURITY_HARDENING.md b/docs/agent-runner-pluginization/SECURITY_HARDENING.md new file mode 100644 index 000000000..de4fd9537 --- /dev/null +++ b/docs/agent-runner-pluginization/SECURITY_HARDENING.md @@ -0,0 +1,211 @@ +# Runner Security Boundary + +本文档记录 agent-runner 插件化后的安全边界和最小护栏。 + +## 状态 + +**当前结论：不采用高强度监管模型。** + +LangBot 的目标不是托管一个强隔离、不可信 code runner 平台。Runner 插件，尤其是 ACP / Claude Code / Codex / OpenCode / Kimi Code 这类外部 harness，默认视为 **operator-owned execution**：用户或部署者显式配置并承担其文件系统、进程、网络、workspace、provider 登录态和 native tool 风险。 + +LangBot 需要负责的是保护 **LangBot 自己持有的资源**，包括模型、知识库、LangBot tools、history、event、state、plugin/workspace storage、sandbox/workspace 文件访问等。只要这些资源访问是 run-scoped、permission-scoped、可校验、可诊断的，当前阶段即可接受。 + +这意味着： + +- 不要求 LangBot 在应用层实现完整 OS sandbox、VM、cgroup、seccomp、CPU / memory / network quota。 +- 不要求为 ACP runner 做复杂审批流；用户选择 ACP runner 即表示显式 opt-in。 +- 不要求在非 Docker 进程部署里做强监管；只要文档明确风险归属即可。 +- Docker / K8s 可以提供部署级隔离，但不是 LangBot agent-runner 协议发布的前置条件。 +- 不能宣传 LangBot 已经提供 managed sandbox；除非未来真的提供受管执行环境。 + +## 责任边界 + +### LangBot Host 负责 + +- **资源授权**：根据 runner manifest permissions、binding resource policy、run scope 生成本次 run 可访问的资源快照。 +- **运行期校验**：所有带 `run_id` 的 SDK / Host action 必须校验 active run session、caller plugin identity、resource id 和 operation。 +- **Scoped projection**：只把授权后的资源摘要、MCP server config、context、attachment/path ref、state snapshot 投影给 runner。 +- **LangBot 文件路径约束**：LangBot 自己 staged 和读取的文件必须限制在声明 root 内，防止 path escape。 +- **基础 secret 策略**：不要主动把 LangBot 持有的 API key / token / secret 投影给 runner；日志和错误里做常见 secret 字段脱敏。 +- **基础运行约束**：提供 timeout、取消传播、输出大小限制或错误映射的基础能力。 +- **audit-lite**：记录 event、run id、runner id、binding、资源授权摘要、关键失败、state/file/transcript 事实。 + +### Runner Plugin 负责 + +- 遵守 Host 下发的 `ctx.resources`、`ctx.context.available_apis`、runner config 和 state policy。 +- 把 LangBot 资源投影成目标平台可消费的形式，例如 MCP config、context prompt、HTTP header、run token。 +- 不绕过 SDK / Host action 直接访问 LangBot 内部资源。 +- 对自己启动的外部进程做合理封装，包括参数构造、timeout、取消、输出解析和错误映射。 +- 清楚记录自身 README 中的 provider 风险、部署假设和限制。 + +### 部署者 / 用户负责 + +- ACP / external harness 的 workspace 内容、文件系统访问、进程权限、网络访问、provider-native tool 权限。 +- Docker / K8s 的 image、volume、secret、network policy、resource limit、namespace、service account 配置。 +- 本机进程部署时的 OS 用户权限、PATH、HOME、CLI 登录态、全局配置和外部 MCP 配置。 +- 是否允许 runner 对某个目录执行真实写操作。 + +### 外部 Harness 负责 + +Claude Code、Codex、OpenCode、Kimi Code、Gemini CLI 等外部工具继续使用自己的权限模型、MCP 加载策略、session/resume、sandbox 或 approval 能力。LangBot 不承诺约束这些工具对其所在容器或宿主 OS 用户本来可访问资源的能力。 + +## 部署场景策略 + +| 场景 | LangBot 策略 | 不由 LangBot 承担 | +| --- | --- | --- | +| 普通进程部署 | 文档提示 operator-owned execution；Host 只保护 LangBot 资源。 | 阻止外部 CLI 读取同一 OS 用户可访问的文件、进程、HOME、全局 CLI 配置。 | +| Docker / K8s 部署 | 继续使用相同 Host 资源边界；容器隔离由部署环境提供。 | 应用层重复实现容器/VM/cgroup/seccomp/network quota。 | +| ACP runner | 用户显式选择 runner 和 workspace；LangBot 注入 scoped MCP / run token。 | ACP CLI native tools、workspace 写入、provider 登录态和外部 MCP 行为。 | +| 外部 SaaS runner，例如 Dify | LangBot 通过 run token / gateway 限制 LangBot 资产访问。 | SaaS 平台内部 agent 执行策略、模型工具消息格式、平台侧日志。 | +| 未来 managed runner | 只有当 LangBot 明确提供受管执行环境时，才需要单独定义强隔离 SLA。 | 当前协议闭环不承诺 managed sandbox。 | + +## 最小护栏 + +以下是当前阶段需要维持的最小要求。它们是保护 LangBot 资源边界的要求，不是完整监管外部进程的要求。 + +### Resource Permission Boundary + +每次 run 前必须冻结授权快照： + +- runner manifest permissions 是资源访问上限。 +- binding resource policy / runner config 决定本次实际授权。 +- runtime action 按 `run_id` + `caller_plugin_identity` + resource id + operation 校验。 +- manifest permissions 只约束 LangBot 持有资源，不约束 external harness native tools。 + +当前实现方向是正确的：`AgentRunSessionRegistry` 保存 run-scoped snapshot，`plugin/handler.py` 对模型、工具、知识库、history、state、storage 等 action 做运行期校验，sandbox/workspace 文件访问由 scoped tool 边界控制。 + +**Skill 读写门控（不可弱化）**：pipeline-visible 的 skill 一次性以 `rw` 挂进同一 sandbox，mount 层不区分「可见」与「已激活」；写类 native 操作（write/edit/exec）只放行 activated skill，读类放行 visible + activated——这层区分等同资产授权语义，必须保留。skill 全 tool 化后尤其注意：「都是 tool」不等于「只控资产授权即可」，native 层的 visible/activated 门控不能砍。可弱化的只是 realpath 越界字符串检查（有 chroot/namespace 兜底）。 + +### MCP / Asset Gateway Boundary + +LangBot MCP / asset gateway 只暴露当前 run 授权的工具面： + +- `langbot_list_assets` +- `langbot_get_current_event` +- `langbot_history_page` +- `langbot_retrieve_knowledge` +- `langbot_get_tool_detail` +- `langbot_call_tool` + +外部平台需要使用短期 `run_token` 或 Authorization bearer token。token 缺失、错误或过期时必须拒绝访问。 + +不要求当前阶段实现 admin 级 MCP allowlist、dangerous tool approval 或复杂审批流。是否注册外部 MCP provider 是部署者/用户行为。 + +### Workspace / Path Boundary + +LangBot 只需要约束自己管理的路径： + +- Host staged 文件必须校验 `realpath` 和 root containment。 +- Attachment/file metadata 不应暴露 Host-only storage key / host path。 +- Context 文件、sandbox/workspace 文件如由 LangBot 创建，应放在可清理的位置。 + +用户配置给 ACP runner 的 workspace 不属于 LangBot 的强监管范围。Docker/K8s 下依赖 volume 挂载边界；普通进程部署下依赖 OS 用户权限和用户自担风险。 + +### Secret Handling + +这里的 secret 指 API key、provider token、run token、MCP token、platform secret、数据库密码等。 + +当前阶段只要求基础策略： + +- LangBot 不主动把自己持有的 secret 投影给 runner，除非这是 runner config 明确需要的外部服务凭据。 +- run token 是短期、run-scoped 的，不应长期保存。 +- 日志、错误、transcript、attachment/file metadata 尽量避免打印常见 secret 字段。 +- 配置 UI / API 返回时继续沿用现有 secret masking 规则。 + +不要求当前阶段实现完整 DLP、全链路敏感数据追踪、secret lineage 或自动轮换体系。 + +### Process / Runtime Bounds + +LangBot 需要提供基本可控性： + +- Host run deadline / runner timeout。 +- runner 侧请求 timeout。 +- generator close / cancel 传播。 +- 输出和 inline payload size 上限。 +- 错误映射为受控 runner failure。 + +不要求 LangBot 为外部 harness 实现 CPU、内存、磁盘、网络、进程树强隔离。需要这些能力时由 Docker/K8s、systemd、容器平台或用户机器策略提供。 + +### UI / Admin Surface + +前端可以展示 runner 权限摘要，但它是信息披露，不是审批系统。 + +权限摘要指 runner manifest 声明的 LangBot 资源权限，例如： + +- `tools.detail` +- `tools.call` +- `knowledge_bases.retrieve` +- `history.page` +- `storage.plugin` + +当前阶段不要求强制弹窗、管理员审批、dangerous tool approval 或生产禁用开关。可以在 runner 配置区展示简短提示：此 runner 能访问哪些 LangBot 资源，外部 harness 执行风险由用户/部署者承担。 + +### Audit Lite + +需要记录足够排查问题的事实： + +- run id、runner id、binding、event。 +- 授权资源摘要。 +- state update、file write/read event、transcript message。 +- MCP / pull API 拒绝时的 warning。 +- steering queued / injected / dropped。 + +不要求当前阶段建立独立安全审计产品、审批记录系统或 SIEM 级事件模型。 + +## 降级后的检查表 + +| 项目 | 当前要求 | 状态判断 | +| --- | --- | --- | +| Path isolation | 只约束 LangBot 管理的 context/sandbox 文件路径；runner workspace 归用户/部署环境。 | Minimal required | +| Permission boundary | 必须保护 LangBot 资源；不约束外部 CLI native 能力。 | Required | +| Secret handling | 基础不投影、基础 masking、run token 短期化。 | Basic required | +| MCP policy | run-scoped token + scoped tool surface；无复杂审批。 | Required | +| Skill access policy | skill 通过 Host 授权 tool 暴露（发现 / activate / register / native exec 走统一 tool 授权）；**native 层 visible（只读）vs activated（可写）门控不可弱化**——所有 pipeline-visible skill 以 `rw` 挂进同一 sandbox，读写区分全靠 native 层；harness-native skill 文件不作为 LangBot 安全边界。 | Required | +| Process isolation | 由 Docker/K8s/用户机器负责。 | Out of scope | +| State lifecycle | scope 隔离、JSON size limit、基础 cleanup primitive。 | Basic required | +| Audit | 记录运行事实和拒绝原因。 | Audit-lite | +| UI / Admin control | 权限摘要可展示；不要求审批流。 | Optional | +| Test matrix | 覆盖 run auth、MCP token、permission deny、timeout、sandbox path、state size。 | Focused tests | + +## 当前实现快照 + +截至 2026-06-15，已有实现覆盖： + +- SDK typed Runner manifest、capabilities、permissions。 +- Host resource builder 按 manifest permissions 和 binding policy 生成 `ctx.resources`。 +- Active run session snapshot 和 `caller_plugin_identity` 校验。 +- History / event / state / tool / knowledge runtime action 的 run-scoped 校验。 +- Sandbox file path `realpath` + root containment。 +- Persistent state scope 隔离和 JSON size limit。 +- SDK-owned MCP bridge 和 long-lived asset gateway。 +- Dify / ACP runner 对 LangBot asset gateway 的接入。 +- Runner timeout、Dify HTTP timeout、ACP startup / initialize / request timeout。 + +仍可继续优化但不阻塞当前发布的事项： + +- 前端展示 runner LangBot 资源权限摘要。 +- 常见 secret 字段 redaction 收敛成统一 helper。 +- Context/sandbox file TTL cleanup 调度。 +- 更完整的 MCP 调用 audit。 +- 更好的文档提示：ACP runner 是 operator-owned execution。 + +## 非目标 + +以下不属于当前 agent-runner pluginization 的安全目标： + +- 防止 ACP / external harness 修改其 workspace。 +- 防止外部 CLI 读取同一容器或 OS 用户本来可读的文件。 +- 管控 external harness 的 provider-native tools、approval、MCP、browser、shell。 +- 在 LangBot 应用层实现 VM / container / cgroup / seccomp / network policy。 +- 为 Docker/K8s 部署替代平台自身的 secret、volume、network、resource limit 管理。 +- 实现企业级审批系统、SIEM、DLP 或安全运营面板。 + +## 发布口径 + +可以对外说明： + +> Runner 插件通过 run-scoped authorization 和 scoped MCP gateway 保护 LangBot 持有资源。外部 code harness 的执行环境由用户或部署平台负责隔离；LangBot 当前不提供 managed sandbox。 + +不能对外说明： + +> LangBot 已经安全沙箱化 Claude Code / Codex / OpenCode 等外部 runner。 diff --git a/docs/agent-runner-pluginization/STATUS.md b/docs/agent-runner-pluginization/STATUS.md new file mode 100644 index 000000000..4d6686e2b --- /dev/null +++ b/docs/agent-runner-pluginization/STATUS.md @@ -0,0 +1,87 @@ +# Runner Pluginization Status + +本文档是 `docs/agent-runner-pluginization/` 的状态事实源。协议 schema 仍以 [PROTOCOL_V1.md](./PROTOCOL_V1.md) 为准；测试步骤以 [RUNNER_QA_GUIDE.md](./RUNNER_QA_GUIDE.md) 为准；安全发布门槛以 [SECURITY_HARDENING.md](./SECURITY_HARDENING.md) 为准。 + +状态快照日期：2026-09-05。代码基线为 LangBot `dev/4.11.x` / `a4d36aa2d` 和 SDK `dev/4.11.x` / `f1da058`；本文是本地检视快照，不代表远端最新状态或正式发布批准。 + +## 当前版本与验收边界 + +- Core 声明版本 `4.11.0`，`pyproject.toml` / `uv.lock` 仍依赖 `langbot-plugin==0.5.3`；SDK 源码声明版本 `0.5.5`。本机 Core 实际导入旁边 SDK 的可编辑源码，不能用这个组合的测试通过证明 registry 安装可复现。 +- 检视时 SDK 有四个未提交文件：`api/agent_tools/asset_gateway.py`、`api/agent_tools/external_tools.py`、`api/entities/builtin/runner/resources.py`（均位于 `src/langbot_plugin/`），以及 `tests/api/test_agent_tools_mcp_bridge.py`。这些改动增加 `platform_tools` 分类发现、可选 schema 和 gateway 引导。下面的 SDK 测试包含这些工作区改动；正式配套版本尚需冻结。 +- 本轮仅重新执行下列定向测试与 TypeScript 检查，没有重跑全量 backend、真实平台、provider、浏览器 E2E 或 Cloud 部署门禁。后文旧日期的成功记录仍是历史证据。 + +| 2026-09-05 验证 | 结果 | +| --- | --- | +| Core：`tests/unit_tests/agent`、`tests/unit_tests/api/service/test_agent_service.py`、`tests/unit_tests/platform/test_routing_rules.py`、`tests/unit_tests/api/service/test_maintenance_service.py` | 561 passed，74 warnings | +| SDK：`tests/api/entities/builtin/runner`、`tests/api/proxies`、`tests/api/test_agent_tools_mcp_bridge.py`、`tests/runtime/plugin/test_mgr_runner.py`、`tests/runtime/plugin/test_dependency_environment.py`、`tests/runtime/plugin/test_restart_coordinator.py` | 362 passed，10 warnings | +| Web：`pnpm exec tsc --noEmit` | pass | +| Web：`pnpm test:unit` | 62 passed，2 failed | + +前端失败为源码形状断言：`oss-cloud-ui-privacy.test.mjs` 仍要求 fieldset 的精确旧 class；`processor-detail-workbench.test.mjs` 仍要求 Agent 的旧四分区。当前表单已增加布局 class，并使用运行器、运行器配置、事件与工具分区。发布前需按当前设计更新断言或修正实现，不能记为全绿。 + +## 近期已集成内容 + +- **EBA 与独立 Agent**：Bot 事件绑定、observer 广播、Pipeline / Agent / discard 单目标分派、Agent CRUD、处理器工作台与路由诊断已在同一分支。逻辑路由器实现在 `pkg/platform/botmgr.py::RuntimeBot`，不是外部独立 EventRouter 服务。 +- **事件感知工具权限**：Core 9 月 4 日提交已实现 `event_*` 自动事件工具、`allowed_platform_tools` 和 `allowed_tools`。Host 冻结事件目标，结合 Runner 权限、适配器能力和运行快照授权，执行时再次检查。详见 [PLATFORM_ACTION_TOOLS.md](./PLATFORM_ACTION_TOOLS.md)。SDK 分类发现的提交状态见上文。 +- **产品流程**：Runner 市场内联安装、安装恢复和健康状态、事件范围选择、统一详情工作台、Agent 调试及 Bot 平台事件调试已实装。Agent 自身已无 enabled 开关；Bot 路由仍有 enabled。具体页面形态见 [处理器页面](../event-based-agents/08-agent-page-and-event-orchestration.md)。 +- **Runtime 与存储**：SDK 已加入 artifact 对应的独立依赖环境、Windows worker 连接路径、安装期间响应性测试；Core/Plugin Runtime/Box 分别报告拥有的存储目录。存储分析是观测，不提供硬配额。 +- **工作空间**：OSS 单工作空间多人协作和资源作用域已合入。Cloud 隔离基础与生产激活分开验收，见 [Cloud 剩余事项](../multi-tenant/cloud-v2-pending-verification.md)。 + +## 实现状态 + +| 领域 | 状态 | 说明 | +| --- | --- | --- | +| SDK manifest schema | Done | `RunnerManifest` 包含 typed `capabilities` / `permissions`；未知 capability / permission key 禁止进入 typed model。 | +| Runner discovery | Done | Runtime 返回 typed manifest；Host registry 校验单个 runner，失败 warning + skip，不影响其它 runner。 | +| Host resource authorization | Done | `ctx.resources` 和 `ctx.context.available_apis` 由 manifest permissions 与 binding policy / run scope 求交后生成。 | +| Run authorization snapshot | Done | active run session 冻结 run-scoped resources 与 available APIs；runtime handler 按 snapshot 校验 pull API。 | +| Result payload validation | Done | Wire 保持 `{type, data}`；Host 对投递/副作用类 payload 严格校验，tool-call telemetry 宽松，未知 type 忽略并 warning。 | +| Old built-in runners | Done | 旧 `src/langbot/pkg/provider/runners/*` 与 `RequestRunner` 路径已从本分支删除。 | +| Official runner manifests | Done | `local-agent`、ACP / Claude Code / Codex 外部 harness runner、外部服务 runner 已重新声明真实生效的 LangBot resource permissions。 | +| Skill 链路 | Unit-pass; WebUI E2E pass | 已按 **skill 全 tool 化** 收敛：发现走 `list_skills` / `langbot_list_assets` 和 skill resources；`activate` / `register_skill` 走统一 tool 授权；`skill_authoring` capability 降级为便捷开关。`activate` 会 best-effort 写入 conversation-scope `host.activated_skills`，后续 run 通过当前 pipeline-visible skill cache 恢复。新注册 Skill 在当前 Query 内立即获得临时可见性；Docker `exec` 产生的宿主侧不可写文件由 `write` / `edit` 回退到 Box 执行。2026-07-15 真实 LocalAgent Debug Chat 已完成创建、注册、同 Query 激活、编辑和执行闭环；非流式 runner turn 只向下游 Pipeline 产出一次，工具中间结果不再拆成额外 Bot 气泡。 | +| Runtime Control Plane v2 foundation | Partial | Host-owned `AgentRun` / `AgentRunEvent` ledger、orchestrator 自动建账、result event persistence、run get/list/event page/cancel/append/finalize actions 已落地；`agent_run:admin` / `runtime:admin` 控制权限、最小 runtime register/heartbeat/list/reconcile 和 run claim/renew/release 原语已落地。完整 Agent Platform 产品形态、daemon supervisor、任务唤醒/长轮询/WebSocket、分布式 runtime 管控仍未完成。 | +| Security boundary | Done | 当前口径降级为轻量边界：LangBot 保护自身持有资源；external harness 的 OS / process / network / workspace 风险由用户或部署环境承担；managed sandbox 不是当前承诺。 | +| Steering control path | Done | claim 异常不再逃逸 consumer loop；queue 有上限；未 pull 的 claimed 输入在 run 结束时写 `steering.dropped` 审计终态。 | +| SDK v1 contract closure | Done | SDK 提供 `AgentAPIError` / `AgentAPIException`、typed `SteeringPullResult`、未知 result type 宽容解析、result `sequence` 注入与取消传播。 | +| EBA processor routing | Done; release gate 5/5 pass | Bot `event_bindings`、Pipeline / Agent 平级路由、WebUI dry-run / 合成测试 / 状态、OneBot 非消息事件到 Agent 及平台回复已闭环；隔离空白实例已验证从 Space 安装并注册 LocalAgent。 | +| Structured interactions | Cross-repo unit-pass; provider E2E pending | Host 已完成 `interaction.requested` 白名单、持久化 callback correlation、TTL/作用域/幂等校验和 Pipeline/Agent 原处理器恢复；六个平台已接入按钮/单选投递，Lark 和 DingTalk 进一步支持原生单字段 `text` / `textarea` / `number` / `select` 控件。SDK typed contract、通用 Runner 脚手架和 DifyAgent `workflow_paused` plugin-storage continuation 已落入对应仓库。 | + +## Spec 与实现已知差距 + +- `action.requested` 是严格白名单协议面：当前只执行 `interaction.requested`；其它 action 仍只记录 telemetry。平台语义动作通过已授权的 `event_*` / `platform_*` 工具执行，不通过任意 result action 或原始 `call_platform_api`。 +- 结构化交互 SDK typed contract 与 DifyAgent continuation 已实现；SDK 正式发布、真实 Dify 凭据 E2E，以及需要长驻双向进程的 Claude Code 权限确认仍是后续验收项。Host 不持有 provider 私有 token。 +- State 与 storage 的长期类型边界仍可继续收窄；当前合同只要求 JSON-safe state 与受控 storage API。 +- `ToolResource.parameters` 已作为 best-effort full schema 由 Host 在构造 `ctx.resources` 时一次塞齐；无 schema 时 runner 仍需兼容 `parameters=None` 或按需调用 detail API。 +- EventLog / Transcript 已提供显式 cleanup primitive；长期 retention 默认值、TTL 调度接入和 sandbox/workspace 文件清理仍是运维收尾项，应在 Runtime Control Plane 产品化前补齐。 +- External harness 的 native shell / filesystem / CLI / MCP 权限不受 manifest permissions 约束；manifest permissions 只约束 LangBot 持有的资源访问。 +- LangBot 当前不承诺 managed sandbox；external harness 的 OS/process/network quota、workspace GC、provider-native tool 权限由用户或部署环境承担。 +- Runtime Control Plane v2 已有 Host 事实源和控制原语，独立 Agent 与处理器 UI 也已存在；仍缺业务任务队列、外部 harness daemon 托管、wakeup channel、跨 Host 分布式锁及 provider 登录态诊断。SDK installation worker 的 supervisor/重启协调已实现，不能与外部 harness 管控混淆。 + +## Runner 历史验收记录（未在本轮重跑） + +| Runner | 状态 | 最近证据 | +| --- | --- | --- | +| `plugin:langbot-team/LocalAgent/default` | Unit-pass; Marketplace UI pass; Debug Chat E2E pass | 2026-07-12 隔离 first-run 实例从真实 Runner catalog 安装 `langbot-team/LocalAgent` 0.1.0，Host 注册 `plugin:langbot-team/LocalAgent/default`，Wizard 自动选中并解锁后续操作。2026-07-15 `2026-07-15-08-44-10-770-08-00-sandbox-skill-authoring-edit-existing-e2e` 使用真实 `gpt-5.5` 完成 Skill 创建、注册、同 Query 激活、已激活包编辑与脚本执行；三阶段 UI、浏览器诊断和结构化文件系统检查全部通过，每阶段恰好新增一个 Bot 气泡，p95 14.6 秒、错误率 0。 | +| `plugin:langbot-team/ACPRunner/default` | Unit-pass; Debug Chat E2E pass | 2026-07-15 从本地 0.1.4 发布包安装并注册 PascalCase runner，remote-ssh Claude ACP 通过反向隧道调用 run-scoped `langbot_get_current_event`，97.8 秒返回可见结果；Host 将增量 delta 和 `message.completed` 聚合为一个完整 Bot 气泡。 | +| `plugin:langbot-team/ClaudeCodeAgent/default` / `plugin:langbot-team/CodexAgent/default` | Unit-pass; E2E pending | 通过 runner 仓库单测覆盖 session、run_id 注入和 LangBot MCP gateway；真实 harness E2E 取决于对应运行环境、CLI/daemon 可用性和 provider 登录态。 | +| Dify | Human-input unit-pass; credential E2E pending | `langbot-agent-runner/dify-agent` 已实现 `workflow_paused`、原子字段/确认交互、plugin-storage continuation、Dify submit/events 恢复与再次暂停；真实 Dify 凭据 E2E 待执行。 | +| n8n / Coze / DashScope / Langflow / Tbox / DeerFlow / WeKnora | Unit-pass; credential smoke optional | 2026-06-13 plugin layout / parser tests 通过；真实服务凭据 smoke 非每轮必跑。 | + +## Host / SDK 历史验收记录 + +| 范围 | 状态 | 最近证据 | +| --- | --- | --- | +| LangBot Runtime Control Plane v2 foundation | Unit-pass; EBA release gate 5/5 pass; Runner preflight pass | 2026-07-12 `eba-functional-20260712-release-gate-rerun` 通过 Quick Start 场景筛选、隔离实例 Runner Marketplace 安装、Runner 健康状态、事件路由 dry-run / 合成派发，以及真实 OneBot `group.member_joined` → Agent → `send_group_msg` 链路。2026-07-15 Runner release preflight 16 项通过、0 warning；fixture contract、5 类 behavior matrix、ledger schema / async DB readiness / 100-run stress / 120-run 8-worker contention / claim-lease-auth concurrency、SDK runtime chaos 探针全部通过。 | +| Host Skill / native tool integration | Unit-pass; WebUI E2E pass | 2026-07-15 provider / native / Skill / monitoring 定向测试 67 项通过，Pipeline / Chat / Wrapper 定向测试 61 项通过，Skills CLI 105 项通过；真实 Debug Chat 验证 `register_skill` 后同 Query `activate` 成功，监控工具调用不再把 SQL 行误取为字符串，结构化 JSON 文件检查不依赖格式空格，非流式多阶段 runner 结果只生成一个最终 Bot 气泡。 | +| SDK Runner control entities / proxy | Unit-pass | 2026-06-23 SDK `tests/api/entities/builtin/runner`、`tests/api/proxies`、`tests/api/test_agent_tools_mcp_bridge.py`、`tests/runtime/plugin/test_mgr_runner.py`、`tests/runtime/test_pull_api_handlers.py`、`tests/runtime/io/handlers/test_plugin_handler.py`、EBA event entities 和 message tests 通过，覆盖 typed entities、RunnerAPIProxy、MCP bridge、runtime manager 与 pull API handlers。 | + +## 历史高价值记录 + +历史报告已合并为本状态页和 QA 指南，不再保留单独进度文档。后续若需要追溯，优先查看 `langbot-skills/reports/` 下的原始执行报告。 + +截至 2026-05-29，已有本地 smoke 证明： + +- `local-agent` 可以通过 Pipeline Debug Chat 走插件化 `AgentRunOrchestrator` 主链路。 +- 外部 harness runner 可以通过同一条 `run(event, binding)` 路径执行；当前官方实现已收敛到 ACP / Claude Code / Codex 等直接 runner 插件。 + +这些记录只证明本地协议闭环可用，不代表 LangBot 提供 managed sandbox 或 external harness OS 级隔离。 diff --git a/docs/agent-runner-pluginization/eba-productization-release.md b/docs/agent-runner-pluginization/eba-productization-release.md new file mode 100644 index 000000000..eaac62e1b --- /dev/null +++ b/docs/agent-runner-pluginization/eba-productization-release.md @@ -0,0 +1,66 @@ +# EBA 产品化与发布计划 + +更新：2026-09-05，适用于合并后的 `dev/4.11.x`。本文维护产品边界和剩余交付顺序；具体实现、提交与验证结果统一记录在 [STATUS.md](./STATUS.md)。原 2026-07-01 草案中的功能缺口和 Phase 编号不再作为当前排期。 + +## 产品模型 + +- **Bot**：平台连接、凭据和事件路由入口。用户在机器人上决定“发生什么时，使用哪个处理器”。 +- **Processor**：可复用处理逻辑的上位概念，当前类型为 Agent 与 Pipeline。 +- **Pipeline**：保留完整 Stage 链的消息处理器，提供预处理、AI、后处理、扩展和输出控制。 +- **Agent**：独立配置对象，选择 Runner 插件并配置事件范围、运行器与工具权限，可被多个 Bot 引用。 +- **Workflow**：后续编排方向，当前尚无完整执行产品。 +- **Solution**：后续分发单元，包含处理器、路由模板、依赖、变量和文档。 + +处理器入口聚合 Agent 与 Pipeline，不转换实体，不复制旧 Pipeline runner 配置生成 Agent。EBA 是内部术语，主要产品流程使用机器人、事件、处理器、流水线和工具等名称。 + +## 已完成的产品化基础 + +| 用户流程 | 当前实现 | +| --- | --- | +| 创建处理器 | Agent/Pipeline 类型选择、独立配置与统一详情工作台 | +| 选择运行器 | 已安装 Runner 动态 metadata、市场安装入口、安装进度/恢复和可用性反馈 | +| 配置 Agent | 运行器、运行器配置、事件与工具；基础信息在详情入口编辑 | +| 配置事件 | 分组事件选择、兼容目标过滤、路由排序、冲突提示和兜底匹配说明 | +| 验证行为 | 路由 dry-run、Agent 调试、Bot 配置中的平台事件调试；合成派发仍需遵循抑制真实出站的后端边界 | +| 理解故障 | 路由匹配/失败轨迹、Runner 状态、模型测试与调试错误反馈 | +| 首次使用 | 场景引导与插件化 Runner 安装流程 | +| 工具权限 | 自动事件工具、显式平台动作、普通工具白名单，运行时再次授权 | +| 存储诊断 | Core、Plugin Runtime、Box 分别报告拥有的目录与不可用状态 | + +这张表表示代码已实现，不代表所有平台/provider 和首次安装组合均完成当前版本真实验收。页面契约见 [处理器与事件编排](../event-based-agents/08-agent-page-and-event-orchestration.md)，动作授权见 [平台工具](./PLATFORM_ACTION_TOOLS.md)。 + +## 4.11 发布收尾 + +| 顺序 | 工作 | 完成条件 | +| --- | --- | --- | +| 1 | 冻结跨仓库依赖 | 提交/确认 SDK 平台工具发现改动，固定配套 SDK 和 Runner 包，更新 Core 声明及 lock；验证 registry 或精确 commit 的干净安装 | +| 2 | 收敛自动化 | 按当前页面设计处理前端旧断言，执行 backend/SDK 定向测试、前端单测、类型检查及发布所需 lint/build/E2E | +| 3 | 验证用户路径 | 空白实例安装 Runner、创建 Agent/Pipeline、连接 Bot、保存路由、消息/非消息执行、工具允许与拒绝、错误诊断 | +| 4 | 补真实交互与平台证据 | 当前版本的选定平台媒体/回调、Dify continuation、外部 harness；记录支持、未支持、阻塞及未执行 | +| 5 | 完成生命周期 | EventLog/Transcript retention 调度、状态/文件清理、取消和重启后的行为有明确策略与测试 | + +验收记录必须写明 Core/SDK/Runner 版本、操作系统、Runtime 连接方式、Box backend 和是否使用 editable 源码。真实平台测试、合成事件与 mock provider 测试分别记账。过去的成功报告不能自动作为当前 HEAD 的通过记录。 + +## Cloud 独立交付边界 + +当前 OSS 支持单 Workspace 多成员及固定 RBAC；Cloud 目录和计费归控制面，Core 负责资源作用域与执行边界。不能用 edition 字段或普通配置开启 OSS 多 Workspace。 + +当前业务隔离单位是 Workspace 与 execution generation；不要求依据旧草案额外创建 Tenant/Workspace/Namespace 三层同构表。目录、安装绑定、存储与执行作用域以 [多租户架构](../multi-tenant/workspace-multi-user-architecture.md) 为准。 + +生产激活仍需关闭： + +- 插件和可配置出站目标的网络/SSRF 策略。 +- Plugin installation 与 Box 各存储面真正的 byte/inode 硬配额。 +- 普通业务写入贯穿提交的 generation fence、业务 outbox 和持久对象引用切换保障。 +- 最终 Linux/cgroup、持久卷、数据库权限、容量、故障恢复与 24 小时 soak 验收。 + +完整清单见 [Cloud 剩余验证](../multi-tenant/cloud-v2-pending-verification.md)。存储统计、局部压力测试或 OSS 功能验收不能替代这些门禁。 + +## 后续产品方向 + +- **完整 Agent 管控面**：业务任务队列、唤醒、外部 harness daemon 管理、provider 登录态诊断和分布式执行。已有 run ledger/heartbeat/claim 是底座。 +- **Workflow 与多 Agent**：先确定独立处理器或 Runner 扩展形式，再定义串并联、失败恢复、状态冲突和副作用幂等。 +- **Solution 导入导出**：使用逻辑依赖和路由模板；导入时选择 Bot、模型和资源，不能保留源实例 UUID、凭据、token 或租户密钥。包格式与更新策略待定。 +- **模型能力与评估**：国内 Provider reasoning 专项回归、reasoning token/生效策略记录、质量与延迟成本基线，见 [模型思考控制](../review/model-reasoning-control-design.md)。 + +这些方向不自动构成 4.11 OSS 发布前置条件，也不承诺旧草案中的 5.0/GA 版本号或日期。 diff --git a/docs/beta-adapter-diagnostics.md b/docs/beta-adapter-diagnostics.md new file mode 100644 index 000000000..2bf6c689e --- /dev/null +++ b/docs/beta-adapter-diagnostics.md @@ -0,0 +1,30 @@ +# Beta 适配器诊断开发说明 + +此诊断用于 4.11 适配器事件与 API 的 Beta 验收。仅受支持的 Beta 版本启用；稳定版、Alpha、RC、开发／本地标记版本以及关闭遥测或 Beta 诊断的实例不采集、不上传。 + +## 采集位置 + +- `pkg/platform/botmgr.py`：完成适配器事件监听器注册后产生能力快照。`listener_registered` 表示注册完成，不代表网络连接可用。 +- 各适配器的 `_dispatch_eba_event`：调用 `diagnostics.adapter_event_received(self, event)`，记录一次事件已收到。必须位于调用监听器之前，避免路由、插件或模型失败改变接收事实。 +- `pkg/telemetry/adapter_diagnostics.py`：识别适配器 API 最外层调用、平台专用 API 名、聊天与媒体类型。通过内部上下文标记抑制嵌套 API 重复计数。 +- 转换失败无法可靠确定事件种类时，保留未知事件失败；转换成功本身不重复计算事件接收。 + +新增适配器或原生事件入口时，应检查事件是否经过统一分发入口。只在转换器上添加埋点不足以覆盖直接构造事件及交互回调。 + +## 数据约束 + +`adapter_evidence` 标记可用于验收的边界记录；`listener_registered` 标记注册事实；`chat_type`、`content_type` 仅允许有限类别。消息正文、API 参数、用户／群 ID、异常正文、媒体地址不进入这些字段。内部嵌套调用标记不得序列化。 + +平台专用动作优先匹配清单，名称为 `platform_api.<动作名>`；标准交互 API 保持其标准名称。未知动作不能借通用 `call_platform_api` 包装方法被计为已验收功能。清单读取显式使用 UTF-8，兼容 Windows 默认编码环境。 + +## 联动 Space + +适配器清单或诊断操作变更后，同步生成 Space 的 `internal/diagnostics/catalog_v1.json`，并运行其目录一致性测试。Space 从清单生成完整矩阵，旧诊断记录不作为新增验收证据。部署时应先更新 Space 接收端与数据库迁移，再更新 Core Beta。 + +Core 回归命令： + +```sh +uv run pytest tests/unit_tests/telemetry -q +``` + +重点测试在 `test_adapter_acceptance.py`：版本与开关、隐私、嵌套计数、专用／交互 API、分发入口覆盖、监听器失败。Space 仓库的 `docs/beta-adapter-acceptance.md` 说明验收界面、统计口径及联调方式。 diff --git a/docs/event-based-agents/00-overview.md b/docs/event-based-agents/00-overview.md new file mode 100644 index 000000000..4a215918f --- /dev/null +++ b/docs/event-based-agents/00-overview.md @@ -0,0 +1,200 @@ +# Event Based Agents 架构设计总览 + +> Product revision (2026-09-07): [Event processors and Pipeline plugin compatibility](./09-event-processors.md) defines an explicitly bound Runner alongside Pipeline and Agent. It supersedes the automatic EBA observer product model below; the new component and UI are planned, not yet implemented. + +> 当前状态（2026-09-05）：平台事件、Bot `event_bindings`、独立 Agent、Pipeline / Agent 平级路由及 WebUI 已集成到 `dev/4.11.x`。实现入口为 `pkg/platform/botmgr.py::RuntimeBot` 与 `pkg/agent/runner/`。下文“当前架构的局限性”“现有架构”描述改造前背景；EventBus / EventRouter 图表示职责划分，不表示存在同名独立服务。当前实现和验收以 [STATUS.md](../agent-runner-pluginization/STATUS.md) 为准，平台动作使用[授权工具](../agent-runner-pluginization/PLATFORM_ACTION_TOOLS.md)。 + +## 1. 背景与动机 + +### 当前架构的局限性 + +LangBot 当前的平台适配器架构围绕**消息事件**单一场景设计： + +- **事件层面**：只监听 `FriendMessage`（私聊消息）和 `GroupMessage`（群消息）两种事件 +- **API 层面**：只暴露 `send_message` 和 `reply_message` 两个平台 API +- **处理层面**：所有消息统一进入 Pipeline 流水线处理，无法为不同事件类型配置不同处理逻辑 +- **适配器结构**：每个适配器是单个 Python 文件（200-800 行），随着功能增加难以维护 + +这导致以下问题： + +1. **无法处理非消息事件**：新成员入群、好友请求、消息撤回、消息编辑等大部分平台都支持的事件被完全忽略 +2. **平台能力未充分利用**：编辑消息、撤回消息、获取群成员列表、管理群组等 API 无法使用 +3. **插件能力受限**：插件只能监听消息事件、只能发送/回复消息，无法实现更丰富的交互 +4. **处理逻辑不灵活**：所有消息走同一条 Pipeline，无法为入群欢迎、好友自动通过等场景配置独立的处理流程 + +### 设计目标 + +Event Based Agents（EBA）架构旨在将 LangBot 从"消息处理平台"升级为"事件驱动的智能代理平台"： + +- **丰富事件**：支持消息、群组、好友、Bot 状态等多种事件类型 +- **丰富 API**：支持消息编辑/撤回、群组管理、用户信息查询等通用 API，以及适配器特有 API 的透传调用 +- **灵活编排**：用户可在 WebUI 上为每个 Bot 的每种事件类型配置不同的处理器 +- **可扩展**：适配器可声明自己支持的事件和 API，平台特有能力通过标准机制暴露 +- **向后兼容**：现有插件无需修改即可在新架构下运行 + +## 2. 架构对比 + +### 现有架构 + +``` +消息平台 (Telegram/Discord/...) + │ + ▼ +平台适配器 (单文件, 只处理消息) + │ FriendMessage / GroupMessage + ▼ +RuntimeBot (注册 on_friend_message / on_group_message 回调) + │ + ▼ +MessageAggregator (消息聚合) + │ + ▼ +QueryPool → Controller → Pipeline (固定阶段链) + │ │ + │ ▼ + │ Runner Host orchestrator + │ ▼ + │ plugin Runner + │ + ▼ +adapter.reply_message() / adapter.send_message() +``` + +关键代码路径： +- 适配器基类：`langbot-plugin-sdk/.../abstract/platform/adapter.py` — `AbstractMessagePlatformAdapter` +- 事件定义：`langbot-plugin-sdk/.../builtin/platform/events.py` — 仅 `FriendMessage` / `GroupMessage` +- Bot 管理：`LangBot/src/langbot/pkg/platform/botmgr.py` — `RuntimeBot` 只注册两个消息回调 +- 流水线控制：`LangBot/src/langbot/pkg/pipeline/controller.py` — 从 QueryPool 消费并执行 Pipeline + +### 新架构（Event Based Agents） + +``` +消息平台 (Telegram/Discord/...) + │ + ▼ +平台适配器 (独立目录, 监听所有事件, 实现丰富 API) + │ MessageReceived / MemberJoined / FriendRequest / ... + ▼ +EventBus (统一事件总线) + │ + ├─→ Plugin EventListener observers（始终广播，不参与响应仲裁） + │ + ▼ +EventRouter (读取 Bot 的 event_bindings) + │ + ├─→ Pipeline target — 完整 Stage 链，仅消息事件 + ├─→ Agent target — 独立 Agent，经插件 Runner 执行 + └─→ discard — 明确丢弃 + │ + ▼ +统一平台 API + send / reply / edit / delete / getGroupInfo / getUserInfo / callPlatformApi / ... +``` + +## 3. 核心概念 + +### 3.1 统一事件体系 + +所有平台事件统一为命名空间式的事件类型： + +| 命名空间 | 事件 | 说明 | +|----------|------|------| +| `message.*` | `message.received`, `message.edited`, `message.deleted`, `message.reaction` | 消息相关 | +| `feedback.*` | `feedback.received` | 用户对 Bot 回复的点赞、点踩、取消反馈等评价事件 | +| `group.*` | `group.member_joined`, `group.member_left`, `group.member_banned`, `group.info_updated` | 群组相关 | +| `friend.*` | `friend.request_received`, `friend.added`, `friend.removed` | 好友相关 | +| `bot.*` | `bot.invited_to_group`, `bot.removed_from_group`, `bot.muted`, `bot.unmuted` | Bot 状态 | +| `platform.*` | `platform.{adapter}.{action}` | 适配器特有事件 | + +详见 [01-event-system.md](./01-event-system.md)。 + +### 3.2 统一平台 API + +扩展适配器基类，提供通用 API + 透传机制： + +| 类别 | API | 必需/可选 | +|------|-----|----------| +| 消息 | `send_message`, `reply_message`, `edit_message`, `delete_message`, `forward_message` | send/reply 必需，其余可选 | +| 群组 | `get_group_info`, `get_group_member_list`, `get_group_member_info`, `mute_member`, `kick_member` | 全部可选 | +| 用户 | `get_user_info`, `get_friend_list` | 全部可选 | +| 媒体 | `upload_file`, `get_file_url` | 全部可选 | +| 透传 | `call_platform_api(action, params)` | 可选 | + +详见 [02-platform-api.md](./02-platform-api.md)。 + +### 3.3 适配器新结构 + +每个适配器从单文件迁移到独立目录： + +``` +pkg/platform/adapters/ +├── _base/ # 基类和通用定义 +│ ├── adapter.py +│ ├── events.py +│ ├── entities.py +│ └── api.py +├── telegram/ +│ ├── __init__.py +│ ├── adapter.py # 主适配器类 +│ ├── event_converter.py # 事件转换（多种事件类型） +│ ├── message_converter.py # 消息链转换 +│ ├── api_impl.py # 通用 API 实现 +│ ├── platform_api.py # 平台特有 API +│ ├── types.py # 平台特有类型 +│ └── manifest.yaml +├── discord/ +│ └── ... +``` + +详见 [03-adapter-structure.md](./03-adapter-structure.md)。 + +### 3.4 事件响应目标与观察者 + +Pipeline 与 Agent 是长期并存、场景不同的同级处理器。Pipeline 保留完整 Stage 链，面向消息处理；Agent 是独立配置对象，选择一个已安装的插件 Runner，并可声明消息或非消息事件能力。Bot 的 `event_bindings` 只负责把事件绑定到既有 Pipeline、独立 Agent 或 `discard`。 + +插件 EventListener 是观察者：事件先广播给有权限的监听器，随后路由器再选择一个响应目标。Webhook、Dify、n8n 等外部执行方式若需要作为响应者，应由对应 Runner 插件表达，而不是增加另一套 Host Handler 主链。 + +现有 Pipeline 不会被转换为 Agent，Pipeline 内的 runner 配置也不会复制到独立 Agent。用户需要 Agent 时自行创建并绑定。 + +详见 [04-event-routing.md](./04-event-routing.md)。 + +### 3.5 插件 SDK 改造 + +- 新事件类型全部暴露给插件 +- 新 API 全部通过 `LangBotAPIProxy` 暴露 +- 兼容层保证现有插件零修改运行 + +详见 [05-plugin-sdk.md](./05-plugin-sdk.md)。 + +## 4. 关键设计决策 + +| # | 决策点 | 选择 | 理由 | +|---|--------|------|------| +| 1 | 事件处理器配置粒度 | 每个 Bot 独立配置 | Bot 是用户操作的核心单元，不同 Bot 可能对接不同业务场景 | +| 2 | 适配器特有 API | 统一抽象 + `call_platform_api` 透传 | 通用 API 覆盖大部分场景，透传机制保证灵活性，避免每个适配器导出独立的类型化 API 包 | +| 3 | 向后兼容策略 | 兼容层适配 | 保留旧事件类型和 API 作为新系统的 alias/wrapper，现有插件无需修改 | +| 4 | 处理器配置存储 | Bot 表使用 `event_bindings`，目标引用原始 Pipeline 或独立 Agent UUID | 路由关系不复制处理器配置，Pipeline/Agent 各自保持事实源 | +| 5 | Agent 处理器定位 | 独立 Agent + 插件 Runner | Host 不再内置具体 runner；不同 Runner 通过统一协议接入 | +| 6 | 事件命名方式 | 命名空间式（`message.received`） | 清晰的分类层级，便于通配匹配（`message.*`），与 WebUI 配置天然对应 | + +## 5. 文档索引 + +| 文档 | 内容 | +|------|------| +| [01-event-system.md](./01-event-system.md) | 统一事件体系：事件分类、定义、生命周期 | +| [02-platform-api.md](./02-platform-api.md) | 统一平台 API：通用 API、透传 API、实体定义 | +| [03-adapter-structure.md](./03-adapter-structure.md) | 适配器新结构：目录布局、基类、注册机制 | +| [04-event-routing.md](./04-event-routing.md) | 事件路由与编排：路由引擎、处理器类型、WebUI 数据模型 | +| [05-plugin-sdk.md](./05-plugin-sdk.md) | 插件 SDK 改造：新事件/API、兼容层 | +| [06-migration-plan.md](./06-migration-plan.md) | 分阶段迁移计划 | +| [07-agent-orchestration.md](./07-agent-orchestration.md) | **产品最终形态（2026-06 修订）**：Pipeline / Agent 同级处理器编排、SDK Agent 组件契约、发布火车 | + +## 6. 涉及的代码仓库 + +| 仓库 | 改动范围 | +|------|----------| +| **langbot-plugin-sdk** | 事件定义、实体模型、API 接口、适配器基类、通信协议扩展 | +| **LangBot**（后端） | 适配器实现、事件路由引擎、Bot/Agent 实体、Runner Host 编排 | +| **LangBot**（前端） | Bot 事件处理器编排面板 | +| **langbot-wiki** | 新架构文档、插件开发指南更新、适配器开发指南 | +| **langbot-plugin-demo** | 示例更新（使用新事件和 API） | diff --git a/docs/event-based-agents/01-event-system.md b/docs/event-based-agents/01-event-system.md new file mode 100644 index 000000000..5216499d3 --- /dev/null +++ b/docs/event-based-agents/01-event-system.md @@ -0,0 +1,562 @@ +# 统一事件体系 + +## 1. 设计原则 + +- **命名空间分类**：事件类型采用 `{namespace}.{action}` 格式，如 `message.received` +- **通用优先**：大部分平台都支持的事件抽象为通用事件，定义统一的字段格式 +- **平台特有事件标准化**：各适配器的独有事件通过 `PlatformSpecificEvent` 承载，保留原始数据 +- **向后兼容**：现有 `FriendMessage` / `GroupMessage` 通过兼容层映射到新的 `message.received` 事件 + +## 2. 事件基类层次 + +``` +Event (事件基类) +├── MessageEvent (消息相关事件) +│ ├── MessageReceivedEvent # message.received +│ ├── MessageEditedEvent # message.edited +│ ├── MessageDeletedEvent # message.deleted +│ └── MessageReactionEvent # message.reaction +├── FeedbackEvent (用户反馈事件) +│ └── FeedbackReceivedEvent # feedback.received +├── GroupEvent (群组相关事件) +│ ├── MemberJoinedEvent # group.member_joined +│ ├── MemberLeftEvent # group.member_left +│ ├── MemberBannedEvent # group.member_banned +│ ├── MemberUnbannedEvent # group.member_unbanned +│ └── GroupInfoUpdatedEvent # group.info_updated +├── FriendEvent (好友相关事件) +│ ├── FriendRequestReceivedEvent # friend.request_received +│ ├── FriendAddedEvent # friend.added +│ └── FriendRemovedEvent # friend.removed +├── BotEvent (Bot 状态事件) +│ ├── BotInvitedToGroupEvent # bot.invited_to_group +│ ├── BotRemovedFromGroupEvent # bot.removed_from_group +│ ├── BotMutedEvent # bot.muted +│ └── BotUnmutedEvent # bot.unmuted +└── PlatformSpecificEvent # platform.{adapter}.{action} +``` + +## 3. 通用事件定义 + +### 3.1 事件基类 + +```python +class Event(pydantic.BaseModel): + """事件基类""" + + type: str + """事件类型标识，如 'message.received'""" + + timestamp: float + """事件发生的时间戳""" + + bot_uuid: str + """接收到此事件的 Bot UUID""" + + adapter_name: str + """产生此事件的适配器名称""" + + source_platform_object: typing.Optional[typing.Any] = None + """原始平台事件对象，供适配器内部使用""" +``` + +### 3.2 消息事件 + +#### MessageReceivedEvent (`message.received`) + +收到新消息。这是最核心的事件，替代现有的 `FriendMessage` / `GroupMessage`。 + +```python +class MessageReceivedEvent(Event): + """收到新消息""" + + type: str = "message.received" + + message_id: typing.Union[int, str] + """消息 ID""" + + message_chain: MessageChain + """消息内容""" + + sender: User + """发送者""" + + chat_type: ChatType # "private" | "group" + """会话类型""" + + chat_id: typing.Union[int, str] + """会话 ID（私聊为对方用户 ID，群聊为群 ID）""" + + group: typing.Optional[Group] = None + """群信息（仅群聊时存在）""" +``` + +与现有类型的映射关系： +- `chat_type == "private"` → 等价于现有 `FriendMessage` +- `chat_type == "group"` → 等价于现有 `GroupMessage` + +`ChatType` 枚举： + +```python +class ChatType(str, Enum): + PRIVATE = "private" + GROUP = "group" +``` + +#### MessageEditedEvent (`message.edited`) + +消息被编辑。 + +```python +class MessageEditedEvent(Event): + """消息被编辑""" + + type: str = "message.edited" + + message_id: typing.Union[int, str] + """被编辑的消息 ID""" + + new_content: MessageChain + """编辑后的新内容""" + + editor: User + """编辑者""" + + chat_type: ChatType + chat_id: typing.Union[int, str] + group: typing.Optional[Group] = None +``` + +#### MessageDeletedEvent (`message.deleted`) + +消息被删除/撤回。 + +```python +class MessageDeletedEvent(Event): + """消息被删除/撤回""" + + type: str = "message.deleted" + + message_id: typing.Union[int, str] + """被删除的消息 ID""" + + operator: typing.Optional[User] = None + """操作者（可能是发送者自己撤回，也可能是管理员删除）""" + + chat_type: ChatType + chat_id: typing.Union[int, str] + group: typing.Optional[Group] = None +``` + +#### MessageReactionEvent (`message.reaction`) + +消息收到表情回应。 + +```python +class MessageReactionEvent(Event): + """消息收到表情回应""" + + type: str = "message.reaction" + + message_id: typing.Union[int, str] + """被回应的消息 ID""" + + user: User + """回应者""" + + reaction: str + """回应的表情标识（emoji 或平台特定表情 ID）""" + + is_add: bool + """True 为添加回应，False 为移除回应""" + + chat_type: ChatType + chat_id: typing.Union[int, str] + group: typing.Optional[Group] = None +``` + +### 3.3 用户反馈事件 + +#### FeedbackReceivedEvent (`feedback.received`) + +用户对 Bot 回复提交反馈。该事件用于承载平台提供的点赞、点踩、取消反馈以及点踩原因等评价信息；典型来源包括企业微信 AI Bot 的 `feedback_event`、飞书卡片按钮回调、Web Embed 的反馈入口等。 + +```python +class FeedbackReceivedEvent(Event): + """收到用户反馈""" + + type: str = "feedback.received" + + feedback_id: str + """平台侧反馈 ID，用于幂等记录或取消反馈""" + + feedback_type: int + """1 = like, 2 = dislike, 3 = cancel/remove feedback""" + + feedback_content: typing.Optional[str] = None + """用户填写的自由文本反馈""" + + inaccurate_reasons: typing.Optional[list[str]] = None + """点踩时平台提供的预设不准确原因""" + + user_id: typing.Optional[str] = None + """提交反馈的用户 ID""" + + session_id: typing.Optional[str] = None + """会话 ID，例如 person_xxx 或 group_xxx""" + + message_id: typing.Optional[str] = None + """被评价的 Bot 回复消息 ID""" + + stream_id: typing.Optional[str] = None + """流式回复 ID，用于关联 streaming response""" +``` + +设计约定： + +- `feedback_id` 是幂等键；同一个 `feedback_id` 的后续事件应更新已有记录。 +- `feedback_type == 3` 表示用户取消/移除反馈，处理器可删除对应记录或标记为取消。 +- 如果平台只能给出原始回调 payload，差异字段保留在 `source_platform_object` 或 `PlatformSpecificEvent.data` 中；通用字段仍优先映射到 `FeedbackReceivedEvent`。 +- 该事件保留向后兼容映射：EBA 事件可转换为旧的 `FeedbackEvent`，字段语义保持一致。 + +### 3.4 群组事件 + +#### MemberJoinedEvent (`group.member_joined`) + +新成员加入群组。 + +```python +class MemberJoinedEvent(Event): + """新成员加入群组""" + + type: str = "group.member_joined" + + group: Group + """群组""" + + member: User + """加入的成员""" + + inviter: typing.Optional[User] = None + """邀请者（如有）""" + + join_type: typing.Optional[str] = None + """加入方式：'invite' / 'request' / 'direct' / None""" +``` + +#### MemberLeftEvent (`group.member_left`) + +成员离开群组。 + +```python +class MemberLeftEvent(Event): + """成员离开群组""" + + type: str = "group.member_left" + + group: Group + member: User + + is_kicked: bool = False + """是否被踢出""" + + operator: typing.Optional[User] = None + """操作者（踢出时为管理员）""" +``` + +#### MemberBannedEvent (`group.member_banned`) + +成员被禁言。 + +```python +class MemberBannedEvent(Event): + """成员被禁言""" + + type: str = "group.member_banned" + + group: Group + member: User + operator: typing.Optional[User] = None + duration: typing.Optional[int] = None + """禁言时长（秒），None 表示永久""" +``` + +#### MemberUnbannedEvent (`group.member_unbanned`) + +成员被解除禁言。 + +```python +class MemberUnbannedEvent(Event): + """成员被解除禁言""" + + type: str = "group.member_unbanned" + + group: Group + member: User + operator: typing.Optional[User] = None +``` + +#### GroupInfoUpdatedEvent (`group.info_updated`) + +群组信息被修改。 + +```python +class GroupInfoUpdatedEvent(Event): + """群组信息被修改""" + + type: str = "group.info_updated" + + group: Group + """更新后的群组信息""" + + operator: typing.Optional[User] = None + """操作者""" + + changed_fields: list[str] = [] + """发生变更的字段名列表，如 ['name', 'description']""" +``` + +### 3.5 好友事件 + +#### FriendRequestReceivedEvent (`friend.request_received`) + +收到好友请求。 + +```python +class FriendRequestReceivedEvent(Event): + """收到好友请求""" + + type: str = "friend.request_received" + + request_id: typing.Union[int, str] + """请求 ID，用于后续 approve/reject 操作""" + + user: User + """请求者""" + + message: typing.Optional[str] = None + """验证消息""" +``` + +#### FriendAddedEvent (`friend.added`) + +成功添加好友。 + +```python +class FriendAddedEvent(Event): + """成功添加好友""" + + type: str = "friend.added" + + user: User + """新好友""" +``` + +#### FriendRemovedEvent (`friend.removed`) + +好友被移除。 + +```python +class FriendRemovedEvent(Event): + """好友被移除""" + + type: str = "friend.removed" + + user: User + """被移除的好友""" +``` + +### 3.6 Bot 状态事件 + +#### BotInvitedToGroupEvent (`bot.invited_to_group`) + +Bot 被邀请加入群组。 + +```python +class BotInvitedToGroupEvent(Event): + """Bot 被邀请加入群组""" + + type: str = "bot.invited_to_group" + + group: Group + inviter: typing.Optional[User] = None + + request_id: typing.Optional[typing.Union[int, str]] = None + """邀请请求 ID，某些平台需要 Bot 确认才加入""" +``` + +#### BotRemovedFromGroupEvent (`bot.removed_from_group`) + +Bot 被移出群组。 + +```python +class BotRemovedFromGroupEvent(Event): + """Bot 被移出群组""" + + type: str = "bot.removed_from_group" + + group: Group + operator: typing.Optional[User] = None +``` + +#### BotMutedEvent / BotUnmutedEvent (`bot.muted` / `bot.unmuted`) + +Bot 被禁言/解除禁言。 + +```python +class BotMutedEvent(Event): + """Bot 被禁言""" + + type: str = "bot.muted" + + group: Group + operator: typing.Optional[User] = None + duration: typing.Optional[int] = None + + +class BotUnmutedEvent(Event): + """Bot 被解除禁言""" + + type: str = "bot.unmuted" + + group: Group + operator: typing.Optional[User] = None +``` + +### 3.7 平台特有事件 + +对于无法抽象为通用事件的平台特有事件，使用统一的 `PlatformSpecificEvent` 承载： + +```python +class PlatformSpecificEvent(Event): + """平台特有事件 + + 适配器无法映射到通用事件类型时，使用此类型承载。 + 插件可以通过 adapter_name + action 来识别和处理。 + """ + + type: str = "platform.specific" + + action: str + """平台特有的事件动作标识，如 'channel_created', 'pin_message'""" + + data: dict = {} + """事件数据，结构由具体适配器定义""" +``` + +事件类型字符串格式为 `platform.{adapter_name}.{action}`，例如： +- `platform.telegram.chat_member_updated` — Telegram 的群成员信息更新 +- `platform.discord.channel_created` — Discord 的频道创建 +- `platform.discord.voice_state_update` — Discord 的语音状态变更 +- `platform.slack.app_home_opened` — Slack 的 App Home 打开 + +## 4. 各平台事件支持矩阵 + +下表标注各通用事件在主要平台上的支持情况： + +| 事件 | Telegram | Discord | OneBot(QQ) | 飞书 | 钉钉 | Slack | 微信 | LINE | KOOK | +|------|----------|---------|-----------|------|------|-------|------|------|------| +| `message.received` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `message.edited` | Y | Y | N | Y | N | Y | N | N | Y | +| `message.deleted` | Y | Y | Y | Y | N | Y | Y | N | Y | +| `message.reaction` | Y | Y | Y | Y | Y | Y | N | N | Y | +| `feedback.received` | N | N | N | Y | N | N | Y | N | N | +| `group.member_joined` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `group.member_left` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `group.member_banned` | Y | Y | Y | N | N | N | N | N | N | +| `group.info_updated` | Y | Y | Y | Y | Y | Y | N | N | Y | +| `friend.request_received` | N | Y | Y | N | N | N | Y | Y | Y | +| `friend.added` | N | Y | Y | N | N | N | Y | Y | N | +| `bot.invited_to_group` | Y | Y | Y | Y | Y | Y | Y | N | Y | +| `bot.removed_from_group` | Y | Y | Y | Y | N | N | Y | N | Y | +| `bot.muted` | Y | N | Y | N | N | N | N | N | N | +| `bot.unmuted` | Y | N | Y | N | N | N | N | N | N | +| `platform.specific` | Y | Y | Y | Y | Y | Y | Y | Y | Y | + +> 注：此表为初步评估，具体以各平台 SDK/API 文档为准，实施时逐个确认。 + +## 5. 事件生命周期 + +``` +1. 平台 SDK 回调触发 + │ +2. 适配器 EventConverter.target2yiri(raw_event) + │ 将平台原生事件转换为统一 Event 对象 + │ 无法映射的事件 → PlatformSpecificEvent + │ +3. 适配器回调注册的 listener(event, adapter) + │ +4. RuntimeBot 接收事件 + │ +5. EventBus 分发 + │ +6. EventBus 向有权限的 Plugin EventListener 广播观察者事件 + │ observer 不占用响应目标，也不参与 priority 仲裁 + │ +7. EventRouter 查询 Bot 的 event_bindings + │ 精确/通配匹配 event_pattern，应用 filters 与 priority + │ 选择一个 Pipeline、Agent 或 discard 目标 + │ +8. 目标处理事件 + │ Pipeline → 进入完整 Pipeline 流水线（仅消息事件） + │ Agent → Host 编排已安装的插件 Runner + │ discard → 不产生响应 + │ +9. 处理器执行完毕，可能通过 Host 授权 API 执行响应动作 + （发消息、编辑消息、踢人、同意好友请求等） +``` + +## 6. 与现有事件类型的兼容映射 + +为保证现有插件不受影响，建立以下映射关系： + +| 新事件 | 条件 | 旧事件 | +|--------|------|--------| +| `MessageReceivedEvent` (chat_type=private) | — | `FriendMessage` | +| `MessageReceivedEvent` (chat_type=group) | — | `GroupMessage` | + +在插件 SDK 层面： + +| 新事件 | 旧插件事件 | +|--------|-----------| +| `MessageReceivedEvent` (chat_type=private, 非命令) | `PersonNormalMessageReceived` | +| `MessageReceivedEvent` (chat_type=group, 非命令) | `GroupNormalMessageReceived` | +| `MessageReceivedEvent` (chat_type=private, 命令) | `PersonCommandSent` | +| `MessageReceivedEvent` (chat_type=group, 命令) | `GroupCommandSent` | +| `MessageReceivedEvent` (处理完毕后) | `NormalMessageResponded` | + +兼容层在事件分发给插件 EventListener 时自动生成旧格式事件，确保监听旧事件类型的插件仍能正常工作。 + +## 7. 事件类型注册表 + +适配器在 manifest.yaml 中声明自己支持的事件类型： + +```yaml +kind: MessagePlatformAdapter +metadata: + name: telegram +spec: + supported_events: + - message.received + - message.edited + - message.deleted + - message.reaction + - feedback.received + - group.member_joined + - group.member_left + - group.member_banned + - group.info_updated + - bot.invited_to_group + - bot.removed_from_group + - bot.muted + - bot.unmuted + - platform.specific + platform_specific_events: + - chat_member_updated + - chat_join_request +``` + +这份声明用于： +1. WebUI 在配置事件处理器时，只显示当前 Bot 的适配器支持的事件类型 +2. EventRouter 在路由时校验事件类型有效性 +3. 文档自动生成 diff --git a/docs/event-based-agents/02-platform-api.md b/docs/event-based-agents/02-platform-api.md new file mode 100644 index 000000000..f67d11f4a --- /dev/null +++ b/docs/event-based-agents/02-platform-api.md @@ -0,0 +1,546 @@ +# 统一平台 API 与实体定义 + +## 1. 设计原则 + +- **通用 API 抽象**：大部分平台都支持的操作（发消息、获取群信息等）定义为通用 API 方法 +- **required / optional 标记**：每个 API 标记为必需或可选，适配器未实现可选 API 时抛出 `NotSupportedError` +- **透传机制**：适配器特有的操作通过 `call_platform_api(action, params)` 统一入口透传调用 +- **能力声明**：适配器在 manifest 中声明自己支持的 API 列表，供 WebUI 和插件查询 +- **实体统一**：通用实体（User、Group 等）在 SDK 层面统一定义，适配器负责转换 + +## 2. 通用实体定义 + +### 2.1 现有实体回顾 + +当前 SDK 已有以下实体（`langbot_plugin/api/entities/builtin/platform/entities.py`）： + +```python +Entity(id) +├── Friend(id, nickname, remark) +├── Group(id, name, permission) +└── GroupMember(id, member_name, permission, group, special_title) +``` + +### 2.2 新实体设计 + +扩展实体体系，保持向后兼容： + +```python +class User(pydantic.BaseModel): + """用户实体（统一表示）""" + + id: typing.Union[int, str] + """用户 ID""" + + nickname: str = "" + """昵称""" + + avatar_url: typing.Optional[str] = None + """头像 URL""" + + is_bot: bool = False + """是否为 Bot""" + + # 以下为可选的扩展信息，不同平台可能部分为空 + username: typing.Optional[str] = None + """用户名（如 Telegram 的 @username）""" + + remark: typing.Optional[str] = None + """备注名""" + + +class Group(pydantic.BaseModel): + """群组实体""" + + id: typing.Union[int, str] + """群组 ID""" + + name: str = "" + """群组名称""" + + description: typing.Optional[str] = None + """群组描述""" + + member_count: typing.Optional[int] = None + """成员数量""" + + avatar_url: typing.Optional[str] = None + """群组头像 URL""" + + owner_id: typing.Optional[typing.Union[int, str]] = None + """群主 ID""" + + +class GroupMember(pydantic.BaseModel): + """群成员实体""" + + user: User + """用户信息""" + + group_id: typing.Union[int, str] + """所属群组 ID""" + + role: MemberRole + """群内角色""" + + display_name: typing.Optional[str] = None + """群内显示名""" + + joined_at: typing.Optional[float] = None + """加入群组的时间戳""" + + title: typing.Optional[str] = None + """群头衔/特殊称号""" + + +class MemberRole(str, Enum): + """群成员角色""" + OWNER = "owner" + ADMIN = "admin" + MEMBER = "member" +``` + +### 2.3 与现有实体的兼容映射 + +| 新实体 | 旧实体 | 映射方式 | +|--------|--------|----------| +| `User` | `Friend` | `User(id=friend.id, nickname=friend.nickname, remark=friend.remark)` | +| `Group` | `Group`（旧） | `Group(id=old.id, name=old.name)` + `permission` 字段弃用 | +| `GroupMember` | `GroupMember`（旧） | `GroupMember(user=User(...), role=..., display_name=old.member_name)` | +| `MemberRole` | `Permission` | `OWNER↔Owner`, `ADMIN↔Administrator`, `MEMBER↔Member` | + +旧实体类保留，标记为 `@deprecated`，内部通过转换方法桥接到新实体。 + +## 3. 通用 API 定义 + +### 3.1 API 方法一览 + +#### 消息 API + +| 方法 | 必需/可选 | 说明 | +|------|----------|------| +| `send_message(target_type, target_id, message)` | **必需** | 主动发送消息 | +| `reply_message(event, message, quote_origin)` | **必需** | 回复一个消息事件 | +| `edit_message(chat_type, chat_id, message_id, new_content)` | 可选 | 编辑已发送的消息 | +| `delete_message(chat_type, chat_id, message_id)` | 可选 | 删除/撤回消息 | +| `forward_message(from_chat, message_id, to_chat_type, to_chat_id)` | 可选 | 转发消息到另一个会话 | +| `get_message(chat_type, chat_id, message_id)` | 可选 | 获取指定消息的内容 | + +#### 群组 API + +| 方法 | 必需/可选 | 说明 | +|------|----------|------| +| `get_group_info(group_id)` | 可选 | 获取群组信息 | +| `get_group_list()` | 可选 | 获取 Bot 加入的群组列表 | +| `get_group_member_list(group_id)` | 可选 | 获取群成员列表 | +| `get_group_member_info(group_id, user_id)` | 可选 | 获取指定群成员信息 | +| `set_group_name(group_id, name)` | 可选 | 修改群名称 | +| `mute_member(group_id, user_id, duration)` | 可选 | 禁言群成员 | +| `unmute_member(group_id, user_id)` | 可选 | 解除禁言 | +| `kick_member(group_id, user_id)` | 可选 | 踢出群成员 | +| `leave_group(group_id)` | 可选 | Bot 退出群组 | + +#### 用户 API + +| 方法 | 必需/可选 | 说明 | +|------|----------|------| +| `get_user_info(user_id)` | 可选 | 获取用户信息 | +| `get_friend_list()` | 可选 | 获取好友列表 | +| `approve_friend_request(request_id, approve, remark)` | 可选 | 处理好友请求 | +| `approve_group_invite(request_id, approve)` | 可选 | 处理入群邀请 | + +#### 媒体 API + +| 方法 | 必需/可选 | 说明 | +|------|----------|------| +| `upload_file(file_data, filename)` | 可选 | 上传文件，返回可引用的文件 ID 或 URL | +| `get_file_url(file_id)` | 可选 | 获取文件下载 URL | + +#### 透传 API + +| 方法 | 必需/可选 | 说明 | +|------|----------|------| +| `call_platform_api(action, params)` | 可选 | 调用适配器特有 API | + +### 3.2 API 方法签名详解 + +```python +class AbstractPlatformAdapter(pydantic.BaseModel, metaclass=abc.ABCMeta): + """平台适配器基类（新版）""" + + # ======== 必需方法 ======== + + @abc.abstractmethod + async def send_message( + self, + target_type: str, # "private" | "group" + target_id: typing.Union[int, str], + message: MessageChain, + ) -> MessageResult: + """主动发送消息 + + Returns: + MessageResult: 包含 message_id 等发送结果 + """ + ... + + @abc.abstractmethod + async def reply_message( + self, + event: MessageReceivedEvent, + message: MessageChain, + quote_origin: bool = False, + ) -> MessageResult: + """回复一个消息事件""" + ... + + # ======== 可选消息方法 ======== + + async def edit_message( + self, + chat_type: str, + chat_id: typing.Union[int, str], + message_id: typing.Union[int, str], + new_content: MessageChain, + ) -> None: + """编辑已发送的消息""" + raise NotSupportedError("edit_message") + + async def delete_message( + self, + chat_type: str, + chat_id: typing.Union[int, str], + message_id: typing.Union[int, str], + ) -> None: + """删除/撤回消息""" + raise NotSupportedError("delete_message") + + async def forward_message( + self, + from_chat_type: str, + from_chat_id: typing.Union[int, str], + message_id: typing.Union[int, str], + to_chat_type: str, + to_chat_id: typing.Union[int, str], + ) -> MessageResult: + """转发消息""" + raise NotSupportedError("forward_message") + + async def get_message( + self, + chat_type: str, + chat_id: typing.Union[int, str], + message_id: typing.Union[int, str], + ) -> MessageReceivedEvent: + """获取指定消息""" + raise NotSupportedError("get_message") + + # ======== 可选群组方法 ======== + + async def get_group_info( + self, + group_id: typing.Union[int, str], + ) -> Group: + """获取群组信息""" + raise NotSupportedError("get_group_info") + + async def get_group_list(self) -> list[Group]: + """获取 Bot 加入的群组列表""" + raise NotSupportedError("get_group_list") + + async def get_group_member_list( + self, + group_id: typing.Union[int, str], + ) -> list[GroupMember]: + """获取群成员列表""" + raise NotSupportedError("get_group_member_list") + + async def get_group_member_info( + self, + group_id: typing.Union[int, str], + user_id: typing.Union[int, str], + ) -> GroupMember: + """获取指定群成员信息""" + raise NotSupportedError("get_group_member_info") + + async def set_group_name( + self, + group_id: typing.Union[int, str], + name: str, + ) -> None: + """修改群名称""" + raise NotSupportedError("set_group_name") + + async def mute_member( + self, + group_id: typing.Union[int, str], + user_id: typing.Union[int, str], + duration: int = 0, + ) -> None: + """禁言群成员，duration 为秒数，0 表示永久""" + raise NotSupportedError("mute_member") + + async def unmute_member( + self, + group_id: typing.Union[int, str], + user_id: typing.Union[int, str], + ) -> None: + """解除禁言""" + raise NotSupportedError("unmute_member") + + async def kick_member( + self, + group_id: typing.Union[int, str], + user_id: typing.Union[int, str], + ) -> None: + """踢出群成员""" + raise NotSupportedError("kick_member") + + async def leave_group( + self, + group_id: typing.Union[int, str], + ) -> None: + """Bot 退出群组""" + raise NotSupportedError("leave_group") + + # ======== 可选用户方法 ======== + + async def get_user_info( + self, + user_id: typing.Union[int, str], + ) -> User: + """获取用户信息""" + raise NotSupportedError("get_user_info") + + async def get_friend_list(self) -> list[User]: + """获取好友列表""" + raise NotSupportedError("get_friend_list") + + async def approve_friend_request( + self, + request_id: typing.Union[int, str], + approve: bool = True, + remark: typing.Optional[str] = None, + ) -> None: + """处理好友请求""" + raise NotSupportedError("approve_friend_request") + + async def approve_group_invite( + self, + request_id: typing.Union[int, str], + approve: bool = True, + ) -> None: + """处理入群邀请""" + raise NotSupportedError("approve_group_invite") + + # ======== 可选媒体方法 ======== + + async def upload_file( + self, + file_data: bytes, + filename: str, + ) -> str: + """上传文件，返回文件 ID 或 URL""" + raise NotSupportedError("upload_file") + + async def get_file_url( + self, + file_id: str, + ) -> str: + """获取文件下载 URL""" + raise NotSupportedError("get_file_url") + + # ======== 透传 API ======== + + async def call_platform_api( + self, + action: str, + params: dict = {}, + ) -> dict: + """调用适配器特有 API + + Args: + action: 平台特有的 API 动作标识 + params: 参数字典 + + Returns: + dict: 返回结果 + + Examples: + # Telegram: pin 消息 + await adapter.call_platform_api("pin_message", { + "chat_id": 123456, + "message_id": 789 + }) + + # Discord: 创建频道 + await adapter.call_platform_api("create_channel", { + "guild_id": "...", + "name": "new-channel", + "type": "text" + }) + """ + raise NotSupportedError("call_platform_api") + + # ======== 流式输出（保留现有机制） ======== + + async def reply_message_chunk( + self, + event: MessageReceivedEvent, + bot_message: dict, + message: MessageChain, + quote_origin: bool = False, + is_final: bool = False, + ): + """流式回复消息""" + raise NotSupportedError("reply_message_chunk") + + async def is_stream_output_supported(self) -> bool: + """是否支持流式输出""" + return False + + # ======== 生命周期方法（保留现有） ======== + + @abc.abstractmethod + async def run_async(self): + """启动适配器""" + ... + + @abc.abstractmethod + async def kill(self) -> bool: + """停止适配器""" + ... + + @abc.abstractmethod + def register_listener(self, event_type, callback): + """注册事件监听器""" + ... + + @abc.abstractmethod + def unregister_listener(self, event_type, callback): + """注销事件监听器""" + ... +``` + +### 3.3 返回值类型 + +```python +class MessageResult(pydantic.BaseModel): + """消息发送结果""" + + message_id: typing.Optional[typing.Union[int, str]] = None + """发送成功后的消息 ID""" + + raw: typing.Optional[dict] = None + """平台原始返回数据""" + + +class NotSupportedError(Exception): + """适配器未实现此 API""" + + def __init__(self, api_name: str): + self.api_name = api_name + super().__init__(f"API not supported by this adapter: {api_name}") +``` + +## 4. API 能力声明 + +适配器在 manifest.yaml 中声明支持的 API： + +```yaml +kind: MessagePlatformAdapter +metadata: + name: telegram +spec: + supported_apis: + required: + - send_message + - reply_message + optional: + - edit_message + - delete_message + - get_group_info + - get_group_member_list + - get_user_info + - upload_file + - get_file_url + - call_platform_api + platform_specific_apis: + - action: pin_message + description: "Pin a message in a chat" + params_schema: + chat_id: { type: "string", required: true } + message_id: { type: "string", required: true } + - action: unpin_message + description: "Unpin a message" + params_schema: + chat_id: { type: "string", required: true } + message_id: { type: "string", required: true } +``` + +用途： +1. **WebUI**：在配置界面展示当前 Bot 可用的 API 能力 +2. **插件**：插件可查询某个 Bot 是否支持特定 API，据此决定行为 +3. **文档**：自动生成各适配器的 API 支持矩阵 + +## 5. 各平台 API 支持矩阵 + +| API | Telegram | Discord | OneBot(QQ) | 飞书 | 钉钉 | Slack | 微信 | LINE | KOOK | +|-----|----------|---------|-----------|------|------|-------|------|------|------| +| `send_message` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `reply_message` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `edit_message` | Y | Y | N | Y | N | Y | N | N | Y | +| `delete_message` | Y | Y | Y | Y | N | Y | Y | N | Y | +| `forward_message` | Y | N | Y | Y | N | N | Y | N | N | +| `get_group_info` | Y | Y | Y | Y | Y | Y | N | Y | Y | +| `get_group_member_list` | Y | Y | Y | Y | Y | Y | N | Y | Y | +| `get_user_info` | Y | Y | Y | Y | Y | Y | N | Y | Y | +| `get_friend_list` | N | Y | Y | N | N | N | Y | N | N | +| `mute_member` | Y | Y | Y | N | N | N | N | N | N | +| `kick_member` | Y | Y | Y | N | N | N | N | N | Y | +| `upload_file` | Y | Y | Y | Y | Y | Y | Y | Y | Y | +| `call_platform_api` | Y | Y | Y | Y | Y | Y | Y | Y | Y | + +> 注：此表为初步评估，具体以各平台 SDK/API 文档为准。 + +## 6. MessageChain 扩展 + +### 6.1 保留的通用组件 + +以下 MessageComponent 类型保持不变，继续作为通用消息元素： + +- `Source` — 消息元信息 +- `Plain` — 纯文本 +- `Quote` — 引用回复 +- `At` / `AtAll` — @提及 +- `Image` — 图片 +- `Voice` — 语音 +- `File` — 文件 +- `Forward` — 合并转发 +- `Face` — 表情 +- `Unknown` — 未知类型 + +### 6.2 平台特有组件处理 + +当前 MessageChain 中存在大量微信特有的组件类型（`WeChatMiniPrograms`, `WeChatEmoji`, `WeChatLink` 等）。在新架构下： + +- 这些类型**继续保留**在 SDK 中以保持兼容 +- 新增的平台特有消息组件统一使用 `PlatformComponent` 基类： + +```python +class PlatformComponent(MessageComponent): + """平台特有的消息组件""" + + type: str = "Platform" + + platform: str + """平台标识""" + + component_type: str + """组件类型""" + + data: dict = {} + """组件数据""" +``` + +适配器在转换消息链时，对于无法映射到通用组件的平台特有内容，使用 `PlatformComponent` 承载。 diff --git a/docs/event-based-agents/03-adapter-structure.md b/docs/event-based-agents/03-adapter-structure.md new file mode 100644 index 000000000..4943b8ad0 --- /dev/null +++ b/docs/event-based-agents/03-adapter-structure.md @@ -0,0 +1,483 @@ +# 适配器新目录结构 + +## 1. 设计目标 + +- **模块化**：每个适配器从单文件拆分到独立目录，各模块职责清晰 +- **可维护**：随着事件和 API 的增加，代码量会显著增长，目录结构有助于管理复杂度 +- **一致性**：所有适配器遵循相同的目录布局和文件命名约定 +- **兼容现有发现机制**：保持 YAML manifest + ComponentDiscoveryEngine 的注册体系 + +## 2. 新目录布局 + +### 2.1 整体结构 + +``` +pkg/platform/ +├── __init__.py +├── botmgr.py # PlatformManager + RuntimeBot（重构） +├── event_bus.py # EventBus（新增） +├── event_router.py # EventRouter（新增） +├── logger.py # EventLogger（保留） +├── webhook_pusher.py # WebhookPusher（重构为 WebhookHandler） +│ +├── adapters/ # 适配器（新目录） +│ ├── __init__.py +│ │ +│ ├── telegram/ +│ │ ├── __init__.py +│ │ ├── adapter.py # TelegramAdapter 主类 +│ │ ├── event_converter.py # 平台事件 → 统一事件 +│ │ ├── message_converter.py # MessageChain 互转 +│ │ ├── api_impl.py # 通用 API 实现 +│ │ ├── platform_api.py # call_platform_api 的动作映射 +│ │ ├── types.py # 平台特有类型定义 +│ │ └── manifest.yaml # 适配器清单 +│ │ +│ ├── discord/ +│ │ ├── __init__.py +│ │ ├── adapter.py +│ │ ├── event_converter.py +│ │ ├── message_converter.py +│ │ ├── api_impl.py +│ │ ├── platform_api.py +│ │ ├── types.py +│ │ ├── voice.py # Discord 语音连接管理（特有） +│ │ └── manifest.yaml +│ │ +│ ├── aiocqhttp/ # OneBot v11 (QQ) +│ │ └── ... +│ ├── qqofficial/ +│ │ └── ... +│ ├── lark/ # 飞书 +│ │ └── ... +│ ├── dingtalk/ +│ │ └── ... +│ ├── slack/ +│ │ └── ... +│ ├── wechatpad/ +│ │ └── ... +│ ├── officialaccount/ # 微信公众号 +│ │ └── ... +│ ├── wecom/ # 企业微信 +│ │ └── ... +│ ├── wecombot/ +│ │ └── ... +│ ├── wecomcs/ +│ │ └── ... +│ ├── kook/ +│ │ └── ... +│ ├── line/ +│ │ └── ... +│ ├── satori/ +│ │ └── ... +│ ├── websocket/ # 内置 WebSocket 适配器 +│ │ ├── __init__.py +│ │ ├── adapter.py +│ │ ├── manager.py # WebSocket 连接管理 +│ │ └── manifest.yaml +│ │ +│ └── legacy/ # 旧版适配器（保留一段时间后移除） +│ ├── gewechat/ +│ ├── nakuru/ +│ └── qqbotpy/ +│ +└── handlers/ # 事件处理器实现（新增） + ├── __init__.py + ├── base.py # AbstractEventHandler 基类 + ├── pipeline_handler.py # PipelineHandler + ├── agent_handler.py # AgentHandler + ├── webhook_handler.py # WebhookHandler + └── plugin_handler.py # PluginHandler +``` + +### 2.2 适配器目录内各文件职责 + +以 Telegram 为例： + +| 文件 | 职责 | 关键类/函数 | +|------|------|------------| +| `adapter.py` | 主入口，继承 `AbstractPlatformAdapter`，组装其他模块 | `TelegramAdapter` | +| `event_converter.py` | 将 Telegram 原生事件转换为统一事件类型 | `TelegramEventConverter` — 支持 Message/Edit/Delete/Reaction/MemberJoin 等所有事件 | +| `message_converter.py` | `MessageChain` 与 Telegram 消息格式互转 | `TelegramMessageConverter.yiri2target()` / `target2yiri()` | +| `api_impl.py` | 实现通用 API 方法（edit_message, delete_message, get_group_info 等） | 各 API 方法的 Telegram 实现 | +| `platform_api.py` | 实现 `call_platform_api` 的动作分发表 | `PLATFORM_API_MAP = {"pin_message": ..., "unpin_message": ...}` | +| `types.py` | 平台特有的类型定义 | Telegram 特有的枚举、配置结构等 | +| `manifest.yaml` | 适配器清单：名称、配置 schema、支持的事件和 API 列表 | — | + +## 3. 新基类设计 + +### 3.1 AbstractPlatformAdapter + +新基类继承自现有 `AbstractMessagePlatformAdapter` 并扩展，位于 `langbot-plugin-sdk` 中： + +```python +# langbot_plugin/api/definition/abstract/platform/adapter.py + +class AbstractPlatformAdapter(pydantic.BaseModel, metaclass=abc.ABCMeta): + """平台适配器基类（EBA 版本） + + 相比旧版 AbstractMessagePlatformAdapter： + - 新增通用 API 方法（edit_message, delete_message, get_group_info 等） + - 新增透传 API（call_platform_api） + - 新增能力声明（get_supported_events, get_supported_apis） + - 事件监听器支持所有事件类型，不仅限于消息事件 + """ + + bot_account_id: str = "" + config: dict + logger: AbstractEventLogger = pydantic.Field(exclude=True) + + class Config: + arbitrary_types_allowed = True + + # ---- 能力声明 ---- + + def get_supported_events(self) -> list[str]: + """返回此适配器支持的事件类型列表 + + 默认实现从 manifest.yaml 读取。 + 适配器也可以 override 此方法动态声明。 + """ + return ["message.received"] + + def get_supported_apis(self) -> list[str]: + """返回此适配器支持的 API 列表 + + 默认实现从 manifest.yaml 读取。 + """ + return ["send_message", "reply_message"] + + # ---- 必需方法（抽象） ---- + + @abc.abstractmethod + async def send_message(self, target_type, target_id, message) -> MessageResult: + ... + + @abc.abstractmethod + async def reply_message(self, event, message, quote_origin=False) -> MessageResult: + ... + + @abc.abstractmethod + async def run_async(self): + ... + + @abc.abstractmethod + async def kill(self) -> bool: + ... + + @abc.abstractmethod + def register_listener(self, event_type, callback): + ... + + @abc.abstractmethod + def unregister_listener(self, event_type, callback): + ... + + # ---- 可选方法（默认抛 NotSupportedError） ---- + # edit_message, delete_message, forward_message, + # get_group_info, get_group_member_list, ... + # call_platform_api, ... + # （完整签名见 02-platform-api.md） + + # ---- 流式输出（保留） ---- + + async def reply_message_chunk(self, event, bot_message, message, + quote_origin=False, is_final=False): + raise NotSupportedError("reply_message_chunk") + + async def is_stream_output_supported(self) -> bool: + return False + + # ---- 消息卡片（保留） ---- + + async def create_message_card(self, message_id, event) -> bool: + return False + + async def is_muted(self, group_id) -> bool: + return False +``` + +### 3.2 AbstractMessagePlatformAdapter 兼容 + +旧的 `AbstractMessagePlatformAdapter` 保留为 `AbstractPlatformAdapter` 的类型别名： + +```python +# 向后兼容 +AbstractMessagePlatformAdapter = AbstractPlatformAdapter +``` + +现有适配器代码中的 `AbstractMessagePlatformAdapter` 引用不需要立即修改。 + +### 3.3 EventConverter 新设计 + +现有 `AbstractEventConverter` 只有 `target2yiri` 和 `yiri2target` 两个静态方法，且只处理消息事件。 + +新设计支持多种事件类型： + +```python +class AbstractEventConverter: + """事件转换器基类（EBA 版本） + + 适配器需要实现此转换器，将平台原生事件转换为统一事件。 + """ + + @staticmethod + def target2yiri(raw_event: typing.Any) -> typing.Optional[Event]: + """将平台原生事件转换为统一事件 + + Args: + raw_event: 平台 SDK 回调传入的原始事件对象 + + Returns: + 统一 Event 对象，如果无法转换或不需要处理则返回 None + """ + raise NotImplementedError + + @staticmethod + def yiri2target(event: Event) -> typing.Any: + """将统一事件转换为平台原生事件（一般不需要）""" + raise NotImplementedError +``` + +具体适配器的 EventConverter 实现会是一个分发式的结构： + +```python +class TelegramEventConverter(AbstractEventConverter): + """Telegram 事件转换器""" + + @staticmethod + def target2yiri(update: telegram.Update) -> typing.Optional[Event]: + # 消息事件 + if update.message: + return TelegramEventConverter._convert_message(update) + # 消息编辑 + if update.edited_message: + return TelegramEventConverter._convert_edited_message(update) + # 成员变动 + if update.chat_member: + return TelegramEventConverter._convert_chat_member(update) + # 回调查询（按钮点击等） + if update.callback_query: + return TelegramEventConverter._convert_callback_query(update) + # 其他 → PlatformSpecificEvent + return TelegramEventConverter._convert_platform_specific(update) + + @staticmethod + def _convert_message(update) -> MessageReceivedEvent: + ... + + @staticmethod + def _convert_edited_message(update) -> MessageEditedEvent: + ... + + @staticmethod + def _convert_chat_member(update) -> typing.Union[ + MemberJoinedEvent, MemberLeftEvent, ... + ]: + ... + + @staticmethod + def _convert_platform_specific(update) -> PlatformSpecificEvent: + ... +``` + +## 4. Manifest 文件格式扩展 + +现有 manifest.yaml 只声明 `kind`, `metadata`, `spec.config`, `execution`。 + +新增 `spec.supported_events` 和 `spec.supported_apis`： + +```yaml +apiVersion: v1 +kind: MessagePlatformAdapter + +metadata: + name: telegram + label: + en_US: Telegram + zh_Hans: Telegram + icon: telegram.svg + description: + en_US: Telegram Bot adapter + zh_Hans: Telegram Bot 适配器 + +spec: + config: + # 现有配置 schema（保持不变） + - key: token + label: { en_US: "Bot Token", zh_Hans: "Bot Token" } + type: string + required: true + sensitive: true + # ... + + supported_events: + - message.received + - message.edited + - message.deleted + - message.reaction + - feedback.received + - group.member_joined + - group.member_left + - group.member_banned + - group.info_updated + - bot.invited_to_group + - bot.removed_from_group + - bot.muted + - bot.unmuted + - platform.specific + + supported_apis: + required: + - send_message + - reply_message + optional: + - edit_message + - delete_message + - get_group_info + - get_group_member_list + - get_group_member_info + - get_user_info + - upload_file + - get_file_url + - call_platform_api + + platform_specific_apis: + - action: pin_message + description: { en_US: "Pin a message", zh_Hans: "置顶消息" } + - action: unpin_message + description: { en_US: "Unpin a message", zh_Hans: "取消置顶" } + - action: get_chat_administrators + description: { en_US: "Get chat admins", zh_Hans: "获取群管理员列表" } + +execution: + python: + path: pkg/platform/adapters/telegram/adapter.py + attr: TelegramAdapter +``` + +## 5. 适配器注册与发现 + +### 5.1 Blueprint 更新 + +`templates/components.yaml` 中更新扫描路径： + +```yaml +kind: Blueprint +spec: + components: + MessagePlatformAdapter: + fromDirs: + - path: pkg/platform/adapters/ # 新路径 +``` + +`ComponentDiscoveryEngine` 的递归扫描逻辑不变——它会扫描所有子目录中的 `.yaml` 文件。因此每个适配器目录下的 `manifest.yaml` 会被自动发现。 + +### 5.2 PlatformManager 适配 + +`PlatformManager.initialize()` 的核心逻辑基本不变： + +```python +async def initialize(self): + # 1. 发现适配器组件（自动扫描新目录结构） + self.adapter_components = self.ap.discover.get_components_by_kind('MessagePlatformAdapter') + + # 2. 动态导入适配器类 + for component in self.adapter_components: + self.adapter_dict[component.metadata.name] = component.get_python_component_class() + + # 3. 从数据库加载 Bot 并实例化适配器（不变） + await self.load_bots_from_db() +``` + +变更点： +- `execution.python.path` 从 `pkg/platform/sources/telegram.py` 变为 `pkg/platform/adapters/telegram/adapter.py` +- `get_python_component_class()` 正常工作，因为它按路径动态导入 + +## 6. RuntimeBot 重构 + +### 6.1 现有问题 + +当前 `RuntimeBot.initialize()` 硬编码注册了两个回调： + +```python +# 现有代码 +self.adapter.register_listener(platform_events.FriendMessage, on_friend_message) +self.adapter.register_listener(platform_events.GroupMessage, on_group_message) +``` + +### 6.2 新设计 + +`RuntimeBot` 改为注册一个通用的事件回调： + +```python +class RuntimeBot: + async def initialize(self): + # 注册通用事件回调，接收所有事件类型 + self.adapter.register_listener(Event, self._on_event) + + async def _on_event( + self, + event: Event, + adapter: AbstractPlatformAdapter, + ): + """统一事件入口""" + + # 1. 设置事件的 bot_uuid 和 adapter_name + event.bot_uuid = self.bot_entity.uuid + event.adapter_name = self.bot_entity.adapter + + # 2. 日志记录 + await self._log_event(event) + + # 3. 提交给 EventBus + await self.ap.event_bus.emit(event, adapter) +``` + +适配器侧的 `register_listener` 实现也需调整： +- 当 `event_type` 为 `Event`（基类）时，注册为"接收所有事件"的通配回调 +- 适配器在收到平台原生事件时，通过 `EventConverter.target2yiri()` 转换后，调用所有匹配的回调 + +## 7. 从现有单文件适配器迁移 + +### 7.1 迁移模式 + +以 Telegram 为例，从 `sources/telegram.py`（445 行）拆分： + +| 原代码位置 | → 新文件 | +|-----------|----------| +| `TelegramMessageConverter` 类 | `telegram/message_converter.py` | +| `TelegramEventConverter` 类 | `telegram/event_converter.py`（扩展，支持更多事件） | +| `TelegramAdapter.__init__` / `run_async` / `kill` / `register_listener` | `telegram/adapter.py` | +| `TelegramAdapter.send_message` / `reply_message` / `reply_message_chunk` | `telegram/adapter.py`（消息方法保留在主类）+ `telegram/api_impl.py`（新增 API） | +| 新增代码 | `telegram/api_impl.py`（edit_message, delete_message, get_group_info 等） | +| 新增代码 | `telegram/platform_api.py`（pin_message, unpin_message 等的映射） | +| `telegram.yaml` | `telegram/manifest.yaml`（扩展 supported_events/apis） | + +### 7.2 迁移顺序建议 + +1. **Telegram** — 功能最完整的适配器之一，适合作为模板 +2. **Discord** — 第二个迁移，验证模式的通用性 +3. **AioCQHTTP (OneBot)** — 国内最常用，确保兼容 +4. **其他适配器** — 按使用频率排序 + +### 7.3 渐进式迁移 + +不需要一次性迁移所有适配器。可以采用渐进策略： + +1. 先在 `adapters/` 下建立新适配器 +2. `Blueprint` 同时扫描 `sources/` 和 `adapters/` 两个目录 +3. 旧适配器在 `sources/` 中继续工作 +4. 逐个迁移到新结构 +5. 全部迁移完成后移除 `sources/` 目录 + +```yaml +# 过渡期的 Blueprint +kind: Blueprint +spec: + components: + MessagePlatformAdapter: + fromDirs: + - path: pkg/platform/sources/ # 旧路径（尚未迁移的适配器） + - path: pkg/platform/adapters/ # 新路径（已迁移的适配器） +``` diff --git a/docs/event-based-agents/04-event-routing.md b/docs/event-based-agents/04-event-routing.md new file mode 100644 index 000000000..e9805b78f --- /dev/null +++ b/docs/event-based-agents/04-event-routing.md @@ -0,0 +1,175 @@ +# 事件路由与编排 + +> Implementation update (2026-09-08): the EventListener observer-broadcast proposal below is superseded by [Event processors](09-event-processors.md). Legacy EventListener hooks run only inside Pipeline. New EBA handlers use explicitly created and bound Runner instances, a third peer processor type alongside Agent and Pipeline. + +> 状态：当前实施模型（2026-07-12）。本文以 Pipeline / Agent 平级并存为准，不再保留早期 `pipeline / agent / webhook / plugin` 四种 Handler 草案。 + +## 1. 路由边界 + +EBA 将事件处理拆成两个互不替代的阶段： + +1. **观察者广播**：EventBus 把事件广播给有权限的插件 EventListener。观察者可记录、同步或执行受控副作用，但不占用响应目标。 +2. **响应者仲裁**：EventRouter 按 Bot 的 `event_bindings` 选择一个 Pipeline、独立 Agent 或 `discard`。 + +Pipeline 与 Agent 是平级处理器： + +| 处理器 | 配置事实源 | 执行路径 | 事件范围 | +| --- | --- | --- | --- | +| Pipeline | Pipeline 表与完整 Stage 配置 | MessageAggregator -> QueryPool -> RuntimePipeline | 消息事件，首版为 `message.received` | +| Agent | Agent 表中的 runner 与 runner config | Runner Host orchestrator -> plugin Runner | Agent/Runner 声明支持的消息或非消息事件 | +| discard | 无处理器配置 | 明确结束路由 | 任意事件 | + +插件 EventListener 不是第三种响应目标。Webhook、Dify、n8n、Coze 等外部系统需要响应事件时，由对应 Runner 插件承接。 + +## 2. 数据模型 + +### 2.1 独立 Agent + +Agent 保存自己的 Runner 选择与配置，不嵌入 Pipeline，也不复制 Pipeline 的 AI stage： + +```python +class Agent(Base): + uuid: str + name: str + description: str + emoji: str + kind: str # 固定为 "agent" + component_ref: str # Runner id + config: dict # runner + runner_config + supported_event_patterns: list[str] +``` + +当前配置形状： + +```json +{ + "runner": { + "id": "plugin:langbot-team/LocalAgent/default" + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "model-uuid", + "fallbacks": [] + } + } + } +} +``` + +Runner id 来自已安装插件的 Runner manifest。Host 不维护 LocalAgent、Dify 或其他具体实现的内置分支。 + +### 2.2 EventBinding + +Bot 维护事件到处理器的引用： + +```python +class EventBinding(BaseModel): + id: str + event_pattern: str # 精确、namespace.* 或 * + target_type: str # agent | pipeline | discard + target_uuid: str | None # Agent/Pipeline 原始 UUID + filters: list[dict] + priority: int + enabled: bool + description: str +``` + +示例： + +```json +[ + { + "id": "binding-message", + "event_pattern": "message.received", + "target_type": "pipeline", + "target_uuid": "pipeline-uuid", + "filters": [], + "priority": 100, + "enabled": true + }, + { + "id": "binding-member-joined", + "event_pattern": "group.member_joined", + "target_type": "agent", + "target_uuid": "agent-uuid", + "filters": [], + "priority": 50, + "enabled": true + } +] +``` + +Binding 只保存引用与路由条件。它不复制 Pipeline 或 Agent 配置。 + +## 3. 匹配与仲裁 + +事件模式支持： + +- 精确匹配：`group.member_joined` +- 命名空间通配：`group.*` +- 全局通配：`*` + +路由按以下顺序处理： + +1. 忽略 `enabled = false` 的 binding。 +2. 检查 `event_pattern` 与结构化 filters。 +3. 校验目标存在且声明支持该事件。 +4. 按 `priority` 从高到低选择；同优先级按稳定列表顺序。 +5. 只执行一个响应目标。 + +Pipeline 目标只能匹配消息事件。非消息事件不得伪装成用户文本塞进 Pipeline。 + +## 4. 执行流程 + +```text +Platform adapter + -> normalized event + -> EventBus + -> authorized Plugin EventListener observers + -> EventRouter + -> Pipeline target -> full Pipeline stage chain + -> Agent target -> Runner Host orchestrator + -> discard -> stop + -> Host delivery/platform API +``` + +### 4.1 Pipeline target + +消息事件按原有方式构造 Query，经 MessageAggregator、QueryPool 和完整 Pipeline Stage 链执行。Pipeline 可以继续使用 Runner 作为 AI stage 的实现，但 Pipeline 本身不会因此变成 Agent。 + +### 4.2 Agent target + +Host 读取独立 Agent 的 Runner id/config，构造 event-first context、run-scoped resources 与 delivery policy，再调用插件 Runner。Runner 输出由 Host 统一归一化、记录和投递。 + +Runner 可通过 SDK/Python `RunnerAPIProxy.call_tool` 或 SDK-owned scoped MCP bridge 回调 Host 能力。两条路径都映射到 `PluginToRuntimeAction.CALL_TOOL`，使用相同的 run authorization、Host execution Query、ToolManager 和 Box session 规则。Box session 是 Host canonical scope 的固定长度安全哈希；同一平台会话稳定、不同 scope 隔离、缺少 identity 时 fail closed，Runner 不配置 sandbox scope。 + +### 4.3 Observer side effects + +观察者与响应者可以同时工作。为避免编辑、reaction 等合成事件重复触发不可逆操作，Host 应按事件能力和授权过滤观察者可用 API，并记录副作用结果。Observer 广播不作为 fallback 响应。 + +## 5. Pipeline 与 Agent 的并存规则 + +1. Pipeline 与 Agent 保留各自的持久化、编辑和执行语义。 +2. 处理器聚合页面可以统一展示二者，但不会创建第三份处理器记录。 +3. 旧 Pipeline 仍是 Pipeline；其 runner config 不迁移、不复制为独立 Agent。 +4. 需要 Agent 的用户新建 Agent、选择已安装 Runner，再建立 event binding。 +5. 一个 Bot 可按不同事件同时绑定 Pipeline 与 Agent。 + +## 6. WebUI 约束 + +处理器入口展示带类型标识的 Agent 与 Pipeline。Bot 事件编排器应： + +- 按 adapter manifest 展示可用事件； +- 非消息事件不提供 Pipeline 目标； +- Agent 目标按 `supported_event_patterns` 过滤； +- 展示 priority、filters、enabled 与 discard； +- 保存前校验目标 UUID 与 `target_type` 一致。 + +Pipeline 的配置、Debug Chat 和 Monitoring 继续使用 Pipeline 页面；Agent 使用独立表单配置 Runner 与事件能力。 + +## 7. 版本与迁移边界 + +此功能按当前 4.x schema 直接实现，不提供 LangBot 3.x 数据库或配置升级路径，也不读取旧 Runner 字段作为 fallback。旧 Pipeline 中的 runner 配置不会生成独立 Agent；用户按新产品模型添加 Agent 即可。 + +详见 [07-agent-orchestration.md](./07-agent-orchestration.md) 与 [08-agent-page-and-event-orchestration.md](./08-agent-page-and-event-orchestration.md)。 diff --git a/docs/event-based-agents/05-plugin-sdk.md b/docs/event-based-agents/05-plugin-sdk.md new file mode 100644 index 000000000..b79591758 --- /dev/null +++ b/docs/event-based-agents/05-plugin-sdk.md @@ -0,0 +1,740 @@ +# 插件 SDK 改造 + +> Implementation update (2026-09-08): the EventListener observer-broadcast proposal below is superseded by [Event processors](09-event-processors.md). Legacy EventListener hooks run only inside Pipeline. New EBA handlers use explicitly created and bound Runner instances, a third peer processor type alongside Agent and Pipeline. + +## 1. 概述 + +插件 SDK 需要配合 EBA 架构进行以下改造： + +1. **新事件类型**：将所有通用事件暴露给插件 +2. **新 API**：将新增的平台 API 通过 `LangBotAPIProxy` 暴露给插件 +3. **兼容层**：保证现有插件零修改运行 +4. **通信协议扩展**：新增 action 枚举支持新 API + +## 2. 新事件类型暴露 + +### 2.1 插件事件模型扩展 + +当前插件 SDK 的事件模型（`api/entities/events.py`）只有消息相关事件。需要新增所有通用事件的插件级包装： + +```python +# api/entities/events.py — 新增事件 + +# ---- 消息事件（扩展） ---- + +class MessageEditedReceived(BaseEventModel): + """消息被编辑事件""" + launcher_type: str + launcher_id: typing.Union[int, str] + message_id: typing.Union[int, str] + editor_id: typing.Union[int, str] + new_content: MessageChain + chat_type: str # "private" | "group" + +class MessageDeletedReceived(BaseEventModel): + """消息被删除/撤回事件""" + launcher_type: str + launcher_id: typing.Union[int, str] + message_id: typing.Union[int, str] + operator_id: typing.Optional[typing.Union[int, str]] = None + chat_type: str + +class MessageReactionReceived(BaseEventModel): + """消息表情回应事件""" + launcher_type: str + launcher_id: typing.Union[int, str] + message_id: typing.Union[int, str] + user_id: typing.Union[int, str] + reaction: str + is_add: bool + +# ---- 用户反馈事件 ---- + +class FeedbackReceived(BaseEventModel): + """用户对 Bot 回复提交反馈""" + feedback_id: str + feedback_type: int # 1=like, 2=dislike, 3=cancel/remove feedback + feedback_content: typing.Optional[str] = None + inaccurate_reasons: typing.Optional[list[str]] = None + user_id: typing.Optional[str] = None + session_id: typing.Optional[str] = None + message_id: typing.Optional[str] = None + stream_id: typing.Optional[str] = None + +# ---- 群组事件 ---- + +class GroupMemberJoined(BaseEventModel): + """新成员加入群组""" + group_id: typing.Union[int, str] + group_name: str + member_id: typing.Union[int, str] + member_name: str + inviter_id: typing.Optional[typing.Union[int, str]] = None + join_type: typing.Optional[str] = None + +class GroupMemberLeft(BaseEventModel): + """成员离开群组""" + group_id: typing.Union[int, str] + group_name: str + member_id: typing.Union[int, str] + member_name: str + is_kicked: bool = False + operator_id: typing.Optional[typing.Union[int, str]] = None + +class GroupMemberBanned(BaseEventModel): + """成员被禁言""" + group_id: typing.Union[int, str] + member_id: typing.Union[int, str] + operator_id: typing.Optional[typing.Union[int, str]] = None + duration: typing.Optional[int] = None + +class GroupMemberUnbanned(BaseEventModel): + """成员被解除禁言""" + group_id: typing.Union[int, str] + member_id: typing.Union[int, str] + operator_id: typing.Optional[typing.Union[int, str]] = None + +class GroupInfoUpdated(BaseEventModel): + """群组信息被修改""" + group_id: typing.Union[int, str] + group_name: str + operator_id: typing.Optional[typing.Union[int, str]] = None + changed_fields: list[str] = [] + +# ---- 好友事件 ---- + +class FriendRequestReceived(BaseEventModel): + """收到好友请求""" + request_id: typing.Union[int, str] + user_id: typing.Union[int, str] + user_name: str + message: typing.Optional[str] = None + +class FriendAdded(BaseEventModel): + """成功添加好友""" + user_id: typing.Union[int, str] + user_name: str + +class FriendRemoved(BaseEventModel): + """好友被移除""" + user_id: typing.Union[int, str] + user_name: str + +# ---- Bot 状态事件 ---- + +class BotInvitedToGroup(BaseEventModel): + """Bot 被邀请加入群组""" + group_id: typing.Union[int, str] + group_name: str + inviter_id: typing.Optional[typing.Union[int, str]] = None + request_id: typing.Optional[typing.Union[int, str]] = None + +class BotRemovedFromGroup(BaseEventModel): + """Bot 被移出群组""" + group_id: typing.Union[int, str] + group_name: str + operator_id: typing.Optional[typing.Union[int, str]] = None + +class BotMuted(BaseEventModel): + """Bot 被禁言""" + group_id: typing.Union[int, str] + operator_id: typing.Optional[typing.Union[int, str]] = None + duration: typing.Optional[int] = None + +class BotUnmuted(BaseEventModel): + """Bot 被解除禁言""" + group_id: typing.Union[int, str] + operator_id: typing.Optional[typing.Union[int, str]] = None + +# ---- 平台特有事件 ---- + +class PlatformSpecificEventReceived(BaseEventModel): + """平台特有事件""" + adapter_name: str + action: str + data: dict = {} +``` + +### 2.2 EventListener 注册方式 + +插件的 EventListener 继续使用 `@self.handler(EventType)` 装饰器注册，只是可以注册的事件类型大幅增加： + +```python +class MyEventListener(EventListener): + def __init__(self, host): + super().__init__(host) + + # 现有方式（继续工作） + @self.handler(PersonNormalMessageReceived) + async def on_person_message(ctx: EventContext): + ... + + # 新事件类型 + @self.handler(GroupMemberJoined) + async def on_member_joined(ctx: EventContext): + group_name = ctx.event.group_name + member_name = ctx.event.member_name + await ctx.reply(MessageChain([ + Plain(f"欢迎 {member_name} 加入 {group_name}！") + ])) + + @self.handler(FriendRequestReceived) + async def on_friend_request(ctx: EventContext): + # 自动通过好友请求 + await ctx.approve_friend_request( + ctx.event.request_id, approve=True + ) + + @self.handler(FeedbackReceived) + async def on_feedback(ctx: EventContext): + if ctx.event.feedback_type == 2: + await self.log_warning( + f"用户点踩了回复: {ctx.event.feedback_content or ''}" + ) + + @self.handler(PlatformSpecificEventReceived) + async def on_platform_event(ctx: EventContext): + if ctx.event.adapter_name == "telegram" and ctx.event.action == "chat_join_request": + ... +``` + +## 3. 新 API 暴露 + +### 3.1 LangBotAPIProxy 扩展 + +在 `LangBotAPIProxy` 中新增以下方法，插件通过 `self.xxx()` 调用（在 BasePlugin 中继承）： + +```python +class LangBotAPIProxy: + # ---- 现有方法（保留） ---- + # get_langbot_version, get_bots, get_bot_info, + # send_message, invoke_llm, get/set/delete_plugin_storage, ... + + # ---- 新增消息 API ---- + + async def edit_message( + self, + bot_uuid: str, + chat_type: str, + chat_id: typing.Union[int, str], + message_id: typing.Union[int, str], + new_content: MessageChain, + ) -> None: + """编辑已发送的消息""" + ... + + async def delete_message( + self, + bot_uuid: str, + chat_type: str, + chat_id: typing.Union[int, str], + message_id: typing.Union[int, str], + ) -> None: + """删除/撤回消息""" + ... + + async def forward_message( + self, + bot_uuid: str, + from_chat_type: str, + from_chat_id: typing.Union[int, str], + message_id: typing.Union[int, str], + to_chat_type: str, + to_chat_id: typing.Union[int, str], + ) -> dict: + """转发消息""" + ... + + async def get_message( + self, + bot_uuid: str, + chat_type: str, + chat_id: typing.Union[int, str], + message_id: typing.Union[int, str], + ) -> dict: + """获取指定消息""" + ... + + # ---- 新增群组 API ---- + + async def get_group_info( + self, + bot_uuid: str, + group_id: typing.Union[int, str], + ) -> dict: + """获取群组信息""" + ... + + async def get_group_list( + self, + bot_uuid: str, + ) -> list[dict]: + """获取 Bot 加入的群组列表""" + ... + + async def get_group_member_list( + self, + bot_uuid: str, + group_id: typing.Union[int, str], + ) -> list[dict]: + """获取群成员列表""" + ... + + async def get_group_member_info( + self, + bot_uuid: str, + group_id: typing.Union[int, str], + user_id: typing.Union[int, str], + ) -> dict: + """获取指定群成员信息""" + ... + + async def mute_member( + self, + bot_uuid: str, + group_id: typing.Union[int, str], + user_id: typing.Union[int, str], + duration: int = 0, + ) -> None: + """禁言群成员""" + ... + + async def unmute_member( + self, + bot_uuid: str, + group_id: typing.Union[int, str], + user_id: typing.Union[int, str], + ) -> None: + """解除禁言""" + ... + + async def kick_member( + self, + bot_uuid: str, + group_id: typing.Union[int, str], + user_id: typing.Union[int, str], + ) -> None: + """踢出群成员""" + ... + + # ---- 新增用户 API ---- + + async def get_user_info( + self, + bot_uuid: str, + user_id: typing.Union[int, str], + ) -> dict: + """获取用户信息""" + ... + + async def get_friend_list( + self, + bot_uuid: str, + ) -> list[dict]: + """获取好友列表""" + ... + + async def approve_friend_request( + self, + bot_uuid: str, + request_id: typing.Union[int, str], + approve: bool = True, + remark: typing.Optional[str] = None, + ) -> None: + """处理好友请求""" + ... + + async def approve_group_invite( + self, + bot_uuid: str, + request_id: typing.Union[int, str], + approve: bool = True, + ) -> None: + """处理入群邀请""" + ... + + # ---- 新增透传 API ---- + + async def call_platform_api( + self, + bot_uuid: str, + action: str, + params: dict = {}, + ) -> dict: + """调用适配器特有 API + + Examples: + # Telegram: pin 消息 + result = await self.call_platform_api( + bot_uuid, "pin_message", + {"chat_id": 123456, "message_id": 789} + ) + + # Discord: 创建频道 + result = await self.call_platform_api( + bot_uuid, "create_channel", + {"guild_id": "...", "name": "new-channel"} + ) + """ + ... + + # ---- 新增能力查询 API ---- + + async def get_supported_events( + self, + bot_uuid: str, + ) -> list[str]: + """获取指定 Bot 的适配器支持的事件类型""" + ... + + async def get_supported_apis( + self, + bot_uuid: str, + ) -> list[str]: + """获取指定 Bot 的适配器支持的 API""" + ... +``` + +### 3.2 QueryBasedAPIProxy 扩展 + +在事件处理上下文中（EventContext），通过 `QueryBasedAPIProxy` 新增便捷方法： + +```python +class QueryBasedAPIProxy: + # ---- 现有方法（保留） ---- + # reply, get_bot_uuid, set_query_var, get_query_var, + # create_new_conversation, ... + + # ---- 新增便捷方法 ---- + + async def edit_message( + self, + message_id: typing.Union[int, str], + new_content: MessageChain, + ) -> None: + """在当前会话中编辑消息（自动使用当前 bot_uuid 和 chat 信息）""" + ... + + async def delete_message( + self, + message_id: typing.Union[int, str], + ) -> None: + """在当前会话中删除消息""" + ... + + async def approve_friend_request( + self, + request_id: typing.Union[int, str], + approve: bool = True, + remark: typing.Optional[str] = None, + ) -> None: + """处理好友请求（上下文中自动获取 bot_uuid）""" + ... + + async def approve_group_invite( + self, + request_id: typing.Union[int, str], + approve: bool = True, + ) -> None: + """处理入群邀请""" + ... + + async def get_group_info(self) -> dict: + """获取当前群组信息（仅群聊事件中可用）""" + ... + + async def get_group_member_list(self) -> list[dict]: + """获取当前群组成员列表（仅群聊事件中可用）""" + ... + + async def call_platform_api( + self, + action: str, + params: dict = {}, + ) -> dict: + """调用平台特有 API（自动使用当前 bot_uuid）""" + ... +``` + +## 4. 兼容层设计 + +### 4.1 事件兼容层 + +当 PluginHandler 将新的 `MessageReceivedEvent` 分发给插件时，需要同时生成旧格式事件： + +```python +class PluginEventCompatLayer: + """插件事件兼容层 + + 将新的统一事件转换为旧的插件事件格式， + 确保监听旧事件类型的插件仍能正常工作。 + """ + + @staticmethod + def convert_to_legacy_events( + event: Event, + ) -> list[BaseEventModel]: + """将统一事件转换为旧插件事件列表 + + 一个统一事件可能生成多个旧插件事件。 + 例如 MessageReceivedEvent 会同时生成： + - PersonMessageReceived / GroupMessageReceived（总是生成） + - PersonNormalMessageReceived / GroupNormalMessageReceived（非命令时） + - PersonCommandSent / GroupCommandSent（命令时） + """ + legacy_events = [] + + if isinstance(event, MessageReceivedEvent): + if event.chat_type == ChatType.PRIVATE: + legacy_events.append( + PersonMessageReceived( + launcher_type="person", + launcher_id=event.chat_id, + sender_id=event.sender.id, + message_event=event.to_legacy_friend_message(), + message_chain=event.message_chain, + ) + ) + # 命令检测后还会生成 PersonNormalMessageReceived + # 或 PersonCommandSent，在 Pipeline 阶段处理 + elif event.chat_type == ChatType.GROUP: + legacy_events.append( + GroupMessageReceived( + launcher_type="group", + launcher_id=event.chat_id, + sender_id=event.sender.id, + message_event=event.to_legacy_group_message(), + message_chain=event.message_chain, + ) + ) + + # 新事件类型没有旧的对应物，不生成兼容事件 + # 只有监听了新事件类型的插件才会收到 + + return legacy_events +``` + +### 4.2 分发流程 + +``` +统一事件 (MessageReceivedEvent) + │ + ├─→ 转换为旧格式 (PersonMessageReceived / GroupMessageReceived) + │ └─→ 分发给监听旧事件类型的插件 EventListener + │ + └─→ 直接分发为新格式 (MessageReceivedEvent → 对应的插件事件) + └─→ 分发给监听新事件类型的插件 EventListener +``` + +插件 Runtime 在分发事件时检查每个 EventListener 注册的事件类型： +- 如果注册的是旧类型（`PersonMessageReceived` 等），发送兼容层生成的旧格式事件 +- 如果注册的是新类型（`GroupMemberJoined` 等），发送新格式事件 +- 两者可以共存，同一个插件可以同时监听新旧类型 + +### 4.3 API 兼容层 + +现有插件使用的 API 不受影响： + +| 现有 API | 新架构行为 | +|---------|----------| +| `self.send_message(bot_uuid, target_type, target_id, message_chain)` | 不变，直接调用适配器的 `send_message` | +| `ctx.reply(message_chain, quote_origin)` | 不变，在 MessageReceivedEvent 上下文中调用适配器的 `reply_message` | +| `self.get_bots()` | 不变 | +| `self.get_bot_info(bot_uuid)` | 不变 | + +新 API 只是额外新增的方法，不影响现有方法。 + +## 5. 通信协议扩展 + +### 5.1 新增 Action 枚举 + +在 `entities/io/actions/enums.py` 中新增 action： + +```python +class PluginToRuntimeAction(str, Enum): + # ---- 现有 actions（保留） ---- + REGISTER_PLUGIN = "register_plugin" + REPLY = "reply" + SEND_MESSAGE = "send_message" + # ... + + # ---- 新增消息 API ---- + EDIT_MESSAGE = "edit_message" + DELETE_MESSAGE = "delete_message" + FORWARD_MESSAGE = "forward_message" + GET_MESSAGE = "get_message" + + # ---- 新增群组 API ---- + GET_GROUP_INFO = "get_group_info" + GET_GROUP_LIST = "get_group_list" + GET_GROUP_MEMBER_LIST = "get_group_member_list" + GET_GROUP_MEMBER_INFO = "get_group_member_info" + MUTE_MEMBER = "mute_member" + UNMUTE_MEMBER = "unmute_member" + KICK_MEMBER = "kick_member" + + # ---- 新增用户 API ---- + GET_USER_INFO = "get_user_info" + GET_FRIEND_LIST = "get_friend_list" + APPROVE_FRIEND_REQUEST = "approve_friend_request" + APPROVE_GROUP_INVITE = "approve_group_invite" + + # ---- 新增透传 API ---- + CALL_PLATFORM_API = "call_platform_api" + + # ---- 新增能力查询 ---- + GET_SUPPORTED_EVENTS = "get_supported_events" + GET_SUPPORTED_APIS = "get_supported_apis" + + +class RuntimeToPluginAction(str, Enum): + # ---- 现有 actions（保留） ---- + EMIT_EVENT = "emit_event" + # ... + # EMIT_EVENT 的 data 结构扩展以支持新事件类型 +``` + +### 5.2 新增 Action 的请求/响应格式 + +以 `EDIT_MESSAGE` 为例： + +```json +// 请求 (Plugin → Runtime) +{ + "action": "edit_message", + "seq_id": 12345, + "data": { + "bot_uuid": "...", + "chat_type": "group", + "chat_id": "123456", + "message_id": "789", + "new_content": [ + { "type": "Plain", "text": "edited message" } + ] + } +} + +// 响应 (Runtime → Plugin) +{ + "seq_id": 12345, + "code": 0, + "message": "ok", + "data": {} +} +``` + +以 `GET_GROUP_MEMBER_LIST` 为例： + +```json +// 请求 +{ + "action": "get_group_member_list", + "seq_id": 12346, + "data": { + "bot_uuid": "...", + "group_id": "123456" + } +} + +// 响应 +{ + "seq_id": 12346, + "code": 0, + "message": "ok", + "data": { + "members": [ + { + "user": { "id": "111", "nickname": "Alice" }, + "group_id": "123456", + "role": "admin", + "display_name": "管理员Alice" + }, + ... + ] + } +} +``` + +以 `CALL_PLATFORM_API` 为例： + +```json +// 请求 +{ + "action": "call_platform_api", + "seq_id": 12347, + "data": { + "bot_uuid": "...", + "action": "pin_message", + "params": { + "chat_id": "123456", + "message_id": "789" + } + } +} + +// 响应 +{ + "seq_id": 12347, + "code": 0, + "message": "ok", + "data": { + "result": { ... } + } +} +``` + +### 5.3 LangBot 侧 Handler 实现 + +在 `ControlConnectionHandler`（LangBot → Runtime 侧）和 `PluginConnectionHandler`（Runtime → Plugin 侧）中新增对应的 action 处理逻辑： + +```python +# PluginConnectionHandler 中新增 +async def _handle_edit_message(self, data): + bot_uuid = data["bot_uuid"] + bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid) + await bot.adapter.edit_message( + chat_type=data["chat_type"], + chat_id=data["chat_id"], + message_id=data["message_id"], + new_content=MessageChain.model_validate(data["new_content"]), + ) + return {} + +async def _handle_call_platform_api(self, data): + bot_uuid = data["bot_uuid"] + bot = await self.ap.platform_mgr.get_bot_by_uuid(bot_uuid) + result = await bot.adapter.call_platform_api( + action=data["action"], + params=data.get("params", {}), + ) + return {"result": result} +``` + +## 6. 插件开发者迁移指南 + +### 6.1 无需迁移（零修改运行） + +以下场景的现有插件**不需要任何修改**： + +- 使用 `PersonNormalMessageReceived` / `GroupNormalMessageReceived` 监听消息 +- 使用 `PersonCommandSent` / `GroupCommandSent` 处理命令 +- 使用 `ctx.reply()` 回复消息 +- 使用 `self.send_message()` 主动发消息 +- 使用 LLM / 存储 / RAG 等现有 API + +### 6.2 推荐迁移（获得新能力） + +如果插件希望利用新功能，可以： + +1. **监听新事件类型**：在 EventListener 中注册新事件类型的 handler +2. **使用新 API**：调用 `self.edit_message()`, `self.get_group_info()` 等 +3. **使用透传 API**：调用 `self.call_platform_api()` 使用平台特有功能 + +### 6.3 SDK 版本号 + +新功能通过提升 SDK minor 版本发布： + +- 现有版本：`langbot-plugin-sdk >= x.y.z` +- 新版本：`langbot-plugin-sdk >= x.(y+1).0` + +插件的 `manifest.yaml` 中的 `min_sdk_version` 决定是否能使用新 API。使用旧 SDK 版本的插件在新 LangBot 上正常运行（兼容层保证），只是无法调用新 API。 diff --git a/docs/event-based-agents/06-migration-plan.md b/docs/event-based-agents/06-migration-plan.md new file mode 100644 index 000000000..7f6475cc7 --- /dev/null +++ b/docs/event-based-agents/06-migration-plan.md @@ -0,0 +1,147 @@ +# EBA 分阶段实施计划 + +> Implementation update (2026-09-08): the EventListener observer-broadcast proposal below is superseded by [Event processors](09-event-processors.md). Legacy EventListener hooks run only inside Pipeline. New EBA handlers use explicitly created and bound Runner instances, a third peer processor type alongside Agent and Pipeline. + +> 更新：2026-09-05。P0–P4 的主要实现已落入 `dev/4.11.x`，P5 仍需按当前版本验收；下文工作项用于维护实现边界，不表示全部待开发。文件名沿用早期设计，但这里的“迁移”仅指代码架构逐步接入 EBA，不代表 LangBot 3.x 数据库或配置升级。当前提交、定向测试及发布缺口见 [STATUS.md](../agent-runner-pluginization/STATUS.md)。 + +## 1. 发布边界 + +EBA 跨越 SDK、平台适配器、LangBot Host、WebUI 与插件生态，按可验证阶段落地。当前发布遵守以下硬边界： + +- LangBot 4.x 不支持从 3.x 数据库或配置升级；不保留 legacy migration chain、旧 JSON 模板或旧 Runner 字段读取。 +- Pipeline 与 Agent 平级且长期并存，分别保留持久化模型与执行链。 +- 现有 Pipeline 不迁移为 Agent，Pipeline 内的 runner config 不复制到 Agent。 +- 用户需要 Agent 时新建独立 Agent并选择已安装的 Runner。 +- Host 不按 LocalAgent id 做运行时、Box 或 WebUI 特判。 +- Runner 的 SDK/Python 与 scoped MCP bridge 回调共享 Host 授权与事件 session 规则。 + +## 2. 阶段总览 + +| 阶段 | 目标 | 主要仓库 | 完成条件 | +| --- | --- | --- | --- | +| P0 | SDK 事件、能力与 Runner 协议 | `langbot-plugin-sdk` | typed entities、manifest、proxy、runtime action 通过测试 | +| P1 | 平台适配器 EBA 化 | LangBot + SDK | 事件转换、能力声明、通用/透传 API 通过 adapter checklist | +| P2 | Host 观察者与响应者路由 | LangBot backend | observer 广播 + Pipeline/Agent/discard 单目标仲裁可运行 | +| P3 | 独立 Agent 与 Runner 注册 | LangBot backend + plugins | Agent CRUD、registry、run authorization、delivery 可运行 | +| P4 | WebUI 处理器与事件编排 | LangBot web | Agent/Pipeline 聚合入口和 Bot binding 编辑器可用 | +| P5 | 发布门禁与文档 | LangBot skills/docs + runner plugins | 单测、UI E2E、真实 adapter smoke 和插件预检通过 | + +## 3. P0：SDK 契约 + +### 工作项 + +- 定义规范化平台事件、actor/subject/conversation/delivery context。 +- 定义 adapter `supported_events`、`supported_apis` 与平台透传 API。 +- 定义 Runner manifest、run context/result、resource handles 和 pull/callback API。 +- 提供 `RunnerAPIProxy` 与 SDK-owned scoped MCP bridge。 +- 保持协议传输与权限校验可测试，不把 Host 私有 Query 对象暴露给插件。 + +### 验收 + +- stdio/WebSocket runtime 对 typed action 的序列化一致。 +- 无 `run_id` 或越权资源调用被拒绝。 +- Python proxy 与 MCP bridge 对同一 Host tool 呈现一致结果/错误形状。 + +## 4. P1：平台适配器 + +每个适配器按自己的能力增量接入，而不是要求所有平台一次性支持全部事件。 + +### 单适配器步骤 + +1. 将原生 SDK callback 转换为规范化事件。 +2. 声明实际支持的事件与 API，不把缺失能力伪装为成功。 +3. 实现 send/reply/edit/delete/group/user 等适用 API 与平台透传。 +4. 对消息链、媒体、reply target 和错误语义写单元测试。 +5. 按 `adapters/acceptance-checklist.md` 记录真实平台 probe。 + +### 验收 + +- `message.received` 保持正常收发。 +- 新事件不会重复转换或产生循环合成事件。 +- `supported_apis` 与实际调用能力一致。 + +## 5. P2：Host 路由 + +### 执行顺序 + +```text +adapter event + -> EventBus observer broadcast + -> EventRouter match event_bindings + -> one target: Pipeline | Agent | discard + -> Host delivery +``` + +### 约束 + +- Plugin EventListener 是 observer，不作为 priority fallback。 +- Pipeline 只处理消息事件并复用完整 Stage 链。 +- Agent 使用独立 Agent 配置和 Runner Host orchestrator。 +- edit/reaction 等事件的 observer 副作用能力按事件和 adapter 能力过滤。 +- dry-run 与合成派发必须使用同一匹配器，避免 UI 预览与真实路由漂移。 + +### 验收 + +- 精确、namespace wildcard、全局 wildcard、filters 与 priority 有单元覆盖。 +- 同一事件最多一个响应目标，但 observer 仍能收到事件。 +- Pipeline 与 Agent 可以在同一个 Bot 的不同 binding 中同时生效。 + +## 6. P3：独立 Agent 与 Runner + +### 工作项 + +- `agents` 只保存 Agent；Pipeline 继续使用自己的表和 API。 +- Agent config 使用 `runner.id` 与 `runner_config[runner_id]`。 +- registry 只展示已安装、有效的插件 Runner。 +- Host 构造 run-scoped resources、state、delivery 与 event log/transcript。 +- SDK/Python `call_tool` 和 scoped MCP bridge 都回到同一个 Host ToolManager。 +- Box session 由 Host 将 instance/workspace/bot/adapter/target/thread scope 规范化并哈希为固定长度 `lb-box-`；同 scope 稳定、不同 scope 隔离、缺少 identity 时 fail closed。 + +### 验收 + +- 安装/卸载 Runner 后 metadata 与表单选项同步。 +- Runner 无法调用未授权 model/tool/knowledge/state。 +- 两种 Host callback transport 不能覆盖 sandbox session id。 +- LocalAgent、ACP/ClaudeCode/Codex 与外部服务 Runner 不需要 Host id 特判。 + +## 7. P4：WebUI + +### 工作项 + +- `/home/agents` 聚合显示 Agent 与 Pipeline，并明确类型。 +- 创建时选择 Agent 或 Pipeline，编辑时进入各自表单。 +- Agent 表单读取动态 Runner metadata，保存当前 `runner` / `runner_config` 形状。 +- Bot 事件编排器编辑 `event_pattern`、target、filters、priority 与 enabled。 +- 非消息事件过滤 Pipeline；Agent 按声明事件能力过滤。 + +### 验收 + +- 页面不出现 LocalAgent 专属 banner、变量隐藏或 Box/Pipeline 注入逻辑。 +- 空 Runner 市场状态给出可安装 Runner 的正常路径。 +- Pipeline Debug Chat/Monitoring 与 Agent 运行日志分别可用。 + +## 8. P5：发布门禁 + +### 自动化 + +- LangBot backend unit/integration tests 与 Ruff。 +- SDK Runner/proxy/MCP bridge tests。 +- Web lint/build 与关键 Playwright cases。 +- `skills/bin/lbs validate`、`skills/bin/lbs index --check`。 +- LocalAgent 与其他官方 Runner plugin package/test gate。 + +### 真实环境 + +- 至少一个消息事件走 Pipeline。 +- 至少一个非消息事件走独立 Agent 并执行平台动作。 +- SDK/Python 和 MCP bridge 各完成一次受权工具调用，并证明二者都进入同一个 `PluginToRuntimeAction.CALL_TOOL` Host handler。 +- Box 可用/不可用的降级路径可观察且无 Runner 特判。 + +## 9. 非目标 + +- 不把 Pipeline 改名或包装成 Agent。 +- 不自动把 Pipeline 内 runner 配置迁移成 Agent。 +- 不为 3.x 保留数据库迁移、旧模板或配置 fallback。 +- 不要求所有 Runner 经 MCP；本地 Python Runner 可以直接使用 SDK。 +- 不允许 Runner 自定义 Box session scope。 +- 不在首版实现多 Agent 串并联；多步骤编排留给后续 workflow。 diff --git a/docs/event-based-agents/07-agent-orchestration.md b/docs/event-based-agents/07-agent-orchestration.md new file mode 100644 index 000000000..fa16602a2 --- /dev/null +++ b/docs/event-based-agents/07-agent-orchestration.md @@ -0,0 +1,187 @@ +# Agent 与 Pipeline 统一编排（产品最终形态） + +> Implementation update (2026-09-08): the EventListener observer-broadcast proposal below is superseded by [Event processors](09-event-processors.md). Legacy EventListener hooks run only inside Pipeline. New EBA handlers use explicitly created and bound Runner instances, a third peer processor type alongside Agent and Pipeline. + +> **状态**：历史方向稿（2026-06-12）；2026-09-05 标记归档用途。本文的示意 schema、5.0 发布火车、SDK 0.5.0aX 配套与多租户“预留”描述不再作为实施合同。当前 4.11 产品形态见 [08-agent-page-and-event-orchestration.md](./08-agent-page-and-event-orchestration.md)，协议见 [PROTOCOL_V1.md](../agent-runner-pluginization/PROTOCOL_V1.md)，已完成与剩余事项见 [STATUS.md](../agent-runner-pluginization/STATUS.md)。保留正文仅用于解释早期设计取舍。 +> +> 本文档修订 [00-overview.md](./00-overview.md) §3.4 与 [04-event-routing.md](./04-event-routing.md) 中"四种 Handler"的编排模型：**所有编排目标统一进入处理器选择与事件绑定界面，但独立 Agent 与现有 Pipeline 保持不同类型**。事件路由的匹配机制、数据迁移策略、WebUI 交互骨架等内容仍以 04 为准，仅 handler 分类法被本文档取代。 + +## 1. 产品最终形态 + +**适配器接收各种事件 → 用户编排处理逻辑 → 处理器统一选择表面**，实现从 0 代码到低代码再到全代码的全层面支持： + +``` +消息平台 (Telegram / Discord / 企微 / ...) + │ 各类平台事件 + ▼ +平台适配器（EBA 新结构，已迁移 12 个） + │ EBAEvent (message.* / group.* / friend.* / bot.* / feedback.* / platform.*) + ▼ +EventRouter（事件 → 处理器绑定） + ├─→ 选中的处理器（响应者，单一仲裁） + │ ├─ Pipeline：保留现有实体和执行链，仅处理消息事件 + │ └─ Agent：用户新建并选择 Runner 插件，可接本地、低代码或外部 runtime + │ + └─→ 插件 EventListener（观察者，N 个广播，可 prevent_default） +``` + +| 编写方式 | 处理器形态 | 代码化程度 | +|----------|-----------|-----------| +| WebUI 配置模型 + 提示词 + 工具 | 独立 Agent + LocalAgent 插件 | 0 代码 | +| 可视化消息编排 | Pipeline（`kind=pipeline`，完整多 Stage 链） | 0 代码 | +| 市场安装 | Agent 插件（市场分发） | 0 代码（使用者视角） | +| 可视化工作流 | 工作流引擎定义的 Agent | 低代码 | +| 对接外部平台 | dify / n8n / coze / webhook 外部 Agent | 集成 | +| SDK 编写 | Agent 插件组件 | 全代码 | + +### 1.1 三条并行工作线与汇合点 + +| 工作线 | 范围 | 在本架构中的位置 | +|--------|------|------------------| +| 适配器改造（refactor/eba，本分支） | 事件体系、适配器结构、平台 API、EventRouter | 事件的**生产侧** + 路由层 | +| Agent 插件化 | Agent 抽象、Agent 组件类型、市场分发 | 事件的**消费侧**统一抽象 | +| 工作流引擎 | 内部低代码工作流 | Agent 的一种**编写方式** | + +**汇合点是 SDK 的 Agent 组件契约（§4）与 event→处理器绑定模型（§3）**。这两个接口冻结后，三条线可彼此 mock 独立推进。契约由本分支（EBA）牵头起草，三线评审后在 langbot-plugin-sdk 落地（发布通道：0.5.0aX pre-release 已打通）。 + +## 2. 从四种 Handler 到统一处理器表面 + +### 2.1 演进理由 + +04 文档中的 pipeline / agent / webhook / plugin 四种 handler_type，本质上都是"对事件作出响应的逻辑"，差别只在编写和部署方式。产品层统一展示和绑定这些处理器，但不会把既有 Pipeline 持久化为 Agent： + +- **产品**：用户只需理解"给 Bot 的事件绑定处理器"，处理器可以是 Pipeline 或 Agent； +- **工程**：路由层按 `target_type` 分发到 Pipeline 或 Agent，Agent 的扩展集中到 Runner 抽象； +- **生态**：Agent 成为市场上可分发、可复用的一等公民。 + +### 2.2 收编映射 + +| 原 handler_type（04 文档） | 收编后 | +|---------------------------|--------| +| `pipeline` | 保留 Pipeline 实体；binding 使用 `target_type=pipeline` 和原 `pipeline_uuid`，进程内直接复用 MessageAggregator → QueryPool → Pipeline 机制 | +| `agent`（RequestRunner） | 用户新建独立 Agent，并选择对应 Runner 插件；不读取或复制旧 Pipeline 内嵌 runner 配置 | +| `webhook` | 外部 Agent 的一种：事件 POST 出去、响应解析为动作（保留 04 §5.4 的请求/响应格式） | +| `plugin`（EventListener 分发） | **不收编**——角色不同，见 §2.3 | + +### 2.3 响应者与观察者的角色切分 + +事件的消费方有两种角色，不应混为一谈： + +- **响应者（Pipeline 或 Agent）**：路由选中**一个**，负责对事件作出回应（回复消息、执行动作）。多条绑定匹配同一事件时按 priority 仲裁，只取最高者。 +- **观察者（插件 EventListener）**：**广播**给所有注册插件，做旁路逻辑（日志、审计、风控、统计）。沿用现有机制不变，包括 `prevent_default()`——观察者可拦截本次事件，使处理器不被调用（与现有"插件拦截流水线"行为完全兼容）。 + +执行顺序：事件到达 → 先广播观察者（按插件优先级）→ 若未被 prevent_default → 分发给选中的处理器。 + +## 3. 数据模型：event → 处理器绑定 + +### 3.1 独立 Agent 与现有 Pipeline + +Agent 与 Pipeline 都是一等处理器。用户创建 Agent、选择已安装的 Runner，再把适合的事件绑定到 Agent；Pipeline 继续保存在 Pipeline 表中，以完整 Stage 链处理消息事件。两者可在同一处理器列表中以不同 `kind` 展示和选择；这种聚合展示不会创建额外记录，也不会在两种模型之间复制配置。 + +```python +class Agent(Base): + """Agent 实例：一个具体配置过的、可被事件绑定的响应者""" + uuid: str # 主键 + name: str + kind: str # 固定为 "agent"；Pipeline 使用自己的持久模型 + component_ref: str # Runner id，例如 plugin:// + config: dict # JSON — runner id、runner config 与资源/状态/投递策略 + # 多租户预留：归属主体字段（tenant/workspace），首版可空 +``` + +Bot 上的绑定配置（替代 04 §2.2 的 EventHandlerConfig，沿用其匹配语义）： + +```python +class EventBinding(pydantic.BaseModel): + event_pattern: str # 精确 / "message.*" / "*"，匹配规则同 04 §4 + target_type: str # "agent" | "pipeline" | "discard" + target_uuid: str # Agent 或 Pipeline 的原始 UUID；discard 时为空 + enabled: bool = True + priority: int = 0 # 多条匹配时取最高者（单一仲裁） + description: str = '' +``` + +`use_pipeline_uuid` 只迁移 Bot 的路由结构：写入 `{"event_pattern": "message.received", "target_type": "pipeline", "target_uuid": <原 pipeline uuid>}`，继续引用原 Pipeline。迁移不会创建独立 Agent，也不会复制 Pipeline 内嵌的 runner 配置；需要 Agent 的用户自行新增 Agent 并修改 binding。观察者广播不需要配置（始终发生），04 中"兜底 plugin 规则"不再需要。 + +## 4. SDK Agent 组件契约（草案） + +Agent 成为插件系统的第七种组件（现有：Command / Tool / EventListener / KnowledgeEngine / Parser / Page）。 + +### 4.1 Manifest + +```yaml +apiVersion: v1 +kind: Agent +metadata: + name: group-assistant + label: { en_US: Group Assistant, zh_Hans: 群助理 } +spec: + handled_events: # 声明可处理的事件类型；绑定 UI 据此过滤 + - message.received + - group.member_joined + config: # 实例化配置 schema，复用现有组件配置体系 + - name: model + type: llm-model-selector + - name: persona + type: prompt-editor +execution: + python: { path: agent.py, attr: GroupAssistant } +``` + +### 4.2 运行时接口 + +```python +class Agent(BaseComponent): + async def handle(self, ctx: AgentContext) -> typing.AsyncGenerator[AgentChunk, None]: + """处理一次事件，流式产出回复与动作。每次事件调用一次。""" + ... + +class AgentContext: + event: EBAEvent # 触发事件（统一事件体系） + bot: BotHandle # 来源 Bot 信息 + session: SessionHandle # 会话句柄：历史消息、会话变量（LangBot 侧管理，Agent 保持无状态） + config: dict # 该 Agent 实例的配置 + + # 能力面（经 runtime RPC 回 LangBot 执行）： + async def reply(self, chain: MessageChain, quote: bool = False): ... + async def send_message(self, target_type: str, target_id: str, chain: MessageChain): ... + async def call_platform_api(self, action: str, params: dict) -> dict: ... + async def invoke_llm(self, model_uuid: str, messages: list, funcs: list = None) -> dict: ... + # + 工具调用 / KB 检索 / 插件存储（沿用 LangBotAPIProxy 既有方法） + +class AgentChunk: + delta_message: MessageChain | None = None # 增量回复（流式） + actions: list[dict] | None = None # 平台动作（同 webhook response_actions 格式） + final: bool = False +``` + +**流式**：复用 SDK 通信协议既有的 `chunk_status: continue/end` 机制，`handle()` 的每次 yield 对应一个 chunk。 +**Pipeline 与 Agent 分流**：Pipeline target 继续走 LangBot 进程内的 Pipeline 执行链；独立 Agent 经 Runner 插件 runtime 分发。路由层通过 binding 的 `target_type` 明确区分二者。 + +### 4.3 执行语义与可靠性 + +| 关注点 | 约定 | +|--------|------| +| 仲裁 | 单响应者：priority 最高的匹配绑定生效，其余忽略 | +| 性能 | Pipeline 继续走进程内链路；插件 Agent 每事件过一次 RPC 边界，消息场景需设延迟预算（评审项：目标 P95 附加延迟） | +| 会话状态 | 归 LangBot 侧（SessionHandle），插件 Agent 原则上无状态，崩溃重启不丢会话 | +| 降级 | Agent 调用失败/超时：可配置 fallback（回错误提示，或指定备用处理器）；不会隐式回退到旧 Pipeline 配置 | +| 多租户预留 | AgentContext / SessionHandle / 存储接口显式携带归属主体标识，禁止新增全局单例状态——为后续轻量 SaaS 多租户铺路 | + +## 5. 发布火车 + +| 版本 | 内容 | 备注 | +|------|------|------| +| 4.11（可选） | 现状成果：12 个 EBA 适配器、插件全事件订阅、`call_platform_api` | 对用户不可见的管道工程 + 插件新能力，不动产品概念 | +| **5.0** | 产品形态首发：EventRouter + event→处理器绑定 + WebUI 编排 + 旧 Bot 路由迁移 + 独立 Agent / Runner 插件 + SDK Agent 组件契约（可标 experimental） | `use_pipeline_uuid` 仅改写为指向原 Pipeline 的 binding，不生成 Agent；配 SDK 0.5.0 正式版；走 beta 周期 | +| 5.x | 工作流 Agent（工作流引擎线挂入）、Agent 市场生态、剩余适配器（satori 等）、Agent 插件化收尾 | 验证开放注册机制 | +| 多租户 | 独立评估：仅数据隔离 → 5.x 部署选项；伴随权限/计费/产品定位变化 → 6.0 | 前置条件是 §4.3 的归属主体预留已落实 | + +## 6. 开放问题（评审清单） + +1. **webhook 的最终定位**：作为外部 Agent（响应者，现方案）之外，是否还需要"纯通知观察者"形态（现 WebhookPusher 的角色）？ +2. **多 Agent 协作**：单一仲裁之外，是否需要"串联/并联多个 Agent"的场景？（建议 5.0 不做，留给工作流引擎表达） +3. **工作流引擎的宿主**：核心内置，还是自身也作为一个插件交付（解释工作流定义的 Agent 插件）？ +4. **插件 Agent 的延迟预算**：消息主链路过 RPC 的 P95 目标值与压测方案。 +5. **Workflow 的处理器边界**：未来 Workflow 应作为与 Pipeline、Agent 平级的处理器，还是作为其中一种处理器的内部编排实现。 +6. **SDK 1.0 时机**：Agent 契约稳定后是否随 LangBot 5.x 给插件生态一个 API 稳定承诺。 diff --git a/docs/event-based-agents/08-agent-page-and-event-orchestration.md b/docs/event-based-agents/08-agent-page-and-event-orchestration.md new file mode 100644 index 000000000..b9e8d5603 --- /dev/null +++ b/docs/event-based-agents/08-agent-page-and-event-orchestration.md @@ -0,0 +1,185 @@ +# 处理器页面与事件编排产品设计 + +> Implementation update (2026-09-08): the EventListener observer-broadcast proposal below is superseded by [Event processors](09-event-processors.md). Legacy EventListener hooks run only inside Pipeline. New EBA handlers use explicitly created and bound Runner instances, a third peer processor type alongside Agent and Pipeline. + +> 状态：当前实现说明（2026-09-05），对应 `dev/4.11.x`。P0–P3 已集成；发布验收见 [STATUS.md](../agent-runner-pluginization/STATUS.md)。 +> +> 本文档修订 [07-agent-orchestration.md](./07-agent-orchestration.md) 中“Agent 替代 Pipeline”的表述。当前产品形态保留两种长期并存的同级处理器：**Agent** 与 **Pipeline**。处理器页面只是共享入口，不改变二者各自的持久化模型和执行语义。 + +## 1. 产品边界 + +LangBot 的处理逻辑分成两种同级形态： + +| 形态 | 定位 | 可处理事件 | 典型用户 | +| --- | --- | --- | --- | +| Agent | runner 驱动的事件优先处理器，承载 Runner / 外部 runner | `message.*`、`group.*`、`friend.*`、`bot.*`、`feedback.*`、`platform.*` 等声明范围 | 需要直接处理多类平台事件或接入外部 agent runtime 的用户 | +| Pipeline | 可视化、可控、可组合的消息处理流水线，执行完整 Stage 链 | 仅 `message.*`，首版等价于 `message.received` | 需要预处理、AI、后处理、扩展和输出控制的消息场景 | + +处理器页面负责统一管理这两种处理单元： + +- 创建时选择 **Agent** 或 **Pipeline**； +- 列表中清晰标注类型与事件能力； +- Pipeline 的编辑、调试、监控继续复用现有能力； +- Agent 保存 runner 配置与事件能力，并通过 Bot 事件绑定进入运行时执行。 + +## 2. 信息架构 + +### 2.1 处理器页面 + +路径：`/home/agents` + +职责： + +1. 展示所有可被事件绑定的处理单元，包括 Agent 与 Pipeline。 +2. 创建时先选择类型： + - Agent：创建一条独立 Agent 配置对象，默认支持其 runner 声明的事件范围； + - Pipeline：创建一条独立 Pipeline，执行完整消息 Stage 链。 +3. 编辑时按类型进入不同表单： + - Pipeline：沿用原 Pipeline 配置页，包括 AI、触发、安全、输出、扩展、Debug、Monitoring； + - Agent：基础信息由详情入口编辑，主配置分为运行器、运行器配置、事件与工具；事件范围、自动事件工具、平台级动作和普通工具白名单在同一配置流程内维护。 + +处理器详情复用 `ProcessorDetailWorkbench`；Agent 与 Pipeline 保留各自的配置、调试和日志语义。`RunnerSelect` 提供已安装 Runner 和市场安装入口，安装状态可恢复；Runner 配置来自动态 metadata，不按 LocalAgent id 定制 Host 表单。调试事件选择位于输入区域；Bot 的平台事件调试位于机器人配置中，路由 dry-run 只解释匹配结果。 + +`/home/pipelines` 继续提供 Pipeline 直接编辑路径；共享处理器入口当前使用 `/home/agents`。URL 是实现路径，不代表 Agent 包含 Pipeline。 + +### 2.2 Bot 的事件编排 + +Bot 上维护“事件 -> 处理单元”的绑定规则： + +```text +Bot + └─ EventBinding[] + ├─ event_pattern: message.received / group.member_joined / group.* / * + ├─ target_type: agent / pipeline / discard + ├─ target_uuid: Agent UUID 或 Pipeline UUID + ├─ filters: 事件字段过滤条件 + ├─ priority: 数字越大越优先 + └─ enabled +``` + +Pipeline 只能被绑定到 `message.*`。如果用户选择非消息事件，目标选择器不展示 Pipeline。 + +## 3. 持久化模型 + +### 3.1 Agent 实例与 Pipeline 实例 + +`agents` 表只保存 Agent；Pipeline 继续保存在 Pipeline 表中。当前物理表名 `legacy_pipelines` 是既有存储名称，不代表 Pipeline 在产品架构中是遗留或过渡形态。 + +```python +class Agent(Base): + workspace_uuid: str # workspace-scoped persistence + uuid: str + name: str + description: str + emoji: str + kind: str # 首版固定为 "agent" + component_ref: str # runner reference; execution config uses runner.id + config: dict # runner, runner_config, allowed_tools, allowed_platform_tools + supported_event_patterns: list[str] + created_at: datetime + updated_at: datetime +``` + +处理器聚合服务把 `agents` 与 Pipeline 表投影成同一个前端列表： + +```json +{ + "uuid": "...", + "name": "...", + "kind": "agent | pipeline", + "capability": { + "supported_event_patterns": ["*"], + "message_only": false + } +} +``` + +Pipeline 投影时固定： + +```json +{ + "kind": "pipeline", + "capability": { + "supported_event_patterns": ["message.*"], + "message_only": true + } +} +``` + +### 3.2 Bot 事件绑定 + +Bot 使用 `event_bindings` JSON 字段持久化路由。当前未引入独立路由表；是否拆表应由查询、审计和多作用域需求决定。 + +```json +[ + { + "id": "uuid", + "event_pattern": "group.member_joined", + "target_type": "agent", + "target_uuid": "...", + "filters": [], + "priority": 100, + "enabled": true, + "description": "Welcome new group members" + } +] +``` + +## 4. 匹配规则 + +事件模式支持三层： + +1. 精确匹配：`group.member_joined` +2. 命名空间通配：`group.*` +3. 全局通配：`*` + +优先级： + +1. `enabled = true` +2. event pattern 命中 +3. filters 全部命中 +4. `priority` 数值高者优先 +5. 同优先级按列表顺序 + +## 5. 并存策略 + +1. Pipeline 与 Agent 长期并存，各自保存配置并执行自己的运行链路。 +2. 数据库升级中的旧路由转换由 Alembic 负责；当前运行时只读取 `event_bindings`，不再读取 `use_pipeline_uuid`。 +3. 旧 `pipeline_routing_rules` 不再作为第二个运行时路由来源；LangBot 4.x 不支持 3.x 数据库或配置升级。 +4. `event_bindings` 允许 `target_type=pipeline|agent|discard`；Pipeline 目标只限 `message.*`。 +5. Pipeline 与 Agent 保留各自的持久化和编辑语义；处理器聚合入口只负责统一展示和选择。 + +## 6. 已集成阶段与维护范围 + +### P0：处理器入口统一 + +- 新增 `/home/agents`。 +- 侧边栏显示“处理器”，列表包含平级的 Agent 与 Pipeline。 +- 创建时选择 Agent 或 Pipeline。 +- `/home/pipelines` 继续作为 Pipeline 直接编辑路径。 + +### P1：配置模型落地 + +- 新增 `agents` 表与 `/api/v1/agents`。 +- Agent 可保存 runner 与 runner_config。 +- Pipeline 继续使用原 Pipeline 表单与 API。 + +### P2：事件编排配置面 + +- Bot 表单新增事件编排编辑器。 +- 读取 adapter manifest 的 `supported_events` 生成事件选项。 +- 根据事件类型过滤可选目标：Pipeline 仅在 `message.*` 可选。 + +### P3：EventRouter 执行接入 + +- EBA 事件先广播插件 observer。 +- 然后按 `event_bindings` 的事件模式、filters、priority 和顺序选择一个处理器。 +- Pipeline 目标通过 MessageAggregator 进入完整 Pipeline Stage 链；Agent 目标直接进入 Runner 链路。 +- 非消息事件只选择声明支持该事件的 Agent，不调用 Pipeline；Runner 输出有平台 reply target 时会投递回平台。 + +## 7. 不做的事 + +- 不把 Pipeline 改名成 Agent，也不删除 Pipeline 的配置模型。 +- 不把 Pipeline 降级为兼容层、Agent 子类型或临时运行入口。 +- 不把非消息事件伪装成用户文本塞入 Pipeline。 +- 不在首版做多 Agent 串并联；需要多步骤处理时留给后续 workflow。 diff --git a/docs/event-based-agents/09-event-processors.md b/docs/event-based-agents/09-event-processors.md new file mode 100644 index 000000000..7c8484f51 --- /dev/null +++ b/docs/event-based-agents/09-event-processors.md @@ -0,0 +1,92 @@ +# Runner components and Pipeline plugin compatibility + +## Product boundary + +| Product | Implementation | Event entry | +| --- | --- | --- | +| Pipeline | Pipeline stages and an agent-capable Runner | Received messages | +| Agent | A Runner with `spec.usages: [agent]` | Configured events | +| Plugin processor | A Runner with `spec.usages: [event]` | Events declared in `spec.events` | + +Runner is the only component for these execution styles. `spec.usages` can contain +both `agent` and `event`; these are selection capabilities, not mutually exclusive +execution modes. An installed component is reusable code. Users create processor +instances, select a Runner and configure it, then bind Bot events to the instance. +Installation alone never subscribes a component to incoming events. + +## Legacy EventListener contract + +EventListener remains a Pipeline extension. Existing plugins retain their import +paths, handler registration syntax, event classes, Query-based APIs, and Pipeline +plugin selection behavior. No conversion of installed listeners into standalone +processor instances takes place. + +Compatibility must cover execution behavior, not just successful deserialization: + +| Hook | Required behavior | +| --- | --- | +| PersonMessageReceived / GroupMessageReceived | Read the returned EventContext before later stages; retain message edits and default prevention | +| PromptPreProcessing | Preserve timing and apply returned default_prompt and prompt | +| PersonNormalMessageReceived / GroupNormalMessageReceived | Preserve user_message_alter, default prevention, and replacement replies | +| PersonCommandSent / GroupCommandSent | Preserve command stage timing, default prevention, and replacement replies | +| NormalMessageResponded | Preserve response-stage timing, default prevention, and replacement message chains, including streaming behavior | +| All hooks | Preserve plugin ordering, prevent_postorder across installations, bound-plugin filtering, Query identity, and Workspace scope | + +RPC responses are new Python objects. Host code must consume returned values +rather than assume mutations reached the original Query by object identity. +Host-only references, including the active Query and raw adapter message, must +remain available for legacy reply APIs without being exposed in serialized +plugin events. + +Preserve source event fields across EBA-to-legacy conversion, including group +member permissions, bot group permissions, and member titles. Missing platform +information must be distinguished from fields that were dropped during conversion. + +Direct Agent and Event processor execution must not synthesize Pipeline lifecycle +hooks. Those hooks describe actual Pipeline stages. + +## SDK and runtime + +`lbp comp Runner` generates `components/runner`. Every component uses +`plugin:author/plugin/name` as its identity. Names are unique within a plugin. +The component can override `async run(ctx)` and yield RunnerResult objects, or +register typed platform callbacks through `@self.handler(EventClass)` in +`initialize()`. Default run dispatches an exact handler, falling back to EBAEvent. +A custom run can delegate to this dispatch with `await super().run(ctx)`. + +Both styles share RunnerContext, invocation-bound ctx.api, logs, replies, deadlines, +cancellation, worker isolation and the run ledger. ctx.event is the envelope; +ctx.platform_event is the typed platform payload. Each invocation owns its context; +never put the current context or run ID on a shared component or plugin instance. + +The runtime emits completion on normal return unless the Runner already emitted a +terminal result. Exceptions fail the run and retain preceding results. Cancelling +the result stream cancels execution. There is no implicit retry or hidden model +loop. Returned text and logs do not send platform messages: replies are explicit +ctx.reply / ctx.reply_stream actions. Pipeline retains its configured output stage. + +`self.plugin` continues to expose ordinary plugin APIs. ctx.api carries run-scoped +resource grants and records tool actions. Workspace and installation authorization +remain Host-enforced. Run identity and API operation scope are separate concepts. + +## Selection and observability + +Both product selectors discover the same Runner catalog and filter by usage. +Validate usage again before execution. Event-capable Runners must declare events; +users can route a subset, but cannot expand the manifest capability. Unconfigured +instances expose no event subscriptions. Workspace ownership, plugin scope and +instance identity are checked for routing, execution, cancellation and run reads. + +Plugin processor details keep event debugging on the left and configuration/logs +on the right. Component settings use the existing schema form. Logs and action +results remain distinct from actual platform delivery; debug delivery is Mock. +Agent-native interactions stay on the Agent product path; typed handlers consume +platform events. Legacy Pipeline lifecycle hooks remain on the Pipeline path. + +## Validation + +SDK tests cover both execution styles, event matrices, concurrent contexts, +termination, cancellation and permissions. Packaged CLI tests generate, build and +execute the published component. Core tests cover usage-filtered discovery, event +routing, Workspace authorization and real plugin-runtime transport. RunnerDemo +provides multi-step actions, configuration isolation and controlled failures. diff --git a/docs/event-based-agents/adapters/00-index.md b/docs/event-based-agents/adapters/00-index.md new file mode 100644 index 000000000..f6cd8b0b7 --- /dev/null +++ b/docs/event-based-agents/adapters/00-index.md @@ -0,0 +1,41 @@ +# EBA Adapter Migration Records + +This directory records adapter-level migration details for the Event-Based Agents architecture. Each adapter document should be kept close to the implementation and must answer four questions: + +1. What changed in the adapter structure. +2. Which configuration fields are required. +3. Which events and APIs are supported. +4. What has been verified end to end. + +## Adapter Documents + +General acceptance checklist: [EBA Adapter Acceptance Checklist](./acceptance-checklist.md) + +Current acceptance report: [EBA Adapter Acceptance Report](./acceptance-report.md) + +| Adapter | Status | Document | +|---------|--------|----------| +| Telegram | Migrated; partial plugin E2E, real UI inbound image/file verified | [Telegram](./telegram.md) | +| Discord | Migrated; partial plugin E2E, media-inbound gaps remain | [Discord](./discord.md) | +| OneBot v11 / aiocqhttp | Migrated; Matcha UI plus protocol-level multi-component coverage | [OneBot v11 / aiocqhttp](./aiocqhttp.md) | +| DingTalk | Migrated; partial plugin E2E, real UI inbound image/file verified; group gap remains | [DingTalk](./dingtalk.md) | +| Lark / Feishu | Migrated; partial live text E2E, media-inbound gap remains | [Lark / Feishu](./lark.md) | +| WeCom | Migrated; private text plugin E2E verified, media/group gaps remain | [WeCom](./wecom.md) | +| WeComBot | Migrated; private text and outbound/API plugin E2E verified, feedback/group gaps remain | [WeComBot](./wecombot.md) | +| Official Account | Migrated; private text plugin E2E verified, proactive outbound not supported | [Official Account](./officialaccount.md) | +| QQ Official API | Migrated; WebSocket inbound reached LangBot, model config blocked reply | [QQ Official API](./qqofficial.md) | +| Slack | Migrated; private text and outbound/API plugin E2E verified | [Slack](./slack.md) | +| WeCom Customer Service | Migrated; customer-side UI text plugin E2E verified, inbound media and platform-API live coverage pending | [WeCom Customer Service](./wecomcs.md) | +| Kook | Migrated; unit/mocked converter and API coverage only, live acceptance pending | [Kook](./kook.md) | + +## Documentation Checklist + +When migrating a new adapter, add one document here with: + +- Configuration table matching the adapter manifest. +- Supported event list. +- Supported common API list. +- Supported `call_platform_api` action list. +- Known unsupported APIs and the reason. +- Live test notes, including platform, channel type, destructive operations, and residual risks. +- A clear distinction between real UI inbound media, protocol-level injected inbound media, and bot outbound media. diff --git a/docs/event-based-agents/adapters/acceptance-checklist.md b/docs/event-based-agents/adapters/acceptance-checklist.md new file mode 100644 index 000000000..14eb9f4fa --- /dev/null +++ b/docs/event-based-agents/adapters/acceptance-checklist.md @@ -0,0 +1,208 @@ +# EBA Adapter Acceptance Checklist + +This checklist is the architecture-level acceptance standard for every Event-Based Agents platform adapter. It is not platform-specific. Adapter migration is not complete until the adapter has a written result against this checklist. + +## Evidence Levels + +Use these evidence levels consistently in adapter records: + +| Level | Meaning | Can Mark Complete | +|-------|---------|-------------------| +| `plugin-e2e-ui` | Real SDK plugin running through standalone runtime, LangBot core, the migrated adapter, and a real platform/simulator UI action. | Yes | +| `plugin-e2e-protocol` | Real SDK plugin running through standalone runtime, LangBot core, and the migrated adapter from a protocol-boundary event injection, such as a OneBot reverse WebSocket event. | Partial; must not be claimed as UI coverage | +| `plugin-e2e-outbound` | Real SDK plugin calls an API and the bot output is visible in the real platform/simulator UI. | Yes for send/API coverage only | +| `adapter-live` | Direct adapter probe connected to a real or simulator platform endpoint, bypassing plugin runtime. | No, auxiliary only | +| `unit` | Unit/API-shape tests with mocked platform SDK objects or mocked APIs. | No, auxiliary only | +| `not-supported` | Platform protocol or SDK has no equivalent capability. Must include reason and source. | Yes, as explicitly unsupported | +| `blocked` | Intended capability could not be verified because of credentials, permissions, endpoint gaps, or simulator gaps. | No | + +The primary acceptance path must be `plugin-e2e-ui` for inbound UI-triggered behavior and `plugin-e2e-outbound` for bot send/API behavior. `adapter-live`, `plugin-e2e-protocol`, and `unit` tests are useful, but they must be labelled precisely. + +## Required Architecture Path + +Every adapter must prove this full path: + +```text +Real platform / simulator UI + -> platform SDK native event + -> adapter event converter + -> unified EBA event/entity/message types + -> LangBot core event dispatch + -> standalone SDK runtime + -> real test plugin listener + -> plugin calls platform APIs through SDK + -> LangBot core API dispatch + -> adapter API implementation + -> real platform / simulator UI +``` + +The test plugin must record JSONL evidence containing: + +- event class and `event.type` +- `bot_uuid` and `adapter_name` as received by the plugin +- adapter name +- chat type and chat ID +- sender/user/group IDs with secrets redacted +- message component list for received messages +- API action name, input summary, result or error +- raw unsupported/blocked reason when an item is skipped + +## Required Message Receive Tests + +For every adapter, inbound message conversion must be tested through `plugin-e2e-ui` for each component the platform can receive. If a protocol-level injection is used, label it `plugin-e2e-protocol`; it proves the adapter/core/plugin path, but it does not prove that the user-facing platform UI can send that component. If the platform UI/simulator cannot create a component, record it as `blocked` with the endpoint limitation. + +| Component | Required Receive Assertion | +|-----------|----------------------------| +| `Source` | Message ID and timestamp are present and stable enough for reply/get/delete APIs. | +| `Plain` | Text is preserved exactly, including spaces and multi-line content. | +| `At` | Mentioned user ID is converted to common `At.target`. | +| `AtAll` | Broadcast mention is converted to common `AtAll`, if platform supports it. | +| `Image` | Image ID, URL, path, or base64 is represented without leaking platform-native segment shape. | +| `Voice` | Voice/audio component is represented as `Voice` when the platform exposes it. | +| `File` | File name, ID/URL, and size are represented as `File` when available. | +| `Quote` | Reply/quote source ID and origin content are represented when the platform exposes it. | +| `Face` | Native emoji/sticker/dice/rps-like components are represented as `Face` or documented as platform-specific. | +| `Forward` | Merged/forwarded messages are represented as `Forward` when the platform exposes structured content. | +| `Unknown` | Unsupported native segments become `Unknown` or `PlatformSpecificEvent` data, not crashes. | +| Mixed chain | A message containing multiple component types preserves order. | + +The plugin must subscribe to `MessageReceivedEvent` and assert that `message_chain` contains common `langbot_plugin.api.entities.builtin.platform.message` components, not platform-native SDK objects. + +## Required Message Send Tests + +For every adapter, outbound message conversion must be tested through `plugin-e2e-outbound` by having the plugin call SDK platform APIs and verifying the platform UI/simulator receives the expected message. + +| Component | Required Send Assertion | +|-----------|-------------------------| +| `Plain` | Text appears exactly on the platform. | +| `At` | User mention renders as a mention or platform equivalent. | +| `AtAll` | Broadcast mention renders or is explicitly unsupported. | +| `Image` | URL, path, or base64 image sends and renders/downloads correctly. | +| `Voice` | Voice/audio sends when supported. | +| `File` | File sends with name and content/link when supported. | +| `Quote` | Quoted reply points to the original message when supported. | +| `Face` | Native emoji/sticker/dice/rps sends or is explicitly unsupported. | +| `Forward` | Forward/merged-forward sends when supported; otherwise fallback behavior is documented. | +| Mixed chain | A mixed chain preserves component order as closely as the platform allows. | + +If a platform supports a component only in one direction, the adapter record must say so explicitly. + +## Required Event Tests + +The plugin must subscribe to every event declared in `manifest.yaml -> spec.supported_events` and record one of `plugin-e2e-ui`, `plugin-e2e-protocol`, `not-supported`, or `blocked`. + +| Event | Required Assertion | +|-------|--------------------| +| `message.received` | Real message reaches plugin as `MessageReceivedEvent`. | +| `message.edited` | Edited message reaches plugin with message ID and new content, if declared. | +| `message.deleted` | Deleted/recalled message reaches plugin with message ID and operator when available, if declared. | +| `message.reaction` | Reaction add/remove reaches plugin with message ID, user, reaction, and direction, if declared. | +| `feedback.received` | Feedback payload reaches plugin with feedback type and message/session IDs, if declared. | +| `group.member_joined` | Join event reaches plugin with group and member. | +| `group.member_left` | Leave/kick event reaches plugin with group, member, and kick flag. | +| `group.member_banned` | Mute/ban event reaches plugin with group, member, operator, and duration. | +| `group.info_updated` | Group metadata update reaches plugin with changed fields, if declared. | +| `friend.request_received` | Friend request reaches plugin with request ID and message. | +| `friend.added` | Friend-added event reaches plugin. | +| `friend.removed` | Friend-removed event reaches plugin, if declared. | +| `bot.invited_to_group` | Bot invite/join request reaches plugin with group and inviter/request ID. | +| `bot.removed_from_group` | Bot removal reaches plugin with group and operator when available. | +| `bot.muted` | Bot mute reaches plugin with duration. | +| `bot.unmuted` | Bot unmute reaches plugin. | +| `platform.specific` | At least one unmapped native event is delivered as structured platform-specific data, if declared. | + +Do not declare an event in the manifest unless there is an implementation path and an acceptance entry. + +## Required Common API Tests + +The plugin must call every common API declared in `manifest.yaml -> spec.supported_apis.required` and `optional`. Each call must be recorded with input summary and result. + +| API | Required Assertion | +|-----|--------------------| +| `send_message` | Plugin sends to private and group/channel targets where supported. | +| `reply_message` | Plugin replies to the triggering message, with quoted mode tested when supported. | +| `edit_message` | Plugin edits a bot-sent message, if declared. | +| `delete_message` | Plugin deletes/recalls a bot-sent message, if declared and permissions allow. | +| `forward_message` | Plugin forwards or emulates forwarding a real message, if declared. | +| `get_message` | Plugin retrieves a real message and receives common `MessageReceivedEvent` shape. | +| `get_group_info` | Plugin receives `UserGroup` with ID/name/count where available. | +| `get_group_list` | Plugin receives joined groups/channels list where supported. | +| `get_group_member_list` | Plugin receives list of `UserGroupMember` where supported. | +| `get_group_member_info` | Plugin receives one member with role/display name where available. | +| `set_group_name` | Plugin changes and restores a disposable group name, if declared. | +| `mute_member` | Plugin mutes a disposable target, if declared. | +| `unmute_member` | Plugin unmutes the same target, if declared. | +| `kick_member` | Plugin kicks a disposable target only in destructive test mode, if declared. | +| `leave_group` | Plugin leaves only in destructive test mode and only at the end, if declared. | +| `get_user_info` | Plugin receives common `User` shape. | +| `get_friend_list` | Plugin receives friend/contact list where supported. | +| `approve_friend_request` | Plugin accepts/rejects a disposable friend request, if declared. | +| `approve_group_invite` | Plugin accepts/rejects a disposable group invite, if declared. | +| `upload_file` | Plugin uploads a real small file, if declared. | +| `get_file_url` | Plugin resolves a real file ID to a URL, if declared. | +| `call_platform_api` | Plugin calls every declared platform-specific action with safe parameters. | + +Destructive APIs must be opt-in and documented with the exact target used. + +The SDK must expose a plugin-side platform API escape hatch for adapter-specific actions. The acceptance plugin should call it from the same EBA event handler that received the real platform event, so the evidence proves both directions of the path: + +```text +plugin -> SDK call_platform_api -> LangBot core -> adapter call_platform_api -> platform SDK/API +``` + +The result must be serialized into JSON-safe values before it is returned to the plugin runtime. + +## Platform-Specific API Tests + +Every action listed in `manifest.yaml -> spec.platform_specific_apis` must have one acceptance entry: + +- `plugin-e2e-ui` or `plugin-e2e-outbound`: called by the plugin against the live/simulator endpoint. +- `plugin-e2e-protocol`: called by the plugin after a protocol-boundary injected event; useful for endpoint-specific simulators but must be labelled. +- `not-supported`: removed from manifest or explained if the platform SDK exposes it but this adapter intentionally does not. +- `blocked`: endpoint did not implement it, permissions missing, or safe fixture unavailable. + +Do not leave a platform-specific API in the manifest without a corresponding test record. + +## Required Compatibility Tests + +Each migrated adapter must also prove: + +- Manifest supported events match `adapter.get_supported_events()`. +- Manifest supported APIs match `adapter.get_supported_apis()`. +- Manifest platform-specific actions match `PLATFORM_API_MAP`. +- Legacy `FriendMessage` / `GroupMessage` listeners still work when the core registers them. +- EBA listener dispatch prefers the most specific event class, then `EBAEvent`, then base `Event`. +- Self-message filtering prevents bot echo loops without dropping edit/delete/moderation events needed for API tests. +- `source_platform_object` is present for reply/debug but not required by plugins for common behavior. + +## Required Documentation Per Adapter + +Each adapter document must include: + +- adapter directory and manifest name +- config table +- supported event table with evidence level per event +- supported common API table with evidence level per API +- platform-specific API table with evidence level per action +- receive component table with evidence level per component +- send component table with evidence level per component +- exact test date +- exact platform endpoint or simulator used +- standalone runtime command +- plugin path/name used for testing +- evidence JSONL path +- destructive operations performed or explicitly skipped +- blocked items and reasons + +## Acceptance Rule + +An adapter can be marked migrated only when: + +1. All declared events have `plugin-e2e-ui`, justified `plugin-e2e-protocol`, or `not-supported` evidence. +2. All declared APIs have `plugin-e2e-outbound` or `not-supported` evidence. +3. All platform-supported receive components have `plugin-e2e-ui` evidence; protocol-only receive coverage keeps the status partial. +4. All platform-supported send components have `plugin-e2e-outbound` evidence. +5. Unit tests cover conversion and API-shape boundaries. +6. The adapter document lists every blocked or skipped item honestly. + +If any declared capability is only covered by `adapter-live` or `unit`, the adapter status must remain partial. diff --git a/docs/event-based-agents/adapters/acceptance-report.md b/docs/event-based-agents/adapters/acceptance-report.md new file mode 100644 index 000000000..ac86b62b7 --- /dev/null +++ b/docs/event-based-agents/adapters/acceptance-report.md @@ -0,0 +1,171 @@ +# EBA Adapter Acceptance Report + +Date: May 10, 2026 + +Scope: + +- `telegram-omni` +- `discord-omni` +- `aiocqhttp-omni` +- `dingtalk-omni` +- `lark-omni` +- `wecom-omni` +- `wecombot-omni` +- `wecomcs-omni` +- `officialaccount-omni` +- `qqofficial-omni` +- `slack-omni` + +This report follows `acceptance-checklist.md`. Evidence levels are intentionally strict: + +- `plugin-e2e-ui`: real platform or simulator UI event reached LangBot, standalone runtime, and `EBAEventProbe`. +- `plugin-e2e-protocol`: real adapter endpoint event reached LangBot, standalone runtime, and `EBAEventProbe`, but the event was injected at the platform protocol boundary rather than sent through the UI. +- `plugin-e2e-outbound`: the plugin called SDK APIs and the resulting bot message was visible on the platform. +- `unit`: mocked converter/API coverage only. +- `blocked`: not completed, either because the platform/simulator/client could not trigger it or because a safe disposable fixture was unavailable. +- `not-supported`: the platform has no equivalent capability. + +## Summary + +| Adapter | Status | Honest acceptance summary | +|---------|--------|---------------------------| +| Telegram | Partial EBA acceptance | Real Telegram UI covered private text, group mention text, bot invite, inbound private image/file, outbound component sweep, safe SDK APIs, and safe Telegram platform APIs. Real UI inbound voice/quote was not completed in the latest plugin run. | +| Discord | Partial EBA acceptance | Real Discord UI covered group text, outbound image/file/quote/mention components, safe SDK APIs, and safe Discord platform APIs. Real UI inbound attachment/image/file/reply/mention was not completed. A later UI retry was blocked because the Discord client kept the send button disabled. | +| OneBot v11 / aiocqhttp | Partial EBA acceptance | Matcha UI covered real group text and outbound supported components/APIs. Multi-component inbound `Source/Plain/At/Face/Image/Voice/File/Quote` was verified through the real OneBot reverse WebSocket adapter endpoint, but not through Matcha UI upload/send. Matcha blocks file-send and merged-forward APIs. | +| DingTalk | Partial EBA acceptance | Real DingTalk UI covered private text, emoji-as-text inbound, private inbound image/file, outbound image/file/quote/mention fallback components, safe SDK APIs, and safe DingTalk platform APIs. Real UI inbound voice/quote and group trigger were not completed. | +| Lark / Feishu | Partial EBA acceptance | EBA adapter structure, self-built/store app config, WebSocket/Webhook mode handling, converters, common APIs, platform APIs, and unit tests are in place. One real LangBot organization WebSocket private text event reached `EBAEventProbe`; outbound component sweep was visible in Feishu. Latest real UI image/file sends did not reach local plugin evidence, so media receive remains blocked. | +| WeCom | Partial EBA acceptance | Regular WeCom application-message adapter is split into the EBA directory with manifest, converters, API mixin, platform API map, and unit tests. Private text reached `EBAEventProbe` through standalone runtime and the real WeCom client; safe plugin APIs passed. Real inbound media and broader event coverage remain pending. | +| WeComBot | Partial EBA acceptance | WeCom AI Bot is split into the EBA directory with WebSocket long connection mode and optional webhook mode, EBA message/feedback/platform-specific conversion, cache-backed common APIs, platform API map, unit tests, and a direct live probe. Private text, outbound component sweep, safe common APIs, and all declared WeComBot platform APIs reached `EBAEventProbe`; group, real inbound media, and feedback callback evidence remain pending. | +| WeCom Customer Service | Partial EBA acceptance | WeCom Customer Service is split into the EBA directory with manifest, converters, API mixin, platform API map, unit tests, docs, and a direct live probe scaffold. Real WeChat customer-side UI text reached `EBAEventProbe`; plugin outbound text/image and safe cache-backed common APIs passed. Inbound media and platform-specific API live coverage remain pending; later fallback text sends were blocked by WeCom `95001 send msg count limit`. | +| Official Account | Partial EBA acceptance | WeChat Official Account is split into the EBA directory with manifest, converters, cache-backed safe APIs, platform API map, unit tests, and a direct live probe scaffold. Real WeChat Official Account UI private text reached `EBAEventProbe`; safe cache-backed common APIs and declared platform APIs passed. Proactive outbound `send_message` is not supported because replies must be tied to inbound webhook windows; inbound image/voice live UI evidence remains pending. | +| QQ Official API | Partial EBA acceptance | QQ Official API is split into the EBA directory with manifest, converters, cache-backed safe APIs, platform API map, unit tests, docs, and a direct live probe scaffold. A real WebSocket-mode QQ Official bot reached the LangBot pipeline on `dev.rockchin.top`; reply/outbound evidence is blocked by the test model provider returning `model_not_found` for `deepseek-v3`. | +| Slack | Partial EBA acceptance | Slack is split into the EBA directory with manifest, converters, cache-backed safe APIs, platform API map, unit tests, docs, and a direct live probe scaffold. Real Slack private text reached `EBAEventProbe`; safe common APIs, outbound component fallback sweep, and declared Slack platform APIs passed. Channel mention and real inbound media evidence remain pending. | + +Telegram and DingTalk now have real user-side UI image/file upload evidence in plugin JSONL. Discord and aiocqhttp do not yet have real UI inbound image/file evidence. + +## Evidence Files + +| Adapter | Endpoint | Evidence | +|---------|----------|----------| +| Telegram private | Telegram Lite, `@rockchinq_bot` private chat | `data/temp/telegram-plugin-e2e-rerun.jsonl` | +| Telegram private media | Telegram Lite, `@rockchinq_bot` private chat | `data/temp/telegram-plugin-e2e-media-ui.jsonl` | +| Telegram group | Telegram Lite, `Rock'sBotGroup` | `data/temp/telegram-plugin-e2e-group.jsonl` | +| Discord | Discord client, LangBot server, `#debugging` | `data/temp/discord-plugin-e2e-20260510-final.jsonl` | +| aiocqhttp UI | local Matcha, group `test group` | `data/temp/aiocqhttp-plugin-e2e-20260510-multiformat.jsonl` | +| aiocqhttp protocol | OneBot reverse WebSocket endpoint `127.0.0.1:2280/ws` | `data/temp/aiocqhttp-plugin-e2e-20260510-multiformat.jsonl` | +| DingTalk | DingTalk Mac, `LangBot Team` org private chat | `data/temp/dingtalk-plugin-e2e-20260510-rerun.jsonl` | +| DingTalk private media | DingTalk Mac, `LangBot Team` org private chat | `data/temp/dingtalk-plugin-e2e-media-ui.jsonl` | +| Lark / Feishu unit | local mocked Feishu SDK/client paths | `tests/unit_tests/platform/test_lark_eba_adapter.py` | +| Lark / Feishu partial live | Feishu Mac, LangBot organization `LangBotDev` private chat | `data/temp/lark-plugin-e2e-ws.jsonl` | +| WeCom Customer Service | WeChat customer-side UI, `客服消息 -> 浪波智能客服` on `dev.rockchin.top` | `/home/wgc/LangBotxg/LangBotEbaTest/data/temp/wecomcs_eba_plugin_probe.jsonl` | +| Official Account | WeChat desktop client, subscribed Official Account on `dev.rockchin.top` | `/home/wgc/LangBotxg/LangBotEbaTest/data/temp/officialaccount_eba_plugin_probe.jsonl` | +| QQ Official API unit | local mocked QQ Official client paths | `tests/unit_tests/platform/test_qqofficial_eba_adapter.py` | +| Slack unit | local mocked Slack client paths | `tests/unit_tests/platform/test_slack_eba_adapter.py` | +| Slack private | Slack workspace private DM on `dev.rockchin.top` | `/home/wgc/LangBotxg/LangBotEbaTest/data/temp/slack_eba_plugin_probe.jsonl` | + +All plugin runs used SDK standalone runtime ports `5400/5401`, LangBot `--standalone-runtime`, and the real plugin at `langbot-plugin-demo/EBAEventProbe`. + +## Unified Shape Verification + +All four adapters deliver common SDK entities to plugins before LangBot core/plugin logic handles the event. + +| Requirement | Telegram | Discord | aiocqhttp | DingTalk | Lark / Feishu | +|-------------|----------|---------|-----------|----------|---------------| +| `bot_uuid` filled | plugin-e2e | plugin-e2e | plugin-e2e | plugin-e2e | live plugin-e2e pending | +| `adapter_name` filled | `telegram` | `discord` | `aiocqhttp` | `dingtalk` | `lark-omni` in current unit/code; older live text evidence recorded `lark` before the naming fix | +| common `MessageChain` delivered | `Plain`, group `At + Plain`, private `Image`, private `File` | `Source + Plain` | UI `Source + Plain`; protocol `Source + Plain + At + Face + Image + Voice + File + Quote + Plain` | `Source + Plain`, private `Source + Image`, private `Source + File` | live private `Source + Plain`; unit `Source + Plain + At/Image/File`; latest live image/file blocked | +| common user/group entities | plugin-e2e | plugin-e2e | plugin-e2e | plugin-e2e private user; group not completed | live private user; unit private/group | +| raw native object isolation | raw data stays in `source_platform_object` | raw data stays in `source_platform_object` | raw data stays in `source_platform_object` | raw data stays in `source_platform_object` | raw data stays in `source_platform_object` | + +## Message Receive Components + +| Component | Telegram | Discord | aiocqhttp | DingTalk | Lark / Feishu | +|-----------|----------|---------|-----------|----------|---------------| +| `Source` | design gap: event has message id but chain omits `Source` | plugin-e2e-ui | plugin-e2e-ui/protocol | plugin-e2e-ui | plugin-e2e-ui private text | +| `Plain` | plugin-e2e-ui private/group | plugin-e2e-ui | plugin-e2e-ui/protocol | plugin-e2e-ui | plugin-e2e-ui private text | +| `At` | plugin-e2e-ui group mention | unit; real UI mention not completed in latest run | plugin-e2e-protocol; unit | unit; group trigger not completed | unit; group trigger not completed | +| `AtAll` | not-supported | unit only | unit only | unit/send fallback only | unit only | +| `Image` | plugin-e2e-ui private | converter/unit; real UI attachment not completed | plugin-e2e-protocol, not Matcha UI | plugin-e2e-ui private | unit; real UI image sent but not observed in plugin evidence | +| `Voice` | converter/unit; real UI inbound not completed | not-supported as native voice; audio is attachment/file | plugin-e2e-protocol, not Matcha UI | converter/unit; real UI inbound not completed | unit; real UI inbound not completed | +| `File` | plugin-e2e-ui private | converter/unit; real UI attachment not completed | plugin-e2e-protocol, not Matcha UI | plugin-e2e-ui private | unit; real UI file sent but not observed in plugin evidence | +| `Quote` | converter/unit; real UI reply not completed | unit; real UI reply not completed | plugin-e2e-protocol | converter/unit; real UI quote not completed | unit/API-backed quote lookup; real UI quote not completed | +| `Face` | not-supported as common `Face` | not-supported as common `Face` | plugin-e2e-protocol | UI emoji becomes `Plain` (`[smile]` text), not `Face` | not-supported as common `Face` | +| `Forward` | not-supported inbound | not-supported inbound | unit; Matcha forward UI/action blocked | not-supported inbound | not-supported inbound | +| Mixed chain | group `At + Plain`; media tested as separate messages | not completed inbound | plugin-e2e-protocol | media tested as separate messages; mixed inbound not completed | unit only | + +## Message Send Components + +| Component | Telegram | Discord | aiocqhttp | DingTalk | Lark / Feishu | +|-----------|----------|---------|-----------|----------|---------------| +| `Plain` | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound | +| `At` | plugin-e2e-outbound equivalent | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound fallback/equivalent | plugin-e2e-outbound | +| `AtAll` | plugin-e2e-outbound fallback | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound fallback | unit; group live not completed | +| `Image` | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound | +| `Voice` | not-supported in current send converter | not-supported as native voice | converter path; not completed against Matcha UI | fallback as file/text depending DingTalk media support | converter path; live not completed | +| `File` | plugin-e2e-outbound | plugin-e2e-outbound | blocked by Matcha endpoint error | plugin-e2e-outbound | plugin-e2e-outbound | +| `Quote` | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound fallback | plugin-e2e-outbound fallback | +| `Face` | not-supported | not-supported | plugin-e2e-outbound attempted in mixed chain | fallback text | not-supported | +| `Forward` | flattened fallback | flattened fallback | blocked by Matcha unsupported action | flattened fallback | plugin-e2e-outbound flattened fallback | +| Mixed chain | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound except blocked file/forward | plugin-e2e-outbound | plugin-e2e-outbound | + +## Event Acceptance + +| Event category | Telegram | Discord | aiocqhttp | DingTalk | +|----------------|----------|---------|-----------|----------| +| `message.received` | plugin-e2e-ui | plugin-e2e-ui | plugin-e2e-ui and plugin-e2e-protocol | plugin-e2e-ui private | +| `message.edited` | implemented/unit, not plugin-e2e-ui | historical/direct only, not latest plugin-e2e | unit | not declared | +| `message.deleted` | implemented/unit, not plugin-e2e-ui | historical/direct only, not latest plugin-e2e | unit | not declared | +| `message.reaction` | implemented/unit, not plugin-e2e-ui | historical/direct only, not latest plugin-e2e | not-supported in standard OneBot message path | not declared | +| member join/left/ban | implemented/unit or blocked without disposable users | blocked without disposable users | unit; Matcha fixture unavailable | not declared | +| bot invited/removed | invite plugin-e2e-ui for Telegram; removal blocked | invite historical/plugin-series; removal blocked | unit; Matcha fixture unavailable | not declared | +| requests/friend events | not applicable | not applicable | unit; Matcha fixture unavailable | not declared | +| `platform.specific` | implemented; not latest plugin-e2e | not latest plugin-e2e | adapter lifecycle observed; plugin focus was message path | declared for fallback; not reproduced in UI run | + +## Common API Acceptance + +| API area | Telegram | Discord | aiocqhttp | DingTalk | +|----------|----------|---------|-----------|----------| +| send/reply | plugin-e2e-outbound | plugin-e2e-outbound | plugin-e2e-outbound, with Matcha file/forward gaps | plugin-e2e-outbound | +| edit/delete | historical/direct or unit; destructive/current UI not repeated | historical/direct; destructive/current UI not repeated | unit/destructive blocked | not declared or blocked | +| message lookup | not-supported | not-supported | plugin-e2e | inbound cache-backed where available; limited live coverage | +| group info/member info | plugin-e2e safe subset | plugin-e2e safe subset | plugin-e2e safe subset | private path only; group not completed | +| user/friend info | plugin-e2e where platform allows | plugin-e2e where platform allows | plugin-e2e | plugin-e2e private user | +| moderation/leave | blocked without disposable safe targets | blocked without disposable safe targets | blocked without disposable safe targets | blocked/not declared | +| `get_file_url` | implemented; latest inbound `File` carried downloadable file data in plugin evidence | URL passthrough for attachments; inbound attachment not completed | not portable/endpoint-dependent | implemented through DingTalk media API; latest inbound `File` carried a platform file URL | +| `call_platform_api` | plugin-e2e safe actions | plugin-e2e safe actions | plugin-e2e safe actions, Matcha gaps documented | plugin-e2e safe `check_access_token` | + +## Platform-Specific API Acceptance + +| Adapter | plugin-e2e verified | Blocked or not reproduced | +|---------|---------------------|---------------------------| +| Telegram | safe chat/admin/member count/chat-action actions | mutating actions and callback-only actions were not repeated | +| Discord | safe channel/guild/role/typing actions | mutating pin/reaction/invite actions were not repeated in the latest plugin run; inbound attachment paths not completed | +| aiocqhttp | safe OneBot actions such as status/version/can-send checks | `get_group_honor_info` unsupported by Matcha; admin/card/title/ban/record/file/forward require better endpoint fixtures | +| DingTalk | `check_access_token`; real inbound file produced a file URL in the common `File` component | separate media-download replay APIs and group actions need a working follow-up fixture | + +## SDK API Acceptance + +`EBAEventProbe` exercised the standalone runtime path for: + +- bot discovery and bot info lookup +- send message +- component sweep where enabled +- platform API sweep where enabled +- plugin storage +- workspace storage +- plugin/command/tool/knowledge-base list APIs + +The probe logs set `ok=true` when the sweep completed with only expected unsupported/blocked items. Individual call details are stored in the JSONL evidence files. + +## Residual Risks And Required Follow-Up + +- Discord still requires real UI inbound image/file upload evidence before it can be called media-complete. +- aiocqhttp has rich inbound component evidence only at the OneBot reverse WebSocket boundary; Matcha UI did not provide image/file upload coverage. +- DingTalk group trigger remains unclosed; current evidence is private chat only. +- Lark / Feishu requires a clean follow-up live pass: the latest LangBot organization WebSocket run connected, but UI-sent text/image/file after the loop-scheduling fix did not append plugin events. +- Discord UI retry on May 10, 2026 was blocked by the client keeping the send button disabled even after text was entered. +- Destructive moderation and leave APIs are intentionally blocked until disposable users/groups are available. + +## Conclusion + +The EBA conversion path is implemented and partially proven for the migrated adapters. Telegram and DingTalk now have real UI private-chat image/file inbound evidence. Discord, aiocqhttp, and Lark / Feishu still have explicit UI-level media gaps, so the overall adapter set remains partial acceptance rather than production-complete media acceptance. diff --git a/docs/event-based-agents/adapters/aiocqhttp.md b/docs/event-based-agents/adapters/aiocqhttp.md new file mode 100644 index 000000000..c056adfbe --- /dev/null +++ b/docs/event-based-agents/adapters/aiocqhttp.md @@ -0,0 +1,162 @@ +# OneBot v11 / aiocqhttp EBA Adapter + +## Status + +OneBot v11 has been migrated to the EBA adapter directory: + +```text +src/langbot/pkg/platform/adapters/aiocqhttp/ +├── adapter.py +├── api_impl.py +├── event_converter.py +├── manifest.yaml +├── message_converter.py +├── platform_api.py +├── types.py +└── onebot.png +``` + +The EBA adapter is registered as `aiocqhttp-omni`. The legacy adapter remains at `src/langbot/pkg/platform/sources/aiocqhttp.py`. + +## Configuration + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `host` | Yes | `0.0.0.0` | Host for the reverse WebSocket server that the OneBot endpoint connects to. | +| `port` | Yes | `2280` | Reverse WebSocket listen port. | +| `access-token` | No | `""` | OneBot access token, if the endpoint is configured to use one. | + +## Events + +The adapter declares these EBA events: + +- `message.received` +- `message.deleted` +- `group.member_joined` +- `group.member_left` +- `group.member_banned` +- `friend.request_received` +- `friend.added` +- `bot.invited_to_group` +- `bot.removed_from_group` +- `bot.muted` +- `bot.unmuted` +- `platform.specific` + +`platform.specific` is used for OneBot notice/request/meta events that do not yet have a common EBA event type, such as group admin changes, group file uploads, pokes, honor changes, and group join requests from non-bot users. + +## Common APIs + +| API | Status | Notes | +|-----|--------|-------| +| `send_message` | Supported | Supports private and group text, mentions, images, voice, files, faces, and flattened forwards. Group merged forwards are sent through OneBot forward APIs when possible. | +| `reply_message` | Supported | Uses the original OneBot event and can prepend a reply segment. | +| `edit_message` | Not supported | OneBot v11 has no standard message edit action. | +| `delete_message` | Supported | Uses `delete_msg`; permission depends on endpoint and group role. | +| `forward_message` | Supported | Emulates forward by fetching the source message with `get_msg` and sending its content to the target chat. | +| `get_message` | Supported | Uses `get_msg` and converts the response into `MessageReceivedEvent`. | +| `get_group_info` | Supported | Uses `get_group_info`. | +| `get_group_list` | Supported | Uses `get_group_list`. | +| `get_group_member_list` | Supported | Uses `get_group_member_list`. | +| `get_group_member_info` | Supported | Uses `get_group_member_info`. | +| `set_group_name` | Supported | Uses `set_group_name`; may be unsupported by mock endpoints. | +| `get_user_info` | Supported | Uses `get_stranger_info`. | +| `get_friend_list` | Supported | Uses `get_friend_list`. | +| `approve_friend_request` | Supported | Uses `set_friend_add_request`. | +| `approve_group_invite` | Supported | Uses `set_group_add_request` with `sub_type=invite`. | +| `upload_file` | Not supported | OneBot v11 has endpoint-specific file upload extensions but no portable standalone upload action. | +| `get_file_url` | Not supported | OneBot v11 file URL resolution is endpoint-specific. Use `call_platform_api("get_image")`, `get_record`, or endpoint extensions when available. | +| `mute_member` | Supported | Uses `set_group_ban`. | +| `unmute_member` | Supported | Uses `set_group_ban` with duration `0`. | +| `kick_member` | Supported | Destructive; test only with disposable members. | +| `leave_group` | Supported | Destructive; should run last in live tests. | +| `call_platform_api` | Supported | See below. | + +## Platform-Specific APIs + +`call_platform_api(action, params)` supports: + +- `get_login_info` +- `get_status` +- `get_version_info` +- `get_group_honor_info` +- `set_group_card` +- `set_group_special_title` +- `set_group_admin` +- `set_group_whole_ban` +- `send_group_forward_msg` +- `get_forward_msg` +- `get_record` +- `get_image` +- `can_send_image` +- `can_send_record` + +## Message Conversion Notes + +Incoming OneBot segments are converted into common `MessageChain` components before LangBot core/plugin dispatch: + +- `text` -> `Plain` +- `at` -> `At` / `AtAll` +- `image` -> `Image` or `Face` for OneBot emoji-package images +- `record` -> `Voice` +- `file` -> `File` +- `reply` -> `Quote` +- `face`, `rps`, `dice` -> `Face` +- unsupported segments -> `Unknown` + +Outgoing `MessageChain` components are converted back into `aiocqhttp.Message` segments. Base64 media strings are normalized to OneBot `base64://...` format. + +## Live Test Record + +The direct live probe is: + +```bash +PYTHONPATH=/Users/qinjunyan/code/projects/langbot/langbot-plugin-sdk/src \ +uv run python tests/e2e/live_aiocqhttp_eba_probe.py --host 127.0.0.1 --port 2280 +``` + +It starts the reverse WebSocket adapter directly, records observed EBA events to `data/temp/aiocqhttp_eba_live_probe.jsonl`, waits for a real Matcha or OneBot message, then tries reply/send/get/delete/group/user/platform API calls as far as the endpoint supports them. + +Verified on May 10, 2026 with local Matcha connected to `ws://127.0.0.1:2280/ws`: + +- Real inbound group message converted to `MessageReceivedEvent`. +- Real lifecycle connection converted to `PlatformSpecificEvent`. +- Real reply API succeeded and rendered a quoted bot reply in Matcha. +- Real proactive send API succeeded and rendered a bot group message in Matcha. +- Real outgoing component sweep succeeded for text, `At`, `AtAll`, `Face`, and base64 `Image`. +- Real `get_message`, `get_group_info`, `get_login_info`, `get_status`, `get_version_info`, `can_send_image`, and `can_send_record` calls succeeded against Matcha. +- Unit conversion and API-shape tests passed for `Plain`, `At`, `AtAll`, `Image`, `Voice`, `File`, `Quote`, `Face`, `rps`, `dice`, `Forward`, `Unknown`, private/group message events, delete notices, group join/leave/ban notices, bot mute notices, friend requests, group invites, friend added notices, dispatch specificity, send, reply, delete, forward, get message, group APIs, user APIs, request approval APIs, moderation APIs, leave group, unsupported file APIs, and all declared `call_platform_api` actions. + +Skipped or residual live-test items: + +- `edit_message`: not implemented because OneBot v11 has no standard edit action. +- `upload_file` and `get_file_url`: not implemented as common APIs because portable OneBot v11 file upload/download URL semantics are endpoint-specific. +- `kick_member` and `leave_group`: destructive; run only with explicit `--destructive` and disposable Matcha/OneBot state. +- `group.info_updated`, message reactions, and message edits are not declared because OneBot v11 does not provide standard equivalents for them. +- Matcha returned `ActionFailed` for outgoing `File` segment rendering and did not support merged-forward actions in this run. The adapter keeps the conversion/API implementations because they are valid OneBot/NapCat-style capabilities, but the Matcha live probe records them as skipped. +- Matcha returned an empty `get_group_member_list` for the test group, so `get_group_member_info`, mute/unmute, kick, and leave were covered by unit/API-shape tests only in this run. + +## Standalone Runtime Plugin E2E Record + +Verified on May 10, 2026 with `EBAEventProbe`, SDK standalone runtime, LangBot `--standalone-runtime`, local Matcha, and group `测试群`. + +Evidence: + +- Plugin JSONL: `data/temp/aiocqhttp-plugin-e2e-20260510-multiformat.jsonl` + +Observed and verified: + +- A real Matcha group message reached the plugin as `MessageReceived` with `bot_uuid=eba-aiocqhttp-matcha`, `adapter_name=aiocqhttp`, common `Source`/`Plain` message components, common sender, and common group identifiers. +- A protocol-level OneBot reverse WebSocket event reached the plugin as `MessageReceived` with a mixed common chain: `Source`, `Plain`, `At`, `Face`, `Image`, `Voice`, `File`, `Quote`, and trailing `Plain`. This proves the real adapter + LangBot + standalone runtime + plugin path for mixed inbound OneBot payloads, but it was not sent through Matcha UI. +- SDK API calls succeeded: `get_langbot_version`, `get_bots`, `get_bot_info`, `send_message`, plugin storage, workspace storage, `list_plugins_manifest`, `list_commands`, `list_tools`, and `list_knowledge_bases`. +- Outbound component sweep succeeded for plain text plus `At`/`Face`, `AtAll`, base64 `Image`, and quoted reply. +- Common APIs succeeded through the plugin path: `get_message`, `get_user_info`, `get_friend_list`, `get_group_info`, `get_group_list`, `get_group_member_list`, and `get_group_member_info`. +- Safe OneBot platform APIs succeeded through `call_platform_api`: `get_login_info`, `get_status`, `get_version_info`, `can_send_image`, and `can_send_record`. + +Documented Matcha limits in this E2E run: + +- Matcha UI did not provide a completed image/file upload/send path for inbound media. The rich inbound media evidence is `plugin-e2e-protocol`, not UI-level media upload evidence. +- Outbound `File` failed in Matcha even after the adapter emitted an official `file` segment shape. +- Outbound `Forward` failed because Matcha returned unsupported action for merged-forward. +- `get_group_honor_info` failed because Matcha returned unsupported action. +- Destructive/admin APIs such as mute, unmute, kick, leave, group rename, card/title/admin/whole-ban changes, and request approvals were not run without disposable fixtures. diff --git a/docs/event-based-agents/adapters/dingtalk.md b/docs/event-based-agents/adapters/dingtalk.md new file mode 100644 index 000000000..5576c783b --- /dev/null +++ b/docs/event-based-agents/adapters/dingtalk.md @@ -0,0 +1,115 @@ +# DingTalk EBA Adapter Migration Record + +Status: migrated with partial plugin E2E evidence. + +Adapter directory: `src/langbot/pkg/platform/adapters/dingtalk/` + +## What Changed + +The DingTalk adapter now has an Event-Based Agents adapter package with: + +- `manifest.yaml` for adapter metadata, configuration, events, common APIs, and platform-specific APIs. +- `adapter.py` for DingTalk client startup, native callback handling, legacy compatibility, and EBA dispatch. +- `event_converter.py` for native DingTalk events and card callbacks to common EBA events. +- `message_converter.py` for DingTalk message payloads to/from common `MessageChain` components. +- `api_impl.py` for common EBA API implementations. +- `platform_api.py` for DingTalk-specific `call_platform_api` actions. + +The legacy DingTalk HTTP client now returns successful JSON response bodies from proactive send methods and raises with response details on non-200 responses. + +## Configuration + +| Field | Required | Notes | +|-------|----------|-------| +| `client-id` | yes | DingTalk robot/client identifier. | +| `client-secret` | yes | DingTalk client secret. | +| `robot-code` | yes | Robot code used for send APIs. | +| `robot-name` | no | Used for bot mention/self filtering and display. | +| `encrypt-key` | no | DingTalk callback encryption key when configured. | +| `verification-token` | no | DingTalk callback verification token when configured. | + +## Supported Events + +| Event | Support | Evidence | +|-------|---------|----------| +| `message.received` | implemented | `plugin-e2e-ui` private text and emoji-as-text. | +| `feedback.received` | unit covered | DingTalk card callback actions with feedback-like values (`like`, `dislike`, `cancel`, or `1`/`2`/`3`) map to `FeedbackReceivedEvent`. Other card actions remain `platform.specific`. | +| `platform.specific` | implemented | Non-feedback card callbacks and unmapped callback/message shapes are emitted as structured platform-specific events. | + +## Receive Components + +| Component | Support | Evidence | +|-----------|---------|----------| +| `Source` | supported | `plugin-e2e-ui` private message. | +| `Plain` | supported | `plugin-e2e-ui` private text. DingTalk emoji currently arrives as plain text such as `[smile]`. | +| `At` | converter path | Group trigger was not completed in the latest run. | +| `AtAll` | fallback/send-side only | Not completed inbound. | +| `Image` | supported | Real DingTalk Mac private-chat image upload reached the plugin as common `Image`. | +| `Voice` | converter path | Real UI inbound voice was not completed. | +| `File` | supported | Real DingTalk Mac private-chat file upload reached the plugin as common `File`. | +| `Quote` | converter path | Real UI inbound quote was not completed. | +| `Face` | not native common mapping | DingTalk emoji was observed as `Plain`, not `Face`. | +| `Forward` | not-supported inbound | DingTalk does not expose a portable structured forward event in this adapter. | + +## Send Components + +| Component | Support | Evidence | +|-----------|---------|----------| +| `Plain` | supported | `plugin-e2e-outbound`. | +| `At` | supported or text fallback | `plugin-e2e-outbound`. | +| `AtAll` | fallback | `plugin-e2e-outbound`. | +| `Image` | supported | `plugin-e2e-outbound`. | +| `File` | supported | `plugin-e2e-outbound`. | +| `Quote` | fallback | `plugin-e2e-outbound`. | +| `Face` | fallback | `plugin-e2e-outbound` as text fallback. | +| `Forward` | flattened fallback | `plugin-e2e-outbound`. | +| `Voice` | fallback/endpoint-dependent | Not separately verified as a native DingTalk voice send. | + +## Common APIs + +| API | Support | Notes | +|-----|---------|-------| +| `send_message` | supported | Verified through `EBAEventProbe`. | +| `reply_message` | supported | Verified through quoted/fallback send path. | +| `get_message` | cache-backed | Requires the message to have been observed by this adapter process. | +| `get_group_info` | cache-backed/API-backed where available | Group path not completed in latest UI run. | +| `get_group_list` | supported where DingTalk API allows | Limited live coverage. | +| `get_group_member_info` | supported where DingTalk API allows | Limited live coverage. | +| `get_user_info` | supported | Private sender path verified. | +| `get_friend_list` | limited | DingTalk does not expose a portable friend-list equivalent. | +| `get_file_url` | supported with media/file identifiers | Real inbound file yielded a platform file URL in the converted `File` component. | +| `call_platform_api` | supported | Safe action `check_access_token` verified. | + +## Platform-Specific APIs + +| Action | Support | Evidence | +|--------|---------|----------| +| `check_access_token` | supported | `plugin-e2e`. | +| `refresh_access_token` | supported | Implemented; not separately reproduced in the latest plugin run. | +| `get_file_url` | supported | Real inbound file yielded a platform file URL in the converted `File` component. | +| `get_audio_base64` | supported | Needs real inbound audio/media ID. | +| `download_image_base64` | supported | Real inbound image reached the plugin as `Image`; separate image-download API replay was not completed. | + +## End-to-End Evidence + +Evidence files: + +- Text/API/component JSONL: `data/temp/dingtalk-plugin-e2e-20260510-rerun.jsonl` +- Real UI inbound media JSONL: `data/temp/dingtalk-plugin-e2e-media-ui.jsonl` + +Verified: + +- DingTalk Mac private chat in the `LangBot Team` organization produced `MessageReceived` through LangBot standalone runtime and `EBAEventProbe`. +- The common chain was `Source + Plain` for normal text. +- DingTalk emoji was received as `Source + Plain`, not common `Face`. +- Real DingTalk Mac private-chat image upload was received as `Source + Image`. +- Real DingTalk Mac private-chat file upload was received as `Source + File`. +- The plugin sent outbound text, mention/fallback, image, quote/fallback, file, and forward/fallback messages visible in DingTalk. +- The plugin called safe SDK and DingTalk platform APIs. + +Not completed: + +- Real UI inbound voice. +- Real UI inbound quote. +- Group trigger with a real robot mention. +- Destructive or organization-mutating APIs. diff --git a/docs/event-based-agents/adapters/discord.md b/docs/event-based-agents/adapters/discord.md new file mode 100644 index 000000000..95e1fb91e --- /dev/null +++ b/docs/event-based-agents/adapters/discord.md @@ -0,0 +1,146 @@ +# Discord EBA Adapter + +## Status + +Discord has been migrated from the legacy source adapter: + +```text +src/langbot/pkg/platform/sources/discord.py +src/langbot/pkg/platform/sources/discord.yaml +``` + +EBA adapter directory: + +```text +src/langbot/pkg/platform/adapters/discord/ +├── adapter.py +├── api_impl.py +├── event_converter.py +├── manifest.yaml +├── message_converter.py +├── platform_api.py +├── types.py +└── voice.py +``` + +The adapter is registered as `discord-omni`. + +## Configuration + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `client_id` | Yes | `""` | Discord application client ID. | +| `token` | Yes | `""` | Discord bot token. | + +The bot needs gateway permissions and intents for the target test server. Message Content intent is required for message bodies, Server Members intent is required for member APIs/events, and reaction events require the Reactions intent and channel permissions. + +## Events + +Discord declares these EBA events: + +- `message.received` +- `message.edited` +- `message.deleted` +- `message.reaction` +- `group.member_joined` +- `group.member_left` +- `bot.invited_to_group` +- `bot.removed_from_group` +- `platform.specific` + +Discord-specific events that do not map cleanly to common events should be surfaced as `platform.specific`. + +## Common APIs + +| API | Status | Notes | +|-----|-----------------|-------| +| `send_message` | Supported | Supports text, image, file, and mixed message chains through Discord messages and attachments. | +| `reply_message` | Supported | Uses Discord message references when replying to a received EBA message event. | +| `edit_message` | Supported | Bot can edit its own messages. File edits are implemented by clearing old attachments and sending replacement files when needed. | +| `delete_message` | Supported | Requires message management permissions for non-bot messages. | +| `forward_message` | Emulated | Discord has no native forward API; the adapter copies content and attachments. | +| `get_group_info` | Supported | Maps Discord guild metadata to EBA group info. | +| `get_group_member_list` | Supported | Requires member cache or the Server Members intent/fetch permission. | +| `get_group_member_info` | Supported | Maps Discord roles/permissions into EBA member roles. | +| `get_user_info` | Supported | Uses Discord user fetch/cache. | +| `upload_file` | Not supported | Discord uploads files as message attachments; standalone upload raises `NotSupportedError`. | +| `get_file_url` | Supported | Discord attachment URLs are already downloadable URLs, so the adapter returns the input URL. | +| `mute_member` | Supported where possible | Uses Discord timeout API and requires guild moderation permission. | +| `unmute_member` | Supported where possible | Clears timeout and requires guild moderation permission. | +| `kick_member` | Supported | Destructive; test only with a disposable account/bot. | +| `leave_group` | Supported | Bot leaves a guild; destructive and should run last. | +| `call_platform_api` | Supported | Discord-specific actions live here. | + +## Platform-Specific APIs + +`call_platform_api(action, params)` supports: + +- `get_channel` +- `get_guild` +- `get_guild_channels` +- `get_guild_roles` +- `create_invite` +- `pin_message` +- `unpin_message` +- `add_reaction` +- `remove_reaction` +- `typing` + +Voice helpers are intentionally kept Discord-specific: + +- `join_voice_channel` +- `leave_voice_channel` +- `get_voice_connection_status` +- `list_active_voice_connections` +- `get_voice_channel_info` + +## Live Test Record + +The live probe is: + +```bash +uv run python tests/e2e/live_discord_eba_probe.py --help +``` + +Verified on May 7, 2026 with a newly created Discord application/bot named `LangBot EBA Test 0507`, the LangBot Discord server, and the `#🐞-debugging` channel: + +- SDK standalone runtime started with WebSocket control/debug ports, and the `EBAEventProbe` plugin connected through `lbp run`. +- Plugin runtime received real Discord events through LangBot: `BotInvitedToGroup`, `MessageReceived`, `MessageReactionReceived` add/remove, `MessageEdited`, and `MessageDeleted`. +- Plugin runtime API calls succeeded through the standalone runtime: `get_langbot_version`, `get_bots`, `get_bot_info`, `send_message`, plugin storage APIs, workspace storage APIs, `list_plugins_manifest`, `list_commands`, `list_tools`, and `list_knowledge_bases`. +- Direct live adapter probe observed `message.received`, `message.edited`, `message.deleted`, and `bot.removed_from_group`. +- Message APIs verified: send, reply, edit, delete, forward, text/image/file mixed message chains. +- User and guild APIs verified: `get_user_info`, `get_group_info`, `get_group_member_list`, `get_group_member_info`. +- Platform-specific APIs verified: `get_channel`, `get_guild`, `get_guild_channels`, `get_guild_roles`, `create_invite`, `typing`, `pin_message`, `unpin_message`, `add_reaction`, `remove_reaction`. +- Unsupported API behavior verified: `upload_file` raises `NotSupportedError`. +- Destructive API verified at the end: `leave_group`, which emitted `bot.removed_from_group`. + +Not verified in the shared LangBot server live run: `mute_member`, `unmute_member`, and `kick_member`, because the run did not use a disposable target member. They are implemented through Discord timeout/kick APIs and should only be exercised against a disposable account or bot. + +The test fixed one real test-fixture issue: `EBAEventProbe` previously assumed `get_bots()` returned UUID strings. The current standalone runtime returns bot dictionaries, so the probe now selects an enabled bot dictionary and passes its `uuid` to `get_bot_info` and `send_message`. The probe also now subscribes to `MessageDeleted`. + +## Standalone Runtime Plugin E2E Record + +Verified again on May 10, 2026 with SDK standalone runtime, LangBot `--standalone-runtime`, Discord web client, the LangBot server, and `#🐞-debugging`. + +Evidence: + +- Main plugin JSONL: `data/temp/discord-plugin-e2e-20260510-final.jsonl` +- LangBot runtime log: `data/temp/discord-langbot-e2e-20260510-rerun.log` + +Observed and verified: + +- A newly invited Discord bot connected to the LangBot server and received a real web-client message in `#🐞-debugging`. +- `MessageReceived` reached the plugin with `bot_uuid=eba-discord-live`, `adapter_name=discord`, common `Source`/`Plain` message components, common `User`, and common `UserGroup` for the guild. +- SDK API calls succeeded: `get_langbot_version`, `get_bots`, `get_bot_info`, `send_message`, plugin storage, workspace storage, `list_plugins_manifest`, `list_commands`, `list_tools`, and `list_knowledge_bases`. +- Outbound component sweep succeeded: plain text plus user mention, `AtAll`/`@everyone`, base64 image, quoted reply, file attachment, and flattened forward fallback. +- Common APIs succeeded: `get_user_info`, `get_group_info`, `get_group_member_list`, and `get_group_member_info`. +- Discord platform APIs succeeded through `call_platform_api`: `get_channel`, `typing`, `get_guild`, `get_guild_channels`, and `get_guild_roles`. + +Documented limits in this E2E run: + +- Real Discord UI inbound attachment/image/file, reply/quote, and fresh mention-chain messages were not completed in the plugin E2E evidence. Outbound image/file attachments from the bot do not prove inbound attachment conversion. +- A later May 10 UI retry could write text into the Discord message box, but the client kept the send button disabled and did not send the message, so it produced no new plugin evidence. +- `get_message`, `get_friend_list`, and `get_group_list` are not supported by this Discord adapter. +- Destructive moderation and guild-leave APIs were not repeated against the shared LangBot server. +- Native Discord voice is not represented as common `Voice`; audio-like payloads are treated as file attachments. +- `create_invite`, pin/unpin, and reaction mutation were covered by prior direct live probes but were not repeated by the final plugin run to avoid extra shared-server side effects. diff --git a/docs/event-based-agents/adapters/kook.md b/docs/event-based-agents/adapters/kook.md new file mode 100644 index 000000000..14e3198a3 --- /dev/null +++ b/docs/event-based-agents/adapters/kook.md @@ -0,0 +1,108 @@ +# KOOK EBA Adapter + +## Status + +KOOK has been migrated to the EBA adapter directory: + +```text +src/langbot/pkg/platform/adapters/kook/ +├── adapter.py +├── api_impl.py +├── event_converter.py +├── manifest.yaml +├── message_converter.py +├── platform_api.py +└── types.py +``` + +The adapter is registered as `kook-omni`. + +## Configuration + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `token` | Yes | `""` | KOOK bot token. | +| `enable-stream-reply` | Yes | `false` | Reserved for shared platform configuration compatibility. | + +## Events + +| Event | Evidence | Notes | +|-------|----------|-------| +| `message.received` | `plugin-e2e-ui` | Real KOOK UI channel message reached `EBAEventProbe` as `MessageReceivedEvent`. | +| `platform.specific` | `plugin-e2e-ui` | KOOK gateway event without a common EBA mapping reached `EBAEventProbe` as `PlatformSpecificEventReceived`. | + +## Common APIs + +| API | Evidence | Notes | +|-----|----------|-------| +| `send_message` | `plugin-e2e-outbound` | Probe plugin sent channel messages through SDK `send_message`; KOOK returned message IDs. | +| `reply_message` | `unit` | Supports `reply_msg_id` and optional quoted replies when the source message ID is available. | +| `get_message` | `plugin-e2e-outbound` | Probe plugin fetched the cached triggering message. | +| `get_group_info` | `plugin-e2e-outbound` | Probe plugin received cached KOOK channel info. | +| `get_group_list` | `plugin-e2e-outbound` | Probe plugin received cached channel/group entities observed by the adapter. | +| `get_group_member_info` | `plugin-e2e-outbound` | Probe plugin received cached sender info as a group member. | +| `get_user_info` | `plugin-e2e-outbound` | Probe plugin received cached sender user info. | +| `get_friend_list` | `plugin-e2e-outbound` | Probe plugin received cached users. | +| `upload_file` | `unit` | Uses KOOK `asset/create` and returns URL/ID. | +| `get_file_url` | `unit` | KOOK media IDs are URL-like in the adapter path; returns the ID unchanged. | +| `delete_message` | `unit` | Calls KOOK delete endpoints. Live permission verification is still required. | +| `forward_message` | `plugin-e2e-outbound` | Probe plugin sent flattened forward content through SDK `send_message`. | +| `call_platform_api` | `plugin-e2e-outbound` | Probe plugin called safe KOOK platform-specific APIs through SDK `call_platform_api`. | + +## Platform-Specific APIs + +| Action | Evidence | Notes | +|--------|----------|-------| +| `get_current_user` | `plugin-e2e-outbound` | Probe plugin called `user/me`. | +| `get_user` | `plugin-e2e-outbound` | Probe plugin called `user/view` for the triggering sender. | +| `get_channel` | `plugin-e2e-outbound` | Probe plugin called `channel/view` for the triggering channel. | +| `get_guild` | `plugin-e2e-outbound` | Probe plugin called `guild/view`; gateway URLs redact token query values. | +| `get_gateway` | `plugin-e2e-outbound` | Probe plugin called `gateway/index`; returned token query values are redacted. | +| `send_direct_message` | `unit` | Calls `direct-message/create`. | + +## Components + +| Component | Receive Evidence | Send Evidence | Notes | +|-----------|------------------|---------------|-------| +| `Source` | `plugin-e2e-ui` | N/A | KOOK message ID and timestamp are preserved. | +| `Plain` | `plugin-e2e-ui` | `plugin-e2e-outbound` | Text and KMarkdown are represented as plain common text. | +| `At` | `plugin-e2e-ui` | `plugin-e2e-outbound` | KOOK `(met)(met)` mentions map to common `At`. | +| `AtAll` | `unit` | `plugin-e2e-outbound` | KOOK `(met)all(met)` maps to common `AtAll`; real inbound UI AtAll was not tested. | +| `Image` | `unit` | `unit` | URL/image ID based path only; live rendering still needs verification. | +| `Voice` | `unit` | `unit` | URL based path only; live rendering still needs verification. | +| `File` | `unit` | `unit` | URL based path only; upload API is exposed separately. | +| `Forward` | `unit` | `unit` | Outbound forwards are flattened; inbound structured forwards are not exposed by current legacy implementation. | +| `Unknown` | `unit` | N/A | Unsupported KOOK message types become `Unknown` or `PlatformSpecificEvent`. | + +## Acceptance Record + +Test date: June 4, 2026. + +Plugin E2E verified on June 4, 2026 with `EBAEventProbe`, SDK standalone runtime, KOOK WebSocket adapter, and a real KOOK channel UI message. + +Evidence: + +- JSONL: `data/temp/kook_eba_plugin_probe.jsonl` +- Plugin log: `data/logs/eba-probe-kook.log` + +Observed and verified: + +- A real KOOK UI channel message reached the plugin as `MessageReceived` with `bot_uuid=7ab5b065-6e4e-4def-95f0-3c265366e26f`, `adapter_name=kook`, common sender/group/chat fields, and common `MessageChain` components. +- KOOK gateway-specific event reached the plugin as `PlatformSpecificEventReceived`. +- Probe plugin called SDK `send_message`; KOOK returned message IDs for text, At, AtAll, image URL/base64 fallback path, quote fallback, file fallback, and flattened forward cases. +- Probe plugin called common API methods through the SDK path: `get_message`, `get_user_info`, `get_friend_list`, `get_group_info`, `get_group_list`, and `get_group_member_info`. +- Probe plugin called safe KOOK platform-specific APIs through SDK `call_platform_api`: `get_current_user`, `get_user`, `get_channel`, `get_gateway`, and `get_guild`. + +Run: + +```bash +uv run pytest tests/unit_tests/platform/test_kook_eba_adapter.py +git diff --check +``` + +Blocked or partial items: + +- `plugin-e2e-ui` inbound coverage for image, file, voice, AtAll, quote, and forward. +- `plugin-e2e-outbound` visual verification in KOOK UI for image/file/voice rendering. KOOK returned message IDs, but UI inspection was not performed in this run. +- `reply_message` and `delete_message` live permission verification. +- Destructive or permission-sensitive APIs were not declared beyond delete; KOOK mute/kick/leave remain explicit `NotSupportedError` paths until a safe fixture is available. diff --git a/docs/event-based-agents/adapters/lark.md b/docs/event-based-agents/adapters/lark.md new file mode 100644 index 000000000..06af00794 --- /dev/null +++ b/docs/event-based-agents/adapters/lark.md @@ -0,0 +1,135 @@ +# Lark / Feishu EBA Adapter Migration Record + +Status: migrated with unit coverage and partial live plugin E2E. WebSocket text reached the standalone runtime once in the LangBot organization test app, but the latest real UI image/file inbound attempts did not reach the local adapter log, so media receive is not release-complete yet. + +Adapter directory: `src/langbot/pkg/platform/adapters/lark/` + +## What Changed + +The Lark/Feishu adapter now has an Event-Based Agents adapter package with: + +- `manifest.yaml` for adapter metadata, configuration, events, common APIs, platform-specific APIs, app type, and communication mode. +- `adapter.py` for self-built/store app token handling, WebSocket long connection startup, Webhook callback handling, card feedback, streaming-card replies, and EBA dispatch. +- `event_converter.py` for native Feishu events to common EBA events. +- `message_converter.py` for Feishu text/post/image/file/audio payloads to/from common `MessageChain` components. +- `api_impl.py` for common EBA API implementations. +- `platform_api.py` for Feishu-specific `call_platform_api` actions. + +The legacy `lark` adapter remains available while the EBA adapter is registered separately as `lark-omni`. + +## Configuration + +| Field | Required | Notes | +|-------|----------|-------| +| `app_id` | yes | Feishu/Lark application App ID. | +| `app_secret` | yes | Feishu/Lark application App Secret. | +| `bot_name` | yes | Must match the bot name so group mentions can be recognized. | +| `enable-webhook` | yes | `false` uses WebSocket long connection; `true` uses Request URL/Webhook callbacks. | +| `webhook_url` | no | Generated callback URL for Webhook mode. | +| `encrypt-key` | no | Webhook decrypt key when event encryption is enabled. | +| `enable-stream-reply` | yes | Enables streaming replies through an updating Feishu card. | +| `app_type` | no | `self` for self-built apps; `isv` for store apps. | +| `bot_added_welcome` | no | Optional group welcome message sent after bot-added events. | + +## Application And Communication Modes + +| Mode | Support | Implementation | +|------|---------|----------------| +| Self-built application | implemented | Uses standard app credentials and tenant token behavior from the Feishu SDK client. | +| Store application | implemented | Builds an ISV client, requests app tickets, and resolves app/tenant access tokens with per-tenant caching. | +| WebSocket long connection | implemented | Registers `im.message.receive_v1` and card-action callbacks through `lark_oapi.ws.Client`. | +| Webhook Request URL | implemented | Handles URL verification, encrypted payloads, message events, app-ticket events, bot-added events, and card-action feedback. | + +## Supported Events + +| Event | Support | Evidence | +|-------|---------|----------| +| `message.received` | implemented | Unit coverage for private and group native events to common EBA events. | +| `bot.invited_to_group` | implemented | Webhook bot-added event maps to common bot invite event and optional welcome send. | +| `platform.specific` | implemented | Unknown callback events are preserved as `platform.specific`. | +| `FeedbackEvent` | compatibility event | Card button feedback is still dispatched through the existing SDK `FeedbackEvent` type. | + +## Receive Components + +| Component | Support | Evidence | +|-----------|---------|----------| +| `Source` | supported | Unit coverage; live private text evidence. | +| `Plain` | supported | Text and post payloads convert to common text; live private text evidence. | +| `At` | supported | Feishu mentions map to common `At` with user ID and display name. | +| `AtAll` | supported | `user_id=all` maps to common `AtAll`. | +| `Image` | supported | Image payloads download through message resource API and map to common `Image`; real UI image send attempted, but not observed in local plugin evidence yet. | +| `Voice` | supported | Audio payloads download through message resource API and map to common `Voice`. | +| `File` | supported | File payloads download through message resource API and map to common `File`; real UI file send attempted, but not observed in local plugin evidence yet. | +| `Quote` | supported | Parent/thread reply lookup maps quoted content into common `Quote`. | +| `Face` | not native common mapping | Feishu emoji/stickers are not exposed as a portable common `Face` component here. | +| `Forward` | not-supported inbound | Feishu does not expose a portable structured forward event in this adapter. | + +## Send Components + +| Component | Support | Evidence | +|-----------|---------|----------| +| `Plain` | supported | Unit coverage; sends Feishu `text`. | +| `At` | supported | Unit coverage; sends Feishu `post` at element. | +| `AtAll` | supported | Unit coverage; sends Feishu `post` at-all element. | +| `Image` | supported | Uploads image resource and sends Feishu `image`. | +| `Voice` | supported | Uploads OPUS/audio resource and sends Feishu `audio`. | +| `File` | supported | Uploads file resource and sends Feishu `file`. | +| `Quote` | supported/fallback | Sends quote marker plus origin content. | +| `Face` | not-supported | No portable send mapping. | +| `Forward` | flattened fallback | Flattens forward nodes into text/media messages. | + +## Common APIs + +| API | Support | Notes | +|-----|---------|-------| +| `send_message` | supported | Supports private/open_id and group/chat_id targets; live plugin outbound component sweep produced visible Feishu messages. | +| `reply_message` | supported | Replies to the source Feishu message; fixed to recover the native Feishu message ID from legacy-wrapped source events. | +| `get_message` | cache-backed/API-backed | Returns cached inbound event where possible and converts uncached Feishu message API items into common `MessageReceivedEvent`. | +| `get_group_info` | supported | Uses cached group or Feishu chat metadata. | +| `get_group_member_info` | limited | Uses cached user data when available. | +| `get_user_info` | limited | Uses cached user data when available. | +| `get_file_url` | limited | Returns `file://` paths from downloaded inbound resources; remote Feishu resource download uses platform-specific API params. | +| `call_platform_api` | supported | See below. | + +## Platform-Specific APIs + +| Action | Support | Evidence | +|--------|---------|----------| +| `check_tenant_access_token` | supported | Unit coverage. | +| `refresh_app_access_token` | supported | Store-app token path implemented. | +| `refresh_tenant_access_token` | supported | Store-app tenant token path implemented. | +| `get_chat` | supported | Feishu chat metadata API wrapper. | +| `get_message` | supported | Feishu message API wrapper with JSON-safe return values for plugin calls. | +| `get_message_resource` | supported | Feishu message resource download wrapper. | + +## End-to-End Evidence + +Current code-level evidence: + +- `tests/unit_tests/platform/test_lark_eba_adapter.py` +- `PYTHONPATH=../langbot-plugin-sdk/src uv run pytest tests/unit_tests/platform/test_lark_eba_adapter.py -q` + +Live evidence collected on May 11, 2026: + +- Standalone runtime: `uv run lbp rt --ws-control-port 5400 --ws-debug-port 5401 --skip-deps-check` +- LangBot: `uv run main.py --standalone-runtime --debug` +- Plugin: `LangBot__EBAEventProbe` +- Feishu org/app: LangBot organization, `LangBotDev` private chat. +- Observed plugin JSONL: one private `MessageReceived` event with `Source + Plain`; plugin API probe then exercised bot discovery, bot info, `send_message`, outbound component sweep, storage/list APIs, and safe platform API calls. +- Real UI sends attempted after the fixes: private text, local file, and image/video image upload. These appeared in the Feishu client but did not append new `EBAEventProbe` records in the local JSONL during this run. +- Fixes from live testing: reply path now extracts the native Feishu `message_id` from legacy-wrapped source events; WebSocket callbacks are scheduled onto the adapter event loop instead of assuming the SDK callback has a running asyncio loop; platform API results are converted to JSON-safe values. + +Live E2E items still required before marking release-complete: + +- WebSocket self-built app in LangBot organization: repeat private text after callback-loop fix, plus private image/file/audio and group mention message received by `EBAEventProbe`. +- Webhook self-built app in LangBot organization: URL verification plus text/image/file message received by `EBAEventProbe`. +- Store app token path: at least token acquisition/tenant-token safe API through `call_platform_api`; full message E2E if a LangBot organization store-app fixture is available. +- Outbound component sweep: text, mention, at-all, image, file, voice where Feishu accepts the fixture, quote/fallback, and forward/fallback. +- Safe platform API sweep: token check, chat metadata, message lookup, and message resource download using real inbound IDs. + +## Known Limits + +- Store-app live E2E requires a real ISV app ticket/tenant installation fixture. +- Current LangBot organization WebSocket run connected successfully but did not deliver the latest UI-sent image/file attempts to local plugin evidence; this blocks release-complete media acceptance. +- Feishu native emoji/sticker semantics are not represented as common `Face`. +- Destructive org or chat mutations are not declared in this adapter. diff --git a/docs/event-based-agents/adapters/officialaccount.md b/docs/event-based-agents/adapters/officialaccount.md new file mode 100644 index 000000000..d5f24d582 --- /dev/null +++ b/docs/event-based-agents/adapters/officialaccount.md @@ -0,0 +1,101 @@ +# OfficialAccount EBA Adapter + +Adapter directory: `src/langbot/pkg/platform/adapters/officialaccount/` + +Manifest name: `officialaccount-omni` + +Status: partial migration. Unit/API-shape coverage is present, and private text `plugin-e2e-ui` plus safe API evidence has been verified against the `dev.rockchin.top` Official Account fixture. Proactive outbound `send_message` remains not supported by this adapter because WeChat Official Account replies must be tied to inbound webhook windows. + +## Config + +| Field | Required | Notes | +| --- | --- | --- | +| `webhook_url` | no | Generated by LangBot and copied into the Official Account callback settings. | +| `token` | yes | WeChat callback token. | +| `EncodingAESKey` | yes | WeChat message encryption key. | +| `AppID` | yes | Official Account app ID. | +| `AppSecret` | yes | Official Account app secret. | +| `Mode` | yes | `drop` waits for an in-callback reply; `passive` returns the loading text first and queues the answer for the user's next message. | +| `LoadingMessage` | no | Only used by `passive` mode. | +| `api_base_url` | no | Optional API base URL for proxy deployments. | + +## Events + +| Event | Evidence | Notes | +| --- | --- | --- | +| `message.received` | plugin-e2e-ui, unit | Text UI message verified through WeChat Official Account on `dev.rockchin.top`; image and voice webhook payloads are covered by unit tests. | +| `platform.specific` | unit | Subscribe/unsubscribe/menu/etc. native events are emitted as structured `PlatformSpecificEvent`. | + +## Common APIs + +| API | Evidence | Notes | +| --- | --- | --- | +| `reply_message` | unit | Queues/passively returns text through the inbound webhook source event. | +| `get_message` | plugin-e2e-ui, unit | Cached inbound message retrieved by `EBAEventProbe` platform API sweep. | +| `get_user_info` | plugin-e2e-ui, unit | Cached inbound sender retrieved by `EBAEventProbe` platform API sweep. | +| `get_friend_list` | plugin-e2e-ui, unit | Cached inbound sender list retrieved by `EBAEventProbe` platform API sweep. | +| `call_platform_api` | plugin-e2e-ui, unit | Safe diagnostic actions verified through `get_mode` and `get_cached_response_status`. | +| `send_message` | not-supported | Official Account customer-service proactive messaging is not implemented by the existing SDK adapter; only webhook reply is supported here. | + +## Platform APIs + +| Action | Evidence | Notes | +| --- | --- | --- | +| `get_mode` | plugin-e2e-ui, unit | Returned `{"mode": "drop", "longer_response": false}` in live probe. | +| `get_cached_response_status` | plugin-e2e-ui, unit | Returned `{"pending": false}` in live probe. | + +## Components + +| Receive Component | Evidence | Notes | +| --- | --- | --- | +| `Source` | plugin-e2e-ui, unit | Uses `MsgId` and `CreateTime`; live UI text message included `Source`. | +| `Plain` | plugin-e2e-ui, unit | Live UI text message mapped to `Plain`. | +| `Image` | unit | `PicUrl` and `MediaId` map to common `Image`. | +| `Voice` | unit | `MediaId` maps to common `Voice`. | +| `Unknown` | unit | Unsupported message/event types do not crash. | +| `At`, `AtAll`, `File`, `Quote`, `Face`, `Forward`, mixed chain | not-supported | WeChat Official Account inbound webhook payloads used by the current SDK do not expose these as common structured components. | + +| Send Component | Evidence | Notes | +| --- | --- | --- | +| `Plain` | unit | Sent as webhook reply text. | +| `Image`, `Voice`, `File`, `Quote`, `At`, `AtAll`, `Face`, `Forward`, mixed chain | not-supported | Existing SDK reply path is text XML only; non-text components degrade to readable placeholders in tests and are not declared as supported outbound components. | + +## Verification Record + +Test date: 2026-05-28 + +Endpoint/simulator: `dev.rockchin.top` with WeChat desktop client and a real subscribed Official Account conversation. The running EBA test stack used SDK standalone runtime ports `5400/5401`, LangBot from `/home/wgc/LangBotxg/LangBotEbaTest`, and `EBAEventProbe`. + +Verified UI message: `EBA officialaccount single probe 2026-05-28 16:53` + +Observed event/API evidence: + +- `MessageReceived`: `bot_uuid=d7c46880-a9f8-431a-9172-5d3e0d663dbc`, `adapter_name=officialaccount-omni`, `chat_type=private`, `chat_id=ovH9L7OW6hNpWZWvp_NMmypVh26w`, `message_chain=[Source, Plain]`. +- Common safe APIs through probe platform sweep: `get_message`, `get_user_info`, `get_friend_list`. +- Platform APIs through `call_platform_api`: `get_mode`, `get_cached_response_status`. +- `send_message` and outbound component sweep returned explicit `NotSupportedError: send_message:official_account_requires_inbound_webhook_reply`, as expected for this adapter. + +Standalone runtime command: + +```bash +cd langbot-plugin-sdk +uv run python -m langbot_plugin.cli.__init__ rt --debug-only --ws-control-port 5400 --ws-debug-port 5401 --skip-deps-check +``` + +Probe plugin: `data/plugins/LangBot__EBAEventProbe` when live credentials are available. + +Adapter live probe: + +```bash +uv run python -m py_compile tests/e2e/live_officialaccount_eba_probe.py +OFFICIALACCOUNT_TOKEN=... OFFICIALACCOUNT_ENCODING_AES_KEY=... OFFICIALACCOUNT_APP_SECRET=... OFFICIALACCOUNT_APP_ID=... uv run python tests/e2e/live_officialaccount_eba_probe.py +``` + +Evidence JSONL path: `/home/wgc/LangBotxg/LangBotEbaTest/data/temp/officialaccount_eba_plugin_probe.jsonl` for plugin E2E, or `data/temp/officialaccount_eba_probe.jsonl` for direct adapter live probe. + +Destructive operations: none. + +Blocked items: + +- `plugin-e2e-outbound`: proactive `send_message` is not supported for this adapter; Official Account responses must be produced through the inbound webhook reply window. +- Inbound image and voice live UI evidence remains pending; webhook conversion is covered by unit tests. diff --git a/docs/event-based-agents/adapters/qqofficial.md b/docs/event-based-agents/adapters/qqofficial.md new file mode 100644 index 000000000..bf1e657a5 --- /dev/null +++ b/docs/event-based-agents/adapters/qqofficial.md @@ -0,0 +1,119 @@ +# QQOfficial EBA Adapter + +Adapter directory: `src/langbot/pkg/platform/adapters/qqofficial/` + +Manifest name: `qqofficial-omni` + +Status: partial migration. The EBA adapter structure, manifest, converters, cache-backed safe APIs, platform API map, unit tests, and direct live probe scaffold are in place. A real QQ Official WebSocket bot on `dev.rockchin.top` received an inbound user message and drove LangBot into the normal pipeline path; the response path was blocked by the test environment model service returning `model_not_found` for `deepseek-v3`. + +## Config + +| Field | Required | Notes | +| --- | --- | --- | +| `appid` | yes | QQ Official app ID. | +| `secret` | yes | QQ Official app secret. | +| `token` | yes | QQ Official callback token. | +| `enable-webhook` | yes | Uses LangBot unified webhook when true; otherwise uses the QQ WebSocket gateway. | +| `enable-stream-reply` | yes | Enables C2C streaming replies when supported by the QQ Official endpoint. | +| `webhook_url` | no | Generated by LangBot and copied into the QQ Official callback settings in webhook mode. | + +## Events + +| Event | Evidence | Notes | +| --- | --- | --- | +| `message.received` | adapter-live, unit | `C2C_MESSAGE_CREATE`, `DIRECT_MESSAGE_CREATE`, `GROUP_AT_MESSAGE_CREATE`, and `AT_MESSAGE_CREATE` map to common `MessageReceivedEvent`. A real WebSocket-mode QQ Official bot reached the LangBot pipeline on `dev.rockchin.top`; plugin JSONL evidence remains pending. | +| `message.reaction` | unit | `MESSAGE_REACTION_ADD` and `MESSAGE_REACTION_REMOVE` map to common `MessageReactionEvent`. Live gateway evidence is pending. | +| `group.member_joined` | unit | `GUILD_MEMBER_ADD` and `GROUP_MEMBER_ADD` map to common `MemberJoinedEvent` when the gateway payload carries a group/guild/channel ID and member openid. | +| `group.member_left` | unit | `GUILD_MEMBER_REMOVE` and `GROUP_MEMBER_REMOVE` map to common `MemberLeftEvent`. Live gateway evidence is pending. | +| `bot.invited_to_group` | unit | `GUILD_CREATE` and `GROUP_ADD_ROBOT` map to common `BotInvitedToGroupEvent`. | +| `bot.removed_from_group` | unit | `GUILD_DELETE` and `GROUP_DEL_ROBOT` map to common `BotRemovedFromGroupEvent`. | +| `platform.specific` | unit | Unmapped gateway events are emitted as structured `PlatformSpecificEvent`; live evidence is pending. | + +## Common APIs + +| API | Evidence | Notes | +| --- | --- | --- | +| `send_message` | unit, blocked | Sends private C2C, group, and text-only channel messages through the existing QQ Official client. Live outbound UI verification is pending because the test pipeline failed before producing a bot response. | +| `reply_message` | unit, blocked | Replies using the source `QQOfficialEvent` message ID when available. Live reply was blocked by the test environment model service returning `model_not_found`. | +| `get_message` | unit | Returns cached inbound `MessageReceivedEvent`. | +| `get_user_info` | unit | Returns cached inbound sender. | +| `get_friend_list` | unit | Returns cached private senders. | +| `get_group_info` | unit | Returns cached group/channel metadata from inbound events. | +| `get_group_member_info` | unit | Returns cached group sender as a common member. | +| `get_group_member_list` | unit | Returns cached group members observed by the adapter. | +| `call_platform_api` | unit, blocked | Safe diagnostic actions are implemented; live calls are pending credentials. | + +## Platform APIs + +| Action | Evidence | Notes | +| --- | --- | --- | +| `check_access_token` | unit, blocked | Calls the existing client token check. | +| `refresh_access_token` | unit, blocked | Forces token refresh. | +| `get_gateway_url` | unit, blocked | Fetches the WebSocket gateway URL. | +| `get_mode` | unit | Returns webhook and stream-reply mode. | + +## Components + +| Receive Component | Evidence | Notes | +| --- | --- | --- | +| `Source` | unit | Uses QQ message/event IDs and timestamp. | +| `Plain` | unit | Preserves text content. | +| `At` | unit | Group and channel mention events insert an adapter bot mention marker. | +| `Image` | unit | QQ image attachment URL is converted to common `Image`; falls back to URL if download fails. | +| `Unknown` | unit | Unsupported/empty native payloads become `Unknown`. | +| `Voice`, `File`, `Quote`, `Face`, `Forward`, mixed chain | blocked | Current native parser only exposes text and image attachments; live endpoint behavior still needs verification. | + +| Send Component | Evidence | Notes | +| --- | --- | --- | +| `Plain` | unit, blocked | Sends through private, group, or channel text APIs. | +| `At`, `AtAll` | unit, blocked | Converted to readable mention text. | +| `Image` | unit, blocked | Sends through the QQ Official rich media upload/send path for C2C and group targets. | +| `Voice` | unit, blocked | Sends through the QQ Official rich media upload/send path for C2C and group targets. | +| `File` | unit, blocked | Sends through the QQ Official rich media upload/send path for C2C and group targets. | +| `Quote`, `Forward`, mixed chain | unit, blocked | Flattened to ordered send payloads where possible. | +| `Face` | not-supported | No common QQ Official face mapping is implemented. | + +## Verification Record + +Test date: 2026-06-02 + +Endpoint/simulator: `dev.rockchin.top` with a real QQ Official WebSocket bot (`qqofficial-omni`, bot UUID `80a5560b-52b1-40e7-b7d6-4a2341eb4780`) and LangBot running from `/home/wgc/LangBotxg/LangBotEbaTest`. + +Observed evidence: + +- The QQ Official WebSocket bot was enabled with `enable-webhook=false`. +- A real user message reached LangBot and entered the standard pipeline path. +- The response path stopped at the model layer with `model_not_found` for `deepseek-v3`; this is a model/provider configuration issue, not an adapter conversion failure. +- `qq-webhook.langbot.dev` was temporarily routed through Caddy to `127.0.0.1:5301` for webhook checks, but the observed EBA bot used WebSocket mode. + +Standalone runtime command: + +```bash +cd langbot-plugin-sdk +uv run python -m langbot_plugin.cli.__init__ rt --debug-only --ws-control-port 5400 --ws-debug-port 5401 --skip-deps-check +``` + +Probe plugin: `data/plugins/LangBot__EBAEventProbe` when live credentials are available. + +Adapter live probe: + +```bash +uv run python -m py_compile tests/e2e/live_qqofficial_eba_probe.py +QQOFFICIAL_APPID=... QQOFFICIAL_SECRET=... QQOFFICIAL_TOKEN=... uv run python tests/e2e/live_qqofficial_eba_probe.py +``` + +Webhook-mode probe: + +```bash +QQOFFICIAL_APPID=... QQOFFICIAL_SECRET=... QQOFFICIAL_TOKEN=... uv run python tests/e2e/live_qqofficial_eba_probe.py --webhook --host 0.0.0.0 --port 5312 +``` + +Evidence JSONL path: `data/temp/qqofficial_eba_probe.jsonl` for direct adapter live probe; plugin E2E evidence should use `data/temp/qqofficial_eba_plugin_probe.jsonl`. + +Destructive operations: none implemented. + +Blocked items: + +- `plugin-e2e-ui`: standalone probe plugin JSONL evidence is still pending; the observed live run reached LangBot core/pipeline but was not recorded by the EBA probe plugin. +- `plugin-e2e-outbound`: waiting for visible QQ client verification of plugin `send_message`/`reply_message` output after a working model/provider is configured. +- Inbound non-text media and platform lifecycle events require endpoint evidence before they can be marked complete. diff --git a/docs/event-based-agents/adapters/slack.md b/docs/event-based-agents/adapters/slack.md new file mode 100644 index 000000000..1a860a26f --- /dev/null +++ b/docs/event-based-agents/adapters/slack.md @@ -0,0 +1,84 @@ +# Slack EBA Adapter + +## Structure + +Slack is migrated into `src/langbot/pkg/platform/adapters/slack/` with the standard EBA adapter layout: + +- `adapter.py` owns lifecycle, listener dispatch, unified webhook handling, outbound send/reply, and event caches. +- `event_converter.py` maps Slack `im` and `app_mention` channel events to `message.received`. +- `message_converter.py` maps common `MessageChain` components to Slack text fallback and maps inbound Slack text/image payloads back to EBA components. +- `api_impl.py` provides cache-backed common read APIs. +- `platform_api.py` declares safe Slack-specific API actions. +- `manifest.yaml` declares `slack-omni`. + +The legacy `src/langbot/pkg/platform/sources/slack.py` adapter is kept unchanged. + +## Configuration + +| Field | Required | Notes | +|-------|----------|-------| +| `webhook_url` | No | Generated by LangBot. Paste it into Slack Event Subscriptions. | +| `bot_token` | Yes | Slack bot token, usually `xoxb-...`. | +| `signing_secret` | Yes | Slack app signing secret. | + +## Events + +| Event | Notes | +|-------|-------| +| `message.received` | Emitted for private `im` messages and channel `app_mention` events. Channel messages are mapped to group chats. | +| `platform.specific` | Reserved for Slack event types that are not converted into common message events. | + +## Common APIs + +Required: + +- `send_message` +- `reply_message` + +Optional: + +- `get_message` +- `get_user_info` +- `get_friend_list` +- `get_group_info` +- `get_group_list` +- `get_group_member_list` +- `get_group_member_info` +- `call_platform_api` + +Cache-backed APIs are only available after the relevant inbound event has been observed. + +## Platform APIs + +| Action | Notes | +|--------|-------| +| `get_mode` | Returns webhook mode and configured bot account id. | +| `auth_test` | Calls Slack `auth.test` with the configured bot token. | + +## Known Limits + +- Slack file/image outbound is currently represented as text fallback because the existing Slack SDK wrapper only exposes `chat_postMessage`. +- Inbound channel coverage follows the legacy adapter behavior: only `app_mention` events are treated as group messages. +- Real live testing requires a public callback URL configured in Slack Event Subscriptions. + +## Verification + +Local mocked unit coverage validates manifest parity, event conversion, legacy listener compatibility, cache-backed APIs, send/reply routing, and declared platform APIs. + +Plugin E2E evidence was captured on June 2, 2026 against `dev.rockchin.top` with Slack private DM input and `EBAEventProbe` through the standalone runtime. + +Evidence file: `/home/wgc/LangBotxg/LangBotEbaTest/data/temp/slack_eba_plugin_probe.jsonl`. + +Observed: + +- Real Slack private text produced `MessageReceived` with `adapter_name=slack-omni`, `Source + Plain`, private chat type, and filled `bot_uuid`. +- Safe common APIs passed: `get_message`, `get_user_info`, `get_friend_list`. +- Outbound component fallback sweep passed through `send_message`: plain/at/face, image, quote, file, and forward. +- Declared Slack platform APIs passed: `get_mode`, `auth_test`. + +Still pending: + +- Channel `app_mention` plugin E2E. +- Real inbound Slack file/image UI evidence. + +Live probe scaffold: `tests/e2e/live_slack_eba_probe.py`. diff --git a/docs/event-based-agents/adapters/telegram.md b/docs/event-based-agents/adapters/telegram.md new file mode 100644 index 000000000..c8cdc2999 --- /dev/null +++ b/docs/event-based-agents/adapters/telegram.md @@ -0,0 +1,139 @@ +# Telegram EBA Adapter + +## Status + +Telegram has been migrated to the EBA adapter directory: + +```text +src/langbot/pkg/platform/adapters/telegram/ +├── adapter.py +├── api_impl.py +├── event_converter.py +├── manifest.yaml +├── message_converter.py +├── platform_api.py +└── types.py +``` + +The adapter is registered as `telegram-omni`. + +## Configuration + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `token` | Yes | `""` | Telegram Bot API token from BotFather. | +| `markdown_card` | No | `true` | Whether to render Markdown card style replies. | +| `enable-stream-reply` | Yes | `false` | Whether to use Telegram streaming reply mode. | + +## Events + +Telegram declares these EBA events: + +- `message.received` +- `message.edited` +- `message.reaction` +- `group.member_joined` +- `group.member_left` +- `group.member_banned` +- `bot.invited_to_group` +- `bot.removed_from_group` +- `bot.muted` +- `bot.unmuted` +- `platform.specific` + +`platform.specific` is currently used for Telegram-only callback and chat-member update payloads that do not yet have a more specific common event type. + +## Common APIs + +| API | Status | Notes | +|-----|--------|-------| +| `send_message` | Supported | Supports text, image, file, and mixed message chains. | +| `reply_message` | Supported | Supports quoted replies through the original message event. | +| `edit_message` | Supported | Uses Telegram message editing APIs. | +| `delete_message` | Supported | Deletes messages where bot permissions allow it. | +| `forward_message` | Supported | Forwards a message between Telegram chats. | +| `get_group_info` | Supported | Uses Telegram chat metadata. | +| `get_group_member_list` | Supported | Telegram only exposes administrators through the Bot API; this returns the available member set. | +| `get_group_member_info` | Supported | Maps Telegram member status to EBA member roles. | +| `get_user_info` | Supported | Uses Telegram `get_chat` for user chat metadata. | +| `upload_file` | Not supported | Telegram has no standalone upload endpoint; files are uploaded as part of messages. The adapter raises `NotSupportedError`. | +| `get_file_url` | Supported | Returns the Bot API file URL. Test output redacts the bot token. | +| `mute_member` | Supported | Requires a supergroup and bot moderation permission. | +| `unmute_member` | Supported | Uses current `telegram.ChatPermissions` fields. | +| `kick_member` | Supported | Destructive; should only be run against disposable users/bots in tests. | +| `leave_group` | Supported | Destructive; should run at the end of a live test. | +| `call_platform_api` | Supported | See below. | + +## Platform-Specific APIs + +`call_platform_api(action, params)` supports: + +- `pin_message` +- `unpin_message` +- `unpin_all_messages` +- `get_chat_administrators` +- `set_chat_title` +- `set_chat_description` +- `get_chat_member_count` +- `send_chat_action` +- `create_chat_invite_link` +- `answer_callback_query` + +## Live Test Record + +The live probe is: + +```bash +uv run python tests/e2e/live_telegram_eba_probe.py --help +``` + +It supports private chat tests, group/supergroup tests, moderation tests, destructive tests, and a callback-only mode. + +Verified on May 7, 2026: + +- Private chat message APIs: send, reply, edit, delete, forward. +- Private chat media APIs: image/file sending and `get_file_url`. +- User API: `get_user_info`. +- Supergroup APIs: group info, member list, member info, administrators, member count, invite link. +- Supergroup mutation APIs: pin, unpin, unpin all, set title, restore title, set description, restore description. +- Moderation APIs: mute and unmute against a non-owner target bot. +- Destructive APIs: kick a disposable target bot, then make the test bot leave the test group. +- Event conversion observed for `message.received`, `group.member_banned`, `group.member_left`, `bot.removed_from_group`, and Telegram-specific chat-member updates. + +The test fixed one real compatibility issue: `unmute_member` previously used Telegram's removed `can_send_media_messages` permission field. It now uses the split media permission fields required by current `python-telegram-bot`. + +## Standalone Runtime Plugin E2E Record + +Verified on May 10, 2026 with `EBAEventProbe`, SDK standalone runtime, Telegram Lite, `@rockchinq_bot`, and `Rock'sBotGroup`. + +Evidence: + +- Private chat JSONL: `data/temp/telegram-plugin-e2e-rerun.jsonl` +- Group chat JSONL: `data/temp/telegram-plugin-e2e-group.jsonl` +- Private media JSONL: `data/temp/telegram-plugin-e2e-media-ui.jsonl` + +Observed and verified: + +- `MessageReceived` reached the plugin with `bot_uuid=eba-telegram-live`, `adapter_name=telegram`, common sender/chat fields, and common `MessageChain` content. +- `BotInvitedToGroup` reached the plugin after adding the bot to `Rock'sBotGroup`. +- SDK API calls succeeded: `get_langbot_version`, `get_bots`, `get_bot_info`, `send_message`, plugin storage, workspace storage, `list_plugins_manifest`, `list_commands`, `list_tools`, and `list_knowledge_bases`. +- Outbound component sweep succeeded in private and group chats: plain text, mention text/equivalent, base64 image, quoted reply, file/document, and flattened forward fallback. Group mode also covered `AtAll` fallback behavior. +- Real Telegram Lite private-chat inbound media was verified through the plugin path: a sent document arrived as common `File`, and a sent photo arrived as common `Image`. +- Telegram platform API sweep succeeded for safe group actions: `get_chat_administrators`, `get_chat_member_count`, and `send_chat_action`. +- Common group/user APIs succeeded in group mode: `get_user_info`, `get_group_info`, `get_group_member_list`, and `get_group_member_info`. + +Documented limits in this E2E run: + +- Real Telegram UI inbound voice, sticker/emoji-as-common-component, and reply/quote messages were not completed in the plugin E2E evidence. +- `get_message`, `get_friend_list`, and `get_group_list` are not supported by this Telegram adapter. +- Mutating/destructive Telegram-specific actions such as pin/unpin, title/description changes, invite-link creation, moderation, kick, and leave were not repeated in the plugin run. They remain opt-in live-probe cases. +- Telegram does not expose a portable common `Face` component for native sticker/emoji semantics in the current adapter. + +## Notes for Future Adapters + +Telegram is the reference implementation for: + +- Keeping platform-specific actions behind `call_platform_api`. +- Treating unsupported common APIs as explicit `NotSupportedError`. +- Marking destructive live test operations behind CLI flags. +- Redacting access tokens from live probe output. diff --git a/docs/event-based-agents/adapters/wecom.md b/docs/event-based-agents/adapters/wecom.md new file mode 100644 index 000000000..fe7730c45 --- /dev/null +++ b/docs/event-based-agents/adapters/wecom.md @@ -0,0 +1,130 @@ +# WeCom EBA Adapter + +## Status + +WeCom application messages now have an EBA adapter directory: + +```text +src/langbot/pkg/platform/adapters/wecom/ +├── adapter.py +├── api_impl.py +├── event_converter.py +├── manifest.yaml +├── message_converter.py +├── platform_api.py +└── types.py +``` + +The adapter is registered as `wecom-omni`. + +This record covers the regular WeCom application-message adapter. WeCom AI Bot (`wecombot-omni`) uses a different protocol flow and is documented separately in `wecombot.md`. WeCom Customer Service (`wecomcs`) remains a separate follow-up migration. + +## Configuration + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `webhook_url` | No | `""` | Unified webhook URL copied into the WeCom application callback settings. | +| `corpid` | Yes | `""` | WeCom corporate ID. | +| `secret` | Yes | `""` | WeCom application secret. | +| `token` | Yes | `""` | WeCom callback token. | +| `EncodingAESKey` | Yes | `""` | WeCom callback encryption key. | +| `contacts_secret` | No | `""` | Contacts secret for contact-list based helper APIs. | +| `api_base_url` | No | `https://qyapi.weixin.qq.com/cgi-bin` | WeCom API base URL, overrideable for proxy/private-network deployments. | + +## Events + +WeCom declares these EBA events: + +- `message.received` +- `platform.specific` + +`message.received` currently covers text and image application callbacks. Other WeCom callback types are surfaced as `platform.specific` so plugins can inspect the raw structured payload without crashing the common message path. + +## Common APIs + +| API | Status | Notes | +|-----|--------|-------| +| `send_message` | Supported | Private/person target only. `target_id` must be `user_id|agent_id`. Supports text, image, voice, file, flattened forward, and quote fallback. | +| `reply_message` | Supported | Replies to the original WeCom sender and application agent from `source_platform_object`. | +| `get_message` | Supported from cache | Returns cached inbound `MessageReceivedEvent` by message ID. | +| `get_user_info` | Supported | Uses cached event users first, then WeCom `user/get`. | +| `get_friend_list` | Partial | Returns users seen by this adapter instance. Full contacts listing is not declared as common coverage. | +| `call_platform_api` | Supported | See below. | +| `edit_message` | Not supported | WeCom application messages do not expose a general edit endpoint for sent messages. | +| `delete_message` | Not supported | WeCom application messages do not expose a general delete endpoint for sent messages. | +| `get_group_info` / member APIs | Not supported | Regular WeCom application callbacks handled here are private user messages, not group-chat bot messages. | +| `upload_file` / `get_file_url` | Not supported as common APIs | WeCom media upload is used internally while sending image/voice/file components; no portable standalone common file URL is exposed. | + +## Platform-Specific APIs + +`call_platform_api(action, params)` supports: + +- `check_access_token` +- `refresh_access_token` +- `get_user_info` +- `send_to_all` + +`send_to_all` requires a configured `contacts_secret` with suitable contact visibility and should be treated as a broad-send operation in live testing. + +## Unit Verification + +Covered by: + +```bash +uv run pytest tests/unit_tests/platform/test_wecom_eba_adapter.py +``` + +The unit tests cover: + +- Manifest events/APIs/platform actions match adapter declarations. +- Outbound component conversion for text, image, voice, file, quote fallback, and byte-safe text splitting. +- Text callback conversion to `MessageReceivedEvent`. +- Legacy `FriendMessage` compatibility. +- EBA listener dispatch and inbound message/user cache. +- `send_message`, `reply_message`, and safe platform API dispatch against a mocked WeCom client. + +## Standalone Runtime Plugin E2E Record + +Verified on May 27, 2026 with `EBAEventProbe`, SDK standalone runtime, LangBot core, and a real WeCom desktop client against the server test environment. + +```bash +cd langbot-plugin-sdk +uv run python -m langbot_plugin.cli.__init__ rt --debug-only --ws-control-port 5400 --ws-debug-port 5401 --skip-deps-check + +cd LangBot +uv run main.py --standalone-runtime + +cd data/plugins/LangBot__EBAEventProbe +EBA_PROBE_API=1 EBA_PROBE_COMPONENT_SWEEP=1 EBA_PROBE_PLATFORM_API=1 \ +uv --project /absolute/path/to/langbot-plugin-sdk run python -m langbot_plugin.cli.__init__ run +``` + +Evidence: + +- JSONL: `data/temp/wecom_eba_plugin_probe.jsonl` +- Bot: `wecom-omni` +- Client: real WeCom desktop client +- Environment: `dev.rockchin.top` test server + +Observed and verified: + +- A real private WeCom user message reached the plugin as `MessageReceived` with `adapter_name=wecom-omni`, common sender/chat fields, and `Source + Plain`. +- SDK API calls succeeded through the standalone runtime, including `get_langbot_version`, `get_bots`, `get_bot_info`, `send_message`, plugin/workspace storage, and manifest/list APIs. +- Safe adapter API checks succeeded through the plugin path for cached message/user data and declared safe platform API actions. + +Still required for stricter acceptance: + +- Send a private image and confirm common `Image` reaches the plugin. +- Have the plugin call `send_message` and `reply_message` for text and one media component, then verify the WeCom client receives the bot output. +- Exercise `send_to_all` only with a disposable visible-contact scope. +- Trigger one non-text/image callback, if available, and confirm it becomes `PlatformSpecificEventReceived`. + +## Current Acceptance + +Current status is **partial EBA acceptance**. + +Blocked items: + +- Real inbound image/voice/file evidence was not completed in this run. +- Inbound voice/file callback parsing is not present in the legacy `WecomClient.get_message()` path, so the EBA adapter does not claim those receive components yet. +- Group/member/moderation APIs do not apply to this regular WeCom application-message adapter. diff --git a/docs/event-based-agents/adapters/wecombot.md b/docs/event-based-agents/adapters/wecombot.md new file mode 100644 index 000000000..23751aec7 --- /dev/null +++ b/docs/event-based-agents/adapters/wecombot.md @@ -0,0 +1,148 @@ +# WeComBot EBA Adapter + +## Status + +WeCom AI Bot now has an EBA adapter directory: + +```text +src/langbot/pkg/platform/adapters/wecombot/ +├── adapter.py +├── api_impl.py +├── event_converter.py +├── manifest.yaml +├── message_converter.py +├── platform_api.py +└── types.py +``` + +The adapter is registered as `wecombot-omni`. + +This is separate from regular WeCom internal applications (`wecom-omni`). WeComBot supports WebSocket long connection mode, which does not require a webhook URL. Webhook mode remains available when `enable-webhook=true`. + +## Configuration + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `BotId` | Yes for WebSocket mode | `""` | WeCom AI Bot ID. | +| `robot_name` | Yes | `""` | Bot display name used to strip bot mentions from incoming group text. | +| `enable-webhook` | Yes | `false` | `false` uses WebSocket long connection mode; `true` uses webhook callback mode. | +| `webhook_url` | No | `""` | Unified webhook URL, only needed when webhook mode is enabled. | +| `Secret` | Yes for WebSocket mode | `""` | WeCom AI Bot secret for long connection mode. | +| `Corpid` | Yes for webhook mode | `""` | WeCom corporate ID for webhook callback mode. | +| `Token` | Yes for webhook mode | `""` | WeCom callback token. | +| `EncodingAESKey` | Yes for webhook mode; optional for WebSocket media decrypt | `""` | Message encryption/decryption key. | +| `enable-stream-reply` | No | `true` | Enables WeComBot streaming replies. | + +## Events + +WeComBot declares these EBA events: + +- `message.received` +- `feedback.received` +- `platform.specific` + +`message.received` covers private and group messages from the WeComBot SDK. `feedback.received` covers WeComBot like/dislike feedback callbacks. Native SDK events without a common EBA equivalent are emitted as `platform.specific`. + +## Common APIs + +| API | Status | Notes | +|-----|--------|-------| +| `send_message` | Supported in WebSocket mode | Sends proactive markdown/text to a person or group chat ID. Webhook mode raises `NotSupportedError` because the platform callback flow has no proactive send path here. | +| `reply_message` | Supported | Replies through native `req_id` in WebSocket mode or stream finalization/cache in webhook mode. | +| `get_message` | Supported from cache | Returns cached inbound `MessageReceivedEvent` by message ID. | +| `get_user_info` | Supported from cache | WeComBot events carry user info; no full user lookup endpoint is declared. | +| `get_friend_list` | Partial | Returns users observed by this adapter instance. | +| `get_group_info` | Supported from cache | Returns groups observed from inbound group messages. | +| `get_group_member_info` | Supported from cache | Returns observed sender/group-member pairs. | +| `get_group_member_list` | Partial | Returns observed members for the cached group only. | +| `call_platform_api` | Supported | See below. | +| `edit_message` / `delete_message` / `forward_message` | Not supported | WeComBot does not expose portable common APIs for these operations in the current SDK wrapper. | +| `upload_file` / `get_file_url` | Not supported as common APIs | Media is represented inside messages; no portable standalone file upload/URL API is declared. | +| moderation / leave APIs | Not supported | WeComBot does not expose equivalent common moderation operations through this adapter. | + +## Platform-Specific APIs + +`call_platform_api(action, params)` supports: + +- `is_websocket_mode` +- `get_stream_session_status` +- `send_markdown` + +`send_markdown` is only available in WebSocket mode. + +## Unit Verification + +Covered by: + +```bash +PYTHONPATH=/Users/wangqiang/code/python/langbot-plugin-sdk/src uv run pytest tests/unit_tests/platform/test_wecombot_eba_adapter.py +``` + +The unit tests cover: + +- Manifest events/APIs/platform actions match adapter declarations. +- Outbound common components flatten to WeComBot markdown/text. +- Private and group native events become `MessageReceivedEvent`. +- Inbound image, file, voice, and quote components map to common `MessageChain`. +- Legacy `FriendMessage`/`GroupMessage` compatibility. +- EBA listener dispatch, message/user/group/member cache, reply, send, streaming chunk, feedback, and platform API calls. + +## Live Probe + +The direct adapter probe is: + +```bash +PYTHONPATH=/absolute/path/to/langbot-plugin-sdk/src uv run python tests/e2e/live_wecombot_eba_probe.py --help +``` + +Default mode is WebSocket long connection and requires: + +- `WECOMBOT_BOT_ID` +- `WECOMBOT_SECRET` +- `WECOMBOT_ROBOT_NAME` +- optional `WECOMBOT_ENCODING_AES_KEY` + +Webhook mode uses `--webhook` and requires: + +- `WECOMBOT_TOKEN` +- `WECOMBOT_ENCODING_AES_KEY` +- `WECOMBOT_CORPID` + +The probe writes JSONL evidence to `data/temp/wecombot_eba_live_probe.jsonl`, waits for a real WeComBot message, records common EBA event fields and message components, then runs safe cached/common/platform API checks. + +## Standalone Runtime Plugin E2E Record + +Verified on May 27, 2026 with `EBAEventProbe`, SDK standalone runtime, LangBot core, and the real WeCom desktop client in a WeCom AI Bot private chat. + +Evidence: + +- JSONL: `data/temp/wecombot_eba_plugin_probe.jsonl` +- Bot UUID: `9f5d4125-7b6d-4c98-8ca2-111111111111` +- Adapter: `wecombot-omni` +- Client: real WeCom desktop client, private `LangBot` BOT chat +- Mode: WebSocket long connection (`enable-webhook=false`) + +Observed and verified: + +- A real user-side message reached the plugin as `MessageReceived` with `adapter_name=wecombot-omni`, common sender/chat fields, and `Source + Plain`. +- SDK API calls succeeded through the standalone runtime: `get_langbot_version`, `get_bots`, `get_bot_info`, `send_message`, plugin/workspace storage, manifest/list APIs, and safe cached common platform APIs. +- Outbound component sweep was visible in the WeCom client and returned `errcode=0`: plain/mention/face fallback, base64 image marker, quote fallback, file marker, and flattened forward fallback. +- Declared WeComBot platform APIs succeeded through `plugin.call_platform_api`: `is_websocket_mode`, `get_stream_session_status`, and `send_markdown`. +- The `send_markdown` platform API produced visible bot output in the WeCom client. + +Not completed: + +- Clicking the visible WeCom AI feedback button did not produce a `FeedbackReceived` JSONL entry in this run, so `feedback.received` remains unverified at plugin E2E level. +- Group chat inbound and group cache/member coverage still need a real group-side trigger. +- Real inbound image/file/voice from the WeCom client was not exercised. + +## Current Acceptance + +Current status is **partial EBA acceptance**. + +Blocked or limited items: + +- `feedback.received` is implemented and unit-covered, but real plugin E2E feedback evidence was not observed from the desktop client click. +- Outbound image/voice/file are flattened as textual markers because the WeComBot SDK reply/proactive path used here is markdown/text oriented. +- Group member APIs are cache-backed and only know members observed in received messages. +- Destructive or moderation APIs are not declared because the current WeComBot protocol surface does not provide safe common equivalents. diff --git a/docs/event-based-agents/adapters/wecomcs.md b/docs/event-based-agents/adapters/wecomcs.md new file mode 100644 index 000000000..04a42dc53 --- /dev/null +++ b/docs/event-based-agents/adapters/wecomcs.md @@ -0,0 +1,161 @@ +# WeCom Customer Service EBA Adapter + +## Status + +WeCom Customer Service now has an EBA adapter directory: + +```text +src/langbot/pkg/platform/adapters/wecomcs/ +├── adapter.py +├── api_impl.py +├── event_converter.py +├── manifest.yaml +├── message_converter.py +├── platform_api.py +└── types.py +``` + +The adapter is registered as `wecomcs-omni`. It is separate from regular WeCom application messages (`wecom-omni`) and WeCom AI Bot (`wecombot-omni`). + +## Configuration + +| Field | Required | Default | Description | +|-------|----------|---------|-------------| +| `webhook_url` | No | `""` | Unified webhook URL copied into the WeCom Customer Service callback settings. | +| `corpid` | Yes | `""` | WeCom corporate ID. | +| `secret` | Yes | `""` | Customer Service secret used for access tokens. | +| `token` | Yes | `""` | Customer Service callback token. | +| `EncodingAESKey` | Yes | `""` | Customer Service callback encryption key. | +| `api_base_url` | No | `https://qyapi.weixin.qq.com/cgi-bin` | WeCom API base URL, overrideable for proxy/private-network deployments. | + +## Events + +| Event | Status | Notes | +|-------|--------|-------| +| `message.received` | Plugin E2E UI covered for text | Text, image, file, and voice payloads convert to common EBA message components in unit tests. Real WeChat customer-side UI text reached `EBAEventProbe` on May 27, 2026. | +| `platform.specific` | Unit covered | Non-message or unknown Customer Service payloads become structured `PlatformSpecificEvent` records. | + +## Common APIs + +| API | Status | Notes | +|-----|--------|-------| +| `send_message` | Plugin E2E outbound covered | Private/person target only. `target_id` must be `external_userid|open_kfid`. Text and image are implemented; voice/file are explicitly unsupported. | +| `reply_message` | Plugin E2E partial | Replies through Customer Service `kf/send_msg` using the original `source_platform_object`. The pipeline reply path reached the send API, but the dev account later hit WeCom `95001 send msg count limit`. | +| `get_message` | Plugin E2E covered from cache | Returns cached inbound `MessageReceivedEvent` by message ID. | +| `get_user_info` | Plugin E2E covered | Uses cached event users first, then Customer Service `customer/batchget`. | +| `get_friend_list` | Plugin E2E covered, partial | Returns customer users seen by this adapter instance. | +| `call_platform_api` | Unit covered | See platform-specific APIs below. | +| `edit_message` / `delete_message` | Not supported | WeCom Customer Service does not expose a general edit/delete endpoint for bot-sent messages in this adapter. | +| Group/member/moderation APIs | Not supported | Customer Service conversations handled here are private customer sessions, not group chats. | +| `upload_file` / `get_file_url` | Not supported | Media upload is used internally for outbound image; no portable file URL common API is exposed. | + +## Platform-Specific APIs + +| Action | Status | Notes | +|--------|--------|-------| +| `check_access_token` | Unit covered | Checks whether the current access token is present. | +| `refresh_access_token` | Unit covered | Refreshes the Customer Service access token. | +| `get_customer_info` | Unit covered | Calls Customer Service customer lookup by `external_userid`. | + +## Message Components + +Receive: + +| Component | Status | Notes | +|-----------|--------|-------| +| `Source` | Unit covered | Uses Customer Service `msgid` and `send_time`. | +| `Plain` | Unit covered | Text payload content is preserved. | +| `Image` | Unit covered | Uses the base64 data URL produced by the existing SDK image download path. | +| `Voice` | Unit covered | Maps exposed voice media ID to common `Voice.voice_id`; live UI evidence pending. | +| `File` | Unit covered | Maps exposed file media ID/name/size to common `File`; live UI evidence pending. | +| `Quote`, `At`, `AtAll`, `Face`, `Forward` | Not supported inbound | The current Customer Service SDK event model does not expose these as structured inbound fields. | +| `Unknown` | Unit covered | Unsupported message types become `Unknown` in message conversion or `platform.specific` at event level. | + +Send: + +| Component | Status | Notes | +|-----------|--------|-------| +| `Plain` | Plugin E2E outbound covered | Sends through `kf/send_msg` text. | +| `Image` | Plugin E2E outbound covered | Uploads media as WeCom image media and sends through `kf/send_msg` image. | +| `Quote`, `At`, `AtAll`, `Forward` | Unit covered fallback, live partially blocked | Flattened to text where possible. In the May 27 sweep, later text sends hit WeCom `95001 send msg count limit` after the successful text/image sends. | +| `Voice`, `File`, `Face` | Not supported | The adapter raises `NotSupportedError`; no tested Customer Service send path is implemented. | + +## Unit Verification + +Covered by: + +```bash +PYTHONPATH=/Users/wangqiang/code/python/langbot-plugin-sdk/src uv run pytest tests/unit_tests/platform/test_wecomcs_eba_adapter.py +``` + +Result on May 27, 2026: `10 passed`. + +The local `PYTHONPATH` is required in this workspace because the installed SDK package in the LangBot venv does not contain the newer `langbot_plugin.api.entities.builtin.platform.errors` module; the existing EBA adapter tests need the same SDK override. + +## Live Probe + +Auxiliary direct adapter probe: + +```bash +PYTHONPATH=/path/to/langbot-plugin-sdk/src uv run python -m py_compile tests/e2e/live_wecomcs_eba_probe.py + +WECOMCS_CORPID=... \ +WECOMCS_SECRET=... \ +WECOMCS_TOKEN=... \ +WECOMCS_ENCODING_AES_KEY=... \ +PYTHONPATH=/path/to/langbot-plugin-sdk/src \ +uv run python tests/e2e/live_wecomcs_eba_probe.py \ + --path /wecomcs/callback \ + --log data/temp/wecomcs_eba_live_probe.jsonl +``` + +This probe is diagnostic only. Final EBA acceptance still requires the standalone SDK runtime plus `EBAEventProbe` plugin path. + +## Standalone Runtime Plugin E2E Record + +Completed partial plugin E2E on May 27, 2026 against `dev.rockchin.top` and the WeChat customer-side UI entry `微信 -> 客服消息 -> 浪波智能客服`. + +Evidence: + +- Server JSONL: `/home/wgc/LangBotxg/LangBotEbaTest/data/temp/wecomcs_eba_plugin_probe.jsonl` +- Trigger text: `EBA wecomcs dedupe probe 2026-05-27` +- `bot_uuid`: `cc810d2c-91f3-4f92-8f27-e1bf9f7b6cb4` +- `adapter_name`: `wecomcs-omni` +- Observed common event: `MessageReceived`, `event.type=message.received` +- Observed message chain: `Source + Plain` +- Observed chat: `chat_type=private`, `chat_id=external_userid|open_kfid` +- Observed sender: customer `User` with nickname/avatar from Customer Service lookup +- Plugin API probe: `send_message`, `get_message`, `get_user_info`, `get_friend_list`, plugin/workspace storage, and manifest/list APIs succeeded +- Component sweep: outbound `Plain` and `Image` succeeded; `Face` and `File` returned explicit `NotSupportedError`; later quote/forward fallback sends were blocked by WeCom `95001 send msg count limit` + +Command shape used: + +```bash +cd langbot-plugin-sdk +uv run python -m langbot_plugin.cli.__init__ rt --debug-only --ws-control-port 5400 --ws-debug-port 5401 --skip-deps-check + +cd LangBot +PYTHONPATH=/absolute/path/to/langbot-plugin-sdk/src uv run main.py --standalone-runtime + +cd data/plugins/LangBot__EBAEventProbe +DEBUG_RUNTIME_WS_URL=ws://127.0.0.1:5401/plugin/ws \ +EBA_PROBE_LOG=/absolute/path/to/LangBot/data/temp/wecomcs_eba_plugin_probe.jsonl \ +EBA_PROBE_API=1 \ +EBA_PROBE_COMPONENT_SWEEP=1 \ +EBA_PROBE_PLATFORM_API=1 \ +uv --project /absolute/path/to/langbot-plugin-sdk run python -m langbot_plugin.cli.__init__ run +``` + +Required real UI trigger: send a Customer Service message from the WeCom/WeChat customer-side UI to the configured `dev.rockchin.top` Customer Service account. + +## Current Acceptance + +Current status is **partial EBA acceptance**. + +Blocked or pending items: + +- Inbound UI media (`Image`, `Voice`, `File`) was not sent from the real WeChat customer UI during this run, so receive-side media remains unit-covered only. +- Pipeline auto-reply reached `kf/send_msg`, but the test account hit WeCom `95001 send msg count limit` after successful plugin outbound text/image sends. This is recorded as an account/platform rate-limit block, not a conversion or API-shape failure. +- The current `EBAEventProbe` run did not call the adapter-specific `call_platform_api` actions (`check_access_token`, `refresh_access_token`, `get_customer_info`); the platform API map remains unit-covered. +- Inbound voice/file depends on whether the real Customer Service callback plus `sync_msg` endpoint returns those fields in the shape the local SDK models. +- Group, member, edit, delete, moderation, and standalone file URL APIs are intentionally not declared because this Customer Service protocol path does not provide tested common equivalents. diff --git a/docs/multi-tenant/cloud-v2-pending-verification.md b/docs/multi-tenant/cloud-v2-pending-verification.md index 7b90856b5..1cb22f28d 100644 --- a/docs/multi-tenant/cloud-v2-pending-verification.md +++ b/docs/multi-tenant/cloud-v2-pending-verification.md @@ -4,6 +4,8 @@ 更新日期：2026-07-29 +> 2026-09-05 文档核对：下列版本、测试数量和镜像 pin 属于 7 月交付记录，不代表当前 `dev/4.11.x` 的依赖状态。当前 Core/SDK 配套与定向测试见 [STATUS.md](../agent-runner-pluginization/STATUS.md)。本次未重新验收闭源控制面、最终部署或 24 小时 soak，因此保留本页生产激活门禁；新增 Runtime 存储统计也不关闭 B-02 硬配额要求。 + 本文是 Cloud v2 首期上线前的剩余验证清单。它只记录尚不能由当前代码审查、 单元测试、集成测试、合成容量探针或短时 Linux 容器实验替代的证据。这里的项目 不属于 2026-07-29 代码与本地测试资源审查的完成条件，也不会让该审查持续保持未完成； diff --git a/docs/multi-tenant/implementation-checklist.md b/docs/multi-tenant/implementation-checklist.md index c74c3dd57..cc8487767 100644 --- a/docs/multi-tenant/implementation-checklist.md +++ b/docs/multi-tenant/implementation-checklist.md @@ -1,5 +1,7 @@ # Multi-tenant implementation checklist +> Status interpretation, 2026-09-05: this checklist retains the original multi-tenant implementation and activation evidence. The kernel is now also present on `dev/4.11.x`; the branch names below are historical workstream labels. SDK installation-worker restart coordination is implemented in `runtime/plugin/restart_coordinator.py`, with focused tests passing on the inspected workspace. An unchecked production gate does not necessarily mean its code is absent. Production fault injection, egress, hard storage quotas, and generation-aware business transaction/outbox gates remain separate. See [current 4.11 status](../agent-runner-pluginization/STATUS.md) and [Cloud activation gates](./cloud-v2-pending-verification.md). + This checklist turns the Workspace architecture into implementation and verification gates. Exact commands and observed results are recorded in the [verification report](./verification-report.md). diff --git a/docs/pipeline-migration-config-map.zh-CN.md b/docs/pipeline-migration-config-map.zh-CN.md new file mode 100644 index 000000000..d3c90077c --- /dev/null +++ b/docs/pipeline-migration-config-map.zh-CN.md @@ -0,0 +1,504 @@ +# 旧流水线迁移：配置去向与语义变更 + +本文对应迁移规划器 v3。原生行为基线：`9b7ba0d64708496ace30a82866f6dbc185f089dc`。 + +## 迁移边界 + +- 迁移只由用户在 UI 选择流水线并确认后执行。打开页面、预览、刷新、安装插件都不触发自动迁移。 +- 流水线 ID、名称、Bot 绑定以及非 AI 配置保持不变。插件凭证、应用 ID、端点和模型配置仍属于本流水线，不迁入共享插件全局设置。 +- 活动 AI 配置规范化为 `ai.runner` 与 `ai.runner_config`，避免编辑器把迁移结果判为混合旧配置。所有原始 AI 段（包括未选中的 Runner 配置）与完整原始配置一起保存到同一事务的持久化备份；预览会列出被归档的字段。 +- 备份不是公开下载，也不意味着已有一键恢复 UI。未知提交结果不显示为已回滚；激活待重试只在新预览、重新选择并确认后执行，且不重复转换配置。 +- “配置迁移”不等于“运行状态无损迁移”。原有远端会话、历史、Box 文件、待处理表单令牌不自动导入；UI 必须在确认前说明新会话/状态重置。自定义 Box 共享作用域与未结束的交互不能被静默清除。 + +## 功能与默认值的取舍 + +- LocalAgent 保留日期提示、结构化提示词、模型回退、模型专属推理配置、工具/知识库/沙箱能力；保留现代上下文预算、检查点和摘要机制，不恢复旧的 `max-round` 截断算法。 +- `max-round` 退役，不把“10 轮”伪装成“50 条消息”等价换算；新上下文参数使用下面列明的默认值。工具执行仍显式采用 `serial`，不偷偷改变有副作用工具的执行顺序。 +- Dify 旧保存项 `timeout` 并非原生 Runner 实际读取的超时控制。迁移采用新插件 30 秒默认值，并提示用户；旧任意数值只保存在备份中。 +- Dify/Coze/N8n 迁移显式选择 `user-id-source=legacy-session`，Tbox 选择 `legacy-bot`；新建插件配置仍默认 `sender`。旧身份来自 Host 的可信会话/事件/机器人上下文，缺失时失败，不猜测为发送者。 +- Dify/N8n 的旧身份使用原 `query.session`，Coze 使用原 `query`；不能在群聊、不同会话策略或恢复交互时把这两者混同。 +- Coze、Langflow 等现代持久化远端会话机制保留；旧远端会话 ID 不跨账户、端点、Workspace 或 Runner 导入。 +- DashScope 引用字段别名、Langflow 输入输出字段别名、Coze 历史开关别名分别归一化；冲突值阻断，而非靠字段出现顺序决定。 +- N8n 的 ignore 响应模式、认证和保留身份字段要维持明确语义。普通业务变量保留支持的 JSON 值，内部/权限/敏感变量不无条件转发。 +- `remove-think` 保留原输出策略，并与支持该字段的 Runner 参数同步；仅用户实际修改才传播，不把表单挂载默认值视为修改。 +- 空字符串、缺失和显式 null 区分处理。WeKnora 的 agent-id 不得在打开/保存其他字段时被默认值替换；未知字段/类型不能靠 truthiness 静默接受。 +- 工具、KB、MCP 的授权由 Host 校验；表单未显示的授权字段仍必须保留，不能因保存而删除或扩大权限。 + +## 插件版本要求 + +以下是本次本地源码的目标版本；本表不代表插件已经发布。旧版本即使名称相同，也不能作为新迁移能力的证明。 + +- `local-agent` → `LocalAgent` **0.1.6**。 +- `dify-service-api` → `DifyAgent` **0.1.7**。 +- `coze-api` → `CozeAgent` **0.1.7**。 +- `dashscope-app-api` → `DashScopeAgent` **0.1.7**。 +- `n8n-service-api` → `N8nAgent` **0.1.7**。 +- `langflow-api` → `LangflowAgent` **0.1.7**。 +- `deerflow-api` → `DeerFlowAgent` **0.1.7**。 +- `tbox-app-api` → `TboxAgent` **0.1.5**。 +- `weknora-api` → `WeKnoraAgent` **0.1.7**。 + +## 逐字段完整清单 + +共 9 类 Runner、104 条字段条目；按消费者展开共享字段。未知部署自定义键不冒充已穷举字段，遇到时阻断并保留原值。 + +目标路径缩写 `R` 表示 `config.ai.runner_config[当前插件 Runner ID]`；`extensions_preferences` 是流水线的 Host 扩展策略，不是插件全局配置。 + +### LocalAgent（28 项） + +- **`ai.local-agent.box-session-id-template`** → `R.box-session-id-template`。 + - 全局、每会话、每用户、每对话上下文、每消息及有效自定义模板均原样保留；缺失时沿用插件默认 `{launcher_type}_{launcher_id}`。插件清单的 `allow_custom: true` 允许非预制值。空值、错误括号、位置参数、格式转换和属性访问会提示修正；公开请求变量保留，但运行时必须有值。复用范围保留不等于迁移旧容器及文件，新运行器按同样规则申请新的沙箱。 + - 所有权：Runner 管理复用策略、申请、绑定及附件传输；Host 管理连接、工作区隔离和配额。沙箱开关沿用 LocalAgent 插件默认 `true`，仅在有沙箱工具授权且平台支持时使用。 +- **`ai.local-agent.enable-all-tools`** → `R.enable-all-tools`。 + - 原样复制有效授权；mcp 字段缺失时取 extensions_preferences 对应值；显式 null/错误类型不回退扩大权限。 + - 所有权：`host_policy_in_pipeline_runner_config`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:34`。 +- **`ai.local-agent.knowledge-base`** → `R.knowledge-bases`。 + - 有效非空 plural 列表优先；plural 缺失/有效空列表且 singular 非空且非__none__→[原UUID]；非法容器/null单独校验，不以truthiness宽容；其他为空→[]。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:280`。 +- **`ai.local-agent.knowledge-bases`** → `R.knowledge-bases`。 + - 有效非空 plural 列表优先；plural 缺失/有效空列表且 singular 非空且非__none__→[原UUID]；非法容器/null单独校验，不以truthiness宽容；其他为空→[]。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:278`。 +- **`ai.local-agent.max-round`** → `退役；不生成同名活动参数`。 + - 活动目标省略 max-round；保留回滚备份和弃用通知；不把10轮映射成50条。 + - 所有权：`obsolete_native_context_control`；原生依据：`src/langbot/pkg/pipeline/msgtrun/truncators/round.py:13`。 +- **`ai.local-agent.mcp-resource-agent-read-enabled`** → `R.mcp-resource-agent-read-enabled`。 + - 原样复制有效授权；mcp 字段缺失时取 extensions_preferences 对应值；显式 null/错误类型不回退扩大权限。 + - 所有权：`host_policy_in_pipeline_runner_config`；原生依据：`src/langbot/pkg/pipeline/pipelinemgr.py:133`。 +- **`ai.local-agent.mcp-resources`** → `R.mcp-resources`。 + - 原样复制有效授权；mcp 字段缺失时取 extensions_preferences 对应值；显式 null/错误类型不回退扩大权限。 + - 所有权：`host_policy_in_pipeline_runner_config`；原生依据：`src/langbot/pkg/pipeline/pipelinemgr.py:129`。 +- **`ai.local-agent.model`** → `R.model`。 + - 旧字符串UUID→{primary:原UUID,fallbacks:[],reasoning:{}}；对象深拷贝 primary/fallbacks/reasoning；不替换失效UUID、不扩大授权。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/localagent.py:283`。 +- **`ai.local-agent.model.fallbacks`** → `R.model.fallbacks`。 + - 保留有序UUID列表，缺失=[]；runtime 去重不是迁移重排。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:93`。 +- **`ai.local-agent.model.primary`** → `R.model.primary`。 + - 逐字复制非空UUID；Host 所有权交集校验，缺失不得选无关模型。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:92`。 +- **`ai.local-agent.model.reasoning`** → `R.model.reasoning`。 + - 复制 UUID→level 映射，缺失={}；缺项保留模型持久默认；显式 provider_default 覆盖持久级别。 + - 所有权：`pipeline_selector_with_host_provider_policy`；原生依据：`src/langbot/pkg/provider/runners/localagent.py:284`。 +- **`ai.local-agent.prompt`** → `R.prompt`。 + - 保留有序消息、合法SDK结构化内容、空消息、name/provider metadata；缺失不拿UI短提示词覆盖seed长提示词；空Host提示列表有权威性。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:122`。 +- **`ai.local-agent.prompt[0].content`** → `R.prompt[0].content`。 + - 保留所有prompt记录的content，这里[0]仅默认记录形状；不把seed较长content替换为UI短默认，合法空content保留。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/templates/metadata/pipeline/ai.yaml:121`。 +- **`ai.local-agent.prompt[0].role`** → `R.prompt[0].role`。 + - 保留所有prompt记录的role，这里[0]仅默认记录形状；不把seed较长content替换为UI短默认，合法空content保留。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/templates/metadata/pipeline/ai.yaml:120`。 +- **`ai.local-agent.rerank-model`** → `R.rerank-model`。 + - 复制UUID；空字符串或__none__代表禁用；Host验证 rerank 授权。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/localagent.py:442`。 +- **`ai.local-agent.rerank-top-k`** → `R.rerank-top-k`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/localagent.py:456`。 +- **`ai.local-agent.tools`** → `R.tools`。 + - 原样复制有效授权；mcp 字段缺失时取 extensions_preferences 对应值；显式 null/错误类型不回退扩大权限。 + - 所有权：`host_policy_in_pipeline_runner_config`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:37`。 +- **`ai.runner.expire-time`** → `config.ai.runner.expire-time`。 + - 原样保留非负整数；缺失=0，禁止用 token budget 替换。 + - 所有权：`pipeline_host_lifecycle`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:131`。 +- **`ai.runner.runner`** → `config.ai.runner.id`。 + - 所有权：`pipeline_selector`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:73`。 +- **`extensions_preferences.enable_all_mcp_servers`** → `extensions_preferences.enable_all_mcp_servers`。 + - 按原存在性和值保留；显式 false/空名单=不授权；无字段的旧有效缺省只有在同一所有权范围内可保留。 + - 所有权：`host_resource_authorization_shared_all_nine`；原生依据：`src/langbot/pkg/pipeline/pipelinemgr.py:127`。 +- **`extensions_preferences.enable_all_plugins`** → `extensions_preferences.enable_all_plugins`。 + - 按原存在性和值保留；显式 false/空名单=不授权；无字段的旧有效缺省只有在同一所有权范围内可保留。 + - 所有权：`host_resource_authorization_shared_all_nine`；原生依据：`src/langbot/pkg/pipeline/pipelinemgr.py:126`。 +- **`extensions_preferences.enable_all_skills`** → `extensions_preferences.enable_all_skills`。 + - 按原存在性和值保留；显式 false/空名单=不授权；无字段的旧有效缺省只有在同一所有权范围内可保留。 + - 所有权：`host_resource_authorization_shared_all_nine`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:339`。 +- **`extensions_preferences.mcp_resource_agent_read_enabled`** → `extensions_preferences.mcp_resource_agent_read_enabled`。 + - 按原存在性和值保留；显式 false/空名单=不授权；无字段的旧有效缺省只有在同一所有权范围内可保留。 + - 所有权：`host_resource_authorization_shared_all_nine`；原生依据：`src/langbot/pkg/pipeline/pipelinemgr.py:135`。 +- **`extensions_preferences.mcp_resources`** → `extensions_preferences.mcp_resources`。 + - 按原存在性和值保留；显式 false/空名单=不授权；无字段的旧有效缺省只有在同一所有权范围内可保留。 + - 所有权：`host_resource_authorization_shared_all_nine`；原生依据：`src/langbot/pkg/pipeline/pipelinemgr.py:131`。 +- **`extensions_preferences.mcp_servers`** → `extensions_preferences.mcp_servers`。 + - 按原存在性和值保留；显式 false/空名单=不授权；无字段的旧有效缺省只有在同一所有权范围内可保留。 + - 所有权：`host_resource_authorization_shared_all_nine`；原生依据：`src/langbot/pkg/pipeline/pipelinemgr.py:149`。 +- **`extensions_preferences.plugins`** → `extensions_preferences.plugins`。 + - 按原存在性和值保留；显式 false/空名单=不授权；无字段的旧有效缺省只有在同一所有权范围内可保留。 + - 所有权：`host_resource_authorization_shared_all_nine`；原生依据：`src/langbot/pkg/pipeline/pipelinemgr.py:142`。 +- **`extensions_preferences.skills`** → `extensions_preferences.skills`。 + - 按原存在性和值保留；显式 false/空名单=不授权；无字段的旧有效缺省只有在同一所有权范围内可保留。 + - 所有权：`host_resource_authorization_shared_all_nine`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:344`。 +- **`output.misc.remove-think`** → `R.remove-think`。 + - 复制严格 bool 到所选 Runner remove-think；同时保留 config.output.misc.remove-think；缺失=false。 + - 所有权：`pipeline_runner_parameters_plus_retained_host_output`；原生依据：`src/langbot/pkg/provider/runners/localagent.py:539`。 + +### DifyAgent（8 项） + +- **`ai.dify-service-api.api-key`** → `R.api-key`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/difysvapi.py:838`。 +- **`ai.dify-service-api.app-type`** → `R.app-type`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/difysvapi.py:833`。 +- **`ai.dify-service-api.base-prompt`** → `R.base-prompt`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/difysvapi.py:984`。 +- **`ai.dify-service-api.base-url`** → `R.base-url`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/difysvapi.py:842`。 +- **`ai.dify-service-api.timeout`** → `R.timeout`。 + - 旧seed timeout=30没有被native Runner读取；native请求显式120。采用新插件30秒默认，旧任意保存值只留备份，不悄悄激活；用户可显式配置新timeout。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/templates/default-pipeline-config.json:65`。 +- **`ai.runner.expire-time`** → `config.ai.runner.expire-time`。 + - 原样保留非负整数；缺失=0，禁止用 token budget 替换。 + - 所有权：`pipeline_host_lifecycle`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:131`。 +- **`ai.runner.runner`** → `config.ai.runner.id`。 + - 所有权：`pipeline_selector`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:73`。 +- **`output.misc.remove-think`** → `R.remove-think`。 + - 复制严格 bool 到所选 Runner remove-think；同时保留 config.output.misc.remove-think；缺失=false。 + - 所有权：`pipeline_runner_parameters_plus_retained_host_output`；原生依据：`src/langbot/pkg/provider/runners/difysvapi.py:859`。 + +### CozeAgent（9 项） + +- **`ai.coze-api.api-base`** → `R.api-base`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/cozeapi.py:35`。 +- **`ai.coze-api.api-key`** → `R.api-key`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/cozeapi.py:31`。 +- **`ai.coze-api.auto-save-history`** → `R.auto-save-history`。 + - 有效 underscore bool 为旧runtime值；仅hyphen有效bool时明确修复UI旧bug并取该值；两者有效且冲突需用户选择；缺失/null旧值明确弃用后采用目标 true；非bool(含空白字符串)拒绝，不bool()。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/templates/metadata/pipeline/ai.yaml:566`。 +- **`ai.coze-api.auto_save_history`** → `R.auto-save-history`。 + - 有效 underscore bool 为旧runtime值；仅hyphen有效bool时明确修复UI旧bug并取该值；两者有效且冲突需用户选择；缺失/null旧值明确弃用后采用目标 true；非bool(含空白字符串)拒绝，不bool()。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/cozeapi.py:34`。 +- **`ai.coze-api.bot-id`** → `R.bot-id`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/cozeapi.py:32`。 +- **`ai.coze-api.timeout`** → `R.timeout`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/cozeapi.py:33`。 +- **`ai.runner.expire-time`** → `config.ai.runner.expire-time`。 + - 原样保留非负整数；缺失=0，禁止用 token budget 替换。 + - 所有权：`pipeline_host_lifecycle`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:131`。 +- **`ai.runner.runner`** → `config.ai.runner.id`。 + - 所有权：`pipeline_selector`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:73`。 +- **`output.misc.remove-think`** → `R.remove-think`。 + - 复制严格 bool 到所选 Runner remove-think；同时保留 config.output.misc.remove-think；缺失=false。 + - 所有权：`pipeline_runner_parameters_plus_retained_host_output`；原生依据：`src/langbot/pkg/provider/runners/cozeapi.py:50`。 + +### DashScopeAgent（8 项） + +- **`ai.dashscope-app-api.api-key`** → `R.api-key`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/dashscopeapi.py:58`。 +- **`ai.dashscope-app-api.app-id`** → `R.app-id`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/dashscopeapi.py:57`。 +- **`ai.dashscope-app-api.app-type`** → `R.app-type`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/dashscopeapi.py:51`。 +- **`ai.dashscope-app-api.references-quote`** → `R.references_quote`。 + - 有效 underscore 字符串优先；只有 hyphen 字符串则明确重命名修复；两者冲突需选择；都缺失采用“参考资料来自:”；显式空串和空白原样保留，null非字符串拒绝或用户明确选择默认。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/templates/default-pipeline-config.json:71`。 +- **`ai.dashscope-app-api.references_quote`** → `R.references_quote`。 + - 有效 underscore 字符串优先；只有 hyphen 字符串则明确重命名修复；两者冲突需选择；都缺失采用“参考资料来自:”；显式空串和空白原样保留，null非字符串拒绝或用户明确选择默认。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/dashscopeapi.py:59`。 +- **`ai.runner.expire-time`** → `config.ai.runner.expire-time`。 + - 原样保留非负整数；缺失=0，禁止用 token budget 替换。 + - 所有权：`pipeline_host_lifecycle`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:131`。 +- **`ai.runner.runner`** → `config.ai.runner.id`。 + - 所有权：`pipeline_selector`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:73`。 +- **`output.misc.remove-think`** → `R.remove-think`。 + - 复制严格 bool 到所选 Runner remove-think；同时保留 config.output.misc.remove-think；缺失=false。 + - 所有权：`pipeline_runner_parameters_plus_retained_host_output`；原生依据：`src/langbot/pkg/provider/runners/dashscopeapi.py:121`。 + +### N8nAgent（13 项） + +- **`ai.n8n-service-api.auth-type`** → `R.auth-type`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:47`。 +- **`ai.n8n-service-api.basic-password`** → `R.basic-password`。 + - 凭证字节原样复制；选用basic时同时显式 basic-encoding="latin1"（native aiohttp默认），插件新建默认utf-8不改；不支持编码字符应报安全错误而非改写。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:52`。 +- **`ai.n8n-service-api.basic-username`** → `R.basic-username`。 + - 凭证字节原样复制；选用basic时同时显式 basic-encoding="latin1"（native aiohttp默认），插件新建默认utf-8不改；不支持编码字符应报安全错误而非改写。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:51`。 +- **`ai.n8n-service-api.header-name`** → `R.header-name`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:57`。 +- **`ai.n8n-service-api.header-value`** → `R.header-value`。 + - 原样复制包括空字符串；active header必须有合法header-name，不省略合法空value；隐藏/inactive字段仍保留。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:58`。 +- **`ai.n8n-service-api.jwt-algorithm`** → `R.jwt-algorithm`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:55`。 +- **`ai.n8n-service-api.jwt-secret`** → `R.jwt-secret`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:54`。 +- **`ai.n8n-service-api.output-key`** → `R.output-key`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:41`。 +- **`ai.n8n-service-api.response-handling`** → `R.response-handling`。 + - reply/ignore逐字复制；缺失reply；ignore现已实现2xx只读header、不读body、不输出；非2xx仍失败；不得留旧缺特性blocker。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:42`。 +- **`ai.n8n-service-api.timeout`** → `R.timeout`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:38`。 +- **`ai.n8n-service-api.webhook-url`** → `R.webhook-url`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/n8nsvapi.py:35`。 +- **`ai.runner.expire-time`** → `config.ai.runner.expire-time`。 + - 原样保留非负整数；缺失=0，禁止用 token budget 替换。 + - 所有权：`pipeline_host_lifecycle`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:131`。 +- **`ai.runner.runner`** → `config.ai.runner.id`。 + - 所有权：`pipeline_selector`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:73`。 + +### LangflowAgent（10 项） + +- **`ai.langflow-api.api-key`** → `R.api-key`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/langflowapi.py:119`。 +- **`ai.langflow-api.base-url`** → `R.base-url`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/langflowapi.py:118`。 +- **`ai.langflow-api.flow-id`** → `R.flow-id`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/langflowapi.py:120`。 +- **`ai.langflow-api.input-type`** → `R.input-type`。 + - underscore存在且合法字符串→对应hyphen；underscore缺失且hyphen缺失/chat→chat；仅hyphen非默认属于旧UI被忽略的意图，用户选修复为该值或维持chat；两者冲突需选择；underscore显式null/空白不能被or chat吞掉。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/templates/metadata/pipeline/ai.yaml:692`。 +- **`ai.langflow-api.input_type`** → `R.input-type`。 + - underscore存在且合法字符串→对应hyphen；underscore缺失且hyphen缺失/chat→chat；仅hyphen非默认属于旧UI被忽略的意图，用户选修复为该值或维持chat；两者冲突需选择；underscore显式null/空白不能被or chat吞掉。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/langflowapi.py:81`。 +- **`ai.langflow-api.output-type`** → `R.output-type`。 + - underscore存在且合法字符串→对应hyphen；underscore缺失且hyphen缺失/chat→chat；仅hyphen非默认属于旧UI被忽略的意图，用户选修复为该值或维持chat；两者冲突需选择；underscore显式null/空白不能被or chat吞掉。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/templates/metadata/pipeline/ai.yaml:702`。 +- **`ai.langflow-api.output_type`** → `R.output-type`。 + - underscore存在且合法字符串→对应hyphen；underscore缺失且hyphen缺失/chat→chat；仅hyphen非默认属于旧UI被忽略的意图，用户选修复为该值或维持chat；两者冲突需选择；underscore显式null/空白不能被or chat吞掉。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/langflowapi.py:82`。 +- **`ai.langflow-api.tweaks`** → `R.tweaks`。 + - 目标接受dict或严格JSON object（递归保留null/false/0）；缺失/null/空串/JSON null→{}；空白串按新便利默认{}并记录；false/0/list拒绝；JSON重复键/NaN/Infinity/过深结构拒绝；坏JSON绝不降级{}。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/langflowapi.py:93`。 +- **`ai.runner.expire-time`** → `config.ai.runner.expire-time`。 + - 原样保留非负整数；缺失=0，禁止用 token budget 替换。 + - 所有权：`pipeline_host_lifecycle`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:131`。 +- **`ai.runner.runner`** → `config.ai.runner.id`。 + - 所有权：`pipeline_selector`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:73`。 + +### DeerFlowAgent（13 项） + +- **`ai.deerflow-api.api-base`** → `R.api-base`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:61`。 +- **`ai.deerflow-api.api-key`** → `R.api-key`。 + - 逐字保留字符串包括空白/空串；null或非string拒绝；仅缺失采用target_default。auth-header是完整Authorization值，非空优先于api-key。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:68`。 +- **`ai.deerflow-api.assistant-id`** → `R.assistant-id`。 + - 逐字保留字符串包括空白/空串；null或非string拒绝；仅缺失采用target_default。auth-header是完整Authorization值，非空优先于api-key。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:70`。 +- **`ai.deerflow-api.auth-header`** → `R.auth-header`。 + - 逐字保留字符串包括空白/空串；null或非string拒绝；仅缺失采用target_default。auth-header是完整Authorization值，非空优先于api-key。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:69`。 +- **`ai.deerflow-api.max-concurrent-subagents`** → `R.max-concurrent-subagents`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:75`。 +- **`ai.deerflow-api.model-name`** → `R.model-name`。 + - 逐字保留字符串包括空白/空串；null或非string拒绝；仅缺失采用target_default。auth-header是完整Authorization值，非空优先于api-key。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:71`。 +- **`ai.deerflow-api.plan-mode`** → `R.plan-mode`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:73`。 +- **`ai.deerflow-api.recursion-limit`** → `R.recursion-limit`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:77`。 +- **`ai.deerflow-api.subagent-enabled`** → `R.subagent-enabled`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:74`。 +- **`ai.deerflow-api.thinking-enabled`** → `R.thinking-enabled`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:72`。 +- **`ai.deerflow-api.timeout`** → `R.timeout`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/deerflowapi.py:76`。 +- **`ai.runner.expire-time`** → `config.ai.runner.expire-time`。 + - 原样保留非负整数；缺失=0，禁止用 token budget 替换。 + - 所有权：`pipeline_host_lifecycle`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:131`。 +- **`ai.runner.runner`** → `config.ai.runner.id`。 + - 所有权：`pipeline_selector`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:73`。 + +### TboxAgent（5 项） + +- **`ai.runner.expire-time`** → `config.ai.runner.expire-time`。 + - 原样保留非负整数；缺失=0，禁止用 token budget 替换。 + - 所有权：`pipeline_host_lifecycle`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:131`。 +- **`ai.runner.runner`** → `config.ai.runner.id`。 + - 所有权：`pipeline_selector`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:73`。 +- **`ai.tbox-app-api.api-key`** → `R.api-key`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/tboxapi.py:59`。 +- **`ai.tbox-app-api.app-id`** → `R.app-id`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/tboxapi.py:58`。 +- **`output.misc.remove-think`** → `R.remove-think`。 + - 复制严格 bool 到所选 Runner remove-think；同时保留 config.output.misc.remove-think；缺失=false。 + - 所有权：`pipeline_runner_parameters_plus_retained_host_output`；原生依据：`src/langbot/pkg/provider/runners/tboxapi.py:114`。 + +### WeKnoraAgent（10 项） + +- **`ai.runner.expire-time`** → `config.ai.runner.expire-time`。 + - 原样保留非负整数；缺失=0，禁止用 token budget 替换。 + - 所有权：`pipeline_host_lifecycle`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:131`。 +- **`ai.runner.runner`** → `config.ai.runner.id`。 + - 所有权：`pipeline_selector`；原生依据：`src/langbot/pkg/pipeline/preproc/preproc.py:73`。 +- **`ai.weknora-api.agent-id`** → `R.agent-id`。 + - 原键缺失：明确写入按旧app-type的默认（chat quick-answer，agent smart-reasoning）；显式null/空串保持(线端省略agent_id)；其余字符串含空白逐字保留，不能因chat而覆盖历史smart ID。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/weknoraapi.py:89`。 +- **`ai.weknora-api.api-key`** → `R.api-key`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/weknoraapi.py:38`。 +- **`ai.weknora-api.app-type`** → `R.app-type`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/weknoraapi.py:336`。 +- **`ai.weknora-api.base-prompt`** → `R.base-prompt`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/weknoraapi.py:65`。 +- **`ai.weknora-api.base-url`** → `R.base-url`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/weknoraapi.py:45`。 +- **`ai.weknora-api.knowledge-base-ids`** → `R.knowledge-base-ids`。 + - 保留远端ID顺序/重复/字节；缺失或null→[]；无效元素/空白ID拒绝，不静默丢弃。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/weknoraapi.py:90`。 +- **`ai.weknora-api.timeout`** → `R.timeout`。 + - 源键存在时深拷贝原值；缺失时按下面 native_defaults 的实际读取语义或显式新默认处理。不得把 null/空字符串统一当作缺失。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/weknoraapi.py:92`。 +- **`ai.weknora-api.web-search-enabled`** → `R.web-search-enabled`。 + - 复制严格bool，缺失false；仅agent线端使用，chat保存true仍不激活。 + - 所有权：`pipeline_runner_parameters`；原生依据：`src/langbot/pkg/provider/runners/weknoraapi.py:91`。 + +## 新增参数与明确采用的默认值 + +这些值是新机制的默认值，不声称与旧行为一一等价。已存在且受支持的有效值按映射规则保留；必填凭证/端点不能由模板伪造。 + +### LocalAgent + +```json +{ + "advanced-settings": false, + "date-grounding": true, + "timeout": 300, + "retrieval-top-k": 5, + "rerank-model": "", + "rerank-top-k": 5, + "max-tool-iterations": 100, + "tool-execution-mode": "serial", + "max-tool-result-chars": 20000, + "context-history-fetch-limit": 50, + "context-window-tokens": 200000, + "context-reserve-tokens": 16384, + "context-keep-recent-tokens": 20000, + "context-summary-tokens": 8000, + "enable-all-tools": true, + "tools": [], + "knowledge-bases": [] +} +``` + +### DifyAgent + +```json +{ + "timeout": 30, + "advanced-settings": false +} +``` + +### CozeAgent + +```json +{ + "auto-save-history": true, + "advanced-settings": false +} +``` + +### DashScopeAgent + +```json +{ + "references_quote": "参考资料来自:", + "timeout": 120, + "advanced-settings": false +} +``` + +### N8nAgent + +```json +{ + "auth-type": "none", + "basic-username": "", + "basic-password": "", + "jwt-secret": "", + "jwt-algorithm": "HS256", + "header-name": "", + "header-value": "", + "timeout": 120, + "output-key": "response", + "response-handling": "reply", + "basic-encoding": "latin1", + "advanced-settings": false +} +``` + +### LangflowAgent + +```json +{ + "input-type": "chat", + "output-type": "chat", + "tweaks": {}, + "advanced-settings": false +} +``` + +### DeerFlowAgent + +```json +{ + "api-key": "", + "auth-header": "", + "assistant-id": "lead_agent", + "model-name": "", + "thinking-enabled": false, + "plan-mode": false, + "subagent-enabled": false, + "max-concurrent-subagents": 3, + "timeout": 300, + "recursion-limit": 1000 +} +``` + +### TboxAgent + +```json +{ + "timeout": 120 +} +``` + +### WeKnoraAgent + +```json +{ + "knowledge-base-ids": [], + "web-search-enabled": false, + "timeout": 120, + "base-prompt": "请回答用户的问题。", + "advanced-settings": false +} +``` + +## 验证与交付状态 + +本清单负责说明所有已识别旧字段的去向和语义；不替代最终的集成验收。源码修改、插件打包、OSS/Cloud 持久化与权限、真实 SDK 调用、浏览器测试、独立审查和发布状态必须分别记录，不能以其中一项通过宣称全部完成。 + +源码库存与每条字段的多处原生证据、默认值/表达式及哈希见工作区的 `411-completion-config-audit.json` 和其标注的原始库存；最终验证以 `411-completion-*` 的实际日志和源码哈希为准。 diff --git a/docs/review/box-architecture.md b/docs/review/box-architecture.md index 2a5e06e65..4a390ebf1 100644 --- a/docs/review/box-architecture.md +++ b/docs/review/box-architecture.md @@ -1,6 +1,6 @@ # Box 系统架构深度分析 -> 更新日期: 2026-06-02 +> 更新日期: 2026-07-12 > 状态更新: 自部署社区版已具备发布条件（box 可选、降级完善、无迁移欠债）；工具调用循环上限、配额遍历异步化、`host_path` 挂载白名单等已落地。剩余多租户 / 安全硬化项见 [SaaS 阻塞项清单](./box-issues.md)。 > 分支: `feat/sandbox` (LangBot + langbot-plugin-sdk) > 相关文档: [SaaS 阻塞项](./box-issues.md) | [Session 作用域](./box-session-scope.md) | [Runtime 对比](./box-vs-plugin-runtime.md) | [测试覆盖](./box-test-coverage.md) | [toB 分析](./box-tob-analysis.md) @@ -13,7 +13,9 @@ ┌──────────────────────────────────────────────────────────────────┐ │ LangBot 主进程 │ │ │ -│ LocalAgentRunner ──> ToolManager ──> NativeToolLoader │ +│ Runner ──> SDK call_tool / scoped MCP bridge │ +│ │ │ │ +│ └────────────────> ToolManager ──> NativeToolLoader │ │ │ │ │ │ │ │ │ exec / read / write / edit │ │ │ │ glob / grep │ @@ -32,7 +34,7 @@ │ ├─ Host mount 校验 (allowed_mount_roots) │ │ ├─ Workspace quota 检查 │ │ ├─ 输出截断 (head+tail) │ -│ ├─ Session ID 模板解析 (resolve_box_session_id) │ +│ ├─ Host scope 哈希 (resolve_box_session_id) │ │ ├─ 技能挂载组装 (build_skill_extra_mounts) │ │ ├─ 重连循环 (_reconnect_loop, 指数退避) │ │ └─ BoxRuntimeConnector │ @@ -85,7 +87,8 @@ **核心设计原则**: - Box Runtime 作为独立进程运行，通过 Action RPC 与 LangBot 主进程通信，两者复用 SDK 的 IO 层（Handler → Connection → Controller） - 一个 session_id 对应一个容器/沙箱实例。同一 session 内可并存多条 mount 与多个 managed process -- Skill / 默认 exec / MCP Server 共享同一个 session 容器（详见 [box-session-scope.md](./box-session-scope.md)） +- Runner 无权指定 session scope。SDK/Python `call_tool` 与 scoped MCP bridge 都发出同一个 `PluginToRuntimeAction.CALL_TOOL`，最终由 Host 的 ToolManager 执行，并使用当前 run 保存的同一个 execution Query +- Box 内托管的 stdio MCP server 使用独立的长期 `mcp-shared` session；它不是 Runner 本次事件的 sandbox session（详见 [box-session-scope.md](./box-session-scope.md)） --- @@ -93,7 +96,7 @@ ### 2.1 BoxService (`pkg/box/service.py`, 722 行) -应用层门面，协调 Profile、安全校验、配额、连接、Skill 挂载与 Session 模板： +应用层门面，协调 Profile、安全校验、配额、连接、Skill 挂载与 Host scope 哈希： 主要公开方法（按定义顺序）： @@ -104,7 +107,7 @@ BoxService ├─ _reconnect_loop(connector) 指数退避重连 ├─ available (property) 连接状态 │ - ├─ resolve_box_session_id(query) 从 pipeline 模板解析 session_id + ├─ resolve_box_session_id(query) 哈希 Host 私有 scope，生成固定长度 session_id ├─ build_skill_extra_mounts(query) 组装 pipeline-bound skill 的挂载列表 │ ├─ execute_tool(parameters, query) Agent 调用 exec 时的入口 @@ -137,6 +140,8 @@ BoxService **输出截断**: 默认 4000 字符上限，保留前 60% + 后 40%，中间插入 `[...truncated...]`。 +**Session 所有权**: `resolve_box_session_id(query)` 只接受 Host 已确定的私有 scope 或 Query launcher/session identity，并输出 `lb-box-` + 64 位小写 SHA-256 十六进制摘要（固定 71 个 ASCII 字符）。哈希输入是 canonical JSON，包含 instance、workspace、bot、platform adapter、target type/id 与 thread；原始用户、群组、conversation 或 event id 不会出现在 Box session id 中。相同 Host scope 稳定复用，不同 target/thread/workspace/bot/adapter/instance 相互隔离；缺少可用 identity 时 fail closed。Pipeline、Agent 或 Runner 配置都不能覆盖该规则。 + **Skill 挂载合并**: `execute_tool()` 调用时，`build_skill_extra_mounts(query)` 会把当前 pipeline-bound 的所有 skill 的 `package_root` 作为 `extra_mounts` 加入 BoxSpec，挂在 `/workspace/.skills/`。LLM 通过 `activate` 工具显式激活某个 skill 后，工具调用才允许引用这个 skill 的虚拟路径。 ### 2.2 BoxRuntimeConnector (`pkg/box/connector.py`, 357 行) @@ -414,7 +419,9 @@ ToolManager.initialize() 1. 验证 skill 已激活 2. 单次 exec 只能引用一个 skill 包 3. 若 skill 是 Python 项目（有 `requirements.txt` 或 `pyproject.toml`），命令会被 venv bootstrap 包裹（在 skill 挂载点内创建 `.venv`） -4. 调用 `box_service.execute_tool()` → 走默认 session_id 与已组装好的 `extra_mounts`，**不再为每 skill 起独立 session** +4. 调用 `box_service.execute_tool()` → 走 Host 从当前事件生成的 session_id 与已组装好的 `extra_mounts`，**不再为每 skill 起独立 session** + +Runner 可以直接通过 SDK/Python `RunnerAPIProxy.call_tool` 调用这些工具，也可以让外部 harness 通过 SDK-owned scoped MCP bridge 回调。两条入口都发送 `PluginToRuntimeAction.CALL_TOOL`，共享同一个 run authorization、Host session 中保存的 execution Query、ToolManager 与 `resolve_box_session_id(query)` 规则；Runner 不能提交自定义 Box session id。Pipeline run 保存原 Query；纯 EBA run 由 Host 构造 `pipeline_config=None`、`pipeline_uuid=None` 的最小 Query。 ### 4.3 MCP-in-Box (`mcp_stdio.py`, 354 行) @@ -422,7 +429,7 @@ ToolManager.initialize() ``` initialize() - 1. 复用/创建共享 session (session_id = _build_box_session_id()) + 1. 复用/创建共享 session (`session_id = mcp-shared`) - persistent=True，长期保持 2. workspace.execute_raw(install_cmd) 安装依赖 (可选) 3. 将每个 MCP server 文件 stage 到 /workspace/.mcp// @@ -435,6 +442,8 @@ initialize() 每条 MCP server 是同一 session 中的一个 managed process，独立的 `process_id`、独立 attach URL，互不阻塞。 +这里的 `mcp-shared` 只承载 LangBot 管理的 stdio MCP server 进程。Runner 的 scoped MCP bridge 是回调 Host 工具的协议入口，不会把事件运行的 exec/read/write 改到 `mcp-shared`。 + --- ## 5. 启动与生命周期 @@ -566,22 +575,14 @@ volumes: | http/sse MCP server | 正常 | 正常(不依赖 Box) | | Skill 列表/读取 (`list_skills`/`get_skill`/`read_skill_file`) | 走 Box runtime | 走 LangBot 本地 `data/skills/` 只读 fallback | | Skill 创建/编辑/安装/写文件 | 走 Box runtime | **HTTP 400** + 明确错误信息(`_require_box_for_write`) | -| Pipeline AI 配置中 `box-session-id-template` | 正常生效 | **前端 banner** 提示字段无效 | | Pipeline 扩展页 `enable_all_skills` / 绑定 skill | 可编辑 | **前端禁用** + banner | | 仪表盘 Box 状态卡片 | 绿点 / "已连接" | 灰点 / "已禁用"(disabled) 或 红点 / "已断开"(failed) | > 后端拒写的边界条件:如果 `ap.box_service` **完全没装**(老式 dev mode,没经过 BuildAppStage),`_require_box_for_write` 视作 no-op,保留 `data/skills/` 本地路径——以兼容历史测试与最小化设置。生产环境总会装 `ap.box_service`,因此该 fallback 不会被触发。 -### Pipeline 配置 (templates/metadata/pipeline/ai.yaml) +### Session scope -`local-agent.config.box-session-id-template` 控制 session 作用域，预设： - -- `{launcher_type}_{launcher_id}` — 每个会话 (推荐，默认) -- `{launcher_type}_{launcher_id}_{sender_id}` — 群聊每个用户 -- `{launcher_type}_{launcher_id}_{conversation_id}` — 每个对话上下文 -- `{query_id}` — 每条消息（完全隔离） - -详见 [box-session-scope.md](./box-session-scope.md)。 +Pipeline 与 Runner 配置不再暴露 sandbox session 模板。Host 将当前平台会话/事件 scope 规范化后哈希成固定长度的 `lb-box-`；相同 scope 稳定复用，不同 scope 隔离，缺少 identity 时拒绝执行。SDK/Python 与 scoped MCP bridge 的工具调用遵守同一规则。详见 [box-session-scope.md](./box-session-scope.md)。 ### REST API diff --git a/docs/review/box-session-scope.md b/docs/review/box-session-scope.md index bb92265de..7d7e47edc 100644 --- a/docs/review/box-session-scope.md +++ b/docs/review/box-session-scope.md @@ -1,402 +1,181 @@ -# Box Session Scope Design +# Box Session Scope -> Date: 2026-04-18 (last reviewed 2026-06-02) -> Status (2026-06-02): the self-hosted community edition is release-ready (box optional, clean degradation, no migration debt). Tool-call loop cap, async quota scan, and the host_path mount allowlist have landed. Remaining multi-tenant / security hardening is tracked in [box-issues.md](./box-issues.md). -> Branch: `feat/sandbox` (LangBot + langbot-plugin-sdk) +> Last reviewed: 2026-07-12 +> Status: implemented Host-owned, hashed execution scope; Runner/Pipeline session templates are removed. > Related: [Box Architecture](./box-architecture.md) | [Box vs Plugin Runtime](./box-vs-plugin-runtime.md) ---- +## 1. Decision -## 0. Implementation Status (2026-05-19) +The LangBot Host owns the Box session used by an event run. A Pipeline, Agent, +or Runner cannot choose a global, per-user, per-conversation, or per-query +sandbox mode. -This document was authored as a design proposal. The current `feat/sandbox` branch -has shipped the design largely as written: +`BoxService.resolve_box_session_id(query)` always returns this shape: -| Item | Status | Notes | -|------|--------|-------| -| `BoxMountSpec` + `BoxSpec.extra_mounts` | ✅ Shipped | SDK `box/models.py` | -| Docker / nsjail / E2B backends apply extra mounts | ✅ Shipped | Last gap closed by SDK commit `0fea9b1` (E2B) | -| `box-session-id-template` in `local-agent` pipeline config | ✅ Shipped | `templates/metadata/pipeline/ai.yaml`, default `{launcher_type}_{launcher_id}` | -| `BoxService.resolve_box_session_id(query)` | ✅ Shipped | `pkg/box/service.py:166` | -| `BoxService.build_skill_extra_mounts(query)` | ✅ Shipped | `pkg/box/service.py:189` | -| Skill exec uses unified container + extra mounts | ✅ Shipped | `pkg/provider/tools/loaders/native.py` skill branch | -| MCP-in-Box uses shared persistent session, multi-process | ✅ Shipped (earlier than originally scoped) | SDK commit `529088e`, LangBot `mcp_stdio.py:_build_box_session_id` | -| `BoxManagedProcessSpec.process_id` + multi-process per session | ✅ Shipped | `BoxRuntime` keeps `managed_processes: dict[pid, _ManagedProcess]` | -| Per-tenant / quota integration with templates | ❌ Not started | See [box-tob-analysis.md](./box-tob-analysis.md) | - -The "Phase 2 deferred" note in §10 is **out of date** — MCP unification went in on -the same line. Pipeline-scoped (not user-scoped) MCP container is the realized -behavior: each pipeline's MCP servers share one `mcp-` session, and -user exec sessions use the template-derived id. - -The remaining open work is multi-tenant overlays (tenant_id in session_id, -quota counters keyed by tenant), tracked in the toB analysis doc rather than here. - ---- - -## 1. Problems - -### 1.1 Default exec: per-message containers - -Currently, `BoxService.execute_tool()` sets `session_id = str(query.query_id)` — an -auto-incrementing integer per incoming message. Every user message creates a new sandbox -container. Dependencies installed and in-container state are lost between messages. - -### 1.2 Three isolated container pools - -Default exec, skills, and MCP servers each manage their own containers with -independent session IDs: - -| Path | Session ID | Container | -|--------------|-----------------------------------------------|-------------| -| Default exec | `str(query_id)` (per message) | Ephemeral | -| Skill exec | `skill-{launcher}_{id}-{skill_name}` | Per skill | -| MCP stdio | `mcp-{server_uuid}` | Per server | - -This means a single logical user interaction can spawn 3+ containers that cannot -share state, see each other's files, or reuse installed dependencies. - -### 1.3 Single bind mount limitation - -`BoxSpec` currently supports only **one** `host_path` → `mount_path` bind mount. -This prevents mounting both a default workspace and skill directories into the -same container. - ---- - -## 2. Concept Model - -``` -Platform Message - → Query (query_id: int, auto-increment, per message) - → Session (launcher_type + launcher_id, per chat window) - → Conversation (uuid, per dialogue context within a Session) +```text +lb-box-<64 lowercase SHA-256 hex characters> ``` -| Concept | Key | Example | Scope | -|---------------|-------------------------------------|----------------------------|------------------------------| -| Query | `query_id` | `42` | Single message | -| Session | `launcher_type` + `launcher_id` | `group_123456` | Chat window (group or PM) | -| Conversation | `conversation_id` (UUID) | `a1b2c3d4-...` | Dialogue context within a Session | -| Sender | `sender_id` | `789` | Individual user | +The result is exactly 71 ASCII characters. Raw platform, user, group, +conversation, thread, and event identifiers never appear in the Box session +id. This avoids unsafe path characters, unbounded identifier length, and +identity leakage through runtime/container metadata. -Note: in a **group chat**, all users share the same Session (keyed by `group_id`). The -individual sender is tracked as `sender_id` but does not affect Session/Conversation routing. +This rule replaces all former concepts of: ---- +- Pipeline or Runner `box-session-id-template` fields; +- a global forced session template; +- API fields that let a caller supply sandbox scope; +- LocalAgent-specific Host injection of Box availability, scope, or Pipeline id. -## 3. Target Scenarios +## 2. Canonical Host scope -| # | Scenario | Box Granularity | Desired `session_id` | -|----|--------------------------------|------------------------------------------|---------------------------------------------------------| -| 1 | Personal assistant | 1 Box per user, long-lived | `{launcher_type}_{launcher_id}` | -| 2 | Customer service | 1 Box per customer, cross-pipeline | `{launcher_type}_{launcher_id}` | -| 3 | Internal employee tool | 1 Box per employee | `{launcher_type}_{launcher_id}` | -| 4 | Group chat shared assistant | 1 Box per group | `{launcher_type}_{launcher_id}` | -| 5 | Group chat isolated per user | 1 Box per user within a group | `{launcher_type}_{launcher_id}_{sender_id}` | -| 6 | Teaching (cross-channel) | 1 Box per student across groups/PMs | `{sender_id}` | -| 7 | One-off execution | 1 Box per message (current behavior) | `{query_id}` | -| 8 | Multi-project development | 1 Box per conversation context | `{launcher_type}_{launcher_id}_{conversation_id}` | +Before hashing, the Host creates a canonical, sorted JSON scope with these +dimensions: -No single fixed granularity covers all scenarios. A template-based approach is needed. +| Dimension | Purpose | +| --- | --- | +| `instance_id` | Isolate separate LangBot installations | +| `workspace_id` | Preserve workspace/tenant boundary when available | +| `bot_id` | Prevent two bots from sharing a sandbox accidentally | +| `platform_adapter` | Separate identical target ids from different adapters | +| `target_type` / `target_id` | Identify the platform session or event target | +| `thread_id` | Isolate threads within a target when available | ---- +The canonical JSON is domain-separated and hashed by the Host. Runner input, +runner config, and tool parameters are not trusted sources for this scope. -## 4. Design Overview +### 2.1 Target identity priority -Two key changes: +The Host resolves `target_type` / `target_id` in this order: -1. **Unified container**: exec, skills, and MCP all share the same container per - session scope. No more separate container pools. -2. **Configurable session scope**: `session_id` is generated from a template with - pipeline variables, configurable per pipeline. +1. For a Pipeline-backed run, use the exact Query launcher tuple. +2. For a pure EBA run, use `delivery.reply_target.target_type/target_id` + (`launcher_type/launcher_id` aliases are accepted). +3. If there is no delivery target, use `conversation_id`. +4. For a non-message event without a conversation, use `event_id`, producing + an event-scoped sandbox. -### 4.1 Unified Container with Multiple Mounts +The adapter class or declared adapter capability supplies platform adapter +identity. The Host includes the active LangBot instance, workspace, bot, and +thread dimensions when they exist. -A single container per session scope is created on first use. It has: +### 2.2 Stability and isolation -- **Primary mount**: default workspace at `/workspace` (from `default_host_workspace`) -- **Skill mounts**: each pipeline-bound skill's `package_root` mounted at - `/workspace/.skills/{skill_name}/` -- **MCP servers**: run as managed processes inside the same container +The same normalized scope always produces the same hash, so repeated runs in +the same platform conversation reuse the same Box workspace. A rotating +transcript/conversation id does not change the scope when an explicit platform +reply target remains the same. -``` -Container (session_id = "group_123456") - /workspace/ ← default workspace (bind mount, rw) - /workspace/.skills/web-search/ ← skill package (bind mount, rw) - /workspace/.skills/data-analysis/ ← skill package (bind mount, rw) - [managed process: mcp-server-a] ← MCP server running inside - [managed process: mcp-server-b] ← MCP server running inside +A different target, thread, workspace, bot, platform adapter, or LangBot +instance changes the hash. If delivery target is unavailable and +`conversation_id` is the fallback, different conversations also produce +different hashes. Event-scoped fallback isolates unrelated non-message events. + +### 2.3 Fail closed + +If the private Host scope marker is present but empty or malformed, Box rejects +execution with `BoxValidationError`. A direct Query without either a valid +Host scope or launcher/session identity is also rejected. There is no +`unknown`, raw query id, global, or caller-selected fallback. + +## 3. Host execution Query + +Runner callbacks need a Host-owned Query view because model/tool loaders +already consume that type. The Query is internal and is never exposed as a +Runner-controlled object. + +- A Pipeline run stores the exact current Query in `AgentRunSession`. +- A pure EBA run builds a minimal Query with a valid Session and + `pipeline_config=None`, `pipeline_uuid=None`. +- The Host attaches canonical `_host_box_scope` and the authorized skill names + in `_pipeline_bound_skills`. +- `PluginToRuntimeAction.CALL_TOOL` restores this Query from the active + `run_id` before dispatching to `ToolManager`. + +This gives Pipeline and pure EBA execution the same Host tool path without +inventing a fake Pipeline for an independent Agent. + +## 4. Runner callback paths + +Runner implementations may use either callback transport: + +1. SDK/Python runners call `RunnerAPIProxy.call_tool`. +2. External harnesses call the SDK-owned scoped MCP bridge. + +Both transports emit the same `PluginToRuntimeAction.CALL_TOOL`. The Host then +validates the same run authorization, restores the same execution Query, and +dispatches to the same ToolManager and BoxService. + +```text +Runner + +-- RunnerAPIProxy.call_tool --------+ + | | + +-- SDK-owned scoped MCP bridge -------+--> PluginToRuntimeAction.CALL_TOOL + --> run authorization + --> execution Query + --> ToolManager + --> BoxService + --> lb-box- ``` -This requires extending `BoxSpec` to support multiple mounts (see §5). +An Runner is not required to use MCP. Local Python runners can use the SDK +directly; code-agent harnesses can use the bridge. The transports do not define +different authorization or sandbox semantics. -### 4.2 Session ID Template +## 5. Skills and mounts -A new field `box-session-id-template` in the `local-agent` pipeline runner config -controls the session scope: +Native exec and skill-backed exec for one Host scope use the same hashed +session. `BoxService.build_skill_extra_mounts(query)` adds visible, authorized +skill packages under `/workspace/.skills/` when the session is created. -```yaml -# templates/metadata/pipeline/ai.yaml (under local-agent.config) -- name: box-session-id-template - label: - en_US: Sandbox Scope - zh_Hans: 沙箱作用域 - description: - en_US: >- - Determines how sandbox environments are shared. Use variables to - control isolation granularity. - zh_Hans: >- - 决定沙箱环境的共享方式。使用变量控制隔离粒度。 - type: select - required: false - default: "{launcher_type}_{launcher_id}" - options: - - value: "{launcher_type}_{launcher_id}" - label: - en_US: Per chat (Recommended) - zh_Hans: 每个会话（推荐） - - value: "{launcher_type}_{launcher_id}_{sender_id}" - label: - en_US: Per user in chat - zh_Hans: 会话中每个用户 - - value: "{launcher_type}_{launcher_id}_{conversation_id}" - label: - en_US: Per conversation context - zh_Hans: 每个对话上下文 - - value: "{query_id}" - label: - en_US: Per message (isolated) - zh_Hans: 每条消息（完全隔离） -``` +Skill activation controls which skill-backed tools and paths are available. It +does not create a different session and does not grant the Runner authority to +change the session id. -Available template variables (populated by PreProcessor in `query.variables`): +## 6. `mcp-shared` is a different session -| Variable | Source | Example | -|---------------------|---------------------------------|----------------------| -| `{launcher_type}` | `query.session.launcher_type` | `person` / `group` | -| `{launcher_id}` | `query.session.launcher_id` | `123456` | -| `{sender_id}` | `query.sender_id` | `789` | -| `{conversation_id}` | `conversation.uuid` | `a1b2c3d4-...` | -| `{query_id}` | `query.query_id` | `42` | +LangBot can host configured stdio MCP servers as managed processes inside Box. +Those long-lived infrastructure processes share the dedicated `mcp-shared` +session and are isolated from one another by `process_id`. -Default `{launcher_type}_{launcher_id}` covers scenarios 1–4 out of the box. +This is separate from the scoped MCP bridge above: ---- +| Path | Purpose | Session rule | +| --- | --- | --- | +| Runner scoped MCP bridge | Call authorized Host tools for one active run | Host-owned `lb-box-` from the run execution Query | +| MCP-in-Box stdio server | Keep configured MCP server processes running | Dedicated persistent `mcp-shared` session | -## 5. SDK Changes: Multi-Mount BoxSpec +Calling a sandbox tool through the Runner bridge never redirects the run +workspace into `mcp-shared`. Conversely, an MCP server's managed-process +lifecycle does not inherit the current event scope. -### 5.1 Model Extension +## 7. Configuration and compatibility -```python -# box/models.py +There is no Box session scope field in Pipeline metadata, Runner config, +or the public Pipeline/Runner API. Operators configure the Box subsystem itself +(`box.enabled`, backend/runtime settings, profiles, mount allowlists, quotas, +and workspace roots), not per-Runner session templates. -class BoxMountSpec(pydantic.BaseModel): - """A single bind mount specification.""" - host_path: str - mount_path: str - mode: BoxHostMountMode = BoxHostMountMode.READ_WRITE +Old configuration containing `box-session-id-template` is unsupported in the +4.x contract. LangBot 4.x does not migrate LangBot 3.x configuration or +databases, so the removed field is not read as a compatibility fallback. -class BoxSpec(pydantic.BaseModel): - # ... existing fields ... - host_path: str | None = None # Primary mount (backward compat) - host_path_mode: BoxHostMountMode = BoxHostMountMode.READ_WRITE - mount_path: str = DEFAULT_BOX_MOUNT_PATH - extra_mounts: list[BoxMountSpec] = [] # NEW: additional mounts -``` +## 8. Regression coverage -`extra_mounts` is additive — the existing `host_path` / `mount_path` pair remains -the primary mount for backward compatibility. +Release tests should prove: -### 5.2 Backend: Apply Extra Mounts - -```python -# box/backend.py — CLISandboxBackend.start_session() - -# Primary mount (unchanged) -if spec.host_path is not None and spec.host_path_mode != BoxHostMountMode.NONE: - args.extend(['-v', f'{spec.host_path}:{spec.mount_path}:{spec.host_path_mode.value}']) - -# Extra mounts (NEW) -for mount in spec.extra_mounts: - if mount.mode != BoxHostMountMode.NONE: - args.extend(['-v', f'{mount.host_path}:{mount.mount_path}:{mount.mode.value}']) -``` - -Same pattern for nsjail backend. - ---- - -## 6. LangBot Changes - -### 6.1 Session ID Resolution - -In `BoxService.execute_tool()`: - -```python -# Before: -spec_payload.setdefault('session_id', str(query.query_id)) - -# After: -template = (query.pipeline_config or {}).get('ai', {}) \ - .get('local-agent', {}).get('box-session-id-template', - '{launcher_type}_{launcher_id}') -variables = query.variables or {} -session_id = template.format_map(collections.defaultdict( - lambda: 'unknown', variables -)) -spec_payload.setdefault('session_id', session_id) -``` - -### 6.2 Skill Exec: Use Same Container - -Currently `native.py:_invoke_exec` creates a separate `BoxWorkspaceSession` per -skill with `host_path=package_root`. Instead: - -1. Use the **same session_id** as default exec (from the template). -2. Pass the skill's `package_root` as an **extra mount** at - `/workspace/.skills/{skill_name}/` instead of replacing `/workspace`. -3. The container already has the default workspace at `/workspace`. - -```python -# native.py — _invoke_exec, skill branch (REVISED) - -# Same session_id as default exec -session_id = resolve_box_session_id(query) - -spec_payload = { - 'cmd': rewritten_command, - 'workdir': rewritten_workdir, - 'session_id': session_id, - 'extra_mounts': [{ - 'host_path': package_root, - 'mount_path': f'/workspace/.skills/{selected_skill_name}', - 'mode': 'rw', - }], -} -result = await self.ap.box_service.execute_spec_payload(spec_payload, query) -``` - -The virtual path `/workspace/.skills/{name}` no longer needs rewriting at the -command level — it maps directly to the bind mount path inside the container. - -### 6.3 MCP: Use Same Container - -MCP servers should run inside the same container as exec and skills. Changes: - -1. `BoxStdioSessionRuntime` uses the pipeline's session_id template instead of - `mcp-{server_uuid}`. -2. MCP server's working directory is a subdirectory (e.g. `/workspace/.mcp/{name}/`). -3. MCP server's dependencies are mounted or installed into that subdirectory. -4. The MCP server runs as a managed process inside the shared container. - -Since MCP servers start at LangBot boot (not per-query), the session must be -created eagerly. The container will be kept alive by the managed process -exemption in TTL reaping (`runtime.py:259`). - -**Note**: MCP sessions are pipeline-scoped (not per-launcher), so their session_id -should be a **fixed identifier per pipeline** rather than the user-facing template. -This means one shared MCP container per pipeline, with user exec sessions separate. - -Alternatively, in a future iteration, MCP managed processes could be launched -lazily into the user's container on first MCP tool call. This is more complex -but maximizes sharing. For V1, keeping MCP containers at pipeline scope is -simpler and more predictable. - ---- - -## 7. Mount Layout Summary - -### Default exec (no skills activated) - -``` -Container (session_id from template) - /workspace/ ← default_host_workspace (rw) -``` - -### Exec with activated skills - -``` -Container (same session_id) - /workspace/ ← default_host_workspace (rw) - /workspace/.skills/web-search/ ← skill package_root (rw) - /workspace/.skills/data-analysis/ ← skill package_root (rw) -``` - -Extra mounts are **additive** — they are added when the container is first -created (or on the first exec that references a skill). Since Docker bind -mounts are specified at container creation time, skills must be known at -creation time. - -**Resolution**: When creating a container, inject `extra_mounts` for **all -pipeline-bound skills** (from `extensions_preferences`), not just the -currently activated one. This way any skill can be activated later without -recreating the container. - -### MCP servers (V1: pipeline-scoped) - -``` -Container (session_id = "mcp-pipeline-{pipeline_uuid}") - /workspace/ ← MCP shared workspace - /workspace/.mcp/server-a/ ← MCP server A files - /workspace/.mcp/server-b/ ← MCP server B files - [managed process: server-a] - [managed process: server-b] -``` - ---- - -## 8. Data Migration - -Existing pipelines do not have `box-session-id-template`. The backend uses -`.get(..., default)` so missing keys fall back to `{launcher_type}_{launcher_id}`. -This changes behavior from per-message to per-launcher for existing pipelines. - -Recommendation: **accept the behavior change** — per-launcher is the more -intuitive default, and the old per-message behavior was rarely desired. - ---- - -## 9. Cloud Quota Implications - -| Scope | Typical concurrent containers | -|-----------------------------------------------|-------------------------------| -| `{query_id}` (per message) | Many, short-lived | -| `{launcher_type}_{launcher_id}` (per chat) | = active chat count | -| `{sender_id}` (per user) | = active user count | -| `{conversation_id}` (per conversation) | Between per-chat and per-msg | - -With the unified container model, each scope value maps to exactly **one** -container (instead of potentially 3+ per-message). This significantly reduces -resource usage. - -Quota enforcement point: `BoxRuntime._get_or_create_session()` in the SDK. - ---- - -## 10. Implementation Phases - -### Phase 1: Session scope + skill unification (this PR) - -1. **SDK**: Extend `BoxSpec` with `extra_mounts: list[BoxMountSpec]`. -2. **SDK**: Update Docker/nsjail backends to apply extra mounts. -3. **LangBot**: Add `box-session-id-template` to `local-agent` YAML metadata - and default pipeline config JSON. -4. **LangBot**: Update `BoxService.execute_tool()` to use template interpolation. -5. **LangBot**: Update `native.py:_invoke_exec` skill branch to use same - session_id + extra mounts instead of separate `BoxWorkspaceSession`. -6. **LangBot**: On container creation, inject extra mounts for all - pipeline-bound skills. -7. **Frontend**: No code change — `DynamicFormComponent` renders `select` fields. -8. **Tests**: Unit tests for template interpolation and multi-mount specs. - -### Phase 2: MCP unification (future) - -1. Refactor `BoxStdioSessionRuntime` to use pipeline-scoped shared container. -2. MCP servers become managed processes in the shared container. -3. Support multiple concurrent managed processes per container. - -MCP unification is deferred because it requires changes to the managed process -model (currently 1 managed process per session) and has startup ordering -concerns (MCP servers start at boot, before any user query determines -a session_id). +- every event-run session id matches `lb-box-[0-9a-f]{64}` and contains no raw + identity; +- the same canonical Host scope is stable while different targets, + conversations, threads, bots, adapters, workspaces, or instances are + isolated; +- Pipeline and pure EBA runs representing the same platform session produce + the same canonical scope; +- missing Host/Query identity fails closed; +- SDK/Python `call_tool` and the scoped MCP bridge both enter + `PluginToRuntimeAction.CALL_TOOL` and restore the run execution Query; +- Runner payload/config cannot override the session id; +- stdio MCP processes remain in `mcp-shared` and are isolated by process id; +- authorized skills are mounted into the hashed run session without creating + per-skill sessions. diff --git a/docs/review/box-test-coverage.md b/docs/review/box-test-coverage.md index 995e6970b..11c53a8d2 100644 --- a/docs/review/box-test-coverage.md +++ b/docs/review/box-test-coverage.md @@ -1,6 +1,6 @@ # Box 系统测试覆盖分析 -> 更新日期: 2026-06-02 +> 更新日期: 2026-07-12 > 状态更新: 自部署社区版已具备发布条件（box 可选、降级完善、无迁移欠债）；工具调用循环上限、配额遍历异步化、`host_path` 挂载白名单等已落地。剩余多租户 / 安全硬化项见 [SaaS 阻塞项清单](./box-issues.md)。 > 分支: `feat/sandbox` (LangBot + langbot-plugin-sdk) @@ -15,13 +15,14 @@ | `tests/unit_tests/box/test_box_connector.py` | 106 | 是 | Connector 传输决策、WS relay URL、dispose、心跳/重连 | | `tests/unit_tests/box/test_box_service.py` | 1224 | 是 | Service 核心逻辑（最全面） | | `tests/unit_tests/box/test_workspace.py` | 147 | 是 | WorkspaceSession 路径重写、payload 构建 | +| `tests/unit_tests/agent/test_execution_context.py` | 144 | 是 | Pipeline/纯 EBA execution Query、canonical Host scope、adapter/instance 隔离 | +| `tests/unit_tests/plugin/test_handler_actions.py` | 1071 | 是 | AgentRun CALL_TOOL 恢复 execution Query、纯 EBA native exec | | `tests/unit_tests/provider/test_mcp_box_integration.py` | 707 | 是 | MCP Box 配置、路径重写、payload、shared-session/multi-process、runtime info | -| `tests/unit_tests/provider/test_localagent_sandbox_exec.py` | 444 | 是 | LocalAgent exec 流程、流式、Skill 激活 (Tool Call) | | `tests/unit_tests/provider/test_tool_manager_native.py` | 249 | 是 | ToolManager 路由、native tool CRUD、路径穿越、6 工具暴露 | | `tests/unit_tests/provider/test_skill_tools.py` | 582 | 是 | Skill 管理、Tool Call 激活、路径、authoring CRUD | | `tests/unit_tests/test_skill_service.py` | 396 | 是 | HTTP service：skill CRUD、zip/GitHub install、文件浏览 | | `tests/unit_tests/test_paths.py` | 23 | 是 | paths 工具 | -| `tests/unit_tests/test_preproc.py` | 134 | 是 | PreProcessor 注入 session 变量、bound skill 解析 | +| `tests/unit_tests/test_preproc.py` | 134 | 是 | PreProcessor 的模型、历史与 bound skill 解析 | | `tests/unit_tests/pipeline/test_chat_handler_logging.py` | 78 | 是 | Chat handler 日志相关回归 | | `tests/integration_tests/box/test_box_integration.py` | 329 | **否** | 真实容器执行、超时、网络隔离 | | `tests/integration_tests/box/test_box_mcp_integration.py` | 368 | **否** | Managed process、WS attach、shared-session 清理 | @@ -35,7 +36,7 @@ | `tests/box/test_e2b_backend.py` | 482 | 是 | E2B SDK mock、session 生命周期、extra_mounts 同步 | | `tests/box/test_skill_store.py` | 88 | 是 | zip preview/install、基础 file CRUD | -**总计**: 17 个测试文件, ~6,500 行测试代码; 其中 2 个集成测试（约 700 行）在 CI 中不运行。 +**说明**: 本表按当前主链列出 Box 相关测试；其中 2 个真实容器集成测试默认不在 CI 中运行。 > 较 2026-04-16 版增加：`test_skill_service.py`、`test_paths.py`、`test_preproc.py`、`test_chat_handler_logging.py` (LangBot)，`test_backend_selection.py`、`test_e2b_backend.py`、`test_skill_store.py` (SDK)。`test_nsjail_backend.py` 增加 CLI 兼容性 case (commit `feed530`)。 @@ -51,7 +52,7 @@ | BoxService workspace quota | 优秀 | 前置/后置配额检查、超额清理 | | BoxService 输出截断 | 优秀 | 短/精确边界/长输出、独立 stderr | | BoxService 可观测性 | 优秀 | 状态报告、error ring buffer、buffer 上限 | -| BoxService session 模板 | 良好 | `resolve_box_session_id` + `build_skill_extra_mounts` 在 service / native / mcp 三处都有覆盖 | +| BoxService Host-owned session | 优秀 | 覆盖 `lb-box-` 固定格式、原始 identity 不泄露、同 scope 稳定、不同 conversation/scope 隔离、缺 identity fail closed | | RPC client/server 协议 | 优秀 | execute/get_sessions/delete/create/conflict error | | BoxRuntimeConnector | 良好 | local/remote 模式、Docker 平台、relay URL、心跳与重连回调 | | BoxWorkspaceSession | 良好 | payload 构建、managed process 路径重写、stage host file | @@ -61,7 +62,7 @@ | Backend selection | 良好 | 显式 backend 优先级、local 探测顺序、配置变更触发 reselect | | MCP Box 集成 | 良好 | config model、路径重写、payload、shared-session 多 process | | Native tool loader | 良好 | 6 工具（exec/read/write/edit/glob/grep）、路径穿越拦截 | -| LocalAgent exec 流程 | 良好 | 完整 tool call 循环、流式、system prompt 注入、Tool Call 激活 | +| Runner 工具入口 | 良好 | SDK proxy 与 MCP bridge 都映射到 `PluginToRuntimeAction.CALL_TOOL`；Host action 测试覆盖 run-scoped execution Query 与纯 EBA native exec | | Skill 系统 | 良好 | 加载、Tool Call 激活、marker、路径解析、authoring CRUD、HTTP service | --- diff --git a/docs/review/mcp-resources-pr-2215-review.md b/docs/review/mcp-resources-pr-2215-review.md index 8e57fc5fa..1d2153212 100644 --- a/docs/review/mcp-resources-pr-2215-review.md +++ b/docs/review/mcp-resources-pr-2215-review.md @@ -3,24 +3,24 @@ > 更新日期: 2026-06-29 > 分支: `mcp_resources` > PR: langbot-app/LangBot#2215 -> 主题: MCP Resources 在 LangBot 中的产品价值、AgentRunner 集成方式与后续架构方向 +> 主题: MCP Resources 在 LangBot 中的产品价值、Runner 集成方式与后续架构方向 ## 结论 PR #2215 对 LangBot 有明确价值：它补齐了 MCP 协议中 Resources 这一重要能力，让 MCP server 不再只暴露 tools，也可以暴露文档、代码片段、配置、日志、图片等上下文资源。管理端可以发现和预览资源，Agent 也可以通过当前实现按需列出和读取资源。 -但当前 AgentRunner 层的接入方式更接近一个可用的第一阶段方案，而不是最终架构。现在 MCP Resources 被包装成两个 synthetic tools： +但当前 Runner 层的接入方式更接近一个可用的第一阶段方案，而不是最终架构。现在 MCP Resources 被包装成两个 synthetic tools： - `langbot_mcp_list_resources` - `langbot_mcp_read_resource` -这让模型可以通过 function calling 主动探索资源，落地成本低，也复用了已有 `ToolManager` / `LocalAgentRunner` 的工具调用链路。不过从 MCP 规范和主流实现来看，Resources 更适合作为一种一等上下文来源，而不是长期隐藏在工具列表里。 +这让模型可以通过 function calling 主动探索资源，落地成本低，也复用了已有 `ToolManager` / `LocalRunner` 的工具调用链路。不过从 MCP 规范和主流实现来看，Resources 更适合作为一种一等上下文来源，而不是长期隐藏在工具列表里。 建议保留当前 synthetic tools 作为探索能力，同时把后续主线设计调整为：MCP Resources 是 pipeline / conversation / message 级别可选择、可固定、可审计的上下文输入。 ## 当前实现判断 -当前 AgentRunner 集成路径如下： +当前 Runner 集成路径如下： ```text Pipeline 绑定 MCP server @@ -28,7 +28,7 @@ Pipeline 绑定 MCP server -> Preproc 为 local-agent 加载工具 -> ToolManager.get_all_tools() -> MCPLoader 注入 synthetic resource tools - -> LocalAgentRunner 将工具 schema 传给模型 + -> LocalRunner 将工具 schema 传给模型 -> 模型发起 list/read tool call -> ToolManager.execute_func_call() -> MCPLoader 调 MCP session.list_resources/read_resource @@ -178,7 +178,7 @@ LangBot 后续应支持模板发现、参数填写、实例化和绑定。否则 - 支持 resource templates。 - 支持资源订阅更新。 - 支持 chunk、summary、RAG 化接入。 -- 为 DifyAgentRunner、LocalAgentRunner 等不同 runner 定义统一资源上下文接口。 +- 为 DifyRunner、LocalRunner 等不同 runner 定义统一资源上下文接口。 ## 最终建议 diff --git a/docs/review/model-reasoning-control-design.md b/docs/review/model-reasoning-control-design.md index 95965ad2d..9238ea8c6 100644 --- a/docs/review/model-reasoning-control-design.md +++ b/docs/review/model-reasoning-control-design.md @@ -68,7 +68,7 @@ Anthropic、Google 和 LiteLLM 的官方文档域名在本次环境中被浏览 - `LLMModel.extra_args` 是 JSON 字段，Web 端已有通用高级参数编辑器。 - `LiteLLMRequester` 会按“模型级 `extra_args`，再调用级 `extra_args`”的顺序合并参数。 - LiteLLM 已统一处理多个 Provider 的 `reasoning_effort`、`thinking` 和返回的 `reasoning_content`。 -- `LocalAgentRunner` 的非流式、流式、工具调用和 fallback 路径都经过 `RuntimeProvider.invoke_llm*()`。 +- `LocalRunner` 的非流式、流式、工具调用和 fallback 路径都经过 `RuntimeProvider.invoke_llm*()`。 - `remove-think` 已能控制 `` 或独立 reasoning 内容是否进入展示文本。 - Gemini 工具调用所需的 `provider_specific_fields` / thought signature 已有保留逻辑和单元测试。 diff --git a/docs/review/omni-mainline-sync-2026-09-10.md b/docs/review/omni-mainline-sync-2026-09-10.md new file mode 100644 index 000000000..add380cc8 --- /dev/null +++ b/docs/review/omni-mainline-sync-2026-09-10.md @@ -0,0 +1,63 @@ +# Legacy and Omni adapter mainline audit + +Audit date: 2026-09-10. + +- Destination: `dev/4.11.x`, starting at `5ba25c39af85844283f7049beddc54ea71afea76`. +- Upstream: refreshed `origin/master`, `ce6b647fe7e620031c3dd9d16a8e8c08ba13b4b1`. +- Scope: adapter changes since the first Omni fork in March 2026, including shared platform clients, configuration manifests, and shutdown/resource handling. +- Existing local changes, including the Omni identifier rename, are preserved. + +The two implementations do not inherit behavior automatically. A commit in `sources/` does not update `adapters/`. The audit compares the current legacy source with master, then checks its actual equivalent in the Omni converter, transport, client, configuration, and lifecycle code. Native interaction handling continues to use `interaction.request` and typed callbacks; old Dify runner-private state is not copied into the new event architecture. + +## Platforms with an Omni adapter + +| Platform | Relevant mainline work | Legacy result | Omni result | +| --- | --- | --- | --- | +| OneBot / aiocqhttp | JSON cards (`c75890874`, `0963fd544`), base64 (`ccc51522c`), group metadata (`cc7a13158`), listener lifecycle (`2c3e52c16`), bounded lookups (`e1ac5e0fc`), original image URLs (`463b12092`) | Already present | Added JSON-card parsing and normalization for image, voice, and file payloads. Reused the cached, timeout-bounded group/member lookup per adapter instance. Preserved native handler registration, typed events, and original image URLs. | +| Telegram | Streaming (`0755beebc`), bounded media/state (`e1ac5e0fc`), token-bearing image URLs (`9df021eb8`) | Already present | Shares the legacy message converter; ported persistent-message streaming, throttling/size fallback, bounded state, and shutdown. Token-bearing download URLs remain absent from the public Image component. | +| Discord | Streaming (`0755beebc`), media/resource limits (`e1ac5e0fc`), original image URLs (`463b12092`) | Already present | Added snapshot-based send/edit streaming with bounded state. Enforced the mainline media limit on Omni image, voice, and file loading. Incoming public attachment URLs are preserved. | +| Feishu / Lark | Files (`e06fac2bb`), proactive sends (`3680a8024`), service domain (`c7cb42bd7`), tables (`1d15798e5`), nonblocking connection discovery (`48952206d`), resource bounds (`e1ac5e0fc`), final duplicate text (`5ca30133a`), feedback association (`f8010a20e`) | Applied final-text fix; retained the newer SDK cache-task cleanup | Added identical domestic/international/custom domain fields and applied the selected domain to both HTTP and WebSocket clients. Reused nonblocking connection discovery and bounded upload helpers; bounded incoming downloads and callback tasks. Added table cards and feedback-to-monitoring association. The final card contains one text element and exits streaming mode; it does not use the legacy duplicate-placeholder layout. | +| DingTalk | Voice recognition (`de4d14fee`), files (`e06fac2bb`), interactive cards (`0755beebc`), task/card bounds (`e1ac5e0fc`), automatic layout (`8cf001550`) | Applied layout fix in the shared client | Shared client fix reaches both `create_and_card` and native interaction cards. Retained typed file/voice conversion and native callbacks, bounded normal stream cards, cleared callback/card state on shutdown, and exposed the template download. | +| QQ Official | Optional token (`cb45807b1`), Markdown (`c87548c0b`), complete stream snapshots (`79634772d`), resource bounds (`e1ac5e0fc`), original image URLs (`463b12092`) | Applied optional-token fix | Removed the token requirement in constructor and manifest. Added Markdown configuration and sending. Stream and non-stream fallback replace full snapshots instead of appending deltas. Restored original image URLs, bounded state, and closed the owned client on shutdown. | +| WeCom application | Media ID key (`d942bfe19`), original image URLs (`463b12092`), client lifetime (`e1ac5e0fc`) | Applied the `media_id` send fix | Media dispatch already used `media_id`; restored original image URLs and client shutdown. | +| WeCom AI Bot | WS/files/feedback (`d9378c3a8`, `14b1e0d33`, `c7efa4dd7`, `83ccb33fd`), sandbox media (`e934f08ad`), WS loading (`e69a80f5e`) | Already present | Outbound conversion now retains media items. Replies and final stream chunks use the shared upload/reply implementation instead of `[Image]` / `[File]` placeholders. Existing native inbound file/voice/quote handling, feedback, and shared WS fixes remain active. | +| WeCom customer service | Proactive text (`13dba887d`), `open_kfid` and unique outbound `msgid` (`cabde423a`), client limits (`e1ac5e0fc`) | Already present | Existing target parsing and generated outbound message IDs already match. Added owned-client shutdown and cache clearing; removed a shadowed obsolete image-send definition, retaining the bounded active implementation. | +| Slack | Original image URLs (`463b12092`), shared HTTP session (`e1ac5e0fc`) | Already present | Restored original image URLs alongside downloaded data. Existing shared-client behavior remains active; bounded retained typed-event caches. | +| KOOK | Bounded gateway decompression, HTTP responses, and shared sessions (`e1ac5e0fc`) | Already present | Reuses the bounded gateway decoder off the event loop and limited JSON response reader. Existing URL-based image conversion does not eagerly download media. | +| WeChat Official Account | Resource/state cleanup (`e1ac5e0fc`) and outbound IP display (`bca710dbd`) | Already present | Calls the shared client's cleanup on shutdown, bounds typed-event caches, and exposes the outbound IP configuration field. | + +All 12 Omni manifests include the corresponding legacy configuration field names and current deployment help links. The Feishu/Lark `domain` and `custom_domain` fields are structurally identical, including labels, descriptions, options, defaults, and conditional display. Display names are unchanged. + +## Platforms with no separate Omni implementation + +| Platform | Result | +| --- | --- | +| Matrix | Applied `fc1c99843` to fix the unbound logout command during re-login. Existing connection/media/resource fixes already match master. | +| Mattermost | Added the complete mainline adapter, manifest, icon, and regression tests from `d6443b10b`. | +| LINE | Stable source-based sessions (`777fe1f20`), mention conversion (`855ae2bdb`), and resource limits already match master. | +| OpenClaw Weixin | Mainline adapter, outbound IP display, and resource/lifecycle changes already present. | +| WeChatPad | Mainline media/download and resource/lifecycle changes already present. | +| Satori | Mainline lifecycle/resource changes already present. | +| HTTP Bot | Mainline standalone HTTP integration and workspace/resource changes already present. | +| WebPage Bot | Delegated stream helpers and embedded-session behavior already match master. | +| WebSocket / browser debug | Mainline session isolation, scoped history, resource bounds, and routing work already present. Retained 4.11's image-reference retention and pipeline metadata additions. | +| Archived Gewechat, Nakuru, QQBotPy sources | Mainline source changes already present; no separate Omni counterpart exists. | + +Documentation-link updates `1cfe87186` and `ec63978ec` were also applied to legacy manifests and matching Omni manifests. + +## Intentional architectural differences + +- New typed events and platform APIs remain in the Omni modules. Legacy Pipeline message/context conversion stays in the established compatibility path. +- Native interaction delivery uses the generic interaction request/callback protocol. The current 4.11 pipeline/provider code no longer emits `_form_data` or `_resume_from_form`; copying old Dify callback internals would reconnect the wrong execution model. +- This audit establishes parity for the mainline changes in scope, not a claim that every pre-fork legacy-only extension has a typed Omni API. For example, Discord voice-channel management still belongs to the legacy source; sending voice attachments and the post-fork streaming/resource fixes are covered here. +- Shared platform clients retain 4.11-specific return values and additional APIs instead of being overwritten with entire master files. + +## Verification + +- Full backend unit suite: **3751 passed, 1 skipped**. This includes platform transport tests, Pipeline/event routing, service validation, plugin actions, workspace isolation, and resource handling. +- Dedicated mainline-parity regression suite: **62 passed**. Coverage for configuration coverage, OneBot JSON/base64/metadata, QQ Markdown/snapshots, WeCom media replies, bounded media/gateway reads, lifecycle cleanup, streaming, and Feishu/Lark region selection. +- Ruff formatting and checks pass on the affected Python files. +- Restarted the local backend on port 5399; `/healthz` returns success and reports the plugin runtime connected. All 12 Omni icon endpoints and the new Mattermost icon return HTTP 200. +- Vendor requests in automated tests are mocked. Real delivery to every external platform has not been tested. The instance contains pre-existing incomplete Feishu and WeCom AI Bot configurations; the same missing-credential errors were present before this audit. + +Tests: `tests/unit_tests/platform/test_omni_mainline_parity.py`, the platform-specific test files, and the imported Mattermost/WeCom/Lark/DingTalk regressions. diff --git a/packaging/fnos/LICENSE b/packaging/fnos/LICENSE index e297f1789..a2028f684 100644 --- a/packaging/fnos/LICENSE +++ b/packaging/fnos/LICENSE @@ -142,4 +142,3 @@ https://github.com/langbot-app/LangBot/blob/master/LICENSE 或 LangBot 源码包内的 LICENSE 文件。 数据收集政策：https://docs.langbot.app/zh/insight/data-collection-policy - diff --git a/pyproject.toml b/pyproject.toml index 04692deb8..ba56f6a37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "langbot" -version = "4.10.11" +version = "4.11.0-beta.3" description = "Production-grade platform for building agentic IM bots" readme = "README.md" license-files = ["LICENSE"] @@ -129,7 +129,7 @@ requires = ["setuptools>=61.0", "wheel"] build-backend = "setuptools.build_meta" [tool.setuptools] -package-data = { "langbot" = ["templates/**", "pkg/provider/modelmgr/requesters/*", "pkg/platform/sources/*", "web/dist/**", "pkg/persistence/alembic/**"] } +package-data = { "langbot" = ["templates/**", "pkg/provider/modelmgr/requesters/*", "pkg/platform/sources/*", "pkg/platform/adapters/**", "web/dist/**", "pkg/persistence/alembic/**"] } [dependency-groups] dev = [ diff --git a/scripts/stamp_build_revision.py b/scripts/stamp_build_revision.py new file mode 100644 index 000000000..1db6c2969 --- /dev/null +++ b/scripts/stamp_build_revision.py @@ -0,0 +1,23 @@ +"""Stamp the exact source revision into distributable artifacts (build time only).""" + +import argparse +from pathlib import Path +import re +import subprocess + + +def stamp(root: Path, revision: str | None = None) -> str: + if revision is None: + revision = subprocess.check_output(['git', 'rev-parse', '--verify', 'HEAD'], cwd=root, text=True).strip() + if not re.fullmatch(r'[0-9a-f]{40}', revision): + raise ValueError('Build revision must be a full lowercase Git SHA') + target = root / 'src/langbot/_build_info.py' + target.write_text(f'"""Exact source revision stamped at artifact build time."""\n\nCORE_REVISION = {revision!r}\n') + return revision + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--revision') + args = parser.parse_args() + print(stamp(Path(__file__).resolve().parents[1], args.revision)) diff --git a/skills/docs/user-guide.md b/skills/docs/user-guide.md index 124d3af36..19eed56b6 100644 --- a/skills/docs/user-guide.md +++ b/skills/docs/user-guide.md @@ -98,6 +98,23 @@ message truncation. Do not treat it as a long-term QA contract. ## 5. Run Gates +Cross-repository contract gate (no model calls): + +```bash +bin/lbs suite plan langbot-workspace-contract-gate +bin/lbs suite run langbot-workspace-contract-gate --run-id langbot-workspace-contract-local +``` + +Top-down workspace release gate (browser workflows plus one complex Agent task): + +```bash +bin/lbs suite plan langbot-workspace-release-gate +bin/lbs suite run langbot-workspace-release-gate --run-id langbot-workspace-release-local --include-manual-check +``` + +Run `agent-run-ledger-audit` immediately after the complex Agent case. Set +`LANGBOT_AGENT_RUN_ID` when the test instance has concurrent operator traffic. + Fast contract gate, no live service required: ```bash diff --git a/skills/package.json b/skills/package.json index 4b9f42be7..e7f3ecdae 100644 --- a/skills/package.json +++ b/skills/package.json @@ -13,7 +13,7 @@ "pretest": "node scripts/bootstrap-lbs.mjs", "precheck": "node scripts/bootstrap-lbs.mjs", "lbs": "node src/lbs.ts", - "test": "node test/lbs-cli.test.ts", + "test": "node test/lbs-cli.test.ts && python3 -m unittest discover -s test -p 'test_*.py'", "validate": "node src/lbs.ts validate", "index": "node src/lbs.ts index", "index:check": "node src/lbs.ts index --check", diff --git a/skills/qa-agent-docs/qa-agent/02-log-guard-plan.md b/skills/qa-agent-docs/qa-agent/02-log-guard-plan.md index a166b1378..da7568c61 100644 --- a/skills/qa-agent-docs/qa-agent/02-log-guard-plan.md +++ b/skills/qa-agent-docs/qa-agent/02-log-guard-plan.md @@ -99,7 +99,7 @@ LangBot 是异步且集成度高的系统，有些问题不会直接表现为页 ```text Action list_plugins call timed out -Action list_agent_runners call timed out +Action list_runners call timed out Action invoke_llm_stream call timed out ``` diff --git a/skills/qa-agent-docs/user-guide.md b/skills/qa-agent-docs/user-guide.md index f19beeb97..62373a159 100644 --- a/skills/qa-agent-docs/user-guide.md +++ b/skills/qa-agent-docs/user-guide.md @@ -61,7 +61,7 @@ bin/lbs fixture check ``` `env doctor` 会检查 URL、路径、代理变量等。代理变量是可选项；只有大小写代理变量互相冲突时才会报错。失败不一定代表仓库坏了，通常说明本地 LangBot 没启动、代理不一致或浏览器 profile 不存在。 -`fixture check` 会检查仓库内测试 fixture 是否存在，例如 MCP stdio server、RAG 文档、多模态图片、qa-plugin-smoke 包和 QA AgentRunner 包。它也会校验 `.lbpkg` 是 zip 包，并检查 QA AgentRunner fixture 的入口文件未漂移。 +`fixture check` 会检查仓库内测试 fixture 是否存在，例如 MCP stdio server、RAG 文档、多模态图片、qa-plugin-smoke 包和 QA Runner 包。它也会校验 `.lbpkg` 是 zip 包，并检查 QA Runner fixture 的入口文件未漂移。 4. 查看已有测试 case： @@ -344,8 +344,8 @@ npx playwright install chromium 脚本会尝试通过 `LANGBOT_PIPELINE_NAME` 从 Pipelines 页面进入目标 pipeline。两者都没有时， 该自动化会返回 `blocked`，不会伪造通过。 -Runner 专用 case 不应复用通用 pipeline 变量。Local Agent、Codex AgentRunner 和 -Claude Code AgentRunner 这类 case 会通过 `automation_pipeline_url_env` / +Runner 专用 case 不应复用通用 pipeline 变量。Local Agent、Codex Runner 和 +Claude Code Runner 这类 case 会通过 `automation_pipeline_url_env` / `automation_pipeline_name_env` 映射到 case-specific env，例如 `LANGBOT_LOCAL_AGENT_PIPELINE_URL`。这些 case 如果缺少专用变量，会返回 `blocked`， 不会退回到 `LANGBOT_PIPELINE_URL`，避免跑错 pipeline 后产生假阳性。 diff --git a/skills/schemas/case.schema.json b/skills/schemas/case.schema.json index 46601142a..4100a1b9c 100644 --- a/skills/schemas/case.schema.json +++ b/skills/schemas/case.schema.json @@ -257,6 +257,22 @@ "type": "string", "enum": ["0", "1", "false", "true"] }, + "automation_debug_chat_load_require_success": { + "type": "string", + "enum": ["0", "1", "false", "true"] + }, + "automation_debug_chat_load_provider_model_thresholds_json": { + "type": "string" + }, + "automation_fake_provider_pipeline_name": { + "type": "string" + }, + "automation_fake_provider_model_name": { + "type": "string" + }, + "automation_fake_provider_fallback_model_names": { + "type": "string" + }, "automation_fake_provider_response_text": { "type": "string" }, @@ -275,6 +291,9 @@ "automation_fake_provider_fail_every_n": { "type": "string" }, + "automation_fake_provider_fail_models": { + "type": "string" + }, "automation_fake_provider_fault_status": { "type": "string" }, @@ -282,6 +301,16 @@ "type": "string", "enum": ["0", "1", "false", "true"] }, + "automation_fake_provider_fail_after_first_chunk_delay_ms": { + "type": "string" + }, + "automation_fake_provider_fail_after_first_chunk_mode": { + "type": "string", + "enum": ["disconnect", "error_event"] + }, + "automation_fake_provider_fail_after_first_chunk_models": { + "type": "string" + }, "automation_fake_provider_dynamic_response": { "type": "string", "enum": ["0", "1", "false", "true"] diff --git a/skills/scripts/e2e/agent-run-ledger-audit.mjs b/skills/scripts/e2e/agent-run-ledger-audit.mjs new file mode 100644 index 000000000..6ef20a966 --- /dev/null +++ b/skills/scripts/e2e/agent-run-ledger-audit.mjs @@ -0,0 +1,86 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { readFile } from "node:fs/promises"; +import { join } from "node:path"; +import { env } from "node:process"; +import { + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + resolveLangBotRepo, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "agent-run-ledger-audit"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const startedAt = new Date(); +const auditPath = join(paths.evidenceDir, "ledger-audit.json"); +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + audited_run: null, + metrics_summary: null, + failures: [], + warnings: [], + evidence: { + ledger_audit_json: auditPath, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["filesystem", "metrics", "api_diagnostic"], +}; + +function run(command, args, cwd) { + return new Promise((resolvePromise) => { + const child = spawn(command, args, { cwd, env, stdio: ["ignore", "pipe", "pipe"] }); + let stderr = ""; + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", (error) => resolvePromise({ status: null, stderr, error })); + child.on("close", (status) => resolvePromise({ status, stderr, error: null })); + }); +} + +try { + const repo = await resolveLangBotRepo(); + const python = join(repo, ".venv", "bin", "python"); + const args = ["scripts/e2e/agent-run-ledger-audit.py", "--repo", repo, "--output", auditPath]; + if (env.LANGBOT_AGENT_RUN_ID) args.push("--run-id", env.LANGBOT_AGENT_RUN_ID); + if (env.LANGBOT_AGENT_TOOL_AUTHORIZATION_MODE) { + args.push("--tool-authorization-mode", env.LANGBOT_AGENT_TOOL_AUTHORIZATION_MODE); + } + const execution = await run(python, args, process.cwd()); + const report = JSON.parse(await readFile(auditPath, "utf8")); + result.status = report.status; + result.reason = report.reason; + result.audited_run = report.run || null; + result.metrics_summary = report.metrics || null; + result.failures = report.failures || []; + result.warnings = report.warnings || []; + if (execution.error && result.status === "pass") { + result.status = "env_issue"; + result.reason = execution.error.message; + } +} catch (error) { + result.status = /ENOENT|not found|No matching/i.test(error.message) ? "env_issue" : "fail"; + result.reason = error.message; +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/agent-run-ledger-audit.py b/skills/scripts/e2e/agent-run-ledger-audit.py new file mode 100644 index 000000000..cce5177b3 --- /dev/null +++ b/skills/scripts/e2e/agent-run-ledger-audit.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +"""Audit one persisted Runner run without exposing authorization secrets.""" + +from __future__ import annotations + +import argparse +import asyncio +import datetime +import json +import pathlib +import re +import sys +import urllib.parse + +import sqlalchemy +import yaml +from sqlalchemy.ext.asyncio import create_async_engine + +from agent_run_ledger_policy import ( + classify_invalid_tool_argument_errors, + classify_tool_authorization, + invalid_tool_argument_error_signal, + load_ledger_json, +) + + +def database_url(repo: pathlib.Path) -> str: + config = yaml.safe_load((repo / 'data/config.yaml').read_text(encoding='utf-8')) or {} + database = config.get('database', {}) + kind = database.get('use', 'sqlite') + if kind == 'sqlite': + path = pathlib.Path(database.get('sqlite', {}).get('path', 'data/langbot.db')) + if not path.is_absolute(): + path = repo / path + return f'sqlite+aiosqlite:///{path}' + if kind in {'postgres', 'postgresql'}: + values = database.get('postgresql', {}) + user = urllib.parse.quote_plus(str(values.get('user', 'postgres'))) + password = urllib.parse.quote_plus(str(values.get('password', 'postgres'))) + host = values.get('host', '127.0.0.1') + port = values.get('port', 5432) + name = values.get('database', 'postgres') + return f'postgresql+asyncpg://{user}:{password}@{host}:{port}/{name}' + raise RuntimeError(f'Unsupported database backend: {kind}') + + +def parse_created_after(value: str | None) -> datetime.datetime | None: + if not value: + return None + parsed = datetime.datetime.fromisoformat(value.replace('Z', '+00:00')) + if parsed.tzinfo is not None: + parsed = parsed.astimezone(datetime.timezone.utc).replace(tzinfo=None) + return parsed + + +def event_matches_tool_call(data_json: str | None, tool_name: str, parameters: dict | None) -> bool: + try: + data = json.loads(data_json or '{}') + except (TypeError, ValueError): + return False + if not isinstance(data, dict) or data.get('tool_name') != tool_name: + return False + return parameters is None or data.get('parameters') == parameters + + +def collect_result_texts(value: object) -> list[str]: + texts: list[str] = [] + if isinstance(value, dict): + for key, item in value.items(): + if key == 'text' and isinstance(item, str): + texts.append(item) + else: + texts.extend(collect_result_texts(item)) + elif isinstance(value, list): + for item in value: + texts.extend(collect_result_texts(item)) + return texts + + +async def audit( + repo: pathlib.Path, + run_id: str | None, + *, + created_after: datetime.datetime | None = None, + expected_tool_name: str | None = None, + expected_parameters: dict | None = None, + expected_result_text: str | None = None, + tool_authorization_mode: str = 'strict', +) -> dict: + engine = create_async_engine(database_url(repo)) + failures: list[dict] = [] + warnings: list[dict] = [] + try: + async with engine.connect() as connection: + if run_id: + run_row = ( + ( + await connection.execute( + sqlalchemy.text('SELECT * FROM agent_run WHERE run_id = :run_id'), + {'run_id': run_id}, + ) + ) + .mappings() + .first() + ) + elif expected_tool_name: + query = 'SELECT * FROM agent_run' + params = {} + if created_after is not None: + query += ' WHERE created_at >= :created_after' + params['created_after'] = created_after + query += ' ORDER BY id DESC LIMIT 100' + candidates = (await connection.execute(sqlalchemy.text(query), params)).mappings().all() + run_row = None + for candidate in candidates: + started_rows = ( + ( + await connection.execute( + sqlalchemy.text( + 'SELECT data_json FROM agent_run_event ' + "WHERE run_id = :run_id AND type = 'tool.call.started' ORDER BY sequence" + ), + {'run_id': str(candidate['run_id'])}, + ) + ) + .mappings() + .all() + ) + if any( + event_matches_tool_call(row.get('data_json'), expected_tool_name, expected_parameters) + for row in started_rows + ): + run_row = candidate + break + else: + run_row = ( + (await connection.execute(sqlalchemy.text('SELECT * FROM agent_run ORDER BY id DESC LIMIT 1'))) + .mappings() + .first() + ) + if run_row is None: + status = 'fail' if expected_tool_name else 'env_issue' + return { + 'status': status, + 'reason': 'No Runner run contains the expected tool call.' + if expected_tool_name + else 'No matching Runner run exists.', + 'failures': [{'kind': 'expected_tool_call_missing'}] if expected_tool_name else [], + 'warnings': [], + } + + selected_run_id = str(run_row['run_id']) + event_rows = ( + ( + await connection.execute( + sqlalchemy.text( + 'SELECT sequence, type, data_json, metadata_json FROM agent_run_event WHERE run_id = :run_id ORDER BY sequence' + ), + {'run_id': selected_run_id}, + ) + ) + .mappings() + .all() + ) + finally: + await engine.dispose() + + authorization = load_ledger_json( + run_row.get('authorization_json'), + field='agent_run.authorization_json', + failures=failures, + ) + tools = authorization.get('resources', {}).get('tools', []) if isinstance(authorization, dict) else [] + allowed_tools: dict[str, dict] = {} + incomplete_tool_metadata: list[dict] = [] + for tool in tools if isinstance(tools, list) else []: + if not isinstance(tool, dict): + incomplete_tool_metadata.append({'tool_name': '', 'missing': ['tool object']}) + continue + name = str(tool.get('tool_name', '')) + missing = [] + if not name: + missing.append('tool_name') + if not str(tool.get('description', '')).strip(): + missing.append('description') + if not isinstance(tool.get('parameters'), dict): + missing.append('parameters') + if not (tool.get('source') or tool.get('tool_type') or tool.get('source_id')): + missing.append('owner') + if missing: + incomplete_tool_metadata.append({'tool_name': name, 'missing': missing}) + if name: + allowed_tools[name] = tool + if incomplete_tool_metadata: + failures.append({'kind': 'incomplete_tool_metadata', 'tools': incomplete_tool_metadata}) + + starts: dict[str, list[dict]] = {} + completions: dict[str, list[dict]] = {} + event_types: list[str] = [] + invalid_event_json = 0 + suspicious_errors: list[dict] = [] + invalid_tool_argument_errors: list[dict] = [] + successful_tool_completion_sequences: list[int] = [] + forbidden_pattern = re.compile( + r'invalid json(?! arguments)|unauthori[sz]ed|permission denied|forbidden|timed?\s*out|timeout', + re.I, + ) + + def error_surface(value: object) -> list[str]: + """Collect diagnostic fields without treating normal tool parameters as errors.""" + collected: list[str] = [] + if not isinstance(value, dict): + return collected + for key, item in value.items(): + normalized = str(key).lower() + if normalized in {'error', 'code', 'status', 'reason', 'error_message'} and item is not None and item != '': + collected.append(str(item)) + if isinstance(item, dict): + collected.extend(error_surface(item)) + return collected + + for row in event_rows: + event_type = str(row['type']) + event_types.append(event_type) + before = len(failures) + data = load_ledger_json( + row.get('data_json'), + field=f'agent_run_event[{row["sequence"]}].data_json', + failures=failures, + ) + invalid_event_json += int(len(failures) > before) + if not isinstance(data, dict): + failures.append({'kind': 'invalid_event_payload', 'sequence': row['sequence'], 'type': event_type}) + continue + if event_type in {'tool.call.started', 'tool.call.completed'}: + call_id = str(data.get('tool_call_id', '')) + item = {'sequence': row['sequence'], 'tool_name': str(data.get('tool_name', '')), 'data': data} + if not call_id: + failures.append({'kind': 'missing_tool_call_id', 'sequence': row['sequence'], 'type': event_type}) + elif event_type == 'tool.call.started': + starts.setdefault(call_id, []).append(item) + else: + completions.setdefault(call_id, []).append(item) + if not data.get('error') and data.get('result') is not None: + successful_tool_completion_sequences.append(row['sequence']) + diagnostic_text = '\n'.join(error_surface(data)) + if event_type == 'run.failed': + diagnostic_text += '\n' + json.dumps(data, ensure_ascii=True) + match = forbidden_pattern.search(diagnostic_text) + if match: + suspicious_errors.append({'sequence': row['sequence'], 'type': event_type, 'signal': match.group(0)}) + elif event_type == 'tool.call.completed': + signal = invalid_tool_argument_error_signal(diagnostic_text) + if signal: + invalid_tool_argument_errors.append({'sequence': row['sequence'], 'type': event_type, 'signal': signal}) + + if run_row['status'] != 'completed': + failures.append({'kind': 'run_status', 'actual': run_row['status'], 'expected': 'completed'}) + if 'run.completed' not in event_types: + failures.append({'kind': 'missing_run_completed_event'}) + if 'run.failed' in event_types: + failures.append({'kind': 'run_failed_event'}) + + all_call_ids = sorted(set(starts) | set(completions)) + unauthorized_calls = [] + for call_id in all_call_ids: + started = starts.get(call_id, []) + completed = completions.get(call_id, []) + if len(started) != 1 or len(completed) != 1: + failures.append( + { + 'kind': 'tool_call_pairing', + 'tool_call_id': call_id, + 'started': len(started), + 'completed': len(completed), + } + ) + continue + if started[0]['tool_name'] != completed[0]['tool_name']: + failures.append({'kind': 'tool_name_mismatch', 'tool_call_id': call_id}) + if started[0]['sequence'] >= completed[0]['sequence']: + failures.append({'kind': 'tool_call_order', 'tool_call_id': call_id}) + if started[0]['tool_name'] not in allowed_tools: + unauthorized_calls.append({'tool_call_id': call_id, 'tool_name': started[0]['tool_name']}) + authorization_failures, authorization_warnings = classify_tool_authorization( + unauthorized_calls, + authorization_mode=tool_authorization_mode, + ) + failures.extend(authorization_failures) + warnings.extend(authorization_warnings) + unrecovered_argument_errors, recovered_argument_warnings = classify_invalid_tool_argument_errors( + invalid_tool_argument_errors, + successful_tool_completion_sequences=successful_tool_completion_sequences, + run_completed=( + run_row['status'] == 'completed' and 'run.completed' in event_types and 'run.failed' not in event_types + ), + ) + if unrecovered_argument_errors: + suspicious_errors.extend(unrecovered_argument_errors) + warnings.extend(recovered_argument_warnings) + if suspicious_errors: + failures.append({'kind': 'forbidden_error_signals', 'events': suspicious_errors}) + if not event_rows: + failures.append({'kind': 'missing_run_events'}) + if not tools: + warnings.append({'kind': 'no_authorized_tools', 'reason': 'The run authorization snapshot exposes no tools.'}) + + expected_call_summary = None + if expected_tool_name: + matching_starts = [ + item + for items in starts.values() + for item in items + if item['tool_name'] == expected_tool_name + and (expected_parameters is None or item['data'].get('parameters') == expected_parameters) + ] + if len(matching_starts) != 1: + failures.append({'kind': 'expected_tool_call_count', 'actual': len(matching_starts), 'expected': 1}) + matching_completions = [] + for started in matching_starts: + call_id = str(started['data'].get('tool_call_id', '')) + matching_completions.extend(completions.get(call_id, [])) + result_text_match = expected_result_text is None or any( + expected_result_text in collect_result_texts(completed['data'].get('result')) + for completed in matching_completions + ) + if expected_result_text is not None and not result_text_match: + failures.append({'kind': 'expected_tool_result_text_missing'}) + expected_call_summary = { + 'tool_name': expected_tool_name, + 'parameters_match_required': expected_parameters is not None, + 'matched_started_count': len(matching_starts), + 'matched_completed_count': len(matching_completions), + 'result_text_match_required': expected_result_text is not None, + 'result_text_match': result_text_match, + } + + metrics = { + 'event_count': len(event_rows), + 'tool_call_started': sum(len(items) for items in starts.values()), + 'tool_call_completed': sum(len(items) for items in completions.values()), + 'tool_call_ids': len(all_call_ids), + 'authorized_tool_count': len(allowed_tools), + 'tool_authorization_mode': tool_authorization_mode, + 'runner_native_tool_call_count': len(unauthorized_calls) if tool_authorization_mode == 'runner-native' else 0, + 'invalid_event_json': invalid_event_json, + 'suspicious_error_count': len(suspicious_errors), + 'recovered_tool_argument_error_count': len(recovered_argument_warnings), + } + return { + 'status': 'pass' if not failures else 'fail', + 'reason': 'Agent run ledger audit passed.' + if not failures + else f'Agent run ledger audit found {len(failures)} invariant failure(s).', + 'run': { + 'run_id': selected_run_id, + 'runner_id': run_row['runner_id'], + 'status': run_row['status'], + 'created_at': str(run_row['created_at']), + 'finished_at': str(run_row['finished_at']), + }, + 'metrics': metrics, + 'expected_tool_call': expected_call_summary, + 'failures': failures, + 'warnings': warnings, + } + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument('--repo', required=True) + parser.add_argument('--run-id') + parser.add_argument('--created-after') + parser.add_argument('--expected-tool-name') + parser.add_argument('--expected-parameters-json') + parser.add_argument('--expected-result-text') + parser.add_argument( + '--tool-authorization-mode', + choices=('strict', 'runner-native'), + default='strict', + ) + parser.add_argument('--output', required=True) + args = parser.parse_args() + try: + expected_parameters = None + if args.expected_parameters_json: + expected_parameters = json.loads(args.expected_parameters_json) + if not isinstance(expected_parameters, dict): + raise ValueError('--expected-parameters-json must decode to an object') + if (expected_parameters is not None or args.expected_result_text) and not args.expected_tool_name: + raise ValueError('--expected-tool-name is required with expected parameters or result text') + report = asyncio.run( + audit( + pathlib.Path(args.repo).resolve(), + args.run_id, + created_after=parse_created_after(args.created_after), + expected_tool_name=args.expected_tool_name, + expected_parameters=expected_parameters, + expected_result_text=args.expected_result_text, + tool_authorization_mode=args.tool_authorization_mode, + ) + ) + except Exception as exc: # noqa: BLE001 - probe must classify environment failures + report = {'status': 'env_issue', 'reason': str(exc), 'failures': [], 'warnings': []} + pathlib.Path(args.output).write_text(json.dumps(report, indent=2) + '\n', encoding='utf-8') + print(json.dumps(report)) + return 0 if report['status'] == 'pass' else 2 if report['status'] == 'env_issue' else 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/skills/scripts/e2e/agent-runner-health-visibility.mjs b/skills/scripts/e2e/agent-runner-health-visibility.mjs new file mode 100644 index 000000000..393ae1a7c --- /dev/null +++ b/skills/scripts/e2e/agent-runner-health-visibility.mjs @@ -0,0 +1,265 @@ +#!/usr/bin/env node + +import { + apiJson, + createBrowser, + ensureAuthenticatedBrowser, + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + safeScreenshot, + scanBrowserDiagnostics, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +const caseId = "agent-runner-health-visibility"; +await loadEnvFiles(); +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const readyScreenshot = paths.screenshot.replace(/\.png$/, "-ready.png"); +const unavailableScreenshot = paths.screenshot.replace( + /\.png$/, + "-unavailable.png", +); + +const startedAt = new Date(); +const frontendUrl = process.env.LANGBOT_FRONTEND_URL || ""; +const backendUrl = process.env.LANGBOT_BACKEND_URL || ""; +const missingRunnerId = "plugin:qa/missing-runner/default"; + +let browser; +let token = ""; +let agentId = ""; +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + url: "", + visible_signals: [], + api: {}, + diagnostics: null, + cleanup: null, + evidence: { + console_log: paths.consoleLog, + network_log: paths.networkLog, + screenshot: paths.screenshot, + ready_screenshot: readyScreenshot, + unavailable_screenshot: unavailableScreenshot, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["ui", "screenshot", "console", "api_diagnostic"], +}; + +function schemaDefaults(items = []) { + return Object.fromEntries( + items + .filter((item) => item.name && Object.hasOwn(item, "default")) + .map((item) => [item.name, item.default]), + ); +} + +async function openRunnerSettings(page, agentUrl) { + await page.goto(agentUrl, { waitUntil: "domcontentloaded" }); + await page + .getByRole("button", { name: /Runner|运行器|ランナー/ }) + .first() + .waitFor({ timeout: 15_000 }); + await page + .getByRole("button", { name: /Runner|运行器|ランナー/ }) + .first() + .click(); +} + +try { + if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured."); + if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured."); + + browser = await createBrowser(paths); + const { page } = browser; + await page.goto(frontendUrl, { waitUntil: "domcontentloaded" }); + const auth = await ensureAuthenticatedBrowser(page, { + frontendUrl, + backendUrl, + }); + if (auth.status !== "pass") { + result.status = auth.status; + throw new Error(auth.reason); + } + + token = await page.evaluate(() => localStorage.getItem("token") || ""); + if (!token) { + result.status = "blocked"; + throw new Error("Authenticated browser has no reusable local token."); + } + + const pluginStatus = await apiJson( + backendUrl, + "/api/v1/system/status/plugin-system", + { token }, + ); + const pluginData = pluginStatus.json.data || {}; + result.api.plugin_status = { + http_status: pluginStatus.status, + code: pluginStatus.json.code ?? null, + is_enable: pluginData.is_enable ?? null, + is_connected: pluginData.is_connected ?? null, + }; + if (pluginStatus.status >= 400 || pluginStatus.json.code !== 0) { + result.status = "env_issue"; + throw new Error(pluginStatus.json.msg || "Plugin status request failed."); + } + if (!pluginData.is_enable || !pluginData.is_connected) { + result.status = "env_issue"; + throw new Error("The plugin runtime is not enabled and connected."); + } + + const metadata = await apiJson(backendUrl, "/api/v1/agents/_/metadata", { + token, + }); + const runnerTab = metadata.json.data?.runner_config; + const runnerStage = runnerTab?.stages?.find( + (stage) => stage.name === "runner", + ); + const runnerOptions = + runnerStage?.config?.find((item) => item.name === "id")?.options || []; + const runner = runnerOptions[0]; + result.api.agent_metadata = { + http_status: metadata.status, + code: metadata.json.code ?? null, + runner_count: runnerOptions.length, + selected_runner: runner?.name || null, + }; + if (metadata.status >= 400 || metadata.json.code !== 0) { + throw new Error(metadata.json.msg || "Agent metadata request failed."); + } + if (!runner?.name) { + result.status = "blocked"; + throw new Error("No registered Runner is available for the UI check."); + } + + const runnerConfigStage = runnerTab.stages.find( + (stage) => stage.name === runner.name, + ); + const create = await apiJson(backendUrl, "/api/v1/agents", { + method: "POST", + token, + body: { + kind: "agent", + name: `Runner Health ${paths.runId.slice(-40)}`, + description: "Temporary Runner health visibility fixture", + emoji: "H", + component_ref: runner.name, + config: { + runner: { id: runner.name, "expire-time": 0 }, + runner_config: { + [runner.name]: schemaDefaults(runnerConfigStage?.config), + }, + }, + enabled: true, + supported_event_patterns: ["message.*"], + }, + }); + agentId = create.json.data?.uuid || ""; + result.api.create_agent = { + http_status: create.status, + code: create.json.code ?? null, + }; + if (create.status >= 400 || create.json.code !== 0 || !agentId) { + throw new Error(create.json.msg || "Failed to create the temporary Agent."); + } + + const agentUrl = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(agentId)}`; + result.url = agentUrl; + await openRunnerSettings(page, agentUrl); + await page + .getByText(/Runner ready|运行器已就绪|Runner の準備完了/, { exact: true }) + .waitFor({ timeout: 15_000 }); + result.visible_signals.push("registered-runner-ready"); + await safeScreenshot(page, readyScreenshot); + + const staleUpdate = await apiJson( + backendUrl, + `/api/v1/agents/${encodeURIComponent(agentId)}`, + { + method: "PUT", + token, + body: { + component_ref: missingRunnerId, + config: { + runner: { id: missingRunnerId, "expire-time": 0 }, + runner_config: { [missingRunnerId]: {} }, + }, + }, + }, + ); + result.api.set_stale_runner = { + http_status: staleUpdate.status, + code: staleUpdate.json.code ?? null, + }; + if (staleUpdate.status >= 400 || staleUpdate.json.code !== 0) { + throw new Error( + staleUpdate.json.msg || "Failed to set the stale runner fixture.", + ); + } + + await openRunnerSettings(page, agentUrl); + await page + .getByText( + /Selected runner is unavailable|所选运行器不可用|選択した Runner は利用できません/, + { exact: true }, + ) + .waitFor({ timeout: 15_000 }); + await page.getByRole("link", { name: /Extensions|扩展|拡張機能/ }).waitFor(); + result.visible_signals.push("stale-runner-unavailable", "recovery-action"); + await safeScreenshot(page, unavailableScreenshot); + await safeScreenshot(page, paths.screenshot); + + result.diagnostics = await scanBrowserDiagnostics(paths); + if (result.diagnostics.status !== "pass") { + throw new Error(result.diagnostics.reason); + } + result.status = "pass"; + result.reason = + "Runner settings visibly distinguished a registered runner from a stale binding."; +} catch (error) { + if (!["blocked", "env_issue"].includes(result.status)) result.status = "fail"; + result.reason = result.reason || error.message; + if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); +} finally { + const cleanup = {}; + if (agentId && token && backendUrl) { + const deletedAgent = await apiJson( + backendUrl, + `/api/v1/agents/${encodeURIComponent(agentId)}`, + { method: "DELETE", token }, + ).catch((error) => ({ + status: 0, + json: { code: null, msg: error.message }, + })); + cleanup.agent_deleted = + deletedAgent.status < 400 && deletedAgent.json.code === 0; + cleanup.agent_http_status = deletedAgent.status; + } + result.cleanup = cleanup; + if (agentId && !cleanup.agent_deleted && result.status === "pass") { + result.status = "fail"; + result.reason = "The temporary runner health Agent was not deleted."; + } + if (browser) await browser.close().catch(() => {}); + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/agent-runner-release-preflight.mjs b/skills/scripts/e2e/agent-runner-release-preflight.mjs index 6ac69f2f1..bd81220c7 100755 --- a/skills/scripts/e2e/agent-runner-release-preflight.mjs +++ b/skills/scripts/e2e/agent-runner-release-preflight.mjs @@ -6,6 +6,7 @@ import { resolve } from "node:path"; import { env } from "node:process"; import { createBrowser, + ensureBrowserWorkspace, ensureEvidence, evidencePaths, exitCode, @@ -23,7 +24,10 @@ function loadEnvDefaults(path) { if (sep === -1) continue; const key = line.slice(0, sep).trim(); if (env[key]) continue; - env[key] = line.slice(sep + 1).trim().replace(/^["']|["']$/g, ""); + env[key] = line + .slice(sep + 1) + .trim() + .replace(/^["']|["']$/g, ""); } } @@ -45,12 +49,16 @@ function redactMessage(text) { return String(text ?? "") .replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]") .replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]") - .replace(/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, "$1=[redacted]"); + .replace( + /(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, + "$1=[redacted]", + ); } function isEnvironmentError(message) { - return /Playwright is not installed|LANGBOT_FRONTEND_URL|LANGBOT_BACKEND_URL|ERR_CONNECTION_REFUSED|ECONNREFUSED|net::ERR_|fetch failed|timed out/i - .test(message); + return /Playwright is not installed|LANGBOT_FRONTEND_URL|LANGBOT_BACKEND_URL|ERR_CONNECTION_REFUSED|ECONNREFUSED|net::ERR_|fetch failed|timed out/i.test( + message, + ); } loadEnvDefaults("skills/.env"); @@ -70,7 +78,7 @@ const startedAt = new Date(); const targets = [ { id: "local-agent", - expected_runner_id: "plugin:langbot/local-agent/default", + expected_runner_id: "plugin:langbot-team/LocalAgent/default", pipeline_url: firstEnv("LANGBOT_LOCAL_AGENT_PIPELINE_URL"), pipeline_name: firstEnv("LANGBOT_LOCAL_AGENT_PIPELINE_NAME"), require_func_call_model: true, @@ -79,9 +87,15 @@ const targets = [ }, { id: "acp-agent-runner", - expected_runner_id: "plugin:langbot/acp-agent-runner/default", - pipeline_url: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", "LANGBOT_AGENT_RUNNER_PIPELINE_URL"), - pipeline_name: firstEnv("LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME", "LANGBOT_AGENT_RUNNER_PIPELINE_NAME"), + expected_runner_id: "plugin:langbot-team/ACPRunner/default", + pipeline_url: firstEnv( + "LANGBOT_ACP_RUNNER_PIPELINE_URL", + "LANGBOT_RUNNER_PIPELINE_URL", + ), + pipeline_name: firstEnv( + "LANGBOT_ACP_RUNNER_PIPELINE_NAME", + "LANGBOT_RUNNER_PIPELINE_NAME", + ), require_func_call_model: false, require_vision_model: false, }, @@ -110,323 +124,458 @@ const result = { automation_result_json: paths.automationResultJson, result_json: paths.resultJson, }, - evidence_collected: ["ui", "screenshot", "console", "network", "api_diagnostic"], + evidence_collected: [ + "ui", + "screenshot", + "console", + "network", + "api_diagnostic", + ], }; async function run() { if (!backendUrl || !frontendUrl) { result.status = "env_issue"; - result.reason = "LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured."; + result.reason = + "LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured."; return; } browser = await createBrowser(paths); const { page } = browser; await page.goto(frontendUrl, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + await page + .waitForLoadState("networkidle", { timeout: 10_000 }) + .catch(() => {}); + const workspace = await ensureBrowserWorkspace(page, backendUrl); + if (workspace.status !== "pass") { + result.status = workspace.status; + result.reason = workspace.reason; + return; + } - const diagnostic = await page.evaluate(async ({ backendUrl, targets, testModels }) => { - const blockers = []; - const envIssues = []; - const warnings = []; - const checks = []; + const diagnostic = await page.evaluate( + async ({ backendUrl, targets, testModels }) => { + const blockers = []; + const envIssues = []; + const warnings = []; + const checks = []; - const addCheck = (name, status, detail = {}) => { - checks.push({ name, status, ...detail }); - if (status === "blocked") blockers.push({ name, ...detail }); - if (status === "env_issue") envIssues.push({ name, ...detail }); - }; - const safeMessage = (value) => String(value ?? "") - .replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]") - .replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]") - .replace(/(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, "$1=[redacted]"); - - const token = localStorage.getItem("token"); - if (!token) { - addCheck("browser-auth", "blocked", { reason: "Browser profile has no localStorage token." }); - return { authenticated: false, blockers, env_issues: envIssues, warnings, checks }; - } - - const headers = { - Authorization: `Bearer ${token}`, - "Content-Type": "application/json", - }; - const getJson = async (path) => { - const response = await fetch(`${backendUrl}${path}`, { headers }); - return { - status: response.status, - json: await response.json().catch(() => ({})), + const addCheck = (name, status, detail = {}) => { + checks.push({ name, status, ...detail }); + if (status === "blocked") blockers.push({ name, ...detail }); + if (status === "env_issue") envIssues.push({ name, ...detail }); }; - }; - const postJson = async (path, body) => { - const response = await fetch(`${backendUrl}${path}`, { - method: "POST", - headers, - body: JSON.stringify(body), - }); - return { - status: response.status, - json: await response.json().catch(() => ({})), - }; - }; + const safeMessage = (value) => + String(value ?? "") + .replace(/\bbearer\s+[A-Za-z0-9._~+/=-]{8,}/gi, "Bearer [redacted]") + .replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]") + .replace( + /(api[_-]?key|authorization|credential|jwt|oauth|password|secret|token)\s*[:=]\s*["']?[^"',\s]+/gi, + "$1=[redacted]", + ); - const tokenCheck = await getJson("/api/v1/user/check-token"); - addCheck( - "browser-auth", - tokenCheck.status < 400 && (tokenCheck.json.code ?? 0) === 0 ? "pass" : "blocked", - { http_status: tokenCheck.status, code: tokenCheck.json.code ?? null, reason: safeMessage(tokenCheck.json.msg || "") }, - ); - - const systemInfo = await getJson("/api/v1/system/info"); - addCheck( - "backend-system-info", - systemInfo.status < 400 ? "pass" : "env_issue", - { - http_status: systemInfo.status, - version: systemInfo.json.data?.version || systemInfo.json.data?.system?.version || "", - }, - ); - - const pluginSystem = await getJson("/api/v1/system/status/plugin-system"); - addCheck( - "plugin-system", - pluginSystem.status < 400 && (pluginSystem.json.code ?? 0) === 0 ? "pass" : "env_issue", - { - http_status: pluginSystem.status, - code: pluginSystem.json.code ?? null, - status: pluginSystem.json.data?.status || pluginSystem.json.data?.state || "", - reason: safeMessage(pluginSystem.json.msg || ""), - }, - ); - - const boxStatus = await getJson("/api/v1/box/status"); - addCheck( - "box-runtime", - boxStatus.status < 400 && (boxStatus.json.code ?? 0) === 0 ? "pass" : "env_issue", - { - http_status: boxStatus.status, - code: boxStatus.json.code ?? null, - status: boxStatus.json.data?.status || "", - backend: boxStatus.json.data?.backend || "", - reason: safeMessage(boxStatus.json.msg || ""), - }, - ); - - const plugins = await getJson("/api/v1/plugins"); - const installedPluginIds = (plugins.json.data?.plugins || []) - .map((plugin) => { - const metadata = plugin.manifest?.manifest?.metadata || plugin.manifest?.metadata || plugin.metadata || {}; - return metadata.author && metadata.name ? `${metadata.author}/${metadata.name}` : ""; - }) - .filter(Boolean); - const requiredPlugins = ["langbot/local-agent", "langbot/acp-agent-runner", "qa/plugin-smoke"]; - const pluginPresence = Object.fromEntries(requiredPlugins.map((id) => [id, installedPluginIds.includes(id)])); - for (const [id, present] of Object.entries(pluginPresence)) { - addCheck(`plugin:${id}`, present ? "pass" : "blocked", { plugin_id: id, reason: present ? "" : "Required plugin is not listed by /api/v1/plugins." }); - } - - const tools = await getJson("/api/v1/tools"); - const toolNames = (tools.json.data?.tools || []) - .map((tool) => tool.name || tool.tool_name || tool.function?.name || "") - .filter(Boolean) - .sort(); - addCheck( - "tool:qa_plugin_echo", - toolNames.includes("qa_plugin_echo") ? "pass" : "blocked", - { reason: toolNames.includes("qa_plugin_echo") ? "" : "qa-plugin-smoke tool qa_plugin_echo is not exposed through /api/v1/tools." }, - ); - if (!toolNames.includes("qa_mcp_echo")) { - warnings.push({ - name: "tool:qa_mcp_echo", - reason: "qa_mcp_echo is not currently exposed. This is acceptable before mcp-stdio-register, but mcp-stdio-tool-call must run after registration.", - }); - } - - const modelResponse = await getJson("/api/v1/provider/models/llm"); - const models = (modelResponse.json.data?.models || []).map((model) => ({ - uuid: model.uuid, - name: model.name, - abilities: Array.isArray(model.abilities) ? model.abilities : [], - provider_uuid: model.provider_uuid || model.provider?.uuid || "", - provider_name: model.provider_name || model.provider?.name || "", - requester: model.requester || model.provider?.requester || "", - })); - addCheck( - "llm-model-list", - modelResponse.status < 400 && (modelResponse.json.code ?? 0) === 0 ? "pass" : "env_issue", - { http_status: modelResponse.status, model_count: models.length, reason: safeMessage(modelResponse.json.msg || "") }, - ); - const modelById = new Map(models.map((model) => [model.uuid, model])); - - const pipelineList = await getJson("/api/v1/pipelines"); - const pipelines = pipelineList.json.data?.pipelines || []; - addCheck( - "pipeline-list", - pipelineList.status < 400 && (pipelineList.json.code ?? 0) === 0 ? "pass" : "blocked", - { http_status: pipelineList.status, pipeline_count: pipelines.length, reason: safeMessage(pipelineList.json.msg || "") }, - ); - - const resolvedPipelines = []; - const modelTested = new Set(); - for (const target of targets) { - let pipelineId = ""; - let matchedBy = ""; - if (target.pipeline_url) { - try { - pipelineId = new URL(target.pipeline_url).searchParams.get("id") || ""; - matchedBy = pipelineId ? "url" : ""; - } catch { - pipelineId = ""; - } - } - if (!pipelineId && target.pipeline_name) { - const match = pipelines.find((pipeline) => pipeline.name === target.pipeline_name); - if (match) { - pipelineId = match.uuid; - matchedBy = "name"; - } - } - if (!pipelineId) { - addCheck(`pipeline:${target.id}`, "blocked", { - target: target.id, - reason: "Required pipeline env is missing or could not resolve to a pipeline id.", + const token = localStorage.getItem("token"); + if (!token) { + addCheck("browser-auth", "blocked", { + reason: "Browser profile has no localStorage token.", }); - continue; + return { + authenticated: false, + blockers, + env_issues: envIssues, + warnings, + checks, + }; } - const response = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`); - const pipeline = response.json.data?.pipeline; - if (response.status >= 400 || !pipeline) { - addCheck(`pipeline:${target.id}`, "blocked", { - target: target.id, - pipeline_id: pipelineId, - http_status: response.status, - reason: safeMessage(response.json.msg || "Could not load pipeline."), + const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-Workspace-Id": + localStorage.getItem("langbot_active_workspace_uuid") || "", + }; + const getJson = async (path) => { + const response = await fetch(`${backendUrl}${path}`, { headers }); + return { + status: response.status, + json: await response.json().catch(() => ({})), + }; + }; + const postJson = async (path, body) => { + const response = await fetch(`${backendUrl}${path}`, { + method: "POST", + headers, + body: JSON.stringify(body), }); - continue; - } - - const config = pipeline.config || {}; - const aiConfig = config.ai && typeof config.ai === "object" ? config.ai : {}; - const runner = aiConfig.runner && typeof aiConfig.runner === "object" ? aiConfig.runner : {}; - const runnerId = runner.id || runner.runner || ""; - const runnerConfigs = aiConfig.runner_config && typeof aiConfig.runner_config === "object" ? aiConfig.runner_config : {}; - const runnerConfig = runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object" ? runnerConfigs[runnerId] : {}; - const pipelineSummary = { - target: target.id, - pipeline_id: pipelineId, - pipeline_name: pipeline.name, - matched_by: matchedBy, - runner_id: runnerId, - expected_runner_id: target.expected_runner_id, - runner_config_keys: Object.keys(runnerConfig).sort(), + return { + status: response.status, + json: await response.json().catch(() => ({})), + }; }; - resolvedPipelines.push(pipelineSummary); + const tokenCheck = await getJson("/api/v1/user/check-token"); addCheck( - `pipeline:${target.id}:runner`, - runnerId === target.expected_runner_id ? "pass" : "blocked", + "browser-auth", + tokenCheck.status < 400 && (tokenCheck.json.code ?? 0) === 0 + ? "pass" + : "blocked", { - ...pipelineSummary, - reason: runnerId === target.expected_runner_id ? "" : `Expected ${target.expected_runner_id}, got ${runnerId || ""}.`, + http_status: tokenCheck.status, + code: tokenCheck.json.code ?? null, + reason: safeMessage(tokenCheck.json.msg || ""), }, ); - if (target.require_func_call_model || target.require_vision_model || (testModels && target.id === "local-agent")) { - const modelConfig = runnerConfig.model; - const primaryModelId = typeof modelConfig === "string" - ? modelConfig - : modelConfig && typeof modelConfig === "object" - ? modelConfig.primary || "" + const systemInfo = await getJson("/api/v1/system/info"); + addCheck( + "backend-system-info", + systemInfo.status < 400 ? "pass" : "env_issue", + { + http_status: systemInfo.status, + version: + systemInfo.json.data?.version || + systemInfo.json.data?.system?.version || + "", + }, + ); + + const pluginSystem = await getJson("/api/v1/system/status/plugin-system"); + addCheck( + "plugin-system", + pluginSystem.status < 400 && (pluginSystem.json.code ?? 0) === 0 + ? "pass" + : "env_issue", + { + http_status: pluginSystem.status, + code: pluginSystem.json.code ?? null, + status: + pluginSystem.json.data?.status || + pluginSystem.json.data?.state || + "", + reason: safeMessage(pluginSystem.json.msg || ""), + }, + ); + + const boxStatus = await getJson("/api/v1/box/status"); + addCheck( + "box-runtime", + boxStatus.status < 400 && (boxStatus.json.code ?? 0) === 0 + ? "pass" + : "env_issue", + { + http_status: boxStatus.status, + code: boxStatus.json.code ?? null, + status: boxStatus.json.data?.status || "", + backend: boxStatus.json.data?.backend || "", + reason: safeMessage(boxStatus.json.msg || ""), + }, + ); + + const plugins = await getJson("/api/v1/plugins"); + const installedPluginIds = (plugins.json.data?.plugins || []) + .map((plugin) => { + const metadata = + plugin.manifest?.manifest?.metadata || + plugin.manifest?.metadata || + plugin.metadata || + {}; + return metadata.author && metadata.name + ? `${metadata.author}/${metadata.name}` : ""; - if (!primaryModelId) { - addCheck(`pipeline:${target.id}:primary-model`, "blocked", { - ...pipelineSummary, - reason: "Local-agent runner config has no primary model.", - }); - continue; - } - const model = modelById.get(primaryModelId); - if (!model) { - addCheck(`pipeline:${target.id}:primary-model`, "blocked", { - ...pipelineSummary, - model_uuid: primaryModelId, - reason: "Primary model is not listed by /api/v1/provider/models/llm.", - }); - continue; - } - addCheck(`pipeline:${target.id}:primary-model`, "pass", { - ...pipelineSummary, - model: { - uuid: model.uuid, - name: model.name, - abilities: model.abilities, - provider_name: model.provider_name, - requester: model.requester, - }, + }) + .filter(Boolean); + const requiredPlugins = [ + "langbot-team/LocalAgent", + "langbot-team/ACPRunner", + "qa/plugin-smoke", + ]; + const pluginPresence = Object.fromEntries( + requiredPlugins.map((id) => [id, installedPluginIds.includes(id)]), + ); + for (const [id, present] of Object.entries(pluginPresence)) { + addCheck(`plugin:${id}`, present ? "pass" : "blocked", { + plugin_id: id, + reason: present + ? "" + : "Required plugin is not listed by /api/v1/plugins.", }); - if (target.require_func_call_model) { - addCheck( - `pipeline:${target.id}:func-call-model`, - model.abilities.includes("func_call") ? "pass" : "env_issue", - { - model_uuid: model.uuid, - model_name: model.name, - abilities: model.abilities, - reason: model.abilities.includes("func_call") ? "" : "Release gate includes tool-call cases; the local-agent primary model must advertise func_call.", - }, - ); + } + + const tools = await getJson("/api/v1/tools"); + const toolNames = (tools.json.data?.tools || []) + .map((tool) => tool.name || tool.tool_name || tool.function?.name || "") + .filter(Boolean) + .sort(); + addCheck( + "tool:qa_plugin_echo", + toolNames.includes("qa_plugin_echo") ? "pass" : "blocked", + { + reason: toolNames.includes("qa_plugin_echo") + ? "" + : "qa-plugin-smoke tool qa_plugin_echo is not exposed through /api/v1/tools.", + }, + ); + if (!toolNames.includes("qa_mcp_echo")) { + warnings.push({ + name: "tool:qa_mcp_echo", + reason: + "qa_mcp_echo is not currently exposed. This is acceptable before mcp-stdio-register, but mcp-stdio-tool-call must run after registration.", + }); + } + + const modelResponse = await getJson("/api/v1/provider/models/llm"); + const models = (modelResponse.json.data?.models || []).map((model) => ({ + uuid: model.uuid, + name: model.name, + abilities: Array.isArray(model.abilities) ? model.abilities : [], + provider_uuid: model.provider_uuid || model.provider?.uuid || "", + provider_name: model.provider_name || model.provider?.name || "", + requester: model.requester || model.provider?.requester || "", + })); + addCheck( + "llm-model-list", + modelResponse.status < 400 && (modelResponse.json.code ?? 0) === 0 + ? "pass" + : "env_issue", + { + http_status: modelResponse.status, + model_count: models.length, + reason: safeMessage(modelResponse.json.msg || ""), + }, + ); + const modelById = new Map(models.map((model) => [model.uuid, model])); + + const pipelineList = await getJson("/api/v1/pipelines"); + const pipelines = pipelineList.json.data?.pipelines || []; + addCheck( + "pipeline-list", + pipelineList.status < 400 && (pipelineList.json.code ?? 0) === 0 + ? "pass" + : "blocked", + { + http_status: pipelineList.status, + pipeline_count: pipelines.length, + reason: safeMessage(pipelineList.json.msg || ""), + }, + ); + + const resolvedPipelines = []; + const modelTested = new Set(); + for (const target of targets) { + let pipelineId = ""; + let matchedBy = ""; + if (target.pipeline_url) { + try { + pipelineId = + new URL(target.pipeline_url).searchParams.get("id") || ""; + matchedBy = pipelineId ? "url" : ""; + } catch { + pipelineId = ""; + } } - if (target.require_vision_model) { - addCheck( - `pipeline:${target.id}:vision-model`, - model.abilities.includes("vision") ? "pass" : "env_issue", - { - model_uuid: model.uuid, - model_name: model.name, - abilities: model.abilities, - reason: model.abilities.includes("vision") ? "" : "Release gate includes multimodal cases; the local-agent primary model must advertise vision.", - }, + if (!pipelineId && target.pipeline_name) { + const match = pipelines.find( + (pipeline) => pipeline.name === target.pipeline_name, ); + if (match) { + pipelineId = match.uuid; + matchedBy = "name"; + } } - if (testModels && !modelTested.has(model.uuid)) { - modelTested.add(model.uuid); - const modelTest = await postJson(`/api/v1/provider/models/llm/${encodeURIComponent(model.uuid)}/test`, { extra_args: {} }); - const passed = modelTest.status < 400 && (modelTest.json.code ?? 0) === 0; - addCheck( - `model-test:${model.name}`, - passed ? "pass" : "env_issue", - { - model_uuid: model.uuid, - model_name: model.name, - http_status: modelTest.status, - code: modelTest.json.code ?? null, - reason: passed ? "" : safeMessage(modelTest.json.msg || modelTest.json.message || "Model test failed."), + if (!pipelineId) { + addCheck(`pipeline:${target.id}`, "blocked", { + target: target.id, + reason: + "Required pipeline env is missing or could not resolve to a pipeline id.", + }); + continue; + } + + const response = await getJson( + `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, + ); + const pipeline = response.json.data?.pipeline; + if (response.status >= 400 || !pipeline) { + addCheck(`pipeline:${target.id}`, "blocked", { + target: target.id, + pipeline_id: pipelineId, + http_status: response.status, + reason: safeMessage( + response.json.msg || "Could not load pipeline.", + ), + }); + continue; + } + + const config = pipeline.config || {}; + const aiConfig = + config.ai && typeof config.ai === "object" ? config.ai : {}; + const runner = + aiConfig.runner && typeof aiConfig.runner === "object" + ? aiConfig.runner + : {}; + const runnerId = runner.id || ""; + const runnerConfigs = + aiConfig.runner_config && typeof aiConfig.runner_config === "object" + ? aiConfig.runner_config + : {}; + const runnerConfig = + runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object" + ? runnerConfigs[runnerId] + : {}; + const pipelineSummary = { + target: target.id, + pipeline_id: pipelineId, + pipeline_name: pipeline.name, + matched_by: matchedBy, + runner_id: runnerId, + expected_runner_id: target.expected_runner_id, + runner_config_keys: Object.keys(runnerConfig).sort(), + }; + resolvedPipelines.push(pipelineSummary); + + addCheck( + `pipeline:${target.id}:runner`, + runnerId === target.expected_runner_id ? "pass" : "blocked", + { + ...pipelineSummary, + reason: + runnerId === target.expected_runner_id + ? "" + : `Expected ${target.expected_runner_id}, got ${runnerId || ""}.`, + }, + ); + + if ( + target.require_func_call_model || + target.require_vision_model || + (testModels && target.id === "local-agent") + ) { + const modelConfig = runnerConfig.model; + const primaryModelId = + typeof modelConfig === "string" + ? modelConfig + : modelConfig && typeof modelConfig === "object" + ? modelConfig.primary || "" + : ""; + if (!primaryModelId) { + addCheck(`pipeline:${target.id}:primary-model`, "blocked", { + ...pipelineSummary, + reason: "Local-agent runner config has no primary model.", + }); + continue; + } + const model = modelById.get(primaryModelId); + if (!model) { + addCheck(`pipeline:${target.id}:primary-model`, "blocked", { + ...pipelineSummary, + model_uuid: primaryModelId, + reason: + "Primary model is not listed by /api/v1/provider/models/llm.", + }); + continue; + } + addCheck(`pipeline:${target.id}:primary-model`, "pass", { + ...pipelineSummary, + model: { + uuid: model.uuid, + name: model.name, + abilities: model.abilities, + provider_name: model.provider_name, + requester: model.requester, }, - ); + }); + if (target.require_func_call_model) { + addCheck( + `pipeline:${target.id}:func-call-model`, + model.abilities.includes("func_call") ? "pass" : "env_issue", + { + model_uuid: model.uuid, + model_name: model.name, + abilities: model.abilities, + reason: model.abilities.includes("func_call") + ? "" + : "Release gate includes tool-call cases; the local-agent primary model must advertise func_call.", + }, + ); + } + if (target.require_vision_model) { + addCheck( + `pipeline:${target.id}:vision-model`, + model.abilities.includes("vision") ? "pass" : "env_issue", + { + model_uuid: model.uuid, + model_name: model.name, + abilities: model.abilities, + reason: model.abilities.includes("vision") + ? "" + : "Release gate includes multimodal cases; the local-agent primary model must advertise vision.", + }, + ); + } + if (testModels && !modelTested.has(model.uuid)) { + modelTested.add(model.uuid); + const modelTest = await postJson( + `/api/v1/provider/models/llm/${encodeURIComponent(model.uuid)}/test`, + { extra_args: {} }, + ); + const passed = + modelTest.status < 400 && (modelTest.json.code ?? 0) === 0; + addCheck( + `model-test:${model.name}`, + passed ? "pass" : "env_issue", + { + model_uuid: model.uuid, + model_name: model.name, + http_status: modelTest.status, + code: modelTest.json.code ?? null, + reason: passed + ? "" + : safeMessage( + modelTest.json.msg || + modelTest.json.message || + "Model test failed.", + ), + }, + ); + } } } - } - return { - authenticated: true, - blockers, - env_issues: envIssues, - warnings, - checks, - resolved_pipelines: resolvedPipelines, - tools: { - required: ["qa_plugin_echo"], - optional_before_register: ["qa_mcp_echo"], - present: toolNames.filter((name) => ["qa_plugin_echo", "qa_mcp_echo"].includes(name)), - }, - models, - }; - }, { backendUrl, targets, testModels }); + return { + authenticated: true, + blockers, + env_issues: envIssues, + warnings, + checks, + resolved_pipelines: resolvedPipelines, + tools: { + required: ["qa_plugin_echo"], + optional_before_register: ["qa_mcp_echo"], + present: toolNames.filter((name) => + ["qa_plugin_echo", "qa_mcp_echo"].includes(name), + ), + }, + models, + }; + }, + { backendUrl, targets, testModels }, + ); - diagnostic.blockers = (diagnostic.blockers || []).map((item) => ({ ...item, reason: redactMessage(item.reason || "") })); - diagnostic.env_issues = (diagnostic.env_issues || []).map((item) => ({ ...item, reason: redactMessage(item.reason || "") })); - await writeFile(diagnosticPath, `${JSON.stringify(diagnostic, null, 2)}\n`, "utf8"); + diagnostic.blockers = (diagnostic.blockers || []).map((item) => ({ + ...item, + reason: redactMessage(item.reason || ""), + })); + diagnostic.env_issues = (diagnostic.env_issues || []).map((item) => ({ + ...item, + reason: redactMessage(item.reason || ""), + })); + await writeFile( + diagnosticPath, + `${JSON.stringify(diagnostic, null, 2)}\n`, + "utf8", + ); await safeScreenshot(page, paths.screenshot); const blockers = diagnostic.blockers || []; @@ -439,31 +588,49 @@ async function run() { result.reason = `Preflight environment issue: ${envIssues.map((item) => item.name).join(", ")}`; } else { result.status = "pass"; - result.reason = "Release gate preflight passed: auth, plugin runtime, required pipelines, runner ids, tools, and local-agent model checks are ready."; + result.reason = + "Release gate preflight passed: auth, plugin runtime, required pipelines, runner ids, tools, and local-agent model checks are ready."; } - result.check_count = Array.isArray(diagnostic.checks) ? diagnostic.checks.length : 0; - result.warning_count = Array.isArray(diagnostic.warnings) ? diagnostic.warnings.length : 0; + result.check_count = Array.isArray(diagnostic.checks) + ? diagnostic.checks.length + : 0; + result.warning_count = Array.isArray(diagnostic.warnings) + ? diagnostic.warnings.length + : 0; } try { await run(); } catch (error) { - const message = redactMessage(error instanceof Error ? error.message : String(error)); + const message = redactMessage( + error instanceof Error ? error.message : String(error), + ); result.status = isEnvironmentError(message) ? "env_issue" : "fail"; result.reason = message; - await writeFile(diagnosticPath, `${JSON.stringify({ - authenticated: false, - blockers: [], - env_issues: result.status === "env_issue" ? [{ name: "preflight-runtime", reason: message }] : [], - warnings: [], - checks: [ + await writeFile( + diagnosticPath, + `${JSON.stringify( { - name: "preflight-runtime", - status: result.status, - reason: message, + authenticated: false, + blockers: [], + env_issues: + result.status === "env_issue" + ? [{ name: "preflight-runtime", reason: message }] + : [], + warnings: [], + checks: [ + { + name: "preflight-runtime", + status: result.status, + reason: message, + }, + ], }, - ], - }, null, 2)}\n`, "utf8").catch(() => {}); + null, + 2, + )}\n`, + "utf8", + ).catch(() => {}); } finally { if (browser) await browser.close().catch(() => {}); const finishedAt = new Date(); diff --git a/skills/scripts/e2e/agent_run_ledger_policy.py b/skills/scripts/e2e/agent_run_ledger_policy.py new file mode 100644 index 000000000..9e4d0fc52 --- /dev/null +++ b/skills/scripts/e2e/agent_run_ledger_policy.py @@ -0,0 +1,77 @@ +"""Policy helpers for classifying Runner ledger error signals.""" + +from __future__ import annotations + +import json +import re + + +_INVALID_TOOL_ARGUMENT_PATTERN = re.compile( + r'invalid json arguments|\b\d+\s+validation errors?\s+for\s+[A-Za-z_][A-Za-z0-9_]*Args\b', + re.IGNORECASE, +) + + +def load_ledger_json(value: str | None, *, field: str, failures: list[dict]) -> object: + """Decode persisted ledger JSON and retain corruption as an invariant failure.""" + if not value: + return {} + try: + return json.loads(value) + except (TypeError, ValueError) as exc: + failures.append({'kind': 'invalid_json', 'field': field, 'reason': str(exc)}) + return {} + + +def invalid_tool_argument_error_signal(value: str) -> str: + """Return the persisted signal for malformed model-supplied tool arguments.""" + match = _INVALID_TOOL_ARGUMENT_PATTERN.search(value) + return match.group(0) if match else '' + + +def classify_invalid_tool_argument_errors( + events: list[dict], + *, + successful_tool_completion_sequences: list[int], + run_completed: bool, +) -> tuple[list[dict], list[dict]]: + """Split malformed tool arguments into recovered warnings and hard failures.""" + failures: list[dict] = [] + warnings: list[dict] = [] + for event in events: + recovered = run_completed and any( + sequence > event['sequence'] for sequence in successful_tool_completion_sequences + ) + if recovered: + warnings.append( + { + 'kind': 'recovered_tool_argument_error', + 'event': event, + 'reason': 'The model continued with a later successful tool call and the run completed.', + } + ) + else: + failures.append(event) + return failures, warnings + + +def classify_tool_authorization( + calls: list[dict], + *, + authorization_mode: str, +) -> tuple[list[dict], list[dict]]: + """Classify tool names absent from the Host authorization snapshot.""" + if not calls: + return [], [] + if authorization_mode == 'runner-native': + return [], [ + { + 'kind': 'runner_native_tool_calls', + 'calls': calls, + 'reason': ( + 'External runner tool telemetry is not a LangBot Host tool call; ' + "the runner's own permission system governs it." + ), + } + ] + return [{'kind': 'unauthorized_tool_calls', 'calls': calls}], [] diff --git a/skills/scripts/e2e/bot-event-routing-product-flow.mjs b/skills/scripts/e2e/bot-event-routing-product-flow.mjs new file mode 100644 index 000000000..d703ed3cd --- /dev/null +++ b/skills/scripts/e2e/bot-event-routing-product-flow.mjs @@ -0,0 +1,380 @@ +#!/usr/bin/env node + +import { + apiJson, + bodyText, + createBrowser, + ensureAuthenticatedBrowser, + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + safeScreenshot, + scanBrowserDiagnostics, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +const caseId = "bot-event-routing-product-flow"; +await loadEnvFiles(); +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const mobileScreenshot = paths.screenshot.replace(/\.png$/, "-mobile.png"); +const scenarioMenuScreenshot = paths.screenshot.replace( + /\.png$/, + "-scenario-menu.png", +); + +const startedAt = new Date(); +const frontendUrl = process.env.LANGBOT_FRONTEND_URL || ""; +const backendUrl = process.env.LANGBOT_BACKEND_URL || ""; +const fixtureName = `EBA Product Flow ${paths.runId}`; + +let browser; +let token = ""; +let botId = ""; +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + url: "", + bot_id: "", + visible_signals: [], + api: {}, + diagnostics: null, + cleanup: null, + evidence: { + console_log: paths.consoleLog, + network_log: paths.networkLog, + screenshot: paths.screenshot, + mobile_screenshot: mobileScreenshot, + scenario_menu_screenshot: scenarioMenuScreenshot, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["ui", "screenshot", "console", "api_diagnostic"], +}; + +try { + if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured."); + if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured."); + + browser = await createBrowser(paths); + const { page } = browser; + await page.goto(frontendUrl, { waitUntil: "domcontentloaded" }); + const auth = await ensureAuthenticatedBrowser(page, { + frontendUrl, + backendUrl, + }); + if (auth.status !== "pass") { + result.status = auth.status; + throw new Error(auth.reason); + } + + token = await page.evaluate(() => localStorage.getItem("token") || ""); + if (!token) { + result.status = "blocked"; + throw new Error("Authenticated browser has no reusable local token."); + } + + await page.goto(`${frontendUrl.replace(/\/$/, "")}/home/bots?id=new`, { + waitUntil: "domcontentloaded", + }); + await page + .getByText(/Create Bot|创建机器人|ボットを作成/) + .first() + .waitFor({ timeout: 15_000 }); + await page.getByRole("combobox").first().click(); + await page + .getByRole("option", { name: /HTTP Bot|HTTP 通用接入|HTTP ボット/ }) + .click(); + await page + .getByText(/Event Routing|事件路由|イベントルーティング/) + .first() + .waitFor(); + const addBehavior = page.getByRole("button", { + name: /Add behavior|添加行为|動作を追加/, + }); + await addBehavior.waitFor(); + await addBehavior.click(); + const messageBehavior = page.getByRole("menuitem", { + name: /Reply to messages|回复收到的消息|受信メッセージに返信/, + }); + const noEventRoutes = page.getByText( + /No event routes|暂无事件路由|イベントルートはありません/, + { exact: true }, + ); + await noEventRoutes.waitFor(); + try { + await messageBehavior.waitFor({ timeout: 5_000 }); + } catch { + // Async adapter fields can rerender once after the first menu click. + await addBehavior.click(); + await messageBehavior.waitFor(); + } + await page.waitForTimeout(250); + await safeScreenshot(page, scenarioMenuScreenshot); + await messageBehavior.click(); + await noEventRoutes.waitFor({ state: "hidden" }); + result.visible_signals.push("create-mode-routing", "scenario-route-added"); + + const create = await apiJson(backendUrl, "/api/v1/platform/bots", { + method: "POST", + token, + body: { + name: fixtureName, + description: "Temporary EBA product-flow fixture", + adapter: "http_bot", + adapter_config: { + inbound_secret: "eba-product-flow-local-only", + callback_url: "http://127.0.0.1:9/langbot-e2e-unused", + outbound_secret: "", + default_session_type: "person", + signature_required: false, + callback_timeout: 1, + callback_max_retries: 0, + }, + enable: true, + event_bindings: [ + { + id: "qa-message-discard", + event_pattern: "message.received", + target_type: "discard", + target_uuid: "", + filters: [], + priority: 0, + enabled: true, + description: "Discard the deterministic QA event", + order: 0, + }, + { + id: "qa-message-discard-shadowed", + event_pattern: "message.received", + target_type: "discard", + target_uuid: "", + filters: [], + priority: 0, + enabled: true, + description: "Verify visible route conflict guidance", + order: 1, + }, + ], + }, + }); + botId = create.json.data?.uuid || ""; + result.api.create_bot = { + http_status: create.status, + code: create.json.code ?? null, + }; + result.bot_id = botId; + if (create.status >= 400 || create.json.code !== 0 || !botId) { + throw new Error(create.json.msg || "Failed to create the temporary Bot."); + } + + const botUrl = `${frontendUrl.replace(/\/$/, "")}/home/bots?id=${encodeURIComponent(botId)}`; + await page.goto(botUrl, { waitUntil: "domcontentloaded" }); + await page + .waitForLoadState("networkidle", { timeout: 10_000 }) + .catch(() => {}); + result.url = page.url(); + + await page + .getByText(/Event Routing|事件路由|イベントルーティング/) + .first() + .waitFor({ timeout: 15_000 }); + await page.getByText(/Supported events|支持的事件|対応イベント/).waitFor(); + await page + .getByText(/Message received|收到消息|メッセージを受信/) + .first() + .waitFor(); + await page + .getByText( + /Some routes overlap|部分路由存在覆盖冲突|一部のルートが重複しています/, + ) + .waitFor(); + await page + .getByText( + /Events that match no route are ignored|未命中任何路由的事件会被忽略|どのルートにも一致しないイベントは無視されます/, + ) + .waitFor(); + result.visible_signals.push( + "event-routing", + "adapter-capabilities", + "friendly-event-name", + "route-conflict-guidance", + "fallback-guidance", + ); + + await page + .getByRole("button", { name: /Check route|检查路由|ルートを確認/ }) + .click(); + await page.getByRole("dialog").waitFor(); + await page + .getByRole("button", { name: /View match|查看匹配结果|一致結果を確認/ }) + .click(); + await page + .getByText(/Route matched|已命中路由|ルートに一致しました/) + .waitFor({ timeout: 15_000 }); + await page + .getByText(/Discard|丢弃|破棄/) + .first() + .waitFor(); + result.visible_signals.push("dry-run-matched", "discard-target"); + + await page + .getByRole("button", { name: /Close|关闭|閉じる/ }) + .first() + .click(); + await page.getByRole("dialog").waitFor({ state: "hidden" }); + + const adapterConfigCard = page.locator('[data-slot="card"]').filter({ + has: page.getByText(/Adapter Configuration|适配器配置|アダプター設定/, { + exact: true, + }), + }); + await adapterConfigCard + .getByRole("button", { + name: /Listen for platform events|监听平台事件|プラットフォームイベントを監視/, + }) + .click(); + const adapterDialog = page.getByRole("dialog"); + await adapterDialog.waitFor(); + await adapterDialog + .getByText(/Listening|正在监听|監視中/, { exact: true }) + .waitFor({ timeout: 15_000 }); + + const inboundText = `adapter event ${paths.runId}`; + const inbound = await apiJson( + backendUrl, + `/bots/${encodeURIComponent(botId)}`, + { + method: "POST", + token, + body: { + session_id: `adapter-debug-${paths.runId}`, + session_type: "person", + sender: { id: "adapter-debug-user", name: "Adapter QA" }, + message: [{ type: "Plain", text: inboundText }], + }, + }, + ); + result.api.adapter_event_webhook = { + http_status: inbound.status, + code: inbound.json.code ?? null, + }; + if (inbound.status >= 400 || inbound.json.code !== 0) { + throw new Error( + inbound.json.msg || "The HTTP Bot adapter rejected the inbound event.", + ); + } + + await adapterDialog + .getByText(/Message received|收到消息|メッセージ受信/, { exact: true }) + .waitFor({ timeout: 15_000 }); + await adapterDialog.getByText("message.received", { exact: true }).waitFor(); + await adapterDialog.getByText(inboundText, { exact: true }).waitFor(); + result.visible_signals.push( + "adapter-event-listening", + "adapter-event-received", + "adapter-event-raw-code", + ); + await adapterDialog + .getByRole("button", { name: /Close|关闭|閉じる/ }) + .click(); + await adapterDialog.waitFor({ state: "hidden" }); + + const text = await bodyText(page); + if (/\bEBA event\b/.test(text)) { + throw new Error( + "The primary route surface exposes an internal EBA runtime message.", + ); + } + + const status = await apiJson( + backendUrl, + `/api/v1/platform/bots/${encodeURIComponent(botId)}/event-routes/status`, + { token }, + ); + const latest = status.json.data?.routes?.find( + (route) => route.binding_id === "qa-message-discard", + ); + result.api.route_status = { + http_status: status.status, + code: status.json.code ?? null, + last_status: latest?.last_status || null, + }; + if ( + status.status >= 400 || + status.json.code !== 0 || + latest?.last_status !== "discarded" + ) { + throw new Error( + "Route status API did not confirm the discarded synthetic event.", + ); + } + + await safeScreenshot(page, paths.screenshot); + await page.setViewportSize({ width: 390, height: 844 }); + await page.waitForTimeout(250); + const horizontalOverflow = await page.evaluate( + () => document.documentElement.scrollWidth - window.innerWidth, + ); + if (horizontalOverflow > 1) { + throw new Error( + `The mobile route editor overflows horizontally by ${horizontalOverflow}px.`, + ); + } + await page + .getByText( + /Some routes overlap|部分路由存在覆盖冲突|一部のルートが重複しています/, + ) + .waitFor(); + await safeScreenshot(page, mobileScreenshot); + result.visible_signals.push("mobile-layout"); + result.diagnostics = await scanBrowserDiagnostics(paths); + if (result.diagnostics.status !== "pass") { + throw new Error(result.diagnostics.reason); + } + result.status = "pass"; + result.reason = + "Bot event routing, dry-run, real adapter input, and visible route status passed in the WebUI."; +} catch (error) { + if (!["blocked", "env_issue"].includes(result.status)) result.status = "fail"; + result.reason = result.reason || error.message; + if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); +} finally { + if (botId && token && backendUrl) { + const cleanup = await apiJson( + backendUrl, + `/api/v1/platform/bots/${encodeURIComponent(botId)}`, + { method: "DELETE", token }, + ).catch((error) => ({ + status: 0, + json: { code: null, msg: error.message }, + })); + result.cleanup = { + http_status: cleanup.status, + code: cleanup.json.code ?? null, + deleted: cleanup.status < 400 && cleanup.json.code === 0, + }; + if (!result.cleanup.deleted && result.status === "pass") { + result.status = "fail"; + result.reason = + cleanup.json.msg || "The temporary Bot could not be deleted."; + } + } + if (browser) await browser.close().catch(() => {}); + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/ensure-acp-agent-runner-pipeline.mjs b/skills/scripts/e2e/ensure-acp-agent-runner-pipeline.mjs index 1278fcd1b..0a83532c7 100755 --- a/skills/scripts/e2e/ensure-acp-agent-runner-pipeline.mjs +++ b/skills/scripts/e2e/ensure-acp-agent-runner-pipeline.mjs @@ -12,7 +12,7 @@ import { writeResult, } from "./lib/langbot-e2e.mjs"; -const RUNNER_ID = "plugin:langbot/acp-agent-runner/default"; +const RUNNER_ID = "plugin:langbot-team/ACPRunner/default"; const DEFAULT_PIPELINE_NAME = "Agent QA ACP Claude Debug Chat"; const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; const caseId = "ensure-acp-agent-runner-pipeline"; @@ -24,13 +24,18 @@ await ensureEvidence(paths); const writeEnv = process.argv.includes("--write-env"); const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; const backendUrl = env.LANGBOT_BACKEND_URL || ""; -const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME; -const sshTarget = env.LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET || "yhh@101.34.71.12"; -const sshConnectTimeout = env.LANGBOT_ACP_AGENT_RUNNER_SSH_CONNECT_TIMEOUT || "8"; -const sshPort = env.LANGBOT_ACP_AGENT_RUNNER_SSH_PORT || "22"; -const sshIdentityFile = env.LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE || ""; -const sshExtraOptions = env.LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS || ""; -const remoteWorkspace = env.LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE || "/home/yhh/langbot-e2e/acp-workspace"; +const pipelineName = + env.LANGBOT_E2E_CREATE_PIPELINE_NAME || + env.LANGBOT_ACP_RUNNER_PIPELINE_NAME || + DEFAULT_PIPELINE_NAME; +const sshTarget = env.LANGBOT_ACP_RUNNER_SSH_TARGET || "yhh@101.34.71.12"; +const sshConnectTimeout = env.LANGBOT_ACP_RUNNER_SSH_CONNECT_TIMEOUT || "8"; +const sshPort = env.LANGBOT_ACP_RUNNER_SSH_PORT || "22"; +const sshIdentityFile = env.LANGBOT_ACP_RUNNER_SSH_IDENTITY_FILE || ""; +const sshExtraOptions = env.LANGBOT_ACP_RUNNER_SSH_EXTRA_OPTIONS || ""; +const remoteWorkspace = + env.LANGBOT_ACP_RUNNER_REMOTE_WORKSPACE || + "/home/yhh/langbot-e2e/acp-workspace"; const envLocalPath = resolve("skills/.env.local"); const result = { @@ -64,7 +69,9 @@ try { const user = env.LANGBOT_E2E_LOGIN_USER || ""; const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; if (!user) { - throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API."); + throw new Error( + "LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.", + ); } const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); @@ -78,6 +85,14 @@ try { provider: "claude-code", location: "remote-ssh", workspace: remoteWorkspace, + "env-json": JSON.stringify({ + ALL_PROXY: "", + all_proxy: "", + HTTP_PROXY: "", + http_proxy: "", + HTTPS_PROXY: "", + https_proxy: "", + }), "ssh-target": sshTarget, "ssh-port": Number.parseInt(sshPort, 10), "ssh-identity-file": sshIdentityFile, @@ -102,19 +117,19 @@ try { }); Object.assign(result, prepared); if (result.pipeline_id) { - result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(result.pipeline_id)}`; + result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(result.pipeline_id)}`; } if (writeEnv && result.pipeline_id) { await upsertEnvLocal(envLocalPath, { LANGBOT_E2E_LOGIN_USER: user, - LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET: sshTarget, - LANGBOT_ACP_AGENT_RUNNER_SSH_PORT: sshPort, - LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE: sshIdentityFile, - LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS: sshExtraOptions, - LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE: remoteWorkspace, - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL: result.pipeline_url, - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName, + LANGBOT_ACP_RUNNER_SSH_TARGET: sshTarget, + LANGBOT_ACP_RUNNER_SSH_PORT: sshPort, + LANGBOT_ACP_RUNNER_SSH_IDENTITY_FILE: sshIdentityFile, + LANGBOT_ACP_RUNNER_SSH_EXTRA_OPTIONS: sshExtraOptions, + LANGBOT_ACP_RUNNER_REMOTE_WORKSPACE: remoteWorkspace, + LANGBOT_ACP_RUNNER_PIPELINE_URL: result.pipeline_url, + LANGBOT_ACP_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName, }); result.wrote_env = true; } @@ -125,10 +140,20 @@ try { console.log(JSON.stringify(result, null, 2)); } -process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); +process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, +); -async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runnerConfig }) { - const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { token }); +async function ensurePipeline({ + backendUrl, + token, + pipelineName, + runnerId, + runnerConfig, +}) { + const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { + token, + }); if (isApiFailure(pipelineList)) { return { status: "fail", @@ -147,7 +172,8 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne token, body: { name: pipelineName, - description: "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.", + description: + "Local QA pipeline for real ACP Claude Runner Debug Chat smoke tests.", emoji: "QA", }, }); @@ -159,7 +185,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }; } const pipelineId = createdResponse.json.data?.uuid || ""; - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, + { token }, + ); pipeline = loaded.json.data?.pipeline || null; created = true; } @@ -171,7 +201,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }; } - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { token }, + ); if (isApiFailure(loaded) || !loaded.json.data?.pipeline) { return { status: "fail", @@ -182,9 +216,15 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne } pipeline = loaded.json.data.pipeline; - const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; + const config = + pipeline.config && typeof pipeline.config === "object" + ? pipeline.config + : {}; const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; - const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {}; + const runnerConfigs = + ai.runner_config && typeof ai.runner_config === "object" + ? ai.runner_config + : {}; const updatedConfig = { ...config, ai: { @@ -201,16 +241,21 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }, }; - const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { - method: "PUT", - token, - body: { - name: pipelineName, - description: "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.", - emoji: "QA", - config: updatedConfig, + const updateResponse = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { + method: "PUT", + token, + body: { + name: pipelineName, + description: + "Local QA pipeline for real ACP Claude Runner Debug Chat smoke tests.", + emoji: "QA", + config: updatedConfig, + }, }, - }); + ); if (isApiFailure(updateResponse)) { return { status: "fail", @@ -222,7 +267,9 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne return { status: "pass", - reason: created ? "ACP AgentRunner pipeline created and configured." : "ACP AgentRunner pipeline updated.", + reason: created + ? "ACP Runner pipeline created and configured." + : "ACP Runner pipeline updated.", pipeline_id: pipeline.uuid, pipeline_name: pipelineName, created, @@ -231,7 +278,12 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne } function isApiFailure(response) { - return response.status >= 400 || (response.json && response.json.code !== undefined && response.json.code !== 0); + return ( + response.status >= 400 || + (response.json && + response.json.code !== undefined && + response.json.code !== 0) + ); } async function upsertEnvLocal(path, values) { diff --git a/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs index 73f2465fd..9960cd0bb 100755 --- a/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs +++ b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs @@ -14,7 +14,7 @@ import { writeResult, } from "./lib/langbot-e2e.mjs"; -const RUNNER_ID = "local-agent"; +const RUNNER_ID = "plugin:langbot-team/LocalAgent/default"; const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; const DEFAULT_PIPELINE_NAME = "LangBot QA Fake Provider Debug Chat"; const DEFAULT_PROVIDER_NAME = "LangBot QA Fake OpenAI Provider"; @@ -41,6 +41,8 @@ const pipelineName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_NAME || DEFAULT_PIPELINE const providerName = env.LANGBOT_FAKE_PROVIDER_NAME || DEFAULT_PROVIDER_NAME; const requester = env.LANGBOT_FAKE_PROVIDER_REQUESTER || DEFAULT_REQUESTER; const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || DEFAULT_MODEL_NAME; +const fallbackModelNames = textList(env.LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_NAMES) + .filter((name) => name !== modelName); const result = { source: "automation", @@ -75,6 +77,7 @@ const result = { test_status: "not_run", test_reason: "", }, + fallback_models: [], pipeline_id: "", pipeline_name: pipelineName, pipeline_url: "", @@ -141,14 +144,26 @@ try { }); result.model = model; + const fallbackModels = []; + for (const fallbackModelName of fallbackModelNames) { + fallbackModels.push(await ensureModel({ + backendUrl, + token: auth.token, + providerUuid: provider.uuid, + name: fallbackModelName, + })); + } + result.fallback_models = fallbackModels; + const pipeline = await ensurePipeline({ backendUrl, token: auth.token, name: pipelineName, modelUuid: model.uuid, + fallbackModelUuids: fallbackModels.map((item) => item.uuid), }); Object.assign(result, pipeline); - result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.pipeline_id)}`; + result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.pipeline_id)}`; const runConfig = await configureFakeProvider(fakeProvider.url, targetFakeProviderConfig(), true); result.fake_provider.config = runConfig.config || targetFakeProviderConfig(); @@ -161,6 +176,7 @@ try { LANGBOT_FAKE_PROVIDER_PID: fakeProvider.pid ? String(fakeProvider.pid) : "", LANGBOT_FAKE_PROVIDER_PROVIDER_UUID: provider.uuid, LANGBOT_FAKE_PROVIDER_MODEL_UUID: model.uuid, + LANGBOT_FAKE_PROVIDER_FALLBACK_MODEL_UUIDS: fallbackModels.map((item) => item.uuid).join(","), LANGBOT_FAKE_PROVIDER_PIPELINE_URL: result.pipeline_url, LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName, }); @@ -168,7 +184,7 @@ try { } result.status = "pass"; - result.reason = `Fake provider pipeline is configured with ${requester}/${modelName}.`; + result.reason = `Fake provider pipeline is configured with ${requester}/${modelName} and ${fallbackModels.length} fallback(s).`; } catch (error) { result.status = result.status === "env_issue" ? "env_issue" : "fail"; result.reason = result.reason || safeReason(error.message); @@ -328,7 +344,11 @@ function healthyFakeProviderConfig() { fault_status: 500, fail_first_n: 0, fail_every_n: 0, + fail_models: [], fail_after_first_chunk: false, + fail_after_first_chunk_delay_ms: 0, + fail_after_first_chunk_mode: "disconnect", + fail_after_first_chunk_models: [], dynamic_response: true, }; } @@ -342,7 +362,14 @@ function targetFakeProviderConfig() { fault_status: httpFaultStatus(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500), fail_first_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0), fail_every_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0), + fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS), fail_after_first_chunk: envBool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false), + fail_after_first_chunk_delay_ms: nonNegativeInteger( + env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS, + 0, + ), + fail_after_first_chunk_mode: streamFaultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE), + fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS), dynamic_response: envBool(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE, true), }; } @@ -425,7 +452,7 @@ async function ensureModel({ backendUrl, token, providerUuid, name }) { const body = { name, provider_uuid: providerUuid, - abilities: [], + abilities: ["func_call", "vision"], context_length: positiveInteger(env.LANGBOT_FAKE_PROVIDER_CONTEXT_LENGTH, 8192), extra_args: {}, prefered_ranking: 0, @@ -476,7 +503,7 @@ async function ensureModel({ backendUrl, token, providerUuid, name }) { }; } -async function ensurePipeline({ backendUrl, token, name, modelUuid }) { +async function ensurePipeline({ backendUrl, token, name, modelUuid, fallbackModelUuids = [] }) { const list = await apiJson(backendUrl, "/api/v1/pipelines", { token }); if (isApiFailure(list)) { throw new Error(list.json.msg || "Failed to list pipelines."); @@ -511,15 +538,17 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) { const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; - const existingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object" - ? ai["local-agent"] + const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object" + ? ai.runner_config + : {}; + const existingLocalAgentConfig = runnerConfigs[RUNNER_ID] && typeof runnerConfigs[RUNNER_ID] === "object" + ? runnerConfigs[RUNNER_ID] : {}; const localAgentConfig = { timeout: 60, prompt: [{ role: "system", content: "You are a deterministic QA assistant. Reply exactly as instructed." }], "remove-think": false, "knowledge-bases": [], - "box-session-id-template": "{launcher_type}_{launcher_id}", "retrieval-top-k": 5, "rerank-model": "", "rerank-top-k": 5, @@ -536,7 +565,7 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) { "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10), model: { primary: modelUuid, - fallbacks: [], + fallbacks: fallbackModelUuids, }, }; const updatedConfig = { @@ -546,10 +575,12 @@ async function ensurePipeline({ backendUrl, token, name, modelUuid }) { runner: { ...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}), id: RUNNER_ID, - runner: RUNNER_ID, "expire-time": 0, }, - "local-agent": localAgentConfig, + runner_config: { + ...runnerConfigs, + [RUNNER_ID]: localAgentConfig, + }, }, }; @@ -601,6 +632,17 @@ function envBool(value, fallback) { return fallback; } +function textList(value) { + return String(value || "") + .split(/\r?\n|,/) + .map((item) => item.trim()) + .filter(Boolean); +} + +function streamFaultMode(value) { + return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect"; +} + function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs b/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs index b2cf9e7b2..8ee7ad6a4 100755 --- a/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs +++ b/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs @@ -4,6 +4,7 @@ import { readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { env } from "node:process"; import { + authenticatedApiHeaders, apiJson, ensureEvidence, evidencePaths, @@ -21,7 +22,7 @@ await ensureEvidence(paths); const backendUrl = env.LANGBOT_BACKEND_URL || ""; const user = env.LANGBOT_E2E_LOGIN_USER || ""; const password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026"; -const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "azalea-cobalt-7421"; +const expectedText = env.LANGBOT_E2E_RAG_EXPECTED_TEXT || "azalea-cobalt-7421"; const query = env.LANGBOT_E2E_RETRIEVE_QUERY || "What is the local agent runner retrieval sentinel?"; const writeEnv = process.argv.includes("--write-env"); const checkOnly = process.argv.includes("--check-only"); @@ -99,7 +100,15 @@ try { } result.store_task_id = store.json.data?.task_id || ""; - const ready = await waitForSentinel(backendUrl, auth.token, result.kb_uuid, query, expectedText, waitMs); + const ready = await waitForSentinel( + backendUrl, + auth.token, + result.kb_uuid, + query, + expectedText, + waitMs, + upload.fileId, + ); result.file_statuses = ready.fileStatuses; if (ready.matched) { result.checked_bases.push(ready.checked); @@ -209,9 +218,7 @@ async function uploadDocument(backendUrl, token, path) { form.append("file", new Blob([bytes], { type: "text/plain" }), "sentinel-doc.txt"); const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/files/documents`, { method: "POST", - headers: { - Authorization: `Bearer ${token}`, - }, + headers: await authenticatedApiHeaders(backendUrl, token, { contentType: "" }), body: form, }); const json = await response.json().catch(() => ({})); @@ -222,13 +229,14 @@ async function uploadDocument(backendUrl, token, path) { return { fileId }; } -async function waitForSentinel(backendUrl, token, kbUuid, query, expectedText, timeoutMs) { +async function waitForSentinel(backendUrl, token, kbUuid, query, expectedText, timeoutMs, uploadedFileId) { const started = Date.now(); let fileStatuses = []; let lastChecked = null; while (Date.now() - started < timeoutMs) { const files = await apiJson(backendUrl, `/api/v1/knowledge/bases/${encodeURIComponent(kbUuid)}/files`, { token }); - fileStatuses = files.json.data?.files || fileStatuses; + const listedFiles = files.json.data?.files || []; + fileStatuses = listedFiles.filter((item) => item.file_name === uploadedFileId); lastChecked = await retrieveSentinel(backendUrl, token, kbUuid, kbName, query, expectedText); if (lastChecked.matched) { return { matched: true, fileStatuses, checked: lastChecked }; diff --git a/skills/scripts/e2e/ensure-local-agent-pipeline.mjs b/skills/scripts/e2e/ensure-local-agent-pipeline.mjs index da4336211..3df691f67 100755 --- a/skills/scripts/e2e/ensure-local-agent-pipeline.mjs +++ b/skills/scripts/e2e/ensure-local-agent-pipeline.mjs @@ -18,12 +18,14 @@ import { writeResult, } from "./lib/langbot-e2e.mjs"; -const RUNNER_ID = "local-agent"; +const RUNNER_ID = "plugin:langbot-team/LocalAgent/default"; const SPACE_PROVIDER_UUID = "00000000-0000-0000-0000-000000000000"; const DEFAULT_PIPELINE_NAME = "Agent QA Local Agent Debug Chat"; const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; const DEFAULT_MODEL_TEST_LIMIT = 8; const DEFAULT_MODEL_FALLBACK_COUNT = 3; +const DEFAULT_FAKE_MODEL_UUID = "langbot-e2e-fake-local-agent-model"; +const DEFAULT_FAKE_PROVIDER_NAME = "LangBot E2E Fake OpenAI Provider"; const caseId = "ensure-local-agent-pipeline"; await loadEnvFiles(); @@ -31,7 +33,10 @@ const paths = evidencePaths(caseId); await ensureEvidence(paths); const writeEnv = process.argv.includes("--write-env"); -const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || DEFAULT_PIPELINE_NAME; +const pipelineName = + env.LANGBOT_E2E_CREATE_PIPELINE_NAME || + env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || + DEFAULT_PIPELINE_NAME; const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; const backendUrl = env.LANGBOT_BACKEND_URL || ""; const envLocalPath = resolve("skills/.env.local"); @@ -51,11 +56,13 @@ const result = { selected_model_id: "", selected_model_name: "", fallback_model_ids: [], + fake_provider: null, model_count: 0, space_model_count: 0, scanned_space_model_count: 0, tested_model_count: 0, model_tests: [], + plugin_setup: null, created: false, updated: false, wrote_env: false, @@ -83,7 +90,9 @@ try { const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; if (!user) { result.status = "env_issue"; - throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API."); + throw new Error( + "LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.", + ); } const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); @@ -93,11 +102,25 @@ try { backend_token_check: auth.check, }; + const pluginSetup = await ensureLocalRunner({ + backendUrl, + token: auth.token, + }); + result.plugin_setup = pluginSetup; + if (pluginSetup.status !== "pass") { + result.status = pluginSetup.status === "env_issue" ? "env_issue" : "fail"; + throw new Error( + pluginSetup.reason || "Failed to prepare the LocalAgent runner plugin.", + ); + } + const wizard = await skipWizard({ backendUrl, token: auth.token }); result.wizard = wizard; if (wizard.status !== "pass") { result.status = "fail"; - throw new Error(wizard.reason || "Failed to mark the local QA wizard as skipped."); + throw new Error( + wizard.reason || "Failed to mark the local QA wizard as skipped.", + ); } const prepared = await ensureLocalAgentPipeline({ @@ -108,7 +131,7 @@ try { }); Object.assign(result, prepared); if (result.pipeline_id) { - result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(result.pipeline_id)}`; + result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(result.pipeline_id)}`; } if (writeEnv && result.pipeline_id) { @@ -118,10 +141,12 @@ try { LANGBOT_PIPELINE_NAME: result.pipeline_name || pipelineName, LANGBOT_LOCAL_AGENT_PIPELINE_URL: result.pipeline_url, LANGBOT_LOCAL_AGENT_PIPELINE_NAME: result.pipeline_name || pipelineName, - ...(result.selected_model_id ? { - LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id, - LANGBOT_E2E_MODEL_UUID: result.selected_model_id, - } : {}), + ...(result.selected_model_id + ? { + LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id, + LANGBOT_E2E_MODEL_UUID: result.selected_model_id, + } + : {}), }); result.wrote_env = true; } @@ -132,12 +157,21 @@ try { const browserCheck = await verifyBrowserToken(page, backendUrl); result.browser_token_check = browserCheck; if (!browserCheck.authenticated) { - throw new Error(browserCheck.reason || "Browser token check failed after setup."); + throw new Error( + browserCheck.reason || "Browser token check failed after setup.", + ); } - await page.goto(result.pipeline_url || frontendUrl, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + await page.goto(result.pipeline_url || frontendUrl, { + waitUntil: "domcontentloaded", + }); + await page + .waitForLoadState("networkidle", { timeout: 10_000 }) + .catch(() => {}); const text = await bodyText(page); - result.page_signal = ["Pipelines", "流水线", pipelineName].find((signal) => text.includes(signal)) || ""; + result.page_signal = + ["Pipelines", "流水线", pipelineName].find((signal) => + text.includes(signal), + ) || ""; } catch (error) { result.status = result.status === "env_issue" ? "env_issue" : "fail"; result.reason = result.reason || error.message; @@ -148,24 +182,281 @@ try { console.log(JSON.stringify(result, null, 2)); } -process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); +process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, +); async function skipWizard({ backendUrl, token }) { - const response = await apiJson(backendUrl, "/api/v1/system/wizard/completed", { - method: "POST", - token, - body: { status: "skipped" }, - }); + const response = await apiJson( + backendUrl, + "/api/v1/system/wizard/completed", + { + method: "POST", + token, + body: { status: "skipped" }, + }, + ); const ok = response.status < 400 && response.json.code === 0; return { status: ok ? "pass" : "fail", http_status: response.status, code: response.json.code ?? null, - reason: ok ? "Wizard marked skipped for local QA." : response.json.msg || "Wizard status update failed.", + reason: ok + ? "Wizard marked skipped for local QA." + : response.json.msg || "Wizard status update failed.", }; } -async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runnerId }) { +async function ensureLocalRunner({ backendUrl, token }) { + const [author, name] = RUNNER_ID.replace(/^plugin:/, "").split("/"); + const existingRunnerIds = await listRunnerIds(backendUrl, token); + if (existingRunnerIds.includes(RUNNER_ID)) { + return { + status: "pass", + reason: "LocalAgent runner is already registered.", + plugin_id: `${author}/${name}`, + runner_id: RUNNER_ID, + installed: false, + }; + } + + const pluginsResponse = await apiJson(backendUrl, "/api/v1/plugins", { + token, + }); + if (isApiFailure(pluginsResponse)) { + return { + status: "env_issue", + reason: + pluginsResponse.json.msg || + "Failed to inspect installed plugins before preparing LocalAgent.", + }; + } + const pluginId = `${author}/${name}`; + const pluginPresent = (pluginsResponse.json.data?.plugins || []).some( + (plugin) => { + const metadata = + plugin.manifest?.manifest?.metadata || + plugin.manifest?.metadata || + plugin.metadata || + {}; + return `${metadata.author}/${metadata.name}` === pluginId; + }, + ); + if (pluginPresent) { + const registered = await waitForRunnerRegistration({ + backendUrl, + token, + runnerId: RUNNER_ID, + timeoutMs: 30_000, + }); + return registered + ? { + status: "pass", + reason: "Installed LocalAgent plugin finished Runner registration.", + plugin_id: pluginId, + runner_id: RUNNER_ID, + installed: false, + } + : { + status: "fail", + reason: `${pluginId} is installed but ${RUNNER_ID} did not register.`, + plugin_id: pluginId, + runner_id: RUNNER_ID, + installed: false, + }; + } + + const spaceUrl = String( + env.LANGBOT_SPACE_URL || "https://space.langbot.app", + ).replace(/\/$/, ""); + let detailResponse; + try { + detailResponse = await fetch( + `${spaceUrl}/api/v1/marketplace/plugins/${encodeURIComponent(author)}/${encodeURIComponent(name)}`, + ); + } catch (error) { + return { + status: "env_issue", + reason: `Could not reach LangBot Space for ${pluginId}: ${error.message}`, + }; + } + const detail = await detailResponse.json().catch(() => ({})); + const version = detail?.data?.plugin?.latest_version || ""; + if (detailResponse.status >= 400 || detail.code !== 0 || !version) { + return { + status: "env_issue", + reason: + detail.msg || + `LangBot Space did not return an installable version for ${pluginId}.`, + }; + } + + const install = await apiJson( + backendUrl, + "/api/v1/plugins/install/marketplace", + { + method: "POST", + token, + body: { + plugin_author: author, + plugin_name: name, + plugin_version: version, + }, + }, + ); + const taskId = install.json.data?.task_id ?? null; + if (isApiFailure(install) || !taskId) { + return { + status: "fail", + reason: + install.json.msg || + `Marketplace install did not create a task for ${pluginId}.`, + plugin_id: pluginId, + version, + }; + } + const task = await waitForTask({ backendUrl, token, taskId }); + if (!taskComplete(task)) { + return { + status: taskFailed(task) ? "fail" : "env_issue", + reason: + task?.runtime?.exception || + task?.error || + `Marketplace install task for ${pluginId} did not complete.`, + plugin_id: pluginId, + version, + task_id: taskId, + }; + } + const registered = await waitForRunnerRegistration({ + backendUrl, + token, + runnerId: RUNNER_ID, + timeoutMs: 60_000, + }); + return registered + ? { + status: "pass", + reason: `Installed ${pluginId} ${version} and registered ${RUNNER_ID}.`, + plugin_id: pluginId, + runner_id: RUNNER_ID, + version, + task_id: taskId, + installed: true, + } + : { + status: "fail", + reason: `Installed ${pluginId} ${version}, but ${RUNNER_ID} did not register.`, + plugin_id: pluginId, + runner_id: RUNNER_ID, + version, + task_id: taskId, + installed: true, + }; +} + +async function listRunnerIds(backendUrl, token) { + const response = await apiJson(backendUrl, "/api/v1/pipelines/_/metadata", { + token, + }); + if (isApiFailure(response)) return []; + return (response.json.data?.configs || []) + .flatMap((section) => section.stages || []) + .flatMap((stage) => stage.config || []) + .filter((item) => item.name === "id") + .flatMap((item) => item.options || []) + .map((option) => option.name || option.value || option.id || "") + .filter(Boolean); +} + +async function waitForRunnerRegistration({ + backendUrl, + token, + runnerId, + timeoutMs, +}) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if ((await listRunnerIds(backendUrl, token)).includes(runnerId)) + return true; + await sleep(1000); + } + return false; +} + +async function waitForTask({ backendUrl, token, taskId }) { + const deadline = + Date.now() + Number(env.LANGBOT_PLUGIN_INSTALL_TIMEOUT_MS || 120_000); + let task = null; + while (Date.now() < deadline) { + const response = await apiJson( + backendUrl, + `/api/v1/system/tasks/${encodeURIComponent(taskId)}`, + { token }, + ); + task = response.json.data || response.json; + if (taskComplete(task) || taskFailed(task)) return task; + await sleep(1000); + } + return task; +} + +function taskComplete(task) { + const status = String(task?.status || task?.state || "").toLowerCase(); + const runtimeStatus = String( + task?.runtime?.status || task?.runtime?.state || "", + ).toLowerCase(); + return ( + ["done", "completed", "success", "succeeded", "finished"].includes( + status, + ) || + ["done", "completed", "success", "succeeded", "finished"].includes( + runtimeStatus, + ) || + task?.done === true || + task?.completed === true || + (task?.runtime?.done === true && !task?.runtime?.exception) + ); +} + +function taskFailed(task) { + const status = String(task?.status || task?.state || "").toLowerCase(); + const runtimeStatus = String( + task?.runtime?.status || task?.runtime?.state || "", + ).toLowerCase(); + return ( + ["failed", "error", "cancelled", "canceled"].includes(status) || + ["failed", "error", "cancelled", "canceled"].includes(runtimeStatus) || + task?.failed === true || + Boolean(task?.error) || + Boolean(task?.runtime?.exception) + ); +} + +function sleep(ms) { + return new Promise((resolvePromise) => setTimeout(resolvePromise, ms)); +} + +async function ensureLocalAgentPipeline({ + backendUrl, + token, + pipelineName, + runnerId, +}) { + // Fake-provider use must be explicit for this pipeline. The generic fake + // provider variables persist after load tests and must not silently replace + // a real LocalAgent model in later QA runs. + const fakeProviderBaseUrl = env.LANGBOT_E2E_FAKE_PROVIDER_BASE_URL || ""; + let fakeModel = null; + if (fakeProviderBaseUrl) { + fakeModel = await ensureFakeProviderModel({ + backendUrl, + token, + baseUrl: fakeProviderBaseUrl, + }); + if (fakeModel.status !== "pass") return fakeModel; + } + const [pipelineList, modelList] = await Promise.all([ apiJson(backendUrl, "/api/v1/pipelines", { token }), apiJson(backendUrl, "/api/v1/provider/models/llm", { token }), @@ -199,7 +490,9 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne .map((item) => item.trim()) .filter(Boolean), ); - const spaceModels = models.filter((model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid)); + const spaceModels = models.filter( + (model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid), + ); const pipelines = pipelineList.json.data?.pipelines || []; let pipeline = pipelines.find((item) => item.name === pipelineName) || null; let created = false; @@ -210,7 +503,7 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne token, body: { name: pipelineName, - description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.", + description: "Local QA pipeline for Runner Debug Chat smoke tests.", emoji: "QA", }, }); @@ -224,7 +517,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne }; } const pipelineId = createdResponse.json.data?.uuid || ""; - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, + { token }, + ); pipeline = loaded.json.data?.pipeline || null; created = true; } @@ -238,7 +535,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne }; } - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { token }, + ); if (isApiFailure(loaded) || !loaded.json.data?.pipeline) { return { status: "fail", @@ -251,32 +552,53 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne } pipeline = loaded.json.data.pipeline; - const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; + const config = + pipeline.config && typeof pipeline.config === "object" + ? pipeline.config + : {}; const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; - const rawExistingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object" - ? ai["local-agent"] - : {}; + const runnerConfigs = + ai.runner_config && typeof ai.runner_config === "object" + ? ai.runner_config + : {}; + const rawExistingLocalAgentConfig = + runnerConfigs[runnerId] && typeof runnerConfigs[runnerId] === "object" + ? runnerConfigs[runnerId] + : {}; const existingLocalAgentConfig = rawExistingLocalAgentConfig; - const existingModel = existingLocalAgentConfig.model && typeof existingLocalAgentConfig.model === "object" - ? existingLocalAgentConfig.model - : {}; - const requestedModelId = env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || ""; - const selected = await selectWorkingSpaceModel({ - backendUrl, - token, - models, - skippedModelIds, - skippedModelNames, - requestedModelId, - existingModelId: existingModel.primary || "", - }); + const existingModel = + existingLocalAgentConfig.model && + typeof existingLocalAgentConfig.model === "object" + ? existingLocalAgentConfig.model + : {}; + const requestedModelId = + env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || ""; + const selected = fakeModel + ? { + status: "pass", + reason: "", + selected_model_id: fakeModel.model_uuid, + selected_model_name: fakeModel.model_name, + fallback_model_ids: [], + scanned_space_model_count: 0, + tested_model_count: 0, + model_tests: [], + } + : await selectWorkingSpaceModel({ + backendUrl, + token, + models, + skippedModelIds, + skippedModelNames, + requestedModelId, + existingModelId: existingModel.primary || "", + }); const selectedModelId = selected.selected_model_id || ""; const localAgentConfig = { timeout: 300, prompt: [{ role: "system", content: "You are a helpful assistant." }], "remove-think": false, "knowledge-bases": [], - "box-session-id-template": "{launcher_type}_{launcher_id}", "retrieval-top-k": 5, "rerank-model": "", "rerank-top-k": 5, @@ -288,7 +610,6 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne "context-reserve-tokens": 16384, "context-keep-recent-tokens": 20000, "context-summary-tokens": 8000, - ...existingLocalAgentConfig, // Current backend truncation still reads this field directly. "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10), model: { @@ -303,23 +624,29 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne runner: { ...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}), id: runnerId, - runner: runnerId, "expire-time": 0, }, - "local-agent": localAgentConfig, + runner_config: { + ...runnerConfigs, + [runnerId]: localAgentConfig, + }, }, }; - const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { - method: "PUT", - token, - body: { - name: pipelineName, - description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.", - emoji: "QA", - config: updatedConfig, + const updateResponse = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { + method: "PUT", + token, + body: { + name: pipelineName, + description: "Local QA pipeline for Runner Debug Chat smoke tests.", + emoji: "QA", + config: updatedConfig, + }, }, - }); + ); if (isApiFailure(updateResponse)) { return { status: "fail", @@ -341,7 +668,8 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne status: selectedModelId ? "pass" : "env_issue", reason: selectedModelId ? `Local-agent pipeline is configured for Debug Chat with Space model ${selected.selected_model_name || selectedModelId} and ${selected.fallback_model_ids.length} fallback(s).` - : selected.reason || "No working Space LLM model is configured in this LangBot instance.", + : selected.reason || + "No working Space LLM model is configured in this LangBot instance.", pipeline_id: pipeline.uuid, pipeline_name: pipelineName, model_count: models.length, @@ -352,21 +680,214 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne selected_model_id: selectedModelId, selected_model_name: selected.selected_model_name, fallback_model_ids: selected.fallback_model_ids, + fake_provider: fakeModel, created, updated: true, }; } +async function ensureFakeProviderModel({ backendUrl, token, baseUrl }) { + const modelUuid = env.LANGBOT_E2E_FAKE_MODEL_UUID || DEFAULT_FAKE_MODEL_UUID; + const modelName = + env.LANGBOT_E2E_FAKE_MODEL_NAME || + env.LANGBOT_FAKE_PROVIDER_MODEL || + env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || + "langbot-e2e-fake-model"; + const providerName = + env.LANGBOT_E2E_FAKE_PROVIDER_NAME || DEFAULT_FAKE_PROVIDER_NAME; + const providerRequester = + env.LANGBOT_E2E_FAKE_PROVIDER_REQUESTER || "openai-chat-completions"; + const apiKey = + env.LANGBOT_E2E_FAKE_PROVIDER_API_KEY || + env.LANGBOT_FAKE_PROVIDER_API_KEY || + "fake-key"; + + const providersResponse = await apiJson( + backendUrl, + "/api/v1/provider/providers", + { token }, + ); + if (isApiFailure(providersResponse)) { + return { + status: "fail", + reason: + providersResponse.json.msg || + "Failed to list providers before creating fake provider.", + provider_status: providersResponse.status, + }; + } + + const normalizedBaseUrl = String(baseUrl || "").replace(/\/$/, ""); + const providers = providersResponse.json.data?.providers || []; + let provider = providers.find( + (item) => + item.name === providerName || + (item.requester === providerRequester && + String(item.base_url || "").replace(/\/$/, "") === normalizedBaseUrl), + ); + const providerBody = { + name: providerName, + requester: providerRequester, + base_url: normalizedBaseUrl, + api_keys: [apiKey], + }; + + if (provider?.uuid) { + const updateResponse = await apiJson( + backendUrl, + `/api/v1/provider/providers/${encodeURIComponent(provider.uuid)}`, + { + method: "PUT", + token, + body: providerBody, + }, + ); + if (isApiFailure(updateResponse)) { + return { + status: "fail", + reason: updateResponse.json.msg || "Failed to update fake provider.", + provider_status: updateResponse.status, + }; + } + } else { + const createProviderResponse = await apiJson( + backendUrl, + "/api/v1/provider/providers", + { + method: "POST", + token, + body: providerBody, + }, + ); + if (isApiFailure(createProviderResponse)) { + return { + status: "fail", + reason: + createProviderResponse.json.msg || "Failed to create fake provider.", + provider_status: createProviderResponse.status, + }; + } + provider = { uuid: createProviderResponse.json.data?.uuid || "" }; + } + + if (!provider?.uuid) { + return { + status: "fail", + reason: "Fake provider did not return a provider uuid.", + }; + } + + let resolvedModelUuid = modelUuid; + let modelResponse = await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, + { + token, + }, + ); + if (modelResponse.status === 404) { + const providerModelsResponse = await apiJson( + backendUrl, + `/api/v1/provider/models/llm?provider_uuid=${encodeURIComponent(provider.uuid)}`, + { token }, + ); + if (!isApiFailure(providerModelsResponse)) { + const existingModel = ( + providerModelsResponse.json.data?.models || [] + ).find((item) => item.name === modelName); + if (existingModel?.uuid) { + resolvedModelUuid = existingModel.uuid; + modelResponse = await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`, + { + token, + }, + ); + } + } + } + + const modelBody = { + name: modelName, + provider_uuid: provider.uuid, + abilities: ["func_call", "vision"], + context_length: 8192, + extra_args: {}, + prefered_ranking: 0, + }; + if (modelResponse.status === 404) { + const createModelResponse = await apiJson( + backendUrl, + "/api/v1/provider/models/llm", + { + method: "POST", + token, + body: { + uuid: modelUuid, + ...modelBody, + }, + }, + ); + if (isApiFailure(createModelResponse)) { + return { + status: "fail", + reason: createModelResponse.json.msg || "Failed to create fake model.", + model_status: createModelResponse.status, + }; + } + resolvedModelUuid = createModelResponse.json.data?.uuid || modelUuid; + } else if (isApiFailure(modelResponse)) { + return { + status: "fail", + reason: modelResponse.json.msg || "Failed to load fake model.", + model_status: modelResponse.status, + }; + } else { + const updateModelResponse = await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(resolvedModelUuid)}`, + { + method: "PUT", + token, + body: modelBody, + }, + ); + if (isApiFailure(updateModelResponse)) { + return { + status: "fail", + reason: updateModelResponse.json.msg || "Failed to update fake model.", + model_status: updateModelResponse.status, + }; + } + } + + return { + status: "pass", + provider_uuid: provider.uuid, + provider_requester: providerRequester, + base_url: normalizedBaseUrl, + model_uuid: resolvedModelUuid, + model_name: modelName, + }; +} + function isApiFailure(response) { - return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0); + return ( + response.status >= 400 || + (response.json.code !== undefined && response.json.code !== 0) + ); } function isSpaceModel(model) { - const provider = model?.provider && typeof model.provider === "object" ? model.provider : {}; - return model?.provider_uuid === SPACE_PROVIDER_UUID - || provider.uuid === SPACE_PROVIDER_UUID - || provider.requester === "space-chat-completions" - || provider.name === "LangBot Models"; + const provider = + model?.provider && typeof model.provider === "object" ? model.provider : {}; + return ( + model?.provider_uuid === SPACE_PROVIDER_UUID || + provider.uuid === SPACE_PROVIDER_UUID || + provider.requester === "space-chat-completions" || + provider.name === "LangBot Models" + ); } async function selectWorkingSpaceModel({ @@ -379,15 +900,24 @@ async function selectWorkingSpaceModel({ existingModelId, }) { const modelTests = []; - const testLimit = positiveInteger(env.LANGBOT_E2E_MODEL_TEST_LIMIT, DEFAULT_MODEL_TEST_LIMIT); - const fallbackCount = positiveInteger(env.LANGBOT_E2E_MODEL_FALLBACK_COUNT, DEFAULT_MODEL_FALLBACK_COUNT); + const testLimit = positiveInteger( + env.LANGBOT_E2E_MODEL_TEST_LIMIT, + DEFAULT_MODEL_TEST_LIMIT, + ); + const fallbackCount = positiveInteger( + env.LANGBOT_E2E_MODEL_FALLBACK_COUNT, + DEFAULT_MODEL_FALLBACK_COUNT, + ); const workingModels = []; - const spaceModels = rankModels(models.filter((model) => ( - model.uuid - && isSpaceModel(model) - && !skippedModelIds.has(model.uuid) - && !skippedModelNames.has(model.name) - ))); + const spaceModels = rankModels( + models.filter( + (model) => + model.uuid && + isSpaceModel(model) && + !skippedModelIds.has(model.uuid) && + !skippedModelNames.has(model.name), + ), + ); const requestedModel = requestedModelId ? spaceModels.find((model) => model.uuid === requestedModelId) || null : null; @@ -396,7 +926,9 @@ async function selectWorkingSpaceModel({ : null; const candidates = uniqueCandidates([ ...(requestedModel ? [existingCandidate(requestedModel, "requested")] : []), - ...(existingModel ? [existingCandidate(existingModel, "existing-pipeline")] : []), + ...(existingModel + ? [existingCandidate(existingModel, "existing-pipeline")] + : []), ...spaceModels.map((model) => existingCandidate(model, "configured-space")), ]); @@ -405,9 +937,16 @@ async function selectWorkingSpaceModel({ scanResult = await scanSpaceModels({ backendUrl, token }); if (scanResult.status === "pass") { const knownNames = new Set(spaceModels.map((model) => model.name)); - candidates.push(...scanResult.models - .filter((model) => model.name && !knownNames.has(model.name) && !skippedModelNames.has(model.name)) - .map((model) => scannedCandidate(model))); + candidates.push( + ...scanResult.models + .filter( + (model) => + model.name && + !knownNames.has(model.name) && + !skippedModelNames.has(model.name), + ) + .map((model) => scannedCandidate(model)), + ); } } @@ -435,14 +974,16 @@ async function selectWorkingSpaceModel({ }; } - const baseReason = unique.length === 0 - ? scanResult.reason || "No Space LLM model candidates are available." - : `No working Space LLM model found after testing ${modelTests.length} candidate(s).`; + const baseReason = + unique.length === 0 + ? scanResult.reason || "No Space LLM model candidates are available." + : `No working Space LLM model found after testing ${modelTests.length} candidate(s).`; return { status: "env_issue", - reason: requestedModelId && !requestedModel - ? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}` - : baseReason, + reason: + requestedModelId && !requestedModel + ? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}` + : baseReason, selected_model_id: "", selected_model_name: "", fallback_model_ids: [], @@ -462,7 +1003,11 @@ async function scanSpaceModels({ backendUrl, token }) { return { status: "env_issue", models: [], - reason: safeReason(response.json.msg || response.json.message || "Failed to scan Space LLM models."), + reason: safeReason( + response.json.msg || + response.json.message || + "Failed to scan Space LLM models.", + ), }; } return { @@ -492,28 +1037,42 @@ async function ensureAndTestModel({ backendUrl, token, candidate }) { if (isApiFailure(create) || !modelUuid) { return modelTestResult(candidate, { status: "fail", - reason: safeReason(create.json.msg || "Failed to create scanned Space model."), + reason: safeReason( + create.json.msg || "Failed to create scanned Space model.", + ), http_status: create.status, }); } created = true; } - const test = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, { - method: "POST", - token, - body: { extra_args: {} }, - }); + const test = await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, + { + method: "POST", + token, + body: { extra_args: {} }, + }, + ); const passed = !isApiFailure(test); if (!passed && created) { - await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, { - method: "DELETE", - token, - }).catch(() => {}); + await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, + { + method: "DELETE", + token, + }, + ).catch(() => {}); } return modelTestResult(candidate, { status: passed ? "pass" : "fail", - reason: passed ? "" : safeReason(test.json.msg || test.json.message || "Space model test failed."), + reason: passed + ? "" + : safeReason( + test.json.msg || test.json.message || "Space model test failed.", + ), http_status: test.status, model_uuid: modelUuid, created, @@ -558,7 +1117,9 @@ function uniqueCandidates(candidates) { const seen = new Set(); const result = []; for (const candidate of candidates) { - const key = candidate.uuid ? `uuid:${candidate.uuid}` : `name:${candidate.name}`; + const key = candidate.uuid + ? `uuid:${candidate.uuid}` + : `name:${candidate.name}`; if (!candidate.name || seen.has(key)) continue; seen.add(key); result.push(candidate); @@ -568,8 +1129,12 @@ function uniqueCandidates(candidates) { function rankModels(models) { return [...models].sort((left, right) => { - const leftRank = Number.isFinite(Number(left.prefered_ranking)) ? Number(left.prefered_ranking) : 9999; - const rightRank = Number.isFinite(Number(right.prefered_ranking)) ? Number(right.prefered_ranking) : 9999; + const leftRank = Number.isFinite(Number(left.prefered_ranking)) + ? Number(left.prefered_ranking) + : 9999; + const rightRank = Number.isFinite(Number(right.prefered_ranking)) + ? Number(right.prefered_ranking) + : 9999; if (leftRank !== rightRank) return leftRank - rightRank; return String(left.name || "").localeCompare(String(right.name || "")); }); @@ -605,5 +1170,9 @@ async function upsertEnvLocal(path, updates) { for (const [key, value] of Object.entries(updates)) { if (!seen.has(key)) next.push(`${key}=${value}`); } - await writeFile(path, `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, "utf8"); + await writeFile( + path, + `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, + "utf8", + ); } diff --git a/skills/scripts/e2e/ensure-qa-agent-runner-pipeline.mjs b/skills/scripts/e2e/ensure-qa-agent-runner-pipeline.mjs index abc43241a..8af91211b 100755 --- a/skills/scripts/e2e/ensure-qa-agent-runner-pipeline.mjs +++ b/skills/scripts/e2e/ensure-qa-agent-runner-pipeline.mjs @@ -24,7 +24,10 @@ await ensureEvidence(paths); const writeEnv = process.argv.includes("--write-env"); const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; const backendUrl = env.LANGBOT_BACKEND_URL || ""; -const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME; +const pipelineName = + env.LANGBOT_E2E_CREATE_PIPELINE_NAME || + env.LANGBOT_QA_RUNNER_PIPELINE_NAME || + DEFAULT_PIPELINE_NAME; const envLocalPath = resolve("skills/.env.local"); const result = { @@ -55,7 +58,9 @@ try { const user = env.LANGBOT_E2E_LOGIN_USER || ""; const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; if (!user) { - throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API."); + throw new Error( + "LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.", + ); } const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); @@ -74,14 +79,14 @@ try { }); Object.assign(result, prepared); if (result.pipeline_id) { - result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(result.pipeline_id)}`; + result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(result.pipeline_id)}`; } if (writeEnv && result.pipeline_id) { await upsertEnvLocal(envLocalPath, { LANGBOT_E2E_LOGIN_USER: user, - LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL: result.pipeline_url, - LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName, + LANGBOT_QA_RUNNER_PIPELINE_URL: result.pipeline_url, + LANGBOT_QA_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName, }); result.wrote_env = true; } @@ -92,10 +97,20 @@ try { console.log(JSON.stringify(result, null, 2)); } -process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); +process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, +); -async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runnerConfig }) { - const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { token }); +async function ensurePipeline({ + backendUrl, + token, + pipelineName, + runnerId, + runnerConfig, +}) { + const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { + token, + }); if (isApiFailure(pipelineList)) { return { status: "fail", @@ -114,7 +129,8 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne token, body: { name: pipelineName, - description: "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.", + description: + "Local QA pipeline for deterministic QA Runner Debug Chat smoke tests.", emoji: "QA", }, }); @@ -126,7 +142,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }; } const pipelineId = createdResponse.json.data?.uuid || ""; - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, + { token }, + ); pipeline = loaded.json.data?.pipeline || null; created = true; } @@ -138,7 +158,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }; } - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { token }, + ); if (isApiFailure(loaded) || !loaded.json.data?.pipeline) { return { status: "fail", @@ -149,9 +173,15 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne } pipeline = loaded.json.data.pipeline; - const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; + const config = + pipeline.config && typeof pipeline.config === "object" + ? pipeline.config + : {}; const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; - const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {}; + const runnerConfigs = + ai.runner_config && typeof ai.runner_config === "object" + ? ai.runner_config + : {}; const updatedConfig = { ...config, ai: { @@ -168,16 +198,21 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }, }; - const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { - method: "PUT", - token, - body: { - name: pipelineName, - description: "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.", - emoji: "QA", - config: updatedConfig, + const updateResponse = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { + method: "PUT", + token, + body: { + name: pipelineName, + description: + "Local QA pipeline for deterministic QA Runner Debug Chat smoke tests.", + emoji: "QA", + config: updatedConfig, + }, }, - }); + ); if (isApiFailure(updateResponse)) { return { status: "fail", @@ -189,7 +224,9 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne return { status: "pass", - reason: created ? "QA AgentRunner pipeline created and configured." : "QA AgentRunner pipeline updated.", + reason: created + ? "QA Runner pipeline created and configured." + : "QA Runner pipeline updated.", pipeline_id: pipeline.uuid, pipeline_name: pipelineName, created, @@ -198,7 +235,12 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne } function isApiFailure(response) { - return response.status >= 400 || (response.json && response.json.code !== undefined && response.json.code !== 0); + return ( + response.status >= 400 || + (response.json && + response.json.code !== undefined && + response.json.code !== 0) + ); } async function upsertEnvLocal(path, values) { diff --git a/skills/scripts/e2e/fake-openai-provider.mjs b/skills/scripts/e2e/fake-openai-provider.mjs index 1cca9c46b..1dd7e9026 100755 --- a/skills/scripts/e2e/fake-openai-provider.mjs +++ b/skills/scripts/e2e/fake-openai-provider.mjs @@ -9,7 +9,43 @@ const args = parseArgs(process.argv.slice(2)); const host = args.host || env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1"; const port = integer(args.port ?? env.LANGBOT_FAKE_PROVIDER_PORT, 0); const stateFile = args["state-file"] || env.LANGBOT_FAKE_PROVIDER_STATE_FILE || ""; -const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || "gpt-4o-mini"; +const modelName = args.model || env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || env.LANGBOT_FAKE_PROVIDER_MODEL || "gpt-4o-mini"; +const COMPACTION_SENTINEL = "qa_compaction_sentinel_7391"; +const COMBO_SENTINEL = "qa_combo_compaction_sentinel_2406"; +const RAG_SENTINEL = "azalea-cobalt-7421"; +const COMBO_TOOL_INPUT = "combo-tool-ok-local-agent"; +const COMBO_TOOL_RESULT = `qa-plugin-smoke:${COMBO_TOOL_INPUT}`; +const COMBO_FINAL = `COMBO_FINAL ${COMBO_SENTINEL} ${RAG_SENTINEL} ${COMBO_TOOL_RESULT}`; +const MULTITOOL_SENTINEL = "qa_multitool_compaction_sentinel_6718"; +const MULTITOOL_TOOL_A_INPUT = "multi-tool-a-local-agent"; +const MULTITOOL_TOOL_B_INPUT = "multi-tool-b-local-agent"; +const MULTITOOL_TOOL_A_RESULT = `qa-plugin-smoke:${MULTITOOL_TOOL_A_INPUT}`; +const MULTITOOL_TOOL_B_RESULT = `qa-plugin-smoke:${MULTITOOL_TOOL_B_INPUT}`; +const MULTITOOL_FINAL = [ + "MULTITOOL_COMBO_FINAL", + MULTITOOL_SENTINEL, + RAG_SENTINEL, + MULTITOOL_TOOL_A_RESULT, + MULTITOOL_TOOL_B_RESULT, +].join(" "); +const PARALLEL_SENTINEL = "qa_parallel_compaction_sentinel_8142"; +const PARALLEL_TOOL_A_INPUT = "parallel-tool-a-local-agent"; +const PARALLEL_TOOL_B_INPUT = "parallel-tool-b-local-agent"; +const PARALLEL_TOOL_A_RESULT = `qa-plugin-smoke:${PARALLEL_TOOL_A_INPUT}`; +const PARALLEL_TOOL_B_RESULT = `qa-plugin-smoke:${PARALLEL_TOOL_B_INPUT}`; +const PARALLEL_FINAL = [ + "PARALLEL_COMBO_FINAL", + PARALLEL_SENTINEL, + RAG_SENTINEL, + PARALLEL_TOOL_A_RESULT, + PARALLEL_TOOL_B_RESULT, +].join(" "); +const LOOP_LIMIT_INPUT = "loop-limit-repeat-local-agent"; +const TOOL_ERROR_INPUT = "tool-error-recovery-local-agent"; +const TOOL_ERROR_FINAL = "TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed"; +const STEERING_FOLLOWUP_SENTINEL = "qa_steering_sentinel_6194"; +const STEERING_SLEEP_INPUT = "steering-e2e-anchor"; +const STEERING_SLEEP_RESULT = `qa-plugin-smoke:sleep:8:${STEERING_SLEEP_INPUT}`; const config = { response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK", first_token_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25), @@ -18,7 +54,11 @@ const config = { fault_status: integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500), fail_first_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0), fail_every_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0), + fail_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_MODELS), fail_after_first_chunk: bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false), + fail_after_first_chunk_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_DELAY_MS, 0), + fail_after_first_chunk_mode: faultMode(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODE), + fail_after_first_chunk_models: textList(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK_MODELS), dynamic_response: !/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || ""), request_log_limit: integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500), }; @@ -88,6 +128,7 @@ const server = createServer(async (request, response) => { created: 1, owned_by: "langbot-qa", type: "llm", + context_length: 8192, }, ], }); @@ -98,15 +139,20 @@ const server = createServer(async (request, response) => { requestCount += 1; const body = await readJson(request); const requestId = `chatcmpl-langbot-fake-${requestCount}`; + const requestModel = String(body.model || modelName); const shouldFail = requestCount <= config.fail_first_n - || (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0); - const replyText = responseTextForBody(body); + || (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0) + || config.fail_models.includes(requestModel); + const failAfterFirstChunk = config.fail_after_first_chunk + || config.fail_after_first_chunk_models.includes(requestModel); + const replyMessage = buildResponse(body); + const replyText = replyMessage.content || ""; requestRecord = recordRequest({ id: requestId, request_number: requestCount, path: url.pathname, stream: Boolean(body.stream), - model: body.model || "", + model: requestModel, message_count: Array.isArray(body.messages) ? body.messages.length : 0, should_fail: shouldFail, status: "running", @@ -140,8 +186,8 @@ const server = createServer(async (request, response) => { await streamCompletion(response, { requestId, model: body.model || modelName, - content: replyText, - failAfterFirstChunk: config.fail_after_first_chunk, + message: replyMessage, + failAfterFirstChunk, requestRecord, startedPerf, }); @@ -150,7 +196,7 @@ const server = createServer(async (request, response) => { sendJson(response, 200, completionPayload({ requestId, model: body.model || modelName, - content: replyText, + message: replyMessage, })); markRequestTiming(requestRecord, "first_chunk", startedPerf); markRequestTiming(requestRecord, "first_content_chunk", startedPerf); @@ -270,8 +316,8 @@ function sendJson(response, status, payload) { response.end(text); } -function completionPayload({ requestId, model, content }) { - const completionTokens = tokenEstimate(content); +function completionPayload({ requestId, model, message }) { + const completionTokens = tokenEstimate(message.content || JSON.stringify(message.tool_calls || [])); return { id: requestId, object: "chat.completion", @@ -280,11 +326,8 @@ function completionPayload({ requestId, model, content }) { choices: [ { index: 0, - message: { - role: "assistant", - content, - }, - finish_reason: "stop", + message, + finish_reason: message.tool_calls?.length ? "tool_calls" : "stop", }, ], usage: { @@ -298,7 +341,7 @@ function completionPayload({ requestId, model, content }) { async function streamCompletion(response, { requestId, model, - content, + message, failAfterFirstChunk: failMidStream, requestRecord, startedPerf, @@ -319,7 +362,56 @@ async function streamCompletion(response, { choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], }); - const chunks = splitContent(content); + const chunks = message.tool_calls?.length ? [] : splitContent(message.content || ""); + if (message.tool_calls?.length) { + await sleep(config.chunk_delay_ms); + markRequestTiming(requestRecord, "first_content_chunk", startedPerf); + requestRecord.content_chunk_count = (requestRecord.content_chunk_count || 0) + 1; + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [ + { + index: 0, + delta: { + tool_calls: message.tool_calls.map((call, index) => ({ + index, + id: call.id, + type: call.type, + function: { + name: call.function.name, + arguments: call.function.arguments, + }, + })), + }, + finish_reason: null, + }, + ], + }); + await sleep(config.chunk_delay_ms); + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [{ index: 0, delta: {}, finish_reason: "tool_calls" }], + usage: { + prompt_tokens: 8, + completion_tokens: tokenEstimate(JSON.stringify(message.tool_calls)), + total_tokens: 8 + tokenEstimate(JSON.stringify(message.tool_calls)), + }, + }); + response.write("data: [DONE]\n\n"); + response.end(); + finishRequestRecord(requestRecord, startedPerf, { + status: "ok", + http_status: 200, + }); + return; + } + for (let index = 0; index < chunks.length; index += 1) { await sleep(config.chunk_delay_ms); if (index === 0) markRequestTiming(requestRecord, "first_content_chunk", startedPerf); @@ -332,6 +424,22 @@ async function streamCompletion(response, { choices: [{ index: 0, delta: { content: chunks[index] }, finish_reason: null }], }); if (failMidStream && index === 0) { + await sleep(config.fail_after_first_chunk_delay_ms); + if (config.fail_after_first_chunk_mode === "error_event") { + response.write(`data: ${JSON.stringify({ + error: { + message: "LangBot fake provider injected a mid-stream error event", + type: "fake_provider_stream_fault", + code: "fake_provider_stream_fault", + }, + })}\n\n`); + response.end(); + finishRequestRecord(requestRecord, startedPerf, { + status: "mid_stream_error_event", + http_status: 200, + }); + return; + } finishRequestRecord(requestRecord, startedPerf, { status: "mid_stream_disconnect", http_status: 200, @@ -342,7 +450,7 @@ async function streamCompletion(response, { } await sleep(config.chunk_delay_ms); - const completionTokens = tokenEstimate(content); + const completionTokens = tokenEstimate(message.content || ""); writeSse(response, { id: requestId, object: "chat.completion.chunk", @@ -399,6 +507,266 @@ function responseTextForBody(body) { return config.response_text; } +function messageText(messages = []) { + return messages + .map((message) => { + const parts = [contentText(message?.content)]; + if (Array.isArray(message?.tool_calls)) { + parts.push( + message.tool_calls + .map((call) => `${call?.function?.name || call?.name || ""} ${call?.function?.arguments || ""}`) + .join("\n"), + ); + } + return parts.filter(Boolean).join("\n"); + }) + .join("\n"); +} + +function contentText(content) { + if (typeof content === "string") return content; + if (Array.isArray(content)) return content.map((part) => contentText(part)).join(""); + if (content && typeof content === "object") { + for (const key of ["text", "content", "message", "error", "value"]) { + if (content[key] !== undefined && content[key] !== null) { + return contentText(content[key]); + } + } + return JSON.stringify(content); + } + return ""; +} + +function latestUserText(messages = []) { + const latest = [...messages].reverse().find((message) => message?.role === "user"); + return contentText(latest?.content); +} + +function toolNames(tools = []) { + return tools + .map((tool) => tool?.function?.name || tool?.name || "") + .filter(Boolean); +} + +function firstToolName(tools = [], candidates = []) { + const names = toolNames(tools); + return candidates.find((candidate) => names.includes(candidate)) || ""; +} + +function isSummarizationRequest(messages = []) { + const text = messageText(messages); + return /context summarization assistant|conversation to summarize|/i.test(text); +} + +function buildSummaryResponse(text) { + const references = []; + if (text.includes(COMPACTION_SENTINEL)) references.push(COMPACTION_SENTINEL); + if (text.includes(COMBO_SENTINEL)) references.push(COMBO_SENTINEL); + if (text.includes(MULTITOOL_SENTINEL)) references.push(MULTITOOL_SENTINEL); + if (text.includes(PARALLEL_SENTINEL)) references.push(PARALLEL_SENTINEL); + if (/RAG_TOOL_COMBO_GOAL/i.test(text)) references.push("RAG_TOOL_COMBO_GOAL"); + if (/MULTITOOL_RAG_GOAL/i.test(text)) references.push("MULTITOOL_RAG_GOAL"); + if (/PARALLEL_RAG_GOAL/i.test(text)) references.push("PARALLEL_RAG_GOAL"); + const critical = references.length + ? references.map((reference) => `- ${reference}`).join("\n") + : "- (none)"; + return { + role: "assistant", + content: [ + "## Goal", + "Preserve deterministic LangBot E2E context across compaction.", + "", + "## Constraints & Preferences", + "- Keep exact sentinel values verbatim.", + "", + "## Progress", + "### Done", + "- [x] Earlier conversation was compacted by the fake provider.", + "", + "### In Progress", + "- [ ] Continue the current Debug Chat run.", + "", + "### Blocked", + "- (none)", + "", + "## Key Decisions", + "- **Deterministic QA**: Return only known sentinel values from compacted input.", + "", + "## Next Steps", + "1. Answer the current user request using preserved context.", + "", + "## Critical Context", + critical, + ].join("\n"), + }; +} + +function buildResponse(payload) { + const text = messageText(payload.messages || []); + const current = latestUserText(payload.messages || []); + const tools = payload.tools || []; + if (isSummarizationRequest(payload.messages || [])) { + return buildSummaryResponse(text); + } + + if (/qa-effective-prompt/i.test(current) && /PROMPT_PREPROCESS_OK/.test(text)) { + return { role: "assistant", content: "PROMPT_PREPROCESS_OK" }; + } + + const pluginTool = firstToolName(tools, ["qa_plugin_echo"]); + const pluginFailTool = firstToolName(tools, ["qa_plugin_fail"]); + const pluginSleepTool = firstToolName(tools, ["qa_plugin_sleep"]); + const mcpTool = firstToolName(tools, ["qa_mcp_echo"]); + const requestedMcpEchoText = current.match( + /qa_mcp_echo[\s\S]*?exactly this text:\s*([A-Za-z0-9_:-]+)/i, + )?.[1] || "mcp-ok-local-agent"; + const expectedMcpEchoResult = `qa_mcp_echo:${requestedMcpEchoText}`; + + if (/STEERING_NO_FOLLOWUP|qa_plugin_sleep|steering-e2e-anchor|qa_steering_sentinel_6194/i.test(current || text)) { + if (text.includes(STEERING_FOLLOWUP_SENTINEL) && text.includes(STEERING_SLEEP_RESULT)) { + return { role: "assistant", content: STEERING_FOLLOWUP_SENTINEL }; + } + if (text.includes(STEERING_SLEEP_RESULT) && !text.includes(STEERING_FOLLOWUP_SENTINEL)) { + return { role: "assistant", content: "STEERING_NO_FOLLOWUP" }; + } + if (pluginSleepTool && !text.includes(STEERING_SLEEP_RESULT)) { + return toolCall("call_qa_plugin_sleep_steering", pluginSleepTool, { seconds: 8, text: STEERING_SLEEP_INPUT }); + } + } + + if (/测试暗号是什么|original sentinel|first.*sentinel/i.test(current)) { + return { + role: "assistant", + content: text.includes(COMPACTION_SENTINEL) ? COMPACTION_SENTINEL : "COMPACTION_SENTINEL_MISSING", + }; + } + if ((current.includes(COMPACTION_SENTINEL) + || current.includes(COMBO_SENTINEL) + || current.includes(MULTITOOL_SENTINEL) + || current.includes(PARALLEL_SENTINEL)) + && /请只回复 MEMORY_SET|only reply MEMORY_SET/i.test(current)) { + return { role: "assistant", content: "MEMORY_SET" }; + } + if (/PARALLEL_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "PARALLEL_CONTEXT_PRESSURE_READY" }; + if (/MULTITOOL_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "MULTITOOL_CONTEXT_PRESSURE_READY" }; + if (/COMBO_CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "COMBO_CONTEXT_PRESSURE_READY" }; + if (/CONTEXT_PRESSURE_READY/.test(current)) return { role: "assistant", content: "CONTEXT_PRESSURE_READY" }; + + if (text.includes(expectedMcpEchoResult)) return { role: "assistant", content: expectedMcpEchoResult }; + if (/qa_mcp_echo|mcp-ok-local-agent/i.test(current || text) && mcpTool && !text.includes(expectedMcpEchoResult)) { + return toolCall("call_qa_mcp_echo", mcpTool, { text: requestedMcpEchoText }); + } + + if (/LOOP_LIMIT|loop-limit-repeat-local-agent|iteration limit/i.test(current || text) && pluginTool) { + return toolCall(`call_qa_plugin_echo_loop_limit_${Date.now()}`, pluginTool, { text: LOOP_LIMIT_INPUT }); + } + + if (/TOOL_ERROR_RECOVERY|tool-error-recovery-local-agent|qa_plugin_fail/i.test(current || text)) { + if (/(?:Error:|Tool execution failed:|ActionCallError:|RuntimeError:)/i.test(text) + && /(?:qa-plugin-smoke forced failure|qa_plugin_fail|tool-error-recovery-local-agent)/i.test(text)) { + return { role: "assistant", content: TOOL_ERROR_FINAL }; + } + if (pluginFailTool) { + return toolCall("call_qa_plugin_fail_recovery", pluginFailTool, { text: TOOL_ERROR_INPUT }); + } + } + + const isParallelComboRequest = /PARALLEL_COMBO|parallel-tool-a-local-agent|parallel-tool-b-local-agent|PARALLEL_RAG_GOAL/i.test(current || text); + if (isParallelComboRequest && pluginTool && !text.includes(PARALLEL_TOOL_A_RESULT) && !text.includes(PARALLEL_TOOL_B_RESULT)) { + return toolCalls([ + ["call_qa_plugin_echo_parallel_a", pluginTool, { text: PARALLEL_TOOL_A_INPUT }], + ["call_qa_plugin_echo_parallel_b", pluginTool, { text: PARALLEL_TOOL_B_INPUT }], + ]); + } + if (isParallelComboRequest && (text.includes(PARALLEL_TOOL_A_RESULT) || text.includes(PARALLEL_TOOL_B_RESULT))) { + return missingOrFinal(text, [ + [PARALLEL_SENTINEL, "parallel-memory"], + [RAG_SENTINEL, "rag"], + [PARALLEL_TOOL_A_RESULT, "tool-a"], + [PARALLEL_TOOL_B_RESULT, "tool-b"], + ], "PARALLEL_COMBO_MISSING", PARALLEL_FINAL); + } + + const isMultiToolComboRequest = /MULTITOOL_COMBO|multi-tool-a-local-agent|multi-tool-b-local-agent|MULTITOOL_RAG_GOAL/i.test(current || text); + if (isMultiToolComboRequest && pluginTool && !text.includes(MULTITOOL_TOOL_A_RESULT)) { + return toolCall("call_qa_plugin_echo_multi_a", pluginTool, { text: MULTITOOL_TOOL_A_INPUT }); + } + if (isMultiToolComboRequest && pluginTool && text.includes(MULTITOOL_TOOL_A_RESULT) && !text.includes(MULTITOOL_TOOL_B_RESULT)) { + return toolCall("call_qa_plugin_echo_multi_b", pluginTool, { text: MULTITOOL_TOOL_B_INPUT }); + } + if (isMultiToolComboRequest && text.includes(MULTITOOL_TOOL_A_RESULT) && text.includes(MULTITOOL_TOOL_B_RESULT)) { + return missingOrFinal(text, [ + [MULTITOOL_SENTINEL, "multi-memory"], + [RAG_SENTINEL, "rag"], + [MULTITOOL_TOOL_A_RESULT, "tool-a"], + [MULTITOOL_TOOL_B_RESULT, "tool-b"], + ], "MULTITOOL_COMBO_MISSING", MULTITOOL_FINAL); + } + + const isComboRequest = /qa_combo|\bCOMBO_FINAL\b|combo-tool-ok-local-agent|RAG_TOOL_COMBO_GOAL/i.test(current || text); + if (isComboRequest && pluginTool && !text.includes(COMBO_TOOL_RESULT)) { + return toolCall("call_qa_plugin_echo_combo", pluginTool, { text: COMBO_TOOL_INPUT }); + } + if (isComboRequest && text.includes(COMBO_TOOL_RESULT)) { + return missingOrFinal(text, [ + [COMBO_SENTINEL, "combo-memory"], + [RAG_SENTINEL, "rag"], + [COMBO_TOOL_RESULT, "tool-result"], + ], "COMBO_MISSING", COMBO_FINAL); + } + + if (/qa-plugin-smoke:plugin-tool-ok-local-agent/.test(text)) { + return { role: "assistant", content: "qa-plugin-smoke:plugin-tool-ok-local-agent" }; + } + if (/qa_plugin_echo|plugin-tool-ok-local-agent/i.test(current || text) && pluginTool && !/qa-plugin-smoke:plugin-tool-ok-local-agent/.test(text)) { + return toolCall("call_qa_plugin_echo", pluginTool, { text: "plugin-tool-ok-local-agent" }); + } + + const e2eTool = firstToolName(tools, ["e2e_lookup"]); + if (/Use the e2e lookup tool|e2e lookup tool|tool loop/i.test(current || text) && e2eTool && !/tool-result:alpha/.test(text)) { + return toolCall("call_e2e_lookup", e2eTool, { query: "alpha" }); + } + if (/tool-result:alpha/.test(text)) return { role: "assistant", content: "Tool loop final answer after tool-result:alpha" }; + if (/RAG_SENTINEL/.test(text)) return { role: "assistant", content: "RAG final answer with RAG_SENTINEL" }; + if (/NONSTREAM_OK/i.test(current || text)) return { role: "assistant", content: "NONSTREAM_OK" }; + if (/IMAGE_OK/i.test(current || text) + && /(?:\[Image\]|langbot_input_attachments|data:image\/|\"type\":\s*\"image\")/i.test(current || text)) { + return { role: "assistant", content: "IMAGE_OK" }; + } + if (/azalea-cobalt-7421/.test(text)) return { role: "assistant", content: "azalea-cobalt-7421" }; + + return { role: "assistant", content: responseTextForBody(payload) }; +} + +function toolCall(id, name, args) { + return toolCalls([[id, name, args]]); +} + +function toolCalls(calls) { + return { + role: "assistant", + content: "", + tool_calls: calls.map(([id, name, args]) => ({ + id, + type: "function", + function: { + name, + arguments: JSON.stringify(args), + }, + })), + }; +} + +function missingOrFinal(text, requirements, prefix, finalText) { + const missing = requirements + .filter(([needle]) => !text.includes(needle)) + .map(([, label]) => label); + return { + role: "assistant", + content: missing.length ? `${prefix}_${missing.join("_")}` : finalText, + }; +} + function flattenContent(content) { if (typeof content === "string") return content; if (Array.isArray(content)) { @@ -471,12 +839,18 @@ function applyConfig(updates) { assignNonNegativeInteger(updates, "chunk_count"); assignNonNegativeInteger(updates, "fail_first_n"); assignNonNegativeInteger(updates, "fail_every_n"); + assignTextList(updates, "fail_models"); assignNonNegativeInteger(updates, "request_log_limit"); if (updates.fault_status !== undefined) { const parsed = Number.parseInt(String(updates.fault_status), 10); if (Number.isInteger(parsed) && parsed >= 400 && parsed <= 599) config.fault_status = parsed; } assignBoolean(updates, "fail_after_first_chunk"); + assignNonNegativeInteger(updates, "fail_after_first_chunk_delay_ms"); + if (updates.fail_after_first_chunk_mode !== undefined) { + config.fail_after_first_chunk_mode = faultMode(updates.fail_after_first_chunk_mode); + } + assignTextList(updates, "fail_after_first_chunk_models"); assignBoolean(updates, "dynamic_response"); } @@ -494,3 +868,21 @@ function assignBoolean(updates, key) { if (updates[key] === undefined) return; config[key] = bool(updates[key], config[key]); } + +function assignTextList(updates, key) { + if (updates[key] === undefined) return; + config[key] = Array.isArray(updates[key]) + ? updates[key].map(String).map((item) => item.trim()).filter(Boolean) + : textList(updates[key]); +} + +function textList(value) { + return String(value || "") + .split(/\r?\n|,/) + .map((item) => item.trim()) + .filter(Boolean); +} + +function faultMode(value) { + return String(value || "").trim().toLowerCase() === "error_event" ? "error_event" : "disconnect"; +} diff --git a/skills/scripts/e2e/install-qa-plugin-smoke.mjs b/skills/scripts/e2e/install-qa-plugin-smoke.mjs index 73b89af69..f88f3b6aa 100755 --- a/skills/scripts/e2e/install-qa-plugin-smoke.mjs +++ b/skills/scripts/e2e/install-qa-plugin-smoke.mjs @@ -4,9 +4,12 @@ import { readFile } from "node:fs/promises"; import { resolve } from "node:path"; import { env } from "node:process"; import { + authenticatedApiHeaders, apiJson, ensureEvidence, evidencePaths, + isTaskComplete, + isTaskFailed, loadEnvFiles, resetAndAuthLocalUser, writeResult, @@ -26,8 +29,14 @@ const packagePath = resolve( || "skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg", ); const expectedPluginId = env.LANGBOT_E2E_EXPECTED_PLUGIN_ID || "qa/plugin-smoke"; -const expectedTool = env.LANGBOT_E2E_EXPECTED_TOOL || (expectedPluginId === "qa/plugin-smoke" ? "qa_plugin_echo" : ""); +const expectedTools = (env.LANGBOT_E2E_EXPECTED_TOOLS || env.LANGBOT_E2E_EXPECTED_TOOL || ( + expectedPluginId === "qa/plugin-smoke" ? "qa_plugin_echo" : "" +)) + .split(",") + .map((item) => item.trim()) + .filter(Boolean); const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || ""; +const forceReinstall = /^(?:1|true|yes|on)$/i.test(env.LANGBOT_E2E_FORCE_REINSTALL || ""); const result = { source: "automation", @@ -40,6 +49,7 @@ const result = { package_preview: null, task_id: null, task: null, + reinstall_reason: "", plugin_present_before: false, plugin_present_after: false, tool_names: [], @@ -64,21 +74,25 @@ try { } result.plugin_present_before = await hasPlugin(backendUrl, auth.token); - if (!result.plugin_present_before) { - const form = new FormData(); - form.set("file", new Blob([bytes]), packagePath.split("/").pop()); - const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, { - method: "POST", - headers: { Authorization: `Bearer ${auth.token}` }, - body: form, - }); - const json = await response.json().catch(() => ({})); - if (response.status >= 400 || json.code !== 0) { - throw new Error(json.msg || `Plugin install request failed with HTTP ${response.status}.`); + if (expectedTools.length > 0) { + result.tool_names = await listToolNames(backendUrl, auth.token); + } + const missingToolsBefore = expectedTools.filter((tool) => !result.tool_names.includes(tool)); + if (result.plugin_present_before && (forceReinstall || missingToolsBefore.length > 0)) { + result.reinstall_reason = forceReinstall + ? "Explicit reinstall requested by LANGBOT_E2E_FORCE_REINSTALL." + : `Installed plugin is missing expected tools: ${missingToolsBefore.join(", ")}`; + const removeTask = await removePlugin(backendUrl, auth.token); + if (!isTaskComplete(removeTask)) { + throw new Error(`Plugin reinstall cleanup did not complete successfully: ${JSON.stringify(removeTask)}`); } - result.task_id = json.data?.task_id ?? null; - if (!result.task_id) throw new Error("Plugin install response did not include task_id."); - result.task = await waitForTask(backendUrl, auth.token, result.task_id); + result.plugin_present_before = false; + await sleep(1000); + } + + if (!result.plugin_present_before) { + result.task = await installPlugin(backendUrl, auth.token, bytes, packagePath); + result.task_id = result.task?.id || result.task_id; if (!isTaskComplete(result.task)) { throw new Error(`Plugin install task did not complete successfully: ${JSON.stringify(result.task)}`); } @@ -87,10 +101,11 @@ try { await sleep(1000); result.plugin_present_after = await hasPlugin(backendUrl, auth.token); if (!result.plugin_present_after) throw new Error(`${expectedPluginId} is not listed by /api/v1/plugins after install.`); - if (expectedTool) { + if (expectedTools.length > 0) { result.tool_names = await listToolNames(backendUrl, auth.token); - if (!result.tool_names.includes(expectedTool)) { - throw new Error(`${expectedTool} is not listed by /api/v1/tools after install.`); + const missingTools = expectedTools.filter((tool) => !result.tool_names.includes(tool)); + if (missingTools.length > 0) { + throw new Error(`${missingTools.join(", ")} is not listed by /api/v1/tools after install.`); } } if (expectedRunnerId) { @@ -121,12 +136,47 @@ async function hasPlugin(backendUrl, token) { }); } +async function installPlugin(backendUrl, token, bytes, packagePath) { + const form = new FormData(); + form.set("file", new Blob([bytes]), packagePath.split("/").pop()); + const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local`, { + method: "POST", + headers: await authenticatedApiHeaders(backendUrl, token, { contentType: "" }), + body: form, + }); + const json = await response.json().catch(() => ({})); + if (response.status >= 400 || json.code !== 0) { + throw new Error(json.msg || `Plugin install request failed with HTTP ${response.status}.`); + } + const taskId = json.data?.task_id ?? null; + if (!taskId) throw new Error("Plugin install response did not include task_id."); + const task = await waitForTask(backendUrl, token, taskId); + return { id: taskId, ...task }; +} + +async function removePlugin(backendUrl, token) { + const [author, pluginName] = expectedPluginId.split("/"); + if (!author || !pluginName) throw new Error(`Invalid expected plugin id: ${expectedPluginId}`); + const response = await apiJson( + backendUrl, + `/api/v1/plugins/${encodeURIComponent(author)}/${encodeURIComponent(pluginName)}?delete_data=false`, + { method: "DELETE", token }, + ); + if (response.status >= 400 || response.json.code !== 0) { + throw new Error(response.json.msg || `Plugin delete request failed with HTTP ${response.status}.`); + } + const taskId = response.json.data?.task_id ?? null; + if (!taskId) throw new Error("Plugin delete response did not include task_id."); + const task = await waitForTask(backendUrl, token, taskId); + return { id: taskId, ...task }; +} + async function previewPackage(backendUrl, token, bytes, packagePath) { const form = new FormData(); form.set("file", new Blob([bytes]), packagePath.split("/").pop()); const response = await fetch(`${backendUrl.replace(/\/$/, "")}/api/v1/plugins/install/local/preview`, { method: "POST", - headers: { Authorization: `Bearer ${token}` }, + headers: await authenticatedApiHeaders(backendUrl, token, { contentType: "" }), body: form, }); const json = await response.json().catch(() => ({})); @@ -167,32 +217,12 @@ async function waitForTask(backendUrl, token, taskId) { while (Date.now() < deadline) { const response = await apiJson(backendUrl, `/api/v1/system/tasks/${encodeURIComponent(taskId)}`, { token }); last = response.json.data || response.json; - if (isTaskComplete(last) || isTaskFailed(last)) return last; + if (isTaskFailed(last) || isTaskComplete(last)) return last; await sleep(1000); } return last; } -function isTaskComplete(task) { - const status = String(task?.status || task?.state || "").toLowerCase(); - const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase(); - return ["done", "completed", "success", "succeeded", "finished"].includes(status) - || ["done", "completed", "success", "succeeded", "finished"].includes(runtimeStatus) - || task?.done === true - || task?.completed === true - || (task?.runtime?.done === true && !task?.runtime?.exception); -} - -function isTaskFailed(task) { - const status = String(task?.status || task?.state || "").toLowerCase(); - const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase(); - return ["failed", "error", "cancelled", "canceled"].includes(status) - || ["failed", "error", "cancelled", "canceled"].includes(runtimeStatus) - || task?.failed === true - || Boolean(task?.error) - || Boolean(task?.runtime?.exception); -} - function sleep(ms) { return new Promise((resolve) => setTimeout(resolve, ms)); } diff --git a/skills/scripts/e2e/langrag-kb-retrieve.mjs b/skills/scripts/e2e/langrag-kb-retrieve.mjs index a2b489a60..bc0778c5f 100755 --- a/skills/scripts/e2e/langrag-kb-retrieve.mjs +++ b/skills/scripts/e2e/langrag-kb-retrieve.mjs @@ -3,6 +3,7 @@ import { bodyText, createBrowser, + ensureBrowserWorkspace, ensureEvidence, evidencePaths, exitCode, @@ -57,6 +58,12 @@ try { const { page } = browser; await page.goto(`${frontendUrl.replace(/\/$/, "")}/home/knowledge`, { waitUntil: "domcontentloaded" }); await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + const workspace = await ensureBrowserWorkspace(page, backendUrl); + if (workspace.status !== "pass") { + result.status = workspace.status; + result.reason = workspace.reason; + throw new Error(result.reason); + } result.url = page.url(); const text = await bodyText(page); @@ -77,6 +84,7 @@ try { headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", }, body: JSON.stringify({ query }), }); diff --git a/skills/scripts/e2e/lib/debug-chat.mjs b/skills/scripts/e2e/lib/debug-chat.mjs index 7d44d00c9..d8717da9e 100755 --- a/skills/scripts/e2e/lib/debug-chat.mjs +++ b/skills/scripts/e2e/lib/debug-chat.mjs @@ -1,6 +1,7 @@ import { bodyText, clickFirstVisible, + clickFirstVisibleLocator, countOccurrences, gotoFrontend, isLoginUrl, @@ -28,6 +29,11 @@ export function findNewFailureSignal(beforeText, afterText, failureSignals = DEB return failureSignals.find((signal) => countOccurrences(afterText, signal) > countOccurrences(beforeText, signal)) || ""; } +export function hasDebugChatOutcome(text, expectedText, minExpectedCount, failureBaselines = []) { + if (countOccurrences(text, expectedText) >= minExpectedCount) return true; + return failureBaselines.some(({ signal, count }) => countOccurrences(text, signal) > count); +} + function findFailureSignalInText(text, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS) { return failureSignals.find((signal) => String(text || "").includes(signal)) || ""; } @@ -45,22 +51,24 @@ function debugChatInput(page) { } async function clickDebugChatTab(page) { - const tabByRole = page.getByRole("tab", { name: /Debug Chat|调试聊天|调试对话|Debug|调试/i }).first(); - if (await tabByRole.isVisible({ timeout: 3_000 }).catch(() => false)) { - await tabByRole.click(); - return true; - } - - const tabBySelector = page.locator('[role="tab"]').filter({ hasText: /Debug Chat|调试聊天|调试对话|Debug|调试/i }).first(); - if (await tabBySelector.isVisible({ timeout: 2_000 }).catch(() => false)) { - await tabBySelector.click(); - return true; - } - - return Boolean(await clickFirstVisible(page, ["Debug Chat", "调试聊天", "调试对话"], 2_000)); + const label = /^(?:Debug Chat|调试聊天|调试对话|对话调试)$/i; + const configuredTimeout = Number.parseInt( + process.env.LANGBOT_E2E_UI_READY_TIMEOUT_MS + || process.env.LANGBOT_E2E_NAVIGATION_TIMEOUT_MS + || "30000", + 10, + ); + const timeout = Number.isFinite(configuredTimeout) && configuredTimeout > 0 + ? configuredTimeout + : 30_000; + return await clickFirstVisibleLocator(page, [ + page.getByRole("tab", { name: label }), + page.locator('[data-slot="tabs-trigger"]').filter({ hasText: label }), + page.getByText(label, { exact: true }), + ], timeout); } -async function waitForDebugChatReady(page, timeout = 20_000) { +export async function waitForDebugChatReady(page, timeout = 20_000) { const input = debugChatInput(page); const visible = await input.isVisible({ timeout }).catch(() => false); if (!visible) { @@ -70,7 +78,13 @@ async function waitForDebugChatReady(page, timeout = 20_000) { }; } - const enabled = await input.isEnabled({ timeout }).catch(() => false); + const deadline = Date.now() + timeout; + let enabled = false; + while (Date.now() < deadline) { + enabled = await input.isEnabled().catch(() => false); + if (enabled) break; + await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now()))); + } if (!enabled) { return { ready: false, @@ -85,14 +99,22 @@ export function classifyDebugChatResult({ beforeText, afterText, expectedText, + expectedTexts = null, prompt, latestExpectedLeaf, latestFailureLeaf, beforeMessages = null, afterMessages = null, latestAssistantText = "", + latestAssistantIsFinal = null, + maxNewAssistantMessages = null, failureSignals = DEBUG_CHAT_FAILURE_SIGNALS, }) { + const requiredExpectedTexts = [...new Set( + (Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText]) + .map(String) + .filter(Boolean), + )]; const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt); const finalCount = countOccurrences(afterText, expectedText); const failureText = findNewFailureSignal(beforeText, afterText, failureSignals); @@ -104,11 +126,25 @@ export function classifyDebugChatResult({ const afterAssistantExpectedCount = hasMessageEvidence ? countExpectedInMessages(afterMessages, expectedText) : null; - const assistantExpectedIncreased = hasMessageEvidence - ? afterAssistantExpectedCount > beforeAssistantExpectedCount - : false; + const beforeAssistantMessageCount = hasMessageEvidence + ? beforeMessages.filter((message) => message.role === "assistant").length + : null; + const afterAssistantMessageCount = hasMessageEvidence + ? afterMessages.filter((message) => message.role === "assistant").length + : null; + const newAssistantMessageCount = hasMessageEvidence + ? afterAssistantMessageCount - beforeAssistantMessageCount + : null; + const assistantMessageEvidence = { + before_assistant_message_count: beforeAssistantMessageCount, + after_assistant_message_count: afterAssistantMessageCount, + new_assistant_message_count: newAssistantMessageCount, + }; if (hasMessageEvidence) { + const missingExpectedTexts = requiredExpectedTexts.filter( + (text) => !String(latestAssistantText).includes(text), + ); const latestAssistantFailure = findFailureSignalInText(latestAssistantText, failureSignals); if (latestAssistantFailure) { return { @@ -119,16 +155,44 @@ export function classifyDebugChatResult({ failure_signal: latestAssistantFailure, before_assistant_expected_count: beforeAssistantExpectedCount, after_assistant_expected_count: afterAssistantExpectedCount, + ...assistantMessageEvidence, }; } - if (assistantExpectedIncreased && String(latestAssistantText).includes(expectedText)) { + if (latestAssistantIsFinal === false) { return { - status: "pass", - reason: `Expected text appeared in a new assistant message: ${expectedText}`, + status: "fail", + reason: "The latest assistant message contained the expected text but was not final.", min_expected_count: minExpectedCount, final_count: finalCount, before_assistant_expected_count: beforeAssistantExpectedCount, after_assistant_expected_count: afterAssistantExpectedCount, + ...assistantMessageEvidence, + latest_assistant_is_final: false, + }; + } + if (maxNewAssistantMessages !== null && newAssistantMessageCount > maxNewAssistantMessages) { + return { + status: "fail", + reason: `Debug Chat created ${newAssistantMessageCount} assistant messages; expected at most ${maxNewAssistantMessages}.`, + min_expected_count: minExpectedCount, + final_count: finalCount, + before_assistant_expected_count: beforeAssistantExpectedCount, + after_assistant_expected_count: afterAssistantExpectedCount, + ...assistantMessageEvidence, + }; + } + if (newAssistantMessageCount > 0 && missingExpectedTexts.length === 0) { + return { + status: "pass", + reason: requiredExpectedTexts.length === 1 + ? `Expected text appeared in a new assistant message: ${expectedText}` + : `All ${requiredExpectedTexts.length} expected text fragments appeared in a new assistant message.`, + min_expected_count: minExpectedCount, + final_count: finalCount, + before_assistant_expected_count: beforeAssistantExpectedCount, + after_assistant_expected_count: afterAssistantExpectedCount, + ...assistantMessageEvidence, + missing_expected_texts: [], }; } if (failureText) { @@ -140,15 +204,20 @@ export function classifyDebugChatResult({ failure_signal: failureText, before_assistant_expected_count: beforeAssistantExpectedCount, after_assistant_expected_count: afterAssistantExpectedCount, + ...assistantMessageEvidence, }; } return { status: "fail", - reason: `Expected text did not appear in a new assistant message. Expected assistant occurrences to increase above ${beforeAssistantExpectedCount}, saw ${afterAssistantExpectedCount}.`, + reason: missingExpectedTexts.length > 0 + ? `A new assistant message was missing expected text: ${missingExpectedTexts.join(", ")}` + : `Expected text did not appear in a new assistant message. Expected assistant occurrences to increase above ${beforeAssistantExpectedCount}, saw ${afterAssistantExpectedCount}.`, min_expected_count: minExpectedCount, final_count: finalCount, before_assistant_expected_count: beforeAssistantExpectedCount, after_assistant_expected_count: afterAssistantExpectedCount, + ...assistantMessageEvidence, + missing_expected_texts: missingExpectedTexts, }; } if (failureText) { @@ -188,8 +257,16 @@ export function classifyDebugChatResult({ export async function openPipelineDebugChat(page, { pipelineUrl, pipelineName, envHint = "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME" }) { if (pipelineUrl) { - await page.goto(pipelineUrl, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + let alreadyAtPipeline = false; + try { + alreadyAtPipeline = new URL(page.url()).href === new URL(pipelineUrl).href; + } catch { + // Invalid URLs are handled by page.goto below. + } + if (!alreadyAtPipeline) { + await page.goto(pipelineUrl, { waitUntil: "commit" }); + await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + } } else { if (!pipelineName) { return { @@ -290,12 +367,37 @@ export async function visibleDebugChatMessages(page) { }); } -export async function waitForExpectedDebugChatText(page, { expectedText, minExpectedCount, timeoutMs }) { +export async function waitForExpectedDebugChatText(page, { + expectedText, + expectedTexts = null, + minExpectedCount, + minExpectedCounts = null, + timeoutMs, + beforeText = "", + failureSignals = DEBUG_CHAT_FAILURE_SIGNALS, +}) { + const requiredExpectedTexts = [...new Set( + (Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText]) + .map(String) + .filter(Boolean), + )]; + const expectedRequirements = requiredExpectedTexts.map((text, index) => ({ + text, + min: Array.isArray(minExpectedCounts) && Number.isFinite(minExpectedCounts[index]) + ? minExpectedCounts[index] + : (text === expectedText ? minExpectedCount : minExpectedOccurrences(beforeText, text, "")), + })); + const failureBaselines = failureSignals.map((signal) => ({ + signal, + count: countOccurrences(beforeText, signal), + })); await page.waitForFunction( - ({ expected, min }) => { - return document.body.innerText.split(expected).length - 1 >= min; + ({ requirements, failures }) => { + const text = document.body.innerText; + if (requirements.every((item) => text.split(item.text).length - 1 >= item.min)) return true; + return failures.some(({ signal, count }) => text.split(signal).length - 1 > count); }, - { expected: expectedText, min: minExpectedCount }, + { requirements: expectedRequirements, failures: failureBaselines }, { timeout: timeoutMs }, ).catch(() => {}); } @@ -316,18 +418,104 @@ export async function waitForDebugChatTextStable(page, { timeoutMs = 5_000, quie } } +async function fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType }) { + if (!backendUrl || !pipelineId || !sessionType) { + return { status: "not_required", messages: [] }; + } + return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => { + const token = localStorage.getItem("token") || ""; + try { + const response = await fetch( + `${backendUrl.replace(/\/$/, "")}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/messages/${encodeURIComponent(sessionType)}`, + { + headers: token + ? { + Authorization: `Bearer ${token}`, + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", + } + : {}, + }, + ); + const json = await response.json().catch(() => ({})); + return { + status: response.ok && json.code === 0 ? "ready" : "fail", + http_status: response.status, + code: json.code ?? null, + messages: json.data?.messages || [], + reason: response.ok && json.code === 0 ? "" : json.msg || `Debug Chat history returned HTTP ${response.status}.`, + }; + } catch (error) { + return { + status: "retryable", + messages: [], + reason: `Debug Chat history request failed: ${error instanceof Error ? error.message : String(error)}`, + }; + } + }, { backendUrl, pipelineId, sessionType }); +} + +async function waitForFinalDebugChatAssistant(page, { + backendUrl, + pipelineId, + sessionType, + beforeAssistantCount, + timeoutMs, +}) { + if (!backendUrl || !pipelineId || !sessionType) { + return { status: "not_required", latest_assistant_is_final: null }; + } + const deadline = Date.now() + Math.max(1, timeoutMs); + let lastHistory = null; + while (Date.now() < deadline) { + lastHistory = await fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType }); + if (lastHistory.status === "fail") return lastHistory; + if (lastHistory.status === "retryable") { + await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now()))); + continue; + } + const assistants = lastHistory.messages.filter((message) => message.role === "assistant"); + const latest = assistants.at(-1); + if (assistants.length > beforeAssistantCount && latest?.is_final === true) { + return { + status: "pass", + latest_assistant_is_final: true, + assistant_message_count: assistants.length, + }; + } + await page.waitForTimeout(Math.min(250, Math.max(1, deadline - Date.now()))); + } + const assistants = (lastHistory?.messages || []).filter((message) => message.role === "assistant"); + return { + status: "fail", + reason: "Timed out waiting for the new assistant message to become final.", + latest_assistant_is_final: assistants.at(-1)?.is_final === true, + assistant_message_count: assistants.length, + }; +} + export async function attachDebugChatImage(page, imagePath) { if (!imagePath) return { status: "not_required", reason: "" }; const input = page.locator('input[type="file"][accept*="image"], input[type="file"]').first(); if (!await input.count()) { return { status: "fail", reason: "Could not find a Debug Chat image upload input." }; } + const previews = page.locator('[data-debug-chat-attachment-preview="true"]'); + const beforePreviewCount = await previews.count(); await input.setInputFiles(imagePath); - await page.locator("img").last().waitFor({ state: "visible", timeout: 10_000 }).catch(() => {}); + const previewVisible = await previews + .nth(beforePreviewCount) + .waitFor({ state: "visible", timeout: 10_000 }) + .then(() => true) + .catch(() => false); + if (!previewVisible) { + return { status: "fail", reason: "Debug Chat did not show the selected image preview." }; + } return { status: "ready", reason: `Attached image fixture: ${imagePath}` }; } export async function sendDebugChatPrompt(page, prompt, imagePath = "") { + const sentImages = page.locator('[data-debug-chat-message-image="true"]'); + const beforeSentImageCount = imagePath ? await sentImages.count() : 0; const imageResult = await attachDebugChatImage(page, imagePath); if (imageResult.status === "fail") return imageResult; @@ -342,25 +530,74 @@ export async function sendDebugChatPrompt(page, prompt, imagePath = "") { const clickedSend = await clickFirstVisible(page, ["Send", "发送", "提交"], 1_500); if (!clickedSend) await page.keyboard.press("Enter"); await page.getByText(prompt, { exact: false }).last().waitFor({ state: "visible", timeout: 10_000 }).catch(() => {}); + if (imagePath) { + const sentImageVisible = await sentImages + .nth(beforeSentImageCount) + .waitFor({ state: "visible", timeout: 15_000 }) + .then(() => true) + .catch(() => false); + if (!sentImageVisible) { + return { status: "fail", reason: "The sent Debug Chat message did not render its image attachment." }; + } + } return true; } -export async function runDebugChatPrompt(page, { prompt, expectedText, responseTimeoutMs, imagePath = "", failureSignals = DEBUG_CHAT_FAILURE_SIGNALS }) { +export async function runDebugChatPrompt(page, { + prompt, + expectedText, + expectedTexts = null, + responseTimeoutMs, + imagePath = "", + backendUrl = "", + pipelineId = "", + sessionType = "person", + maxNewAssistantMessages = null, + failureSignals = DEBUG_CHAT_FAILURE_SIGNALS, +}) { const beforeText = await bodyText(page); const beforeMessages = await visibleDebugChatMessages(page); + const beforeHistory = await fetchDebugChatHistory(page, { backendUrl, pipelineId, sessionType }); + const beforeHistoryAssistantCount = beforeHistory.messages.filter((message) => message.role === "assistant").length; + const requiredExpectedTexts = [...new Set( + (Array.isArray(expectedTexts) && expectedTexts.length > 0 ? expectedTexts : [expectedText]) + .map(String) + .filter(Boolean), + )]; const minExpectedCount = minExpectedOccurrences(beforeText, expectedText, prompt); + const minExpectedCounts = requiredExpectedTexts.map( + (text) => minExpectedOccurrences(beforeText, text, prompt), + ); const sent = await sendDebugChatPrompt(page, prompt, imagePath); if (sent !== true) { if (sent && typeof sent === "object" && typeof sent.reason === "string") return sent; return { status: "fail", reason: "Could not find a Debug Chat text input." }; } - await waitForExpectedDebugChatText(page, { + const responseStartedAt = Date.now(); + const expectedTextPromise = waitForExpectedDebugChatText(page, { expectedText, + expectedTexts: requiredExpectedTexts, minExpectedCount, + minExpectedCounts, prompt, timeoutMs: responseTimeoutMs, + beforeText, + failureSignals, }); + const finalAssistantPromise = waitForFinalDebugChatAssistant(page, { + backendUrl, + pipelineId, + sessionType, + beforeAssistantCount: beforeHistoryAssistantCount, + timeoutMs: Math.max(1, responseTimeoutMs - (Date.now() - responseStartedAt)), + }); + if (backendUrl && pipelineId && sessionType) { + await Promise.race([expectedTextPromise, finalAssistantPromise]); + } else { + await expectedTextPromise; + } + const finalAssistant = await finalAssistantPromise; await waitForDebugChatTextStable(page); const afterText = await bodyText(page); @@ -370,18 +607,27 @@ export async function runDebugChatPrompt(page, { prompt, expectedText, responseT const failureText = findNewFailureSignal(beforeText, afterText, failureSignals); const latestFailureLeaf = failureText ? await latestVisibleLeafText(page, [failureText]) : ""; - return classifyDebugChatResult({ + const classified = classifyDebugChatResult({ beforeText, afterText, expectedText, + expectedTexts: requiredExpectedTexts, prompt, latestExpectedLeaf, latestFailureLeaf, beforeMessages, afterMessages, latestAssistantText, + latestAssistantIsFinal: finalAssistant.latest_assistant_is_final, + maxNewAssistantMessages, failureSignals, }); + return { + ...classified, + latest_assistant_is_final: finalAssistant.latest_assistant_is_final, + final_assistant_wait_status: finalAssistant.status, + final_assistant_wait_reason: finalAssistant.reason || "", + }; } export async function setDebugChatStreamOutput(page, desired) { diff --git a/skills/scripts/e2e/lib/isolated-langbot-instance.mjs b/skills/scripts/e2e/lib/isolated-langbot-instance.mjs new file mode 100644 index 000000000..4d75784c9 --- /dev/null +++ b/skills/scripts/e2e/lib/isolated-langbot-instance.mjs @@ -0,0 +1,200 @@ +import { execFile } from "node:child_process"; +import { closeSync, openSync } from "node:fs"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { promisify } from "node:util"; +import { spawn } from "node:child_process"; +import { pathExists, resolveLangBotRepo } from "./langbot-e2e.mjs"; + +const execFileAsync = promisify(execFile); +const proxyKeys = [ + "ALL_PROXY", + "all_proxy", + "HTTP_PROXY", + "http_proxy", + "HTTPS_PROXY", + "https_proxy", +]; + +function withoutProxy(source = process.env) { + const next = { ...source }; + for (const key of proxyKeys) delete next[key]; + next.NO_PROXY = "127.0.0.1,localhost"; + next.no_proxy = "127.0.0.1,localhost"; + return next; +} + +async function getFreePort() { + const server = createServer(); + await new Promise((resolvePromise, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolvePromise); + }); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + await new Promise((resolvePromise) => server.close(resolvePromise)); + if (!port) throw new Error("Could not allocate an isolated LangBot port."); + return port; +} + +async function waitForHttp(url, child, timeoutMs, label) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (child.exitCode !== null) { + throw new Error(`${label} exited before becoming ready (code ${child.exitCode}).`); + } + try { + const response = await fetch(url, { redirect: "manual" }); + if (response.status < 500) return response; + } catch { + // The service is still starting. + } + await new Promise((resolvePromise) => setTimeout(resolvePromise, 500)); + } + throw new Error(`${label} did not become ready within ${timeoutMs}ms.`); +} + +function spawnLogged(command, args, { cwd, env, logPath }) { + const logFd = openSync(logPath, "a"); + try { + return spawn(command, args, { + cwd, + env, + detached: true, + stdio: ["ignore", logFd, logFd], + }); + } finally { + closeSync(logFd); + } +} + +async function stopProcess(child) { + if (!child || child.exitCode !== null) return; + const closed = new Promise((resolvePromise) => child.once("close", resolvePromise)); + try { + process.kill(-child.pid, "SIGTERM"); + } catch { + child.kill("SIGTERM"); + } + const graceful = await Promise.race([ + closed.then(() => true), + new Promise((resolvePromise) => setTimeout(() => resolvePromise(false), 5_000)), + ]); + if (graceful) return; + try { + process.kill(-child.pid, "SIGKILL"); + } catch { + child.kill("SIGKILL"); + } + await closed.catch(() => {}); +} + +async function initializeUser(backendUrl, user, password) { + const response = await fetch(`${backendUrl}/api/v1/user/init`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ user, password }), + }); + const payload = await response.json().catch(() => ({})); + if (response.status >= 400 || ![0, 1].includes(payload.code)) { + throw new Error(payload.msg || `Could not initialize isolated user (HTTP ${response.status}).`); + } +} + +export async function startIsolatedLangBotInstance({ evidenceDir }) { + const repo = await resolveLangBotRepo(); + const webRepo = process.env.LANGBOT_WEB_REPO || join(repo, "web"); + const python = join(repo, ".venv", "bin", "python"); + const configScript = resolve( + dirname(fileURLToPath(import.meta.url)), + "../prepare-isolated-langbot-config.py", + ); + if (!(await pathExists(python))) throw new Error(`LangBot virtualenv Python is missing: ${python}`); + if (!(await pathExists(join(webRepo, "node_modules")))) { + throw new Error(`LangBot frontend dependencies are missing: ${join(webRepo, "node_modules")}`); + } + + const [backendPort, frontendPort, pluginDebugPort] = await Promise.all([ + getFreePort(), + getFreePort(), + getFreePort(), + ]); + const instanceRoot = await mkdtemp(join(tmpdir(), "langbot-clean-catalog-")); + const backendUrl = `http://127.0.0.1:${backendPort}`; + const frontendUrl = `http://127.0.0.1:${frontendPort}`; + const backendLog = join(evidenceDir, "isolated-backend.log"); + const frontendLog = join(evidenceDir, "isolated-frontend.log"); + const user = "langbot-e2e@example.invalid"; + const password = "LangBotIsolatedE2E!2026"; + let backend; + let frontend; + + const stop = async () => { + await stopProcess(frontend); + await stopProcess(backend); + await rm(instanceRoot, { recursive: true, force: true }); + }; + + try { + await execFileAsync( + python, + [ + configScript, + "--instance-root", + instanceRoot, + "--port", + String(backendPort), + "--plugin-debug-port", + String(pluginDebugPort), + ], + { cwd: repo, env: withoutProxy(), timeout: 30_000 }, + ); + + backend = spawnLogged(python, ["-m", "langbot"], { + cwd: instanceRoot, + env: { + ...withoutProxy(), + PYTHONPATH: join(repo, "src"), + LANGBOT_DATA_ROOT: join(instanceRoot, "data"), + API__PORT: String(backendPort), + API__WEBHOOK_PREFIX: backendUrl, + SPACE__DISABLE_TELEMETRY: "true", + SPACE__DISABLE_MODELS_SERVICE: "true", + }, + logPath: backendLog, + }); + await waitForHttp(`${backendUrl}/api/v1/system/info`, backend, 180_000, "Isolated LangBot backend"); + + const configText = await readFile(join(instanceRoot, "data", "config.yaml"), "utf8"); + const recoveryKey = configText.match(/^\s*recovery_key:\s*['"]?([^'"\s#]+)['"]?\s*$/m)?.[1] || ""; + if (!recoveryKey) throw new Error("Isolated LangBot did not generate a recovery key."); + await initializeUser(backendUrl, user, password); + + frontend = spawnLogged( + "pnpm", + ["exec", "vite", "--host", "127.0.0.1", "--port", String(frontendPort), "--strictPort"], + { + cwd: webRepo, + env: { ...withoutProxy(), VITE_API_BASE_URL: backendUrl }, + logPath: frontendLog, + }, + ); + await waitForHttp(frontendUrl, frontend, 60_000, "Isolated LangBot frontend"); + + return { + backendUrl, + frontendUrl, + recoveryKey, + user, + password, + logs: { backend: backendLog, frontend: frontendLog }, + stop, + }; + } catch (error) { + await stop(); + throw error; + } +} diff --git a/skills/scripts/e2e/lib/langbot-e2e.mjs b/skills/scripts/e2e/lib/langbot-e2e.mjs index a7584c904..5d637a9a4 100755 --- a/skills/scripts/e2e/lib/langbot-e2e.mjs +++ b/skills/scripts/e2e/lib/langbot-e2e.mjs @@ -1,8 +1,11 @@ import { appendFile, mkdir, readFile, stat, writeFile } from "node:fs/promises"; -import { join, resolve } from "node:path"; +import { createHash } from "node:crypto"; +import { basename, join, resolve } from "node:path"; import { env } from "node:process"; const secretRe = /(?:authorization|bearer|token|secret|password|api[_-]?key|jwt|oauth)\s*[:=]\s*["']?[^"',\s]+/gi; +const ACTIVE_WORKSPACE_STORAGE_KEY = "langbot_active_workspace_uuid"; +const workspaceUuidCache = new Map(); export function redact(text) { return String(text ?? "") @@ -30,6 +33,19 @@ export function localIsoWithOffset(date = new Date()) { return `${yyyy}-${mm}-${dd}T${hh}:${mi}:${ss}.${ms}${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`; } +export function boxWorkspaceNamespace(instanceUuid, workspaceUuid) { + const instance = String(instanceUuid || "").trim(); + const workspace = String(workspaceUuid || "").trim(); + if (!instance || !workspace) { + throw new Error("Box Workspace namespace requires instance and Workspace UUIDs."); + } + const digest = createHash("sha256") + .update(`${instance}\0${workspace}`, "utf8") + .digest("hex") + .slice(0, 24); + return `ws-${digest}`; +} + export function evidencePaths(caseId) { const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join("reports", "evidence", runId)); @@ -50,6 +66,38 @@ export async function ensureEvidence(paths) { await appendFile(paths.networkLog, "", "utf8"); } +export async function beginBackendLogCapture(evidenceDir, sourcePath = env.LANGBOT_BACKEND_LOG || "") { + if (!sourcePath) return null; + const source = resolve(sourcePath); + try { + const info = await stat(source); + return { + source, + start_offset: info.size, + target: resolve(evidenceDir, "backend.log"), + }; + } catch { + return null; + } +} + +export async function finishBackendLogCapture(capture) { + if (!capture) return null; + try { + const content = await readFile(capture.source); + const start = content.length >= capture.start_offset ? capture.start_offset : 0; + const window = content.subarray(start); + if (window.length === 0) return null; + await writeFile(capture.target, window); + return { + path: capture.target, + bytes: window.length, + }; + } catch { + return null; + } +} + export async function pathExists(path) { try { await stat(path); @@ -71,6 +119,107 @@ export async function writeResult(paths, result) { } } +export function isTaskFailed(task) { + const status = String(task?.status || task?.state || "").toLowerCase(); + const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase(); + return ["failed", "error", "cancelled", "canceled"].includes(status) + || ["failed", "error", "cancelled", "canceled"].includes(runtimeStatus) + || task?.failed === true + || Boolean(task?.error) + || Boolean(task?.runtime?.exception); +} + +export function isTaskComplete(task) { + if (isTaskFailed(task)) return false; + const status = String(task?.status || task?.state || "").toLowerCase(); + const runtimeStatus = String(task?.runtime?.status || task?.runtime?.state || "").toLowerCase(); + return ["done", "completed", "success", "succeeded", "finished"].includes(status) + || ["done", "completed", "success", "succeeded", "finished"].includes(runtimeStatus) + || task?.done === true + || task?.completed === true + || task?.runtime?.done === true; +} + +function browserDiagnosticFindings(source, text) { + const findings = []; + const lines = String(text || "").split(/\r?\n/); + for (let index = 0; index < lines.length; index += 1) { + const line = lines[index]; + if (!line) continue; + const lineNumber = index + 1; + + if (source === "console") { + const checks = [ + ["pageerror", /\[pageerror\]/i], + ["frontend_uncaught_error", /\[error\].*(?:\bUncaught\b|Unhandled(?: promise rejection|Rejection)|TypeError|ReferenceError)/i], + ["http_5xx", /Failed to load resource: the server responded with a status of 5\d\d/i], + ["api_server_error", /\[error\].*Server error:/i], + ["plugin_runtime_timeout", /\[error\].*Action [A-Za-z0-9_]+ call timed out/i], + ]; + for (const [kind, regex] of checks) { + if (!regex.test(line)) continue; + findings.push({ + source, + severity: "fail", + kind, + line: lineNumber, + excerpt: redact(line.trim()), + }); + break; + } + continue; + } + + if (source === "network") { + if (/\[response\]\s+5\d\d\b/i.test(line)) { + findings.push({ + source, + severity: "fail", + kind: "http_5xx", + line: lineNumber, + excerpt: redact(line.trim()), + }); + continue; + } + if (/\[requestfailed\]/i.test(line) && !/net::ERR_ABORTED/i.test(line)) { + findings.push({ + source, + severity: "warning", + kind: "request_failed", + line: lineNumber, + excerpt: redact(line.trim()), + }); + } + } + } + return findings; +} + +export async function scanBrowserDiagnostics(paths) { + const sources = [ + ["console", paths.consoleLog], + ["network", paths.networkLog], + ]; + const findings = []; + for (const [source, path] of sources) { + let text = ""; + try { + text = await readFile(path, "utf8"); + } catch { + continue; + } + findings.push(...browserDiagnosticFindings(source, text)); + } + const hasFailure = findings.some((finding) => finding.severity === "fail"); + return { + status: hasFailure ? "fail" : "pass", + findings, + reason: hasFailure + ? `Browser diagnostics found ${findings.filter((finding) => finding.severity === "fail").length} failing signal(s).` + : "No failing browser diagnostics found.", + }; +} + export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) { const processEnvKeys = new Set(Object.keys(env)); for (const path of paths) { @@ -92,18 +241,125 @@ export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) } } -export async function readRecoveryKey(repo = env.LANGBOT_REPO || "../LangBot") { - const configPath = resolve(repo, "data/config.yaml"); +export async function resolveLangBotRepo(repo = env.LANGBOT_REPO || "", cwd = process.cwd()) { + if (repo) return resolve(repo); + + const candidates = [ + resolve(cwd), + basename(cwd) === "skills" ? resolve(cwd, "..") : "", + resolve(cwd, "../LangBot"), + resolve(cwd, "LangBot"), + ].filter(Boolean); + + const seen = new Set(); + for (const candidate of candidates) { + if (seen.has(candidate)) continue; + seen.add(candidate); + if (await pathExists(resolve(candidate, "data/config.yaml"))) return candidate; + } + + return resolve(cwd, "../LangBot"); +} + +export async function readRecoveryKey(repo = env.LANGBOT_REPO || "") { + const configPath = resolve(await resolveLangBotRepo(repo), "data/config.yaml"); const config = await readFile(configPath, "utf8"); const match = config.match(/^\s*recovery_key:\s*['"]?([^'"\s#]+)['"]?\s*$/m); return match?.[1] || ""; } -export async function apiJson(backendUrl, path, { method = "GET", token = "", body } = {}) { +function workspaceCacheKey(backendUrl, token) { + return `${backendUrl.replace(/\/$/, "")}\0${token}`; +} + +function isAccountScopedApi(path, method) { + const pathname = path.split("?", 1)[0]; + if (pathname === "/api/v1/workspaces/bootstrap") return true; + if (pathname === "/api/v1/workspaces" && method === "GET") return true; + return [ + "/api/v1/user/check-token", + "/api/v1/user/info", + "/api/v1/user/account-info", + "/api/v1/user/change-password", + ].includes(pathname); +} + +export async function resolveWorkspaceUuid( + backendUrl, + token, + preferredWorkspaceUuid = env.LANGBOT_WORKSPACE_UUID || "", +) { + if (!token) throw new Error("A user token is required to resolve the active Workspace."); + + const normalizedBackendUrl = backendUrl.replace(/\/$/, ""); + const normalizedPreferred = preferredWorkspaceUuid.trim(); + const cacheKey = workspaceCacheKey(normalizedBackendUrl, token); + const cached = workspaceUuidCache.get(cacheKey); + if (cached && (!normalizedPreferred || cached === normalizedPreferred)) return cached; + + const response = await fetch(`${normalizedBackendUrl}/api/v1/workspaces/bootstrap`, { + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + }, + }); + const json = await response.json().catch(() => ({})); + if (response.status >= 400 || json.code !== 0) { + throw new Error(json.msg || `Workspace bootstrap failed with HTTP ${response.status}.`); + } + + const workspaces = json.data?.workspaces || []; + const workspaceUuids = workspaces + .map((entry) => entry.workspace?.uuid || entry.uuid || "") + .filter(Boolean); + let workspaceUuid = normalizedPreferred; + if (workspaceUuid && !workspaceUuids.includes(workspaceUuid)) { + throw new Error(`Configured Workspace ${workspaceUuid} is not available to the authenticated Account.`); + } + if (!workspaceUuid && workspaceUuids.length === 1) { + [workspaceUuid] = workspaceUuids; + } + if (!workspaceUuid && workspaceUuids.length === 0) { + throw new Error("The authenticated Account has no available Workspace."); + } + if (!workspaceUuid) { + throw new Error( + "The authenticated Account has multiple Workspaces; set LANGBOT_WORKSPACE_UUID for this QA run.", + ); + } + + workspaceUuidCache.set(cacheKey, workspaceUuid); + return workspaceUuid; +} + +export async function authenticatedApiHeaders( + backendUrl, + token, + { contentType = "application/json", workspaceUuid = "" } = {}, +) { + const resolvedWorkspaceUuid = await resolveWorkspaceUuid(backendUrl, token, workspaceUuid); + return { + Authorization: `Bearer ${token}`, + ...(contentType ? { "Content-Type": contentType } : {}), + "X-Workspace-Id": resolvedWorkspaceUuid, + }; +} + +export async function apiJson( + backendUrl, + path, + { method = "GET", token = "", body, skipWorkspace = false, workspaceUuid = "" } = {}, +) { + const normalizedMethod = method.toUpperCase(); const headers = { "Content-Type": "application/json" }; - if (token) headers.Authorization = `Bearer ${token}`; + if (token) { + headers.Authorization = `Bearer ${token}`; + if (!skipWorkspace && !isAccountScopedApi(path, normalizedMethod)) { + headers["X-Workspace-Id"] = await resolveWorkspaceUuid(backendUrl, token, workspaceUuid); + } + } const response = await fetch(`${backendUrl.replace(/\/$/, "")}${path}`, { - method, + method: normalizedMethod, headers, body: body === undefined ? undefined : JSON.stringify(body), }); @@ -169,6 +425,38 @@ export async function setBrowserToken(page, frontendUrl, token) { await page.evaluate((value) => localStorage.setItem("token", value), token); } +export async function ensureBrowserWorkspace( + page, + backendUrl, + preferredWorkspaceUuid = env.LANGBOT_WORKSPACE_UUID || "", +) { + const browserContext = await page.evaluate((storageKey) => ({ + token: localStorage.getItem("token") || "", + workspaceUuid: localStorage.getItem(storageKey) || "", + }), ACTIVE_WORKSPACE_STORAGE_KEY); + if (!browserContext.token) { + return { status: "blocked", reason: "Browser profile has no localStorage token.", workspace_uuid: "" }; + } + + try { + const workspaceUuid = await resolveWorkspaceUuid( + backendUrl, + browserContext.token, + preferredWorkspaceUuid || browserContext.workspaceUuid, + ); + await page.evaluate(({ storageKey, workspaceUuid: value }) => { + localStorage.setItem(storageKey, value); + }, { storageKey: ACTIVE_WORKSPACE_STORAGE_KEY, workspaceUuid }); + return { + status: "pass", + reason: "Browser Workspace selection is initialized.", + workspace_uuid: workspaceUuid, + }; + } catch (error) { + return { status: "blocked", reason: error.message, workspace_uuid: "" }; + } +} + export async function verifyBrowserToken(page, backendUrl) { return await page.evaluate(async (baseUrl) => { const token = localStorage.getItem("token"); @@ -202,6 +490,87 @@ export async function verifyBrowserToken(page, backendUrl) { }, backendUrl); } +export async function ensureAuthenticatedBrowser(page, { + frontendUrl = env.LANGBOT_FRONTEND_URL || "", + backendUrl = env.LANGBOT_BACKEND_URL || "", + user = env.LANGBOT_E2E_LOGIN_USER || "", + password = env.LANGBOT_E2E_LOGIN_PASSWORD || "LangBotE2ELocalPass!2026", + recoveryKey = "", +} = {}) { + if (!frontendUrl) return { status: "env_issue", reason: "LANGBOT_FRONTEND_URL is not configured." }; + if (!backendUrl) return { status: "env_issue", reason: "LANGBOT_BACKEND_URL is not configured." }; + + const current = await verifyBrowserToken(page, backendUrl).catch((error) => ({ + authenticated: false, + reason: error.message, + })); + if (current.authenticated) { + const workspace = await ensureBrowserWorkspace(page, backendUrl); + if (workspace.status !== "pass") { + return { + status: workspace.status, + reason: workspace.reason, + backend_token_check: null, + browser_token_check: current, + workspace, + injected: false, + }; + } + return { + status: "pass", + reason: "Existing browser token is valid.", + backend_token_check: null, + browser_token_check: current, + workspace, + injected: false, + }; + } + + if (!user) { + return { + status: "blocked", + reason: "Browser profile is not authenticated for LANGBOT_FRONTEND_URL, and LANGBOT_E2E_LOGIN_USER is not configured for automatic local login.", + backend_token_check: null, + browser_token_check: current, + injected: false, + }; + } + + const auth = await resetAndAuthLocalUser({ backendUrl, user, password, recoveryKey }); + await setBrowserToken(page, frontendUrl, auth.token); + const browserCheck = await verifyBrowserToken(page, backendUrl); + if (!browserCheck.authenticated) { + return { + status: "blocked", + reason: browserCheck.reason || "Browser token check failed after automatic local login.", + backend_token_check: auth.check, + browser_token_check: browserCheck, + injected: true, + }; + } + + const workspace = await ensureBrowserWorkspace(page, backendUrl); + if (workspace.status !== "pass") { + return { + status: workspace.status, + reason: workspace.reason, + backend_token_check: auth.check, + browser_token_check: browserCheck, + workspace, + injected: true, + }; + } + + return { + status: "pass", + reason: "Browser token injected from local recovery login.", + backend_token_check: auth.check, + browser_token_check: browserCheck, + workspace, + injected: true, + }; +} + export function exitCode(status) { if (status === "pass") return 0; if (status === "blocked" || status === "env_issue") return 2; @@ -240,6 +609,10 @@ export async function createBrowser(paths) { context = await browser.newContext({ viewport: { width: 1440, height: 960 } }); } const page = context.pages()[0] || await context.newPage(); + const navigationTimeoutMs = Number.parseInt(env.LANGBOT_E2E_NAVIGATION_TIMEOUT_MS || "30000", 10); + if (Number.isFinite(navigationTimeoutMs) && navigationTimeoutMs > 0) { + page.setDefaultNavigationTimeout(navigationTimeoutMs); + } page.on("console", (message) => { appendLine(paths.consoleLog, `[${message.type()}] ${message.text()}`).catch(() => {}); @@ -295,29 +668,42 @@ export function countOccurrences(haystack, needle) { return String(haystack).split(needle).length - 1; } -export async function clickFirstVisible(page, labels, timeout = 2_000) { - for (const label of labels) { - const roleButton = page.getByRole("button", { name: label }).first(); - if (await roleButton.isVisible({ timeout }).catch(() => false)) { - await roleButton.click(); - return label; +async function clickVisibleCandidate(page, candidates, timeout) { + const deadline = Date.now() + Math.max(1, timeout); + do { + for (const candidate of candidates) { + const count = await candidate.locator.count().catch(() => 0); + for (let index = 0; index < count; index += 1) { + const element = candidate.locator.nth(index); + if (!await element.isVisible().catch(() => false)) continue; + const remaining = Math.max(1, deadline - Date.now()); + const clicked = await element.click({ timeout: Math.min(1_000, remaining) }) + .then(() => true) + .catch(() => false); + if (clicked) return candidate.value; + } } - - const roleLink = page.getByRole("link", { name: label }).first(); - if (await roleLink.isVisible({ timeout }).catch(() => false)) { - await roleLink.click(); - return label; - } - - const text = page.getByText(label, { exact: false }).first(); - if (await text.isVisible({ timeout }).catch(() => false)) { - await text.click(); - return label; - } - } + const remaining = deadline - Date.now(); + if (remaining <= 0) break; + await page.waitForTimeout(Math.min(100, remaining)); + } while (Date.now() < deadline); return null; } +export async function clickFirstVisibleLocator(page, locators, timeout = 2_000) { + const candidates = locators.map((locator) => ({ locator, value: true })); + return Boolean(await clickVisibleCandidate(page, candidates, timeout)); +} + +export async function clickFirstVisible(page, labels, timeout = 2_000) { + const candidates = labels.flatMap((label) => [ + { locator: page.getByRole("button", { name: label }), value: label }, + { locator: page.getByRole("link", { name: label }), value: label }, + { locator: page.getByText(label, { exact: false }), value: label }, + ]); + return await clickVisibleCandidate(page, candidates, timeout); +} + export async function fillFirstTextInput(page, value) { const candidates = [ page.getByRole("textbox").last(), diff --git a/skills/scripts/e2e/local-agent-steering-debug-chat.mjs b/skills/scripts/e2e/local-agent-steering-debug-chat.mjs index dae2e265c..56d389d21 100755 --- a/skills/scripts/e2e/local-agent-steering-debug-chat.mjs +++ b/skills/scripts/e2e/local-agent-steering-debug-chat.mjs @@ -10,10 +10,13 @@ import { waitForDebugChatTextStable, } from "./lib/debug-chat.mjs"; import { + beginBackendLogCapture, createBrowser, + ensureAuthenticatedBrowser, ensureEvidence, evidencePaths, exitCode, + finishBackendLogCapture, localIsoWithOffset, loadEnvFiles, pathExists, @@ -26,11 +29,12 @@ await loadEnvFiles(); const caseId = env.LBS_CASE_ID || "local-agent-steering-debug-chat"; const paths = evidencePaths(caseId); await ensureEvidence(paths); +const backendLogCapture = await beginBackendLogCapture(paths.evidenceDir); const backendUrl = (env.LANGBOT_BACKEND_URL || "").replace(/\/$/, ""); const pipelineUrl = env.LANGBOT_E2E_PIPELINE_URL || env.LANGBOT_LOCAL_AGENT_PIPELINE_URL || env.LANGBOT_PIPELINE_URL || ""; const pipelineName = env.LANGBOT_E2E_PIPELINE_NAME || env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || env.LANGBOT_PIPELINE_NAME || ""; -const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "plugin:langbot/local-agent/default"; +const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || "plugin:langbot-team/LocalAgent/default"; const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "qa_steering_sentinel_6194"; const responseTimeoutMs = positiveInt(env.LANGBOT_E2E_RESPONSE_TIMEOUT_MS, 240000); const followupDelayMs = 1000; @@ -74,6 +78,7 @@ const result = { pipeline_config: null, debug_chat_reset: null, tool_diagnostic: null, + browser_auth: null, steering: null, evidence: { console_log: paths.consoleLog, @@ -95,6 +100,18 @@ try { browser = await createBrowser(paths); const { page } = browser; + const authDiagnostic = await ensureAuthenticatedBrowser(page, { + frontendUrl: env.LANGBOT_FRONTEND_URL || "", + backendUrl, + }); + result.browser_auth = authDiagnostic; + if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic"); + if (authDiagnostic.status === "env_issue" || authDiagnostic.status === "blocked" || authDiagnostic.status === "fail") { + result.status = authDiagnostic.status; + result.reason = authDiagnostic.reason || "Browser authentication failed."; + throw new Error(result.reason); + } + const openResult = await openPipelineDebugChat(page, { pipelineUrl, pipelineName, @@ -144,7 +161,8 @@ try { result.status = resetDiagnostic.status; result.reason = resetDiagnostic.reason || "Debug Chat reset failed."; } else { - await page.waitForTimeout(1000); + await page.reload({ waitUntil: "commit" }); + await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); const reopenResult = await openPipelineDebugChat(page, { pipelineUrl, pipelineName, @@ -176,6 +194,12 @@ try { } finally { if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); if (browser) await browser.close().catch(() => {}); + const backendLog = await finishBackendLogCapture(backendLogCapture); + if (backendLog) { + result.evidence.backend_log = backendLog.path; + result.backend_log = backendLog; + if (!result.evidence_collected.includes("backend_log")) result.evidence_collected.push("backend_log"); + } const finishedAt = new Date(); result.finished_at = finishedAt.toISOString(); result.finished_at_local = localIsoWithOffset(finishedAt); @@ -383,6 +407,7 @@ async function inspectPipeline(page, { backendUrl, pipelineUrl, pipelineName, ex headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", }, }); return { @@ -432,7 +457,7 @@ async function inspectPipeline(page, { backendUrl, pipelineUrl, pipelineName, ex } const config = pipeline.config || {}; const runner = config.ai?.runner || {}; - const runnerId = runner.id || runner.runner || ""; + const runnerId = runner.id || ""; if (!runnerId) { return { status: "blocked", @@ -441,7 +466,7 @@ async function inspectPipeline(page, { backendUrl, pipelineUrl, pipelineName, ex pipeline_id: pipelineId, pipeline_name: pipeline.name, matched_by: matchedBy, - reason: "Pipeline has no ai.runner.id or legacy ai.runner.runner.", + reason: "Pipeline has no ai.runner.id.", }; } if (expectedRunnerId && runnerId !== expectedRunnerId) { @@ -485,6 +510,7 @@ async function inspectToolNames(page, { backendUrl }) { headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", }, }); const json = await response.json().catch(() => ({})); @@ -522,6 +548,7 @@ async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionTyp headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", }, }, ); diff --git a/skills/scripts/e2e/mcp-stdio-fixture.mjs b/skills/scripts/e2e/mcp-stdio-fixture.mjs index 894101301..eaf432604 100755 --- a/skills/scripts/e2e/mcp-stdio-fixture.mjs +++ b/skills/scripts/e2e/mcp-stdio-fixture.mjs @@ -35,22 +35,15 @@ await ensureEvidence(paths); const startedAt = new Date(); const fixturePath = resolve(env.LANGBOT_MCP_FIXTURE_PATH || "skills/langbot-testing/fixtures/mcp/qa_mcp_echo_server.py"); const langbotRepo = env.LANGBOT_REPO ? resolve(env.LANGBOT_REPO) : ""; -const uvCandidates = [ - env.LANGBOT_MCP_FIXTURE_UV, - "uv", -].filter(Boolean); -const uv = uvCandidates.find((candidate) => candidate === "uv" || existsSync(candidate)); const pythonCandidates = [ env.LANGBOT_MCP_FIXTURE_PYTHON, langbotRepo ? `${langbotRepo}/.venv/bin/python` : "", "python3", ].filter(Boolean); const python = pythonCandidates.find((candidate) => candidate === "python3" || existsSync(candidate)); -const command = langbotRepo && uv - ? { executable: uv, args: ["run", "python", fixturePath], cwd: langbotRepo, mode: "uv" } - : python - ? { executable: python, args: [fixturePath], cwd: resolve("."), mode: "python" } - : null; +const command = python + ? { executable: python, args: [fixturePath], cwd: resolve("."), mode: "python" } + : null; const expectedText = "qa_mcp_echo:mcp-stdio-fixture-ok"; const result = { @@ -94,7 +87,7 @@ async function request(child, id, method, params) { async function run() { if (!command) { result.status = "env_issue"; - result.reason = "No uv or Python interpreter found. Set LANGBOT_REPO, LANGBOT_MCP_FIXTURE_UV, or LANGBOT_MCP_FIXTURE_PYTHON."; + result.reason = "No Python interpreter found. Set LANGBOT_REPO or LANGBOT_MCP_FIXTURE_PYTHON."; return; } if (!existsSync(fixturePath)) { diff --git a/skills/scripts/e2e/mcp-stdio-register.mjs b/skills/scripts/e2e/mcp-stdio-register.mjs index e2e31a9ad..aa6a53838 100755 --- a/skills/scripts/e2e/mcp-stdio-register.mjs +++ b/skills/scripts/e2e/mcp-stdio-register.mjs @@ -1,11 +1,12 @@ #!/usr/bin/env node import { existsSync, readFileSync } from "node:fs"; -import { writeFile } from "node:fs/promises"; +import { readFile, writeFile } from "node:fs/promises"; import { resolve } from "node:path"; import { env } from "node:process"; import { createBrowser, + ensureBrowserWorkspace, ensureEvidence, evidencePaths, exitCode, @@ -46,6 +47,8 @@ const startupTimeoutSec = Number(env.LANGBOT_MCP_STARTUP_TIMEOUT_SEC || "300"); const readyTimeoutMs = Number(env.LANGBOT_MCP_READY_TIMEOUT_MS || "360000"); const backendUrl = env.LANGBOT_BACKEND_URL || ""; const apiDiagnosticPath = resolve(paths.evidenceDir, "api-diagnostic.json"); +const envLocalPath = resolve("skills/.env.local"); +const serverUuidEnvKey = env.LANGBOT_MCP_SERVER_UUID_ENV_KEY || "LANGBOT_MCP_QA_STDIO_SERVER_UUID"; let browser; const result = { @@ -69,7 +72,8 @@ const result = { automation_result_json: paths.automationResultJson, result_json: paths.resultJson, }, - evidence_collected: ["api_diagnostic"], + evidence_collected: ["screenshot", "console", "network", "api_diagnostic"], + wrote_env: false, }; async function run() { @@ -88,6 +92,12 @@ async function run() { const { page } = browser; await page.goto(env.LANGBOT_FRONTEND_URL, { waitUntil: "domcontentloaded" }); await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + const workspace = await ensureBrowserWorkspace(page, backendUrl); + if (workspace.status !== "pass") { + result.status = workspace.status; + result.reason = workspace.reason; + return; + } const diagnostic = await page.evaluate(async ({ backendUrl, @@ -98,6 +108,7 @@ async function run() { fixtureArgs, startupTimeoutSec, readyTimeoutMs, + workspaceUuid, }) => { const token = localStorage.getItem("token"); if (!token) { @@ -117,6 +128,7 @@ async function run() { const headers = { Authorization: `Bearer ${token}`, "Content-Type": "application/json", + "X-Workspace-Id": workspaceUuid, }; const serverConfig = { name: serverName, @@ -159,6 +171,7 @@ async function run() { const deadline = Date.now() + readyTimeoutMs; let lastTools = []; let lastRuntime = null; + let lastServer = null; while (Date.now() < deadline) { await new Promise((resolveReady) => setTimeout(resolveReady, 500)); const tools = await getJson("/api/v1/tools"); @@ -167,7 +180,8 @@ async function run() { .map((tool) => tool.name || tool.tool_name || tool.function?.name || "") .filter(Boolean) .sort(); - lastRuntime = server.json.data?.server?.runtime_info || null; + lastServer = server.json.data?.server || null; + lastRuntime = lastServer?.runtime_info || null; if (lastTools.includes(expectedTool)) break; } @@ -177,6 +191,7 @@ async function run() { save_status: save.status, save_code: save.json.code ?? null, save_msg: save.json.msg || "", + server_uuid: lastServer?.uuid || save.json.data?.uuid || "", tool_names: lastTools, has_expected_tool: lastTools.includes(expectedTool), runtime_status: lastRuntime?.status || null, @@ -187,7 +202,17 @@ async function run() { runtime_tool_count: lastRuntime?.tool_count ?? null, runtime_error: lastRuntime?.error_message || "", }; - }, { backendUrl, serverName, expectedTool, fixturePath, fixtureCommand, fixtureArgs, startupTimeoutSec, readyTimeoutMs }); + }, { + backendUrl, + serverName, + expectedTool, + fixturePath, + fixtureCommand, + fixtureArgs, + startupTimeoutSec, + readyTimeoutMs, + workspaceUuid: workspace.workspace_uuid, + }); await writeFile(apiDiagnosticPath, `${JSON.stringify(diagnostic, null, 2)}\n`, "utf8"); await safeScreenshot(page, paths.screenshot); @@ -212,11 +237,47 @@ async function run() { result.reason = `MCP server ${serverName} did not expose ${expectedTool}. See ${apiDiagnosticPath}.`; return; } + if (!diagnostic.server_uuid) { + result.status = "fail"; + result.reason = `MCP server ${serverName} exposed ${expectedTool}, but the server UUID was not returned. See ${apiDiagnosticPath}.`; + return; + } + + await upsertEnvLocal(envLocalPath, { + [serverUuidEnvKey]: diagnostic.server_uuid, + }); + result.wrote_env = true; + result.server_uuid = diagnostic.server_uuid; + result.server_uuid_env_key = serverUuidEnvKey; result.status = "pass"; result.reason = `MCP server ${serverName} is connected and exposes ${expectedTool} through LangBot /api/v1/tools.`; } +async function upsertEnvLocal(path, updates) { + let text = ""; + try { + text = await readFile(path, "utf8"); + } catch { + text = ""; + } + + const lines = text ? text.split(/\r?\n/) : []; + const seen = new Set(); + const updated = lines.map((line) => { + const match = line.match(/^([A-Z][A-Z0-9_]*)=/); + if (!match || !Object.prototype.hasOwnProperty.call(updates, match[1])) return line; + seen.add(match[1]); + return `${match[1]}=${updates[match[1]]}`; + }); + + for (const [key, value] of Object.entries(updates)) { + if (!seen.has(key)) updated.push(`${key}=${value}`); + } + + await writeFile(path, `${updated.filter((line, index) => line || index < updated.length - 1).join("\n")}\n`, "utf8"); +} + try { await run(); } catch (error) { diff --git a/skills/scripts/e2e/onebot-runtime-probe.py b/skills/scripts/e2e/onebot-runtime-probe.py new file mode 100644 index 000000000..a080571fa --- /dev/null +++ b/skills/scripts/e2e/onebot-runtime-probe.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Drive one deterministic event through an enabled OneBot reverse WebSocket.""" + +from __future__ import annotations + +import argparse +import asyncio +import json +import time + +import websockets + + +async def run(port: int) -> None: + uri = f'ws://127.0.0.1:{port}/ws' + headers = { + 'X-Self-ID': '900001', + 'X-Client-Role': 'Universal', + 'User-Agent': 'LangBot-E2E-OneBot/1.0', + } + connect_deadline = time.monotonic() + 15 + connection = None + while time.monotonic() < connect_deadline: + try: + connection = await websockets.connect(uri, additional_headers=headers) + break + except OSError: + await asyncio.sleep(0.25) + if connection is None: + raise RuntimeError(f'OneBot reverse WebSocket did not open on port {port}') + + actions: list[str] = [] + event_deadline = time.monotonic() + 20 + async with connection as ws: + await ws.send( + json.dumps( + { + 'post_type': 'notice', + 'notice_type': 'group_increase', + 'sub_type': 'invite', + 'time': int(time.time()), + 'self_id': 900001, + 'group_id': 20001, + 'operator_id': 10002, + 'user_id': 10003, + } + ) + ) + + delivered = False + while time.monotonic() < event_deadline and not delivered: + try: + raw = await asyncio.wait_for(ws.recv(), timeout=2) + except asyncio.TimeoutError: + continue + request = json.loads(raw) + action = request.get('action', '') + actions.append(action) + if action == 'get_group_info': + data = {'group_id': 20001, 'group_name': 'LangBot Runtime QA'} + elif action == 'get_group_member_info': + data = { + 'group_id': 20001, + 'user_id': 10003, + 'nickname': 'Runtime QA Member', + 'card': 'Runtime QA Member', + } + elif action == 'send_group_msg': + data = {'message_id': 70001} + delivered = True + else: + data = {} + await ws.send( + json.dumps( + { + 'status': 'ok', + 'retcode': 0, + 'data': data, + 'echo': request.get('echo'), + } + ) + ) + + if not delivered: + raise RuntimeError(f'Agent output was not delivered; actions={actions}') + print(json.dumps({'connected': True, 'actions': actions, 'delivered': True})) + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--port', type=int, required=True) + args = parser.parse_args() + asyncio.run(run(args.port)) diff --git a/skills/scripts/e2e/pipeline-debug-chat.mjs b/skills/scripts/e2e/pipeline-debug-chat.mjs index 4b20f7757..6e6b3cca6 100755 --- a/skills/scripts/e2e/pipeline-debug-chat.mjs +++ b/skills/scripts/e2e/pipeline-debug-chat.mjs @@ -10,19 +10,24 @@ import { setDebugChatStreamOutput, } from "./lib/debug-chat.mjs"; import { + beginBackendLogCapture, createBrowser, + ensureAuthenticatedBrowser, ensureEvidence, evidencePaths, exitCode, + finishBackendLogCapture, localIsoWithOffset, pathExists, safeScreenshot, + scanBrowserDiagnostics, writeResult, } from "./lib/langbot-e2e.mjs"; const caseId = env.LBS_CASE_ID || "pipeline-debug-chat"; const paths = evidencePaths(caseId); await ensureEvidence(paths); +const backendLogCapture = await beginBackendLogCapture(paths.evidenceDir); const expectedText = env.LANGBOT_E2E_EXPECTED_TEXT || "OK"; const prompt = env.LANGBOT_E2E_PROMPT || `请只回复 ${expectedText}，用于前端调试测试。`; @@ -50,15 +55,20 @@ const pipelineName = pipelineRequired const expectedRunnerId = env.LANGBOT_E2E_EXPECTED_RUNNER_ID || ""; const resetDebugChat = boolFromEnv(env.LANGBOT_E2E_RESET_DEBUG_CHAT, false); const restoreRunnerConfig = boolFromEnv(env.LANGBOT_E2E_RESTORE_RUNNER_CONFIG, true); +const restoreExtensions = boolFromEnv(env.LANGBOT_E2E_RESTORE_EXTENSIONS, true); const debugChatSessionType = env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person"; +const maxNewAssistantMessages = Number.parseInt(env.LANGBOT_E2E_MAX_NEW_ASSISTANT_MESSAGES || "1", 10); const pipelineConfigDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-diagnostic.json"); +const pipelineExtensionsDiagnosticPath = resolve(paths.evidenceDir, "pipeline-extensions-diagnostic.json"); const debugChatResetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json"); const pipelineConfigRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-restore-diagnostic.json"); +const pipelineExtensionsRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-extensions-restore-diagnostic.json"); const metricsPath = resolve(paths.evidenceDir, "metrics.json"); const startedAt = new Date(); let browser; -let restorePlan = null; +let restoreConfigPlan = null; +let restoreExtensionsPlan = null; let result = { source: "automation", case_id: caseId, @@ -106,7 +116,9 @@ function parseJsonEnv(key, fallback) { } function positiveNumberEnv(key, fallback) { - const value = Number(env[key] || ""); + const raw = env[key]; + if (raw === undefined || raw === "") return fallback; + const value = Number(raw); return Number.isFinite(value) && value >= 0 ? value : fallback; } @@ -131,22 +143,28 @@ function stats(values) { function promptStepsFromEnv() { const rawSteps = parseJsonEnv("LANGBOT_E2E_PROMPTS_JSON", null); if (rawSteps === null) { - return [{ prompt, expectedText, responseTimeoutMs: safeResponseTimeoutMs }]; + return [{ prompt, expectedText, expectedTexts: [expectedText], responseTimeoutMs: safeResponseTimeoutMs }]; } if (!Array.isArray(rawSteps) || rawSteps.length === 0) { throw new Error("LANGBOT_E2E_PROMPTS_JSON must be a non-empty JSON array."); } return rawSteps.map((item, index) => { if (typeof item === "string") { - return { prompt: item, expectedText, responseTimeoutMs: safeResponseTimeoutMs }; + return { prompt: item, expectedText, expectedTexts: [expectedText], responseTimeoutMs: safeResponseTimeoutMs }; } if (!item || typeof item !== "object" || typeof item.prompt !== "string" || !item.prompt) { throw new Error(`LANGBOT_E2E_PROMPTS_JSON[${index}] must be a string or an object with a prompt string.`); } const stepTimeout = Number.parseInt(String(item.response_timeout_ms || item.responseTimeoutMs || safeResponseTimeoutMs), 10); + const stepExpectedText = String(item.expected_text || item.expectedText || expectedText); + const additionalExpectedTexts = item.expected_texts || item.expectedTexts || []; + if (!Array.isArray(additionalExpectedTexts)) { + throw new Error(`LANGBOT_E2E_PROMPTS_JSON[${index}].expected_texts must be an array.`); + } return { prompt: item.prompt, - expectedText: String(item.expected_text || item.expectedText || expectedText), + expectedText: stepExpectedText, + expectedTexts: [...new Set([stepExpectedText, ...additionalExpectedTexts.map(String)].filter(Boolean))], responseTimeoutMs: Number.isFinite(stepTimeout) && stepTimeout > 0 ? stepTimeout : safeResponseTimeoutMs, }; }); @@ -231,6 +249,7 @@ async function runFilesystemChecks(checks) { } const contains = textList(check.contains); const notContains = textList(check.not_contains || check.notContains); + const expectedJson = check.json_equals ?? check.jsonEquals; const expectedExitCode = Number.isInteger(check.exit_code) ? check.exit_code : Number.isInteger(check.expected_exit_code) @@ -249,18 +268,31 @@ async function runFilesystemChecks(checks) { } const missing = contains.filter((needle) => !text.includes(needle)); const forbidden = notContains.filter((needle) => text.includes(needle)); + let jsonError = ""; + let jsonMatches = null; + if (expectedJson !== undefined) { + try { + jsonMatches = JSON.stringify(sortJson(JSON.parse(text))) === JSON.stringify(sortJson(expectedJson)); + if (!jsonMatches) jsonError = "JSON content does not match json_equals."; + } catch (error) { + jsonMatches = false; + jsonError = `Invalid JSON: ${error.message}`; + } + } + const failed = missing.length > 0 || forbidden.length > 0 || jsonMatches === false; results.push({ index, - status: missing.length || forbidden.length ? "fail" : "pass", + status: failed ? "fail" : "pass", type: "file", path, missing, forbidden, + json_matches: jsonMatches, reason: missing.length ? `Missing expected text: ${missing.join(", ")}` : forbidden.length ? `Found forbidden text: ${forbidden.join(", ")}` - : "", + : jsonError, }); continue; } @@ -302,6 +334,16 @@ async function runFilesystemChecks(checks) { }; } +function sortJson(value) { + if (Array.isArray(value)) return value.map(sortJson); + if (!value || typeof value !== "object") return value; + return Object.fromEntries( + Object.keys(value) + .sort() + .map((key) => [key, sortJson(value[key])]), + ); +} + function pipelineIdFromUrl(url) { if (!url) return ""; try { @@ -317,6 +359,11 @@ function sanitizePipelineDiagnostic(diagnostic) { return safe; } +function sanitizePipelineExtensionsDiagnostic(diagnostic) { + const { restore_extensions: _restoreExtensions, ...safe } = diagnostic || {}; + return safe; +} + async function prepareImageFixture(paths) { if (imagePathEnv) return resolve(imagePathEnv); if (!imageBase64Path) return ""; @@ -354,6 +401,7 @@ async function inspectAndPatchPipelineConfig(page, { const headers = { Authorization: `Bearer ${token}`, "Content-Type": "application/json", + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", }; const getJson = async (path) => { const response = await fetch(`${backendUrl}${path}`, { headers }); @@ -418,7 +466,7 @@ async function inspectAndPatchPipelineConfig(page, { const config = JSON.parse(JSON.stringify(pipeline.config || {})); const aiConfig = config.ai && typeof config.ai === "object" ? config.ai : {}; const runner = aiConfig.runner && typeof aiConfig.runner === "object" ? aiConfig.runner : {}; - const runnerId = runner.id || runner.runner || ""; + const runnerId = runner.id || ""; if (!runnerId) { return { status: "blocked", @@ -427,7 +475,7 @@ async function inspectAndPatchPipelineConfig(page, { pipeline_id: pipelineId, pipeline_name: pipeline.name, matched_by: matchedBy, - reason: "Pipeline has no ai.runner.id or legacy ai.runner.runner.", + reason: "Pipeline has no ai.runner.id.", }; } if (expectedRunnerId && runnerId !== expectedRunnerId) { @@ -451,6 +499,31 @@ async function inspectAndPatchPipelineConfig(page, { ? runnerConfigs[runnerId] : {}; const patchKeys = Object.keys(runnerConfigPatch || {}); + const sensitiveConfigKeyRe = /(?:api[_-]?key|authorization|bearer|credential|jwt|oauth|password|secret)/i; + const sanitizeConfigValue = (key, value, depth = 0) => { + if (sensitiveConfigKeyRe.test(String(key))) return "[redacted]"; + if (typeof value === "string") { + return value.length > 1200 ? `${value.slice(0, 1200)}...[truncated ${value.length - 1200} chars]` : value; + } + if (Array.isArray(value)) { + const items = value.slice(0, 50).map((item) => sanitizeConfigValue(key, item, depth + 1)); + if (value.length > 50) items.push(`[truncated ${value.length - 50} items]`); + return items; + } + if (value && typeof value === "object") { + if (depth >= 3) return "[object truncated]"; + return Object.fromEntries( + Object.entries(value).map(([childKey, childValue]) => [ + childKey, + sanitizeConfigValue(childKey, childValue, depth + 1), + ]), + ); + } + return value; + }; + const pickPatchValues = (config) => Object.fromEntries( + patchKeys.map((key) => [key, sanitizeConfigValue(key, config?.[key])]), + ); const baseDiagnostic = { status: "ready", authenticated: true, @@ -462,6 +535,7 @@ async function inspectAndPatchPipelineConfig(page, { expected_runner_id: expectedRunnerId || "", patch_keys: patchKeys, runner_config_before_keys: Object.keys(currentRunnerConfig), + runner_config_patch_before: pickPatchValues(currentRunnerConfig), patched: patchKeys.length > 0, }; @@ -506,6 +580,7 @@ async function inspectAndPatchPipelineConfig(page, { put_status: update.status, put_code: update.json.code ?? null, runner_config_after_keys: Object.keys(updatedRunnerConfig), + runner_config_patch_after: pickPatchValues(updatedRunnerConfig), restore_config: config, }; }, { @@ -533,6 +608,7 @@ async function restorePipelineConfig(page, { backendUrl, pipelineId, config }) { headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", }, body: JSON.stringify({ config }), }); @@ -548,6 +624,203 @@ async function restorePipelineConfig(page, { backendUrl, pipelineId, config }) { }, { backendUrl, pipelineId, config }); } +async function inspectAndPatchPipelineExtensions(page, { + backendUrl, + pipelineId, + extensionsPatch, +}) { + return await page.evaluate(async ({ + backendUrl, + pipelineId, + extensionsPatch, + }) => { + const token = localStorage.getItem("token"); + if (!token) { + return { + status: "blocked", + authenticated: false, + pipeline_id: pipelineId, + reason: "Browser profile has no localStorage token.", + }; + } + + const headers = { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", + }; + const getJson = async (path) => { + const response = await fetch(`${backendUrl}${path}`, { headers }); + return { + status: response.status, + json: await response.json().catch(() => ({})), + }; + }; + const putJson = async (path, body) => { + const response = await fetch(`${backendUrl}${path}`, { + method: "PUT", + headers, + body: JSON.stringify(body), + }); + return { + status: response.status, + json: await response.json().catch(() => ({})), + }; + }; + + if (!pipelineId) { + return { + status: "blocked", + authenticated: true, + pipeline_resolved: false, + reason: "Pipeline id is required before patching extensions.", + }; + } + + const before = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`); + let extensions = before.json.data || {}; + let getStatus = before.status; + let getCode = before.json.code ?? null; + let fallbackReason = ""; + if (before.status >= 400 || before.json.code !== 0) { + const pipelineResponse = await getJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}`); + const pipeline = pipelineResponse.json.data?.pipeline || {}; + const prefs = pipeline.extensions_preferences || {}; + if (pipelineResponse.status >= 400 || pipelineResponse.json.code !== 0 || !pipeline.uuid) { + return { + status: "fail", + authenticated: true, + pipeline_id: pipelineId, + get_status: before.status, + get_code: before.json.code ?? null, + fallback_pipeline_status: pipelineResponse.status, + fallback_pipeline_code: pipelineResponse.json.code ?? null, + reason: before.json.msg || "Could not load pipeline extensions.", + }; + } + fallbackReason = before.json.msg || "Could not load pipeline extensions; restored from pipeline preferences."; + extensions = { + enable_all_plugins: prefs.enable_all_plugins ?? true, + enable_all_mcp_servers: prefs.enable_all_mcp_servers ?? true, + enable_all_skills: prefs.enable_all_skills ?? true, + bound_plugins: prefs.plugins || [], + bound_mcp_servers: prefs.mcp_servers || [], + bound_skills: prefs.skills || [], + }; + } + + const patchKeys = Object.keys(extensionsPatch || {}); + const restoreExtensions = { + enable_all_plugins: extensions.enable_all_plugins ?? true, + enable_all_mcp_servers: extensions.enable_all_mcp_servers ?? true, + enable_all_skills: extensions.enable_all_skills ?? true, + bound_plugins: extensions.bound_plugins || [], + bound_mcp_servers: extensions.bound_mcp_servers || [], + bound_skills: extensions.bound_skills || [], + }; + const baseDiagnostic = { + status: "ready", + authenticated: true, + pipeline_id: pipelineId, + patch_keys: patchKeys, + patched: patchKeys.length > 0, + get_status: getStatus, + get_code: getCode, + fallback_reason: fallbackReason, + extensions_before: { + enable_all_plugins: restoreExtensions.enable_all_plugins, + enable_all_mcp_servers: restoreExtensions.enable_all_mcp_servers, + enable_all_skills: restoreExtensions.enable_all_skills, + bound_plugins: restoreExtensions.bound_plugins, + bound_mcp_servers: restoreExtensions.bound_mcp_servers, + bound_skills: restoreExtensions.bound_skills, + }, + }; + + if (patchKeys.length === 0) { + return baseDiagnostic; + } + + const updateBody = { + enable_all_plugins: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_plugins") + ? Boolean(extensionsPatch.enable_all_plugins) + : restoreExtensions.enable_all_plugins, + enable_all_mcp_servers: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_mcp_servers") + ? Boolean(extensionsPatch.enable_all_mcp_servers) + : restoreExtensions.enable_all_mcp_servers, + enable_all_skills: Object.prototype.hasOwnProperty.call(extensionsPatch, "enable_all_skills") + ? Boolean(extensionsPatch.enable_all_skills) + : restoreExtensions.enable_all_skills, + bound_plugins: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_plugins") + ? extensionsPatch.bound_plugins + : restoreExtensions.bound_plugins, + bound_mcp_servers: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_mcp_servers") + ? extensionsPatch.bound_mcp_servers + : restoreExtensions.bound_mcp_servers, + bound_skills: Object.prototype.hasOwnProperty.call(extensionsPatch, "bound_skills") + ? extensionsPatch.bound_skills + : restoreExtensions.bound_skills, + }; + + const update = await putJson(`/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`, updateBody); + if (update.status >= 400 || update.json.code !== 0) { + return { + ...baseDiagnostic, + status: "fail", + put_status: update.status, + put_code: update.json.code ?? null, + reason: update.json.msg || "Pipeline extensions update failed.", + }; + } + + return { + ...baseDiagnostic, + put_status: update.status, + put_code: update.json.code ?? null, + extensions_after: updateBody, + restore_extensions: restoreExtensions, + }; + }, { + backendUrl, + pipelineId, + extensionsPatch, + }); +} + +async function restorePipelineExtensions(page, { backendUrl, pipelineId, extensions }) { + return await page.evaluate(async ({ backendUrl, pipelineId, extensions }) => { + const token = localStorage.getItem("token"); + if (!token) { + return { + status: "blocked", + authenticated: false, + pipeline_id: pipelineId, + reason: "Browser profile has no localStorage token.", + }; + } + const response = await fetch(`${backendUrl}/api/v1/pipelines/${encodeURIComponent(pipelineId)}/extensions`, { + method: "PUT", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", + }, + body: JSON.stringify(extensions), + }); + const json = await response.json().catch(() => ({})); + return { + status: response.status >= 400 || json.code !== 0 ? "fail" : "ready", + authenticated: true, + pipeline_id: pipelineId, + put_status: response.status, + put_code: json.code ?? null, + reason: response.status >= 400 || json.code !== 0 + ? json.msg || "Pipeline extensions restore failed." + : "Pipeline extensions restored.", + }; + }, { backendUrl, pipelineId, extensions }); +} + async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionType }) { return await page.evaluate(async ({ backendUrl, pipelineId, sessionType }) => { const token = localStorage.getItem("token"); @@ -567,6 +840,7 @@ async function resetPipelineDebugChat(page, { backendUrl, pipelineId, sessionTyp headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", + "X-Workspace-Id": localStorage.getItem("langbot_active_workspace_uuid") || "", }, }, ); @@ -590,11 +864,13 @@ try { const promptSteps = promptStepsFromEnv(); const filesystemChecks = parseJsonEnv("LANGBOT_E2E_FILESYSTEM_CHECKS_JSON", []); const runnerConfigPatch = parseJsonEnv("LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON", {}); + const extensionsPatch = parseJsonEnv("LANGBOT_E2E_EXTENSIONS_PATCH_JSON", {}); const runnerPatchKeys = Object.keys(runnerConfigPatch); - if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) { + const extensionsPatchKeys = Object.keys(extensionsPatch); + if (runnerPatchKeys.length > 0 || extensionsPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) { if (!backendUrl) { result.status = "env_issue"; - result.reason = "LANGBOT_BACKEND_URL is required for runner config patch, runner assertion, or Debug Chat reset."; + result.reason = "LANGBOT_BACKEND_URL is required for runner config patch, extensions patch, runner assertion, or Debug Chat reset."; throw new Error(result.reason); } } @@ -602,13 +878,30 @@ try { result.prompt = promptSteps.length === 1 ? promptSteps[0].prompt : `${promptSteps.length} prompts`; result.expected_text = promptSteps.at(-1)?.expectedText || expectedText; - const openResult = await openPipelineDebugChat(page, { - pipelineUrl, - pipelineName, - envHint: pipelineRequired - ? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME" - : "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME", + const authDiagnostic = await ensureAuthenticatedBrowser(page, { + frontendUrl: pipelineUrl || env.LANGBOT_FRONTEND_URL || "", + backendUrl, }); + result.browser_auth = authDiagnostic; + if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic"); + const authFailed = authDiagnostic.status === "env_issue" || authDiagnostic.status === "blocked" || authDiagnostic.status === "fail"; + if (authFailed) { + result.status = authDiagnostic.status; + result.reason = authDiagnostic.reason || "Browser authentication failed."; + } else { + result.status = "running"; + result.reason = ""; + } + + const openResult = authFailed + ? { opened: false, status: result.status, reason: result.reason } + : await openPipelineDebugChat(page, { + pipelineUrl, + pipelineName, + envHint: pipelineRequired + ? "case-specific pipeline env mapped to LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME" + : "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME", + }); result.url = page.url(); if (!openResult.opened) { @@ -617,7 +910,7 @@ try { } else { result.status = "running"; result.reason = ""; - if (runnerPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) { + if (runnerPatchKeys.length > 0 || extensionsPatchKeys.length > 0 || resetDebugChat || expectedRunnerId) { const pipelineDiagnostic = await inspectAndPatchPipelineConfig(page, { backendUrl, pipelineUrl, @@ -636,13 +929,35 @@ try { result.reason = pipelineDiagnostic.reason || "Pipeline config preparation failed."; } else { if (pipelineDiagnostic.restore_config && restoreRunnerConfig) { - restorePlan = { + restoreConfigPlan = { backendUrl, pipelineId: pipelineDiagnostic.pipeline_id, config: pipelineDiagnostic.restore_config, }; } - if (resetDebugChat) { + if (extensionsPatchKeys.length > 0) { + const extensionsDiagnostic = await inspectAndPatchPipelineExtensions(page, { + backendUrl, + pipelineId: pipelineDiagnostic.pipeline_id, + extensionsPatch, + }); + const safeExtensionsDiagnostic = sanitizePipelineExtensionsDiagnostic(extensionsDiagnostic); + await writeFile(pipelineExtensionsDiagnosticPath, `${JSON.stringify(safeExtensionsDiagnostic, null, 2)}\n`, "utf8"); + result.evidence.pipeline_extensions_diagnostic_json = pipelineExtensionsDiagnosticPath; + result.pipeline_extensions = safeExtensionsDiagnostic; + if (!result.evidence_collected.includes("api_diagnostic")) result.evidence_collected.push("api_diagnostic"); + if (extensionsDiagnostic.status === "fail" || extensionsDiagnostic.status === "blocked") { + result.status = extensionsDiagnostic.status; + result.reason = extensionsDiagnostic.reason || "Pipeline extensions preparation failed."; + } else if (extensionsDiagnostic.restore_extensions && restoreExtensions) { + restoreExtensionsPlan = { + backendUrl, + pipelineId: extensionsDiagnostic.pipeline_id, + extensions: extensionsDiagnostic.restore_extensions, + }; + } + } + if (!["fail", "blocked", "env_issue"].includes(result.status) && resetDebugChat) { const resetDiagnostic = await resetPipelineDebugChat(page, { backendUrl, pipelineId: pipelineDiagnostic.pipeline_id, @@ -655,7 +970,8 @@ try { result.status = resetDiagnostic.status; result.reason = resetDiagnostic.reason || "Debug Chat reset failed."; } else { - await page.waitForTimeout(1000); + await page.reload({ waitUntil: "commit" }); + await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); const reopenResult = await openPipelineDebugChat(page, { pipelineUrl, pipelineName, @@ -687,14 +1003,22 @@ try { const chatResult = await runDebugChatPrompt(page, { prompt: step.prompt, expectedText: step.expectedText, + expectedTexts: step.expectedTexts, responseTimeoutMs: step.responseTimeoutMs, imagePath: index === 0 ? imagePath : "", + backendUrl, + pipelineId: result.pipeline_config?.pipeline_id || pipelineIdFromUrl(pipelineUrl), + sessionType: debugChatSessionType, + maxNewAssistantMessages: Number.isFinite(maxNewAssistantMessages) && maxNewAssistantMessages >= 0 + ? maxNewAssistantMessages + : 1, failureSignals: failureSignals.length > 0 ? failureSignals : undefined, }); const promptDurationMs = Date.now() - promptStartedAt; result.chat_results.push({ index, expected_text: step.expectedText, + expected_texts: step.expectedTexts, status: chatResult.status, reason: chatResult.reason, response_duration_ms: promptDurationMs, @@ -702,7 +1026,14 @@ try { final_count: chatResult.final_count, before_assistant_expected_count: chatResult.before_assistant_expected_count, after_assistant_expected_count: chatResult.after_assistant_expected_count, + before_assistant_message_count: chatResult.before_assistant_message_count, + after_assistant_message_count: chatResult.after_assistant_message_count, + new_assistant_message_count: chatResult.new_assistant_message_count, + latest_assistant_is_final: chatResult.latest_assistant_is_final, + final_assistant_wait_status: chatResult.final_assistant_wait_status, + final_assistant_wait_reason: chatResult.final_assistant_wait_reason, failure_signal: chatResult.failure_signal || "", + missing_expected_texts: chatResult.missing_expected_texts || [], }); result.status = chatResult.status; result.reason = `Prompt ${index + 1}/${promptSteps.length}: ${chatResult.reason}`; @@ -720,6 +1051,15 @@ try { result.reason = filesystemResult.reason || "Filesystem checks failed."; } } + + if (result.status === "pass" && !boolFromEnv(env.LANGBOT_E2E_ALLOW_BROWSER_ERRORS, false)) { + const browserDiagnostics = await scanBrowserDiagnostics(paths); + result.browser_diagnostics = browserDiagnostics; + if (browserDiagnostics.status === "fail") { + result.status = "fail"; + result.reason = browserDiagnostics.reason; + } + } } } catch (error) { if (!["env_issue", "blocked", "fail", "pass"].includes(result.status) || !result.reason) { @@ -728,10 +1068,20 @@ try { result.reason = result.reason || error.message; } finally { if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); - if (browser?.page && restorePlan) { - const restoreDiagnostic = await restorePipelineConfig(browser.page, restorePlan).catch((error) => ({ + if (browser?.page && restoreExtensionsPlan) { + const restoreDiagnostic = await restorePipelineExtensions(browser.page, restoreExtensionsPlan).catch((error) => ({ status: "fail", - pipeline_id: restorePlan.pipelineId, + pipeline_id: restoreExtensionsPlan.pipelineId, + reason: error.message, + })); + await writeFile(pipelineExtensionsRestoreDiagnosticPath, `${JSON.stringify(restoreDiagnostic, null, 2)}\n`, "utf8"); + result.evidence.pipeline_extensions_restore_diagnostic_json = pipelineExtensionsRestoreDiagnosticPath; + result.pipeline_extensions_restore = restoreDiagnostic; + } + if (browser?.page && restoreConfigPlan) { + const restoreDiagnostic = await restorePipelineConfig(browser.page, restoreConfigPlan).catch((error) => ({ + status: "fail", + pipeline_id: restoreConfigPlan.pipelineId, reason: error.message, })); await writeFile(pipelineConfigRestoreDiagnosticPath, `${JSON.stringify(restoreDiagnostic, null, 2)}\n`, "utf8"); @@ -739,6 +1089,12 @@ try { result.pipeline_config_restore = restoreDiagnostic; } if (browser) await browser.close().catch(() => {}); + const backendLog = await finishBackendLogCapture(backendLogCapture); + if (backendLog) { + result.evidence.backend_log = backendLog.path; + result.backend_log = backendLog; + if (!result.evidence_collected.includes("backend_log")) result.evidence_collected.push("backend_log"); + } const finishedAt = new Date(); result.finished_at = finishedAt.toISOString(); result.finished_at_local = localIsoWithOffset(finishedAt); diff --git a/skills/scripts/e2e/prepare-isolated-langbot-config.py b/skills/scripts/e2e/prepare-isolated-langbot-config.py new file mode 100644 index 000000000..596baa3d0 --- /dev/null +++ b/skills/scripts/e2e/prepare-isolated-langbot-config.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +import yaml + + +PROJECT_ROOT = Path(__file__).resolve().parents[3] +sys.path.insert(0, str(PROJECT_ROOT)) + +from tests.e2e.utils.config_factory import ( # noqa: E402 + create_minimal_config, + create_test_directories, +) + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument('--instance-root', required=True, type=Path) + parser.add_argument('--port', required=True, type=int) + parser.add_argument('--plugin-debug-port', required=True, type=int) + args = parser.parse_args() + + config_path = create_minimal_config(args.instance_root, port=args.port) + create_test_directories(args.instance_root) + + with config_path.open('r', encoding='utf-8') as file: + config = yaml.safe_load(file) + + config['plugin']['enable'] = True + config['plugin']['enable_marketplace'] = True + config['plugin']['display_plugin_debug_url'] = ( + f'ws://127.0.0.1:{args.plugin_debug_port}/plugin/debug/ws' + ) + + with config_path.open('w', encoding='utf-8') as file: + yaml.safe_dump(config, file, allow_unicode=True, sort_keys=False) + + +if __name__ == '__main__': + main() diff --git a/skills/scripts/e2e/reset-complex-agent-task.mjs b/skills/scripts/e2e/reset-complex-agent-task.mjs new file mode 100644 index 000000000..65f17d18d --- /dev/null +++ b/skills/scripts/e2e/reset-complex-agent-task.mjs @@ -0,0 +1,115 @@ +#!/usr/bin/env node + +import { cp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { env } from "node:process"; +import { spawnSync } from "node:child_process"; +import { + apiJson, + boxWorkspaceNamespace, + ensureEvidence, + evidencePaths, + loadEnvFiles, + resetAndAuthLocalUser, + resolveWorkspaceUuid, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; + +await loadEnvFiles(); +const paths = evidencePaths("reset-complex-agent-task"); +await ensureEvidence(paths); + +const scriptDir = dirname(fileURLToPath(import.meta.url)); +const source = resolve( + scriptDir, + "../../skills/langbot-testing/fixtures/complex-agent-task/workspace", +); +const repo = env.LANGBOT_REPO || ""; +let target = ""; +const result = { + source: "setup_automation", + case_id: "reset-complex-agent-task", + run_id: paths.runId, + status: "fail", + reason: "", + target, + initial_test_exit_code: null, + evidence_collected: ["filesystem"], +}; + +try { + if (!repo) throw new Error("LANGBOT_REPO is required."); + const backendUrl = env.LANGBOT_BACKEND_URL || ""; + const user = env.LANGBOT_E2E_LOGIN_USER || ""; + const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; + if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is required."); + if (!user) throw new Error("LANGBOT_E2E_LOGIN_USER is required."); + + const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); + const workspaceUuid = await resolveWorkspaceUuid(backendUrl, auth.token); + const bootstrap = await apiJson(backendUrl, "/api/v1/workspaces/bootstrap", { + token: auth.token, + skipWorkspace: true, + }); + const workspaceAccess = (bootstrap.json.data?.workspaces || []).find( + (entry) => (entry.workspace?.uuid || entry.uuid) === workspaceUuid, + ); + const instanceUuid = workspaceAccess?.workspace?.instance_uuid || workspaceAccess?.instance_uuid || ""; + if (!instanceUuid) throw new Error("Workspace bootstrap did not include instance_uuid."); + + const boxWorkspace = env.LANGBOT_BOX_WORKSPACE_HOST_PATH || resolve( + repo, + "data/box/default/tenants", + boxWorkspaceNamespace(instanceUuid, workspaceUuid), + ); + target = resolve(boxWorkspace, "order-orchestrator"); + result.target = target; + result.workspace_uuid = workspaceUuid; + await rm(target, { recursive: true, force: true }); + await mkdir(dirname(target), { recursive: true }); + await cp(source, target, { recursive: true }); + + const baseline = spawnSync( + "python3", + ["-m", "unittest", "discover", "-s", "tests", "-v"], + { cwd: target, encoding: "utf8", timeout: 60_000 }, + ); + result.initial_test_exit_code = baseline.status; + result.initial_failure_preview = `${baseline.stdout || ""}\n${baseline.stderr || ""}`.slice(0, 4000); + if (baseline.error) throw baseline.error; + if (baseline.status === 0) throw new Error("Complex task baseline unexpectedly passes; the fixture must start failing."); + + await upsertEnvLocal(resolve("skills/.env.local"), { + LANGBOT_COMPLEX_AGENT_WORKSPACE: target, + }); + + result.status = "pass"; + result.reason = "Complex task workspace reset and failing baseline confirmed."; +} catch (error) { + result.status = /required|ENOENT/.test(error.message) ? "env_issue" : "fail"; + result.reason = error.message; +} + +await writeResult(paths, result); +console.log(JSON.stringify(result, null, 2)); +process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); + +async function upsertEnvLocal(path, updates) { + let content = ""; + try { + content = await readFile(path, "utf8"); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + const lines = content ? content.split(/\r?\n/) : []; + for (const [key, value] of Object.entries(updates)) { + const replacement = `${key}=${value}`; + const index = lines.findIndex((line) => line.startsWith(`${key}=`)); + if (index >= 0) lines[index] = replacement; + else lines.push(replacement); + } + await writeFile(path, `${lines.filter(Boolean).join("\n")}\n`, "utf8"); +} diff --git a/skills/scripts/e2e/reset-sandbox-skill.mjs b/skills/scripts/e2e/reset-sandbox-skill.mjs new file mode 100644 index 000000000..6480d7181 --- /dev/null +++ b/skills/scripts/e2e/reset-sandbox-skill.mjs @@ -0,0 +1,76 @@ +#!/usr/bin/env node + +import { env } from "node:process"; +import { + apiJson, + ensureEvidence, + evidencePaths, + loadEnvFiles, + resetAndAuthLocalUser, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; +const caseId = "reset-sandbox-skill"; + +await loadEnvFiles(); +const paths = evidencePaths(caseId); +await ensureEvidence(paths); + +const nameArgument = process.argv.find((argument) => argument.startsWith("--name=")); +const skillName = nameArgument?.slice("--name=".length).trim() || ""; +const backendUrl = env.LANGBOT_BACKEND_URL || ""; +const result = { + source: "setup_automation", + case_id: caseId, + run_id: paths.runId, + status: "fail", + reason: "", + skill_name: skillName, + existed: false, + deleted: false, + evidence_collected: ["api_diagnostic"], +}; + +try { + if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured."); + if (!skillName || !/^[A-Za-z0-9_-]+$/.test(skillName)) { + throw new Error("--name must contain only letters, numbers, hyphens, or underscores."); + } + + const user = env.LANGBOT_E2E_LOGIN_USER || ""; + if (!user) throw new Error("LANGBOT_E2E_LOGIN_USER is required."); + const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; + const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); + const listed = await apiJson(backendUrl, "/api/v1/skills", { token: auth.token }); + if (listed.status >= 400 || listed.json.code !== 0) { + throw new Error(listed.json.msg || `Skill listing failed with HTTP ${listed.status}.`); + } + + result.existed = (listed.json.data?.skills || []).some((skill) => skill?.name === skillName); + if (result.existed) { + const deleted = await apiJson( + backendUrl, + `/api/v1/skills/${encodeURIComponent(skillName)}`, + { method: "DELETE", token: auth.token }, + ); + if (deleted.status >= 400 || deleted.json.code !== 0) { + throw new Error(deleted.json.msg || `Skill deletion failed with HTTP ${deleted.status}.`); + } + result.deleted = true; + } + + result.status = "pass"; + result.reason = result.deleted + ? `Removed stale sandbox skill ${skillName}.` + : `Sandbox skill ${skillName} was already absent.`; +} catch (error) { + result.status = /ECONNREFUSED|fetch failed|not configured|required/.test(error.message) + ? "env_issue" + : "fail"; + result.reason = error.message; +} + +await writeResult(paths, result); +console.log(JSON.stringify(result, null, 2)); +process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); diff --git a/skills/scripts/e2e/wizard-onebot-agent-runtime.mjs b/skills/scripts/e2e/wizard-onebot-agent-runtime.mjs new file mode 100644 index 000000000..dc94efe31 --- /dev/null +++ b/skills/scripts/e2e/wizard-onebot-agent-runtime.mjs @@ -0,0 +1,373 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { createServer } from "node:net"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; +import { + createBrowser, + ensureAuthenticatedBrowser, + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + pathExists, + resolveLangBotRepo, + safeScreenshot, + scanBrowserDiagnostics, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +const caseId = "wizard-onebot-agent-runtime"; +await loadEnvFiles(); +const paths = evidencePaths(caseId); +await ensureEvidence(paths); + +const startedAt = new Date(); +const frontendUrl = process.env.LANGBOT_FRONTEND_URL || ""; +const backendUrl = process.env.LANGBOT_BACKEND_URL || ""; +let browser; +let botId = ""; +let agentId = ""; +let token = ""; + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + url: "", + bot_id: "", + agent_id: "", + visible_signals: [], + api: {}, + runtime: {}, + cleanup: {}, + diagnostics: null, + evidence: { + screenshot: paths.screenshot, + console_log: paths.consoleLog, + network_log: paths.networkLog, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["ui", "screenshot", "console", "api_diagnostic"], +}; + +async function getFreePort() { + const server = createServer(); + await new Promise((resolve, reject) => { + server.once("error", reject); + server.listen(0, "127.0.0.1", resolve); + }); + const address = server.address(); + const port = typeof address === "object" && address ? address.port : 0; + await new Promise((resolve) => server.close(resolve)); + if (!port) throw new Error("Could not allocate a temporary OneBot port."); + return port; +} + +async function api(page, path, options = {}) { + return await page.evaluate( + async ({ baseUrl, path, options, token }) => { + const response = await fetch(`${baseUrl.replace(/\/$/, "")}${path}`, { + method: options.method || "GET", + headers: { + Authorization: `Bearer ${token}`, + "Content-Type": "application/json", + "X-Workspace-Id": + localStorage.getItem("langbot_active_workspace_uuid") || "", + }, + body: + options.body === undefined ? undefined : JSON.stringify(options.body), + }); + return { + status: response.status, + json: await response.json().catch(() => ({})), + }; + }, + { baseUrl: backendUrl, path, options, token }, + ); +} + +async function runProbe(port) { + const repo = await resolveLangBotRepo(); + const python = join(repo, ".venv", "bin", "python"); + const script = join( + dirname(fileURLToPath(import.meta.url)), + "onebot-runtime-probe.py", + ); + if (!(await pathExists(python))) { + throw new Error(`LangBot virtualenv Python is missing: ${python}`); + } + return await new Promise((resolve, reject) => { + const child = spawn(python, [script, "--port", String(port)], { + cwd: repo, + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => (stdout += chunk)); + child.stderr.on("data", (chunk) => (stderr += chunk)); + child.on("error", reject); + child.on("close", (code) => { + if (code === 0) { + resolve(JSON.parse(stdout.trim().split("\n").at(-1))); + } else { + reject( + new Error( + stderr.trim() || stdout.trim() || `OneBot probe exited ${code}`, + ), + ); + } + }); + }); +} + +try { + if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured."); + if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured."); + const port = await getFreePort(); + + browser = await createBrowser(paths); + const { page } = browser; + await page.goto(frontendUrl, { waitUntil: "domcontentloaded" }); + const auth = await ensureAuthenticatedBrowser(page, { + frontendUrl, + backendUrl, + }); + if (auth.status !== "pass") { + result.status = auth.status; + throw new Error(auth.reason); + } + token = await page.evaluate(() => localStorage.getItem("token") || ""); + if (!token) { + result.status = "blocked"; + throw new Error("Authenticated browser token is unavailable."); + } + + // Keep this case isolated from the instance's real onboarding state. + await page.route("**/api/v1/system/wizard/progress", async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ code: 0, msg: "ok", data: {} }), + }); + }); + + await page.goto(`${frontendUrl.replace(/\/$/, "")}/wizard`, { + waitUntil: "domcontentloaded", + }); + await page + .getByRole("button", { + name: /Welcome new members|欢迎新成员|新しいメンバーを歓迎/, + }) + .click(); + await page.getByText("OneBot v11", { exact: true }).last().click(); + + const botResponse = page.waitForResponse( + (response) => + response.request().method() === "POST" && + /\/api\/v1\/platform\/bots$/.test(new URL(response.url()).pathname), + ); + await page + .getByRole("button", { + name: /Confirm, Create Bot|确定，创建机器人|確定、ボットを作成/, + }) + .click(); + const botPayload = await (await botResponse).json(); + botId = botPayload.data?.uuid || ""; + if (!botId) throw new Error("Wizard did not create a Bot."); + result.bot_id = botId; + result.api.create_bot = { code: botPayload.code ?? null }; + + await page + .getByText(/Configure Your Bot|配置机器人|ボットを設定/) + .first() + .waitFor({ timeout: 15_000 }); + const portInput = page.getByRole("spinbutton").first(); + await portInput.fill(String(port)); + await portInput.press("Tab"); + await page + .getByRole("button", { + name: /Save & Enable Bot|保存并启用|保存して有効化/, + }) + .click(); + await page + .getByText( + /Bot configuration saved and enabled|机器人配置已保存并启用|ボット設定が保存され、有効になりました/, + ) + .waitFor({ timeout: 15_000 }); + result.visible_signals.push("bot-created", "adapter-enabled"); + + await page.getByRole("button", { name: /Next|下一步|次へ/ }).click(); + const localAgentTitle = page.getByText(/^(Local Agent|本地 Agent)$/).first(); + const localAgentCard = localAgentTitle.locator( + 'xpath=ancestor::*[@data-slot="card"][1]', + ); + await localAgentCard + .getByRole("button", { + name: /Use This Runner|使用此运行器|この Runner を使用/, + }) + .click(); + await page.waitForFunction( + () => + Array.from(document.querySelectorAll("textarea")).some((element) => { + const value = element.value || ""; + return [ + "Welcome new group members", + "欢迎新群成员", + "新しいグループメンバー", + ].some((expected) => value.includes(expected)); + }), + null, + { timeout: 15_000 }, + ); + result.visible_signals.push("scenario-prompt-visible"); + + await page + .getByText(/QA Deterministic Runner|QA 确定性 Runner/) + .first() + .click(); + + const agentResponse = page.waitForResponse( + (response) => + response.request().method() === "POST" && + /\/api\/v1\/agents$/.test(new URL(response.url()).pathname), + ); + await page + .getByRole("button", { + name: /Create & Deploy|创建并部署|作成＆デプロイ/, + }) + .click(); + const agentPayload = await (await agentResponse).json(); + agentId = agentPayload.data?.uuid || ""; + if (!agentId) throw new Error("Wizard did not create an Agent."); + result.agent_id = agentId; + result.api.create_agent = { code: agentPayload.code ?? null }; + await page.getByText(/All Set!|一切就绪！/).waitFor({ timeout: 15_000 }); + result.visible_signals.push("agent-created", "wizard-complete"); + + const savedBot = await api(page, `/api/v1/platform/bots/${botId}`); + const route = savedBot.json.data?.bot?.event_bindings?.[0]; + result.api.saved_route = { + http_status: savedBot.status, + event_pattern: route?.event_pattern || null, + target_type: route?.target_type || null, + target_matches: route?.target_uuid === agentId, + }; + if ( + route?.event_pattern !== "group.member_joined" || + route?.target_type !== "agent" || + route?.target_uuid !== agentId + ) { + throw new Error("Wizard did not persist the expected Agent route."); + } + result.visible_signals.push("route-persisted"); + + result.runtime.onebot = await runProbe(port); + let routeStatus; + for (let attempt = 0; attempt < 30; attempt += 1) { + routeStatus = await api( + page, + `/api/v1/platform/bots/${botId}/event-routes/status`, + ); + if (routeStatus.json.data?.routes?.[0]?.last_status === "delivered") break; + await page.waitForTimeout(250); + } + const latest = routeStatus?.json?.data?.routes?.[0]; + result.runtime.route_status = latest || null; + if (latest?.last_status !== "delivered") { + throw new Error( + `Runtime route did not reach delivered status: ${latest?.last_status || "none"}`, + ); + } + result.visible_signals.push( + "platform-event-converted", + "agent-ran", + "reply-delivered", + ); + + const botUrl = `${frontendUrl.replace(/\/$/, "")}/home/bots?id=${encodeURIComponent(botId)}`; + await page.goto(botUrl, { waitUntil: "domcontentloaded" }); + result.url = page.url(); + const deliveredStatus = page + .getByText(/Delivered|已投递|配信済み/, { exact: true }) + .first(); + await deliveredStatus.waitFor({ timeout: 15_000 }); + await deliveredStatus.scrollIntoViewIfNeeded(); + await page.waitForTimeout(250); + result.visible_signals.push("delivered-status-visible"); + await safeScreenshot(page, paths.screenshot); + + result.diagnostics = await scanBrowserDiagnostics(paths); + if (result.diagnostics.status !== "pass") { + throw new Error(result.diagnostics.reason); + } + result.status = "pass"; + result.reason = + "Quick Start created an enabled OneBot scenario that converted a platform event, ran an Agent, delivered its reply, and showed the final status."; +} catch (error) { + if (!["blocked", "env_issue"].includes(result.status)) result.status = "fail"; + result.reason = result.reason || error.message; + if (browser?.page && token && botId) { + const [routeStatus, botLogs] = await Promise.all([ + api( + browser.page, + `/api/v1/platform/bots/${encodeURIComponent(botId)}/event-routes/status`, + ).catch(() => ({ status: 0, json: {} })), + api( + browser.page, + `/api/v1/platform/bots/${encodeURIComponent(botId)}/logs`, + { + method: "POST", + body: { from_index: -1, max_count: 50 }, + }, + ).catch(() => ({ status: 0, json: {} })), + ]); + result.runtime.failure_route_status = routeStatus.json.data || null; + result.runtime.failure_bot_logs = botLogs.json.data || null; + } + if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); +} finally { + if (browser?.page && token) { + if (botId) { + const deleted = await api( + browser.page, + `/api/v1/platform/bots/${encodeURIComponent(botId)}`, + { method: "DELETE" }, + ).catch(() => ({ status: 0, json: {} })); + result.cleanup.bot_deleted = + deleted.status < 400 && deleted.json.code === 0; + } + if (agentId) { + const deleted = await api( + browser.page, + `/api/v1/agents/${encodeURIComponent(agentId)}`, + { method: "DELETE" }, + ).catch(() => ({ status: 0, json: {} })); + result.cleanup.agent_deleted = + deleted.status < 400 && deleted.json.code === 0; + } + } + const cleanupFailed = + (botId && !result.cleanup.bot_deleted) || + (agentId && !result.cleanup.agent_deleted); + if (cleanupFailed && result.status === "pass") { + result.status = "fail"; + result.reason = "The temporary OneBot wizard resources were not deleted."; + } + if (browser) await browser.close().catch(() => {}); + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/wizard-runner-marketplace-catalog.mjs b/skills/scripts/e2e/wizard-runner-marketplace-catalog.mjs new file mode 100644 index 000000000..548af706f --- /dev/null +++ b/skills/scripts/e2e/wizard-runner-marketplace-catalog.mjs @@ -0,0 +1,411 @@ +#!/usr/bin/env node + +import { + apiJson, + createBrowser, + ensureAuthenticatedBrowser, + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + safeScreenshot, + scanBrowserDiagnostics, + writeResult, +} from "./lib/langbot-e2e.mjs"; +import { startIsolatedLangBotInstance } from "./lib/isolated-langbot-instance.mjs"; + +const caseId = "wizard-runner-marketplace-catalog"; +await loadEnvFiles(); +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const mobileScreenshot = paths.screenshot.replace(/\.png$/, "-mobile.png"); +const installedScreenshot = paths.screenshot.replace( + /\.png$/, + "-installed.png", +); + +const startedAt = new Date(); +let frontendUrl = ""; +let backendUrl = ""; +let browser; +let isolated; +let token = ""; +let botId = ""; + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + url: "", + visible_signals: [], + api: {}, + marketplace_request: null, + marketplace_response: null, + diagnostics: null, + cleanup: {}, + evidence: { + screenshot: paths.screenshot, + mobile_screenshot: mobileScreenshot, + installed_screenshot: installedScreenshot, + console_log: paths.consoleLog, + network_log: paths.networkLog, + isolated_backend_log: `${paths.evidenceDir}/isolated-backend.log`, + isolated_frontend_log: `${paths.evidenceDir}/isolated-frontend.log`, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["ui", "screenshot", "console", "api_diagnostic"], +}; + +try { + isolated = await startIsolatedLangBotInstance({ + evidenceDir: paths.evidenceDir, + }); + frontendUrl = isolated.frontendUrl; + backendUrl = isolated.backendUrl; + result.api.isolated_instance = { + backend_url: backendUrl, + frontend_url: frontendUrl, + }; + + browser = await createBrowser(paths); + const { page } = browser; + page.on("request", (request) => { + if ( + !/\/api\/v1\/marketplace\/(extensions|plugins)\/search$/.test( + new URL(request.url()).pathname, + ) + ) { + return; + } + try { + const payload = request.postDataJSON(); + if (payload?.component_filter === "Runner") { + result.marketplace_request = { + endpoint: new URL(request.url()).pathname, + component_filter: payload.component_filter, + type_filter: payload.type_filter || null, + page_size: payload.page_size || null, + }; + } + } catch { + // The assertion below reports a missing or malformed catalog request. + } + }); + page.on("response", async (response) => { + const pathname = new URL(response.url()).pathname; + if ( + !/\/api\/v1\/marketplace\/(extensions|plugins)\/search$/.test(pathname) + ) { + return; + } + try { + const payload = await response.json(); + const entries = payload?.data?.extensions || payload?.data?.plugins || []; + const localAgent = entries.find( + (entry) => + `${entry.author}/${entry.name}` === "langbot-team/LocalAgent", + ); + result.marketplace_response = { + endpoint: pathname, + total: payload?.data?.total ?? entries.length, + local_agent_present: Boolean(localAgent), + local_agent_version: localAgent?.latest_version || null, + }; + } catch { + // The UI assertions below surface malformed Marketplace responses. + } + }); + + await page.goto(frontendUrl, { waitUntil: "domcontentloaded" }); + const auth = await ensureAuthenticatedBrowser(page, { + frontendUrl, + backendUrl, + user: isolated.user, + password: isolated.password, + recoveryKey: isolated.recoveryKey, + }); + if (auth.status !== "pass") { + result.status = auth.status; + throw new Error(auth.reason); + } + token = await page.evaluate(() => localStorage.getItem("token") || ""); + if (!token) { + result.status = "blocked"; + throw new Error("Authenticated browser token is unavailable."); + } + + const [plugins, metadata, system] = await Promise.all([ + apiJson(backendUrl, "/api/v1/plugins", { token }), + apiJson(backendUrl, "/api/v1/pipelines/_/metadata", { token }), + apiJson(backendUrl, "/api/v1/system/info", { token }), + ]); + const runnerStage = metadata.json.data?.configs + ?.find((config) => config.name === "ai") + ?.stages?.find((stage) => stage.name === "runner"); + const runnerOptions = + runnerStage?.config?.find((item) => item.name === "id")?.options || []; + const installedPlugins = plugins.json.data?.plugins || []; + result.api.clean_state = { + plugin_count: installedPlugins.length, + runner_count: runnerOptions.length, + wizard_status: system.json.data?.wizard_status || null, + }; + if (installedPlugins.length !== 0 || runnerOptions.length !== 0) { + result.status = "blocked"; + throw new Error( + `This case requires zero plugins and zero runners; found ${installedPlugins.length} plugins and ${runnerOptions.length} runners.`, + ); + } + if (system.json.data?.wizard_status !== "none") { + result.status = "blocked"; + throw new Error( + `This case requires a first-run instance; wizard status is ${system.json.data?.wizard_status}.`, + ); + } + + const suffix = paths.runId.slice(-40); + const bot = await apiJson(backendUrl, "/api/v1/platform/bots", { + method: "POST", + token, + body: { + name: `Runner Catalog Wizard ${suffix}`, + description: "Temporary clean-runner catalog fixture", + adapter: "aiocqhttp", + adapter_config: { + host: "127.0.0.1", + port: 2280, + "access-token": "", + }, + enable: false, + event_bindings: [], + }, + }); + botId = bot.json.data?.uuid || ""; + if (bot.status >= 400 || bot.json.code !== 0 || !botId) { + throw new Error(bot.json.msg || "Failed to create the temporary Bot."); + } + + const progress = await apiJson(backendUrl, "/api/v1/system/wizard/progress", { + method: "PUT", + token, + body: { + step: 2, + selected_scenario: "message_reply", + selected_adapter: "aiocqhttp", + created_bot_uuid: botId, + bot_saved: true, + selected_runner: null, + }, + }); + if (progress.status >= 400 || progress.json.code !== 0) { + throw new Error(progress.json.msg || "Failed to prepare Wizard progress."); + } + + await page.goto(`${frontendUrl.replace(/\/$/, "")}/wizard`, { + waitUntil: "domcontentloaded", + }); + result.url = page.url(); + await page + .getByText(/Select an AI Engine|选择 AI 引擎|AIエンジンを選択/, { + exact: true, + }) + .waitFor({ timeout: 15_000 }); + const localAgentIdentity = page.getByText("langbot-team/LocalAgent", { + exact: true, + }); + await localAgentIdentity.waitFor({ timeout: 30_000 }); + const localAgentCard = localAgentIdentity.locator( + "xpath=ancestor::*[@data-slot='card'][1]", + ); + const installButton = localAgentCard.getByRole("button", { + name: /Install & Continue|安装并继续|インストールして続行/, + }); + await installButton.waitFor(); + const browseLink = page.getByRole("link", { + name: /Browse Runner Extensions|浏览运行器扩展|Runner 拡張機能を見る/, + }); + await browseLink.waitFor(); + const href = await browseLink.getAttribute("href"); + if (href !== "/home/extensions?type=plugin&component=Runner") { + throw new Error(`Unexpected Runner marketplace URL: ${href}`); + } + const nextButton = page.getByRole("button", { + name: /Create & Deploy|创建并部署|作成＆デプロイ/, + }); + if (!(await nextButton.isDisabled())) { + throw new Error("Wizard allowed continuing without an installed Runner."); + } + if (!result.marketplace_request) { + throw new Error("Wizard did not request the Runner Marketplace catalog."); + } + if ( + !result.marketplace_response?.local_agent_present || + !result.marketplace_response?.local_agent_version + ) { + throw new Error( + "The Marketplace catalog did not return an installable langbot-team/LocalAgent version.", + ); + } + + result.visible_signals.push( + "first-run-ai-engine-step", + "published-local-agent-runner", + "install-and-continue-action", + "runner-marketplace-link", + "next-disabled-without-runner", + ); + await safeScreenshot(page, paths.screenshot); + + await page.setViewportSize({ width: 390, height: 844 }); + await page.waitForTimeout(250); + const overflow = await page.evaluate( + () => document.documentElement.scrollWidth - window.innerWidth, + ); + if (overflow > 1) { + throw new Error(`Runner catalog Wizard overflows mobile by ${overflow}px.`); + } + await browseLink.scrollIntoViewIfNeeded(); + await safeScreenshot(page, mobileScreenshot); + result.visible_signals.push("mobile-layout"); + + await page.setViewportSize({ width: 1440, height: 1000 }); + await installButton.click(); + const installingButton = localAgentCard.getByRole("button", { + name: /Installing\.\.\.|正在安装\.\.\.|インストール中\.\.\./, + }); + await installingButton.waitFor({ timeout: 15_000 }); + const installOutcome = await Promise.race([ + nextButton + .click({ trial: true, timeout: 180_000 }) + .then(() => "registered"), + installButton + .waitFor({ state: "visible", timeout: 180_000 }) + .then(() => "failed"), + ]); + if (installOutcome === "failed") { + throw new Error( + "LocalAgent installation failed before Runner registration.", + ); + } + if (await nextButton.isDisabled()) { + throw new Error( + "Create & Deploy remained disabled after LocalAgent installation.", + ); + } + + const [installedPluginsResponse, installedMetadataResponse] = + await Promise.all([ + apiJson(backendUrl, "/api/v1/plugins", { token }), + apiJson(backendUrl, "/api/v1/pipelines/_/metadata", { token }), + ]); + const postInstallPlugins = installedPluginsResponse.json.data?.plugins || []; + const installedRunnerStage = installedMetadataResponse.json.data?.configs + ?.find((config) => config.name === "ai") + ?.stages?.find((stage) => stage.name === "runner"); + const installedRunnerOptions = + installedRunnerStage?.config?.find((item) => item.name === "id")?.options || + []; + const localAgentInstalled = postInstallPlugins.some((plugin) => { + const metadata = plugin.manifest?.manifest?.metadata || {}; + return `${metadata.author}/${metadata.name}` === "langbot-team/LocalAgent"; + }); + const localAgentRegistered = installedRunnerOptions.some( + (option) => option.name === "plugin:langbot-team/LocalAgent/default", + ); + result.api.installed_state = { + plugin_count: postInstallPlugins.length, + runner_count: installedRunnerOptions.length, + local_agent_installed: localAgentInstalled, + local_agent_registered: localAgentRegistered, + }; + if (!localAgentInstalled || !localAgentRegistered) { + throw new Error( + "LocalAgent installation completed in the UI but the plugin or Runner registration is missing.", + ); + } + await safeScreenshot(page, installedScreenshot); + result.visible_signals.push( + "local-agent-installed", + "local-agent-runner-registered", + "local-agent-selected", + "create-and-deploy-enabled", + ); + + result.diagnostics = await scanBrowserDiagnostics(paths); + if (result.diagnostics.status !== "pass") { + throw new Error(result.diagnostics.reason); + } + result.status = "pass"; + result.reason = + "A clean first-run instance discovered LocalAgent in the Runner catalog, installed and registered it, selected it, and enabled Create & Deploy."; +} catch (error) { + if (!["blocked", "env_issue"].includes(result.status)) result.status = "fail"; + result.reason = result.reason || error.message; + if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); +} finally { + const cleanup = {}; + if (token && backendUrl) { + const resetProgress = await apiJson( + backendUrl, + "/api/v1/system/wizard/progress", + { + method: "PUT", + token, + body: { + step: 0, + selected_scenario: null, + selected_adapter: null, + created_bot_uuid: null, + bot_saved: false, + selected_runner: null, + }, + }, + ).catch(() => ({ status: 0, json: {} })); + cleanup.progress_reset = + resetProgress.status < 400 && resetProgress.json.code === 0; + } + if (botId && token && backendUrl) { + const deletedBot = await apiJson( + backendUrl, + `/api/v1/platform/bots/${encodeURIComponent(botId)}`, + { method: "DELETE", token }, + ).catch(() => ({ status: 0, json: {} })); + cleanup.bot_deleted = deletedBot.status < 400 && deletedBot.json.code === 0; + } + result.cleanup = cleanup; + if ( + result.status === "pass" && + (!cleanup.progress_reset || (botId && !cleanup.bot_deleted)) + ) { + result.status = "fail"; + result.reason = "The clean Wizard fixtures were not fully reset."; + } + if (browser) await browser.close().catch(() => {}); + if (isolated) { + try { + await isolated.stop(); + cleanup.isolated_instance_stopped = true; + } catch (error) { + cleanup.isolated_instance_stopped = false; + if (result.status === "pass") { + result.status = "fail"; + result.reason = `Could not stop the isolated LangBot instance: ${error.message}`; + } + } + } + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/wizard-scenario-routing.mjs b/skills/scripts/e2e/wizard-scenario-routing.mjs new file mode 100644 index 000000000..a0e0fc2a5 --- /dev/null +++ b/skills/scripts/e2e/wizard-scenario-routing.mjs @@ -0,0 +1,349 @@ +#!/usr/bin/env node + +import { + apiJson, + createBrowser, + ensureAuthenticatedBrowser, + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + safeScreenshot, + scanBrowserDiagnostics, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +const caseId = "wizard-scenario-routing"; +await loadEnvFiles(); +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const messageScreenshot = paths.screenshot.replace( + /\.png$/, + "-message-scenario.png", +); +const mobileScreenshot = paths.screenshot.replace(/\.png$/, "-mobile.png"); + +const startedAt = new Date(); +const frontendUrl = process.env.LANGBOT_FRONTEND_URL || ""; +const backendUrl = process.env.LANGBOT_BACKEND_URL || ""; + +let browser; +let token = ""; +let botId = ""; +let agentId = ""; +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + url: "", + visible_signals: [], + api: {}, + diagnostics: null, + cleanup: null, + evidence: { + console_log: paths.consoleLog, + network_log: paths.networkLog, + screenshot: paths.screenshot, + message_scenario_screenshot: messageScreenshot, + mobile_screenshot: mobileScreenshot, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["ui", "screenshot", "console", "api_diagnostic"], +}; + +try { + if (!frontendUrl) throw new Error("LANGBOT_FRONTEND_URL is not configured."); + if (!backendUrl) throw new Error("LANGBOT_BACKEND_URL is not configured."); + + browser = await createBrowser(paths); + const { page } = browser; + await page.goto(frontendUrl, { waitUntil: "domcontentloaded" }); + const auth = await ensureAuthenticatedBrowser(page, { + frontendUrl, + backendUrl, + }); + if (auth.status !== "pass") { + result.status = auth.status; + throw new Error(auth.reason); + } + + token = await page.evaluate(() => localStorage.getItem("token") || ""); + if (!token) { + result.status = "blocked"; + throw new Error("Authenticated browser has no reusable local token."); + } + + const adapters = await apiJson(backendUrl, "/api/v1/platform/adapters", { + token, + }); + const catalog = adapters.json.data?.adapters || []; + const httpBot = catalog.find((adapter) => adapter.name === "http_bot"); + const welcomeAdapters = catalog.filter((adapter) => + (adapter.spec?.supported_events || []).includes("group.member_joined"), + ); + result.api.adapters = { + http_status: adapters.status, + code: adapters.json.code ?? null, + http_bot_supports_message: + !httpBot?.spec?.supported_events?.length || + httpBot.spec.supported_events.includes("message.received"), + http_bot_supports_member_joined: + httpBot?.spec?.supported_events?.includes("group.member_joined") || false, + welcome_adapter_count: welcomeAdapters.length, + }; + if (adapters.status >= 400 || adapters.json.code !== 0) { + throw new Error(adapters.json.msg || "Adapter discovery failed."); + } + if (!httpBot || welcomeAdapters.length === 0) { + throw new Error( + "The adapter catalog does not contain the fixtures required by this case.", + ); + } + if (result.api.adapters.http_bot_supports_member_joined) { + throw new Error( + "HTTP Bot unexpectedly declares group.member_joined support; update the case expectation.", + ); + } + + const progressUrl = `${backendUrl.replace(/\/$/, "")}/api/v1/system/wizard/progress`; + await page.route(progressUrl, async (route) => { + await route.fulfill({ + status: 200, + contentType: "application/json", + body: JSON.stringify({ code: 0, msg: "ok", data: {} }), + }); + }); + + await page.goto(`${frontendUrl.replace(/\/$/, "")}/wizard`, { + waitUntil: "domcontentloaded", + }); + result.url = page.url(); + await page + .getByText( + /What should this bot do\?|这个机器人要完成什么？|このボットで何を実現しますか？/, + ) + .waitFor({ timeout: 15_000 }); + + await page + .getByRole("button", { + name: /Reply to messages|回复收到的消息|受信メッセージに返信/, + }) + .click(); + await page + .getByText(/HTTP Bot|HTTP 通用接入|HTTP ボット/, { exact: true }) + .waitFor(); + result.visible_signals.push("scenario-first", "message-http-compatible"); + await safeScreenshot(page, messageScreenshot); + + await page + .getByRole("button", { + name: /Welcome new members|欢迎新成员|新しいメンバーを歓迎/, + }) + .click(); + await page + .getByText(/Discord/, { exact: true }) + .first() + .waitFor(); + const httpBotCount = await page + .getByText(/HTTP Bot|HTTP 通用接入|HTTP ボット/, { exact: true }) + .count(); + if (httpBotCount !== 0) { + throw new Error("HTTP Bot remained visible for group.member_joined."); + } + const selectedScenario = page.getByRole("button", { + name: /Welcome new members|欢迎新成员|新しいメンバーを歓迎/, + }); + if ( + !(await selectedScenario.getByText("Agent", { exact: true }).isVisible()) + ) { + throw new Error( + "The non-message scenario is not visibly labeled as Agent.", + ); + } + result.visible_signals.push( + "welcome-compatible-channel", + "http-filtered", + "agent-behavior-label", + ); + + await safeScreenshot(page, paths.screenshot); + await page.setViewportSize({ width: 390, height: 844 }); + await page.waitForTimeout(250); + const horizontalOverflow = await page.evaluate( + () => document.documentElement.scrollWidth - window.innerWidth, + ); + if (horizontalOverflow > 1) { + throw new Error( + `The mobile wizard overflows horizontally by ${horizontalOverflow}px.`, + ); + } + await selectedScenario.scrollIntoViewIfNeeded(); + await page + .getByText(/Discord/, { exact: true }) + .first() + .waitFor(); + await safeScreenshot(page, mobileScreenshot); + result.visible_signals.push("mobile-layout"); + + const fixtureSuffix = paths.runId.slice(-48); + const agent = await apiJson(backendUrl, "/api/v1/agents", { + method: "POST", + token, + body: { + kind: "agent", + name: `Wizard Scenario Agent ${fixtureSuffix}`, + description: "Temporary wizard scenario routing fixture", + emoji: "W", + enabled: true, + supported_event_patterns: ["group.member_joined"], + }, + }); + agentId = agent.json.data?.uuid || ""; + result.api.create_agent = { + http_status: agent.status, + code: agent.json.code ?? null, + }; + if (agent.status >= 400 || agent.json.code !== 0 || !agentId) { + throw new Error(agent.json.msg || "Failed to create the temporary Agent."); + } + + const bot = await apiJson(backendUrl, "/api/v1/platform/bots", { + method: "POST", + token, + body: { + name: `Wizard Scenario Bot ${fixtureSuffix}`, + description: "Temporary disabled wizard scenario routing fixture", + adapter: "aiocqhttp", + adapter_config: { + host: "127.0.0.1", + port: 2280, + "access-token": "", + }, + enable: false, + event_bindings: [], + }, + }); + botId = bot.json.data?.uuid || ""; + result.api.create_bot = { + http_status: bot.status, + code: bot.json.code ?? null, + }; + if (bot.status >= 400 || bot.json.code !== 0 || !botId) { + throw new Error(bot.json.msg || "Failed to create the temporary Bot."); + } + + const update = await apiJson( + backendUrl, + `/api/v1/platform/bots/${encodeURIComponent(botId)}`, + { + method: "PUT", + token, + body: { + event_bindings: [ + { + event_pattern: "group.member_joined", + target_type: "agent", + target_uuid: agentId, + filters: [], + priority: 0, + enabled: true, + description: "Welcome new members", + }, + ], + }, + }, + ); + result.api.bind_route = { + http_status: update.status, + code: update.json.code ?? null, + }; + if (update.status >= 400 || update.json.code !== 0) { + throw new Error(update.json.msg || "Failed to bind the temporary route."); + } + + const savedBot = await apiJson( + backendUrl, + `/api/v1/platform/bots/${encodeURIComponent(botId)}`, + { token }, + ); + const savedRoute = savedBot.json.data?.bot?.event_bindings?.[0]; + result.api.read_route = { + http_status: savedBot.status, + code: savedBot.json.code ?? null, + event_pattern: savedRoute?.event_pattern || null, + target_type: savedRoute?.target_type || null, + target_matches: savedRoute?.target_uuid === agentId, + }; + if ( + savedBot.status >= 400 || + savedBot.json.code !== 0 || + savedRoute?.event_pattern !== "group.member_joined" || + savedRoute?.target_type !== "agent" || + savedRoute?.target_uuid !== agentId + ) { + throw new Error("The saved Bot did not return the expected Agent route."); + } + result.visible_signals.push("agent-route-persisted"); + + result.diagnostics = await scanBrowserDiagnostics(paths); + if (result.diagnostics.status !== "pass") { + throw new Error(result.diagnostics.reason); + } + result.status = "pass"; + result.reason = + "Quick Start visibly filtered channels by scenario at desktop and mobile widths."; +} catch (error) { + if (!["blocked", "env_issue"].includes(result.status)) result.status = "fail"; + result.reason = result.reason || error.message; + if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); +} finally { + const cleanup = {}; + if (botId && token && backendUrl) { + const deletedBot = await apiJson( + backendUrl, + `/api/v1/platform/bots/${encodeURIComponent(botId)}`, + { method: "DELETE", token }, + ).catch((error) => ({ + status: 0, + json: { code: null, msg: error.message }, + })); + cleanup.bot_deleted = deletedBot.status < 400 && deletedBot.json.code === 0; + cleanup.bot_http_status = deletedBot.status; + } + if (agentId && token && backendUrl) { + const deletedAgent = await apiJson( + backendUrl, + `/api/v1/agents/${encodeURIComponent(agentId)}`, + { method: "DELETE", token }, + ).catch((error) => ({ + status: 0, + json: { code: null, msg: error.message }, + })); + cleanup.agent_deleted = + deletedAgent.status < 400 && deletedAgent.json.code === 0; + cleanup.agent_http_status = deletedAgent.status; + } + result.cleanup = cleanup; + const cleanupFailed = + (botId && !cleanup.bot_deleted) || (agentId && !cleanup.agent_deleted); + if (cleanupFailed && result.status === "pass") { + result.status = "fail"; + result.reason = "The temporary wizard routing fixtures were not deleted."; + } + if (browser) await browser.close().catch(() => {}); + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/workspace-compatibility-preflight.mjs b/skills/scripts/e2e/workspace-compatibility-preflight.mjs new file mode 100644 index 000000000..6649bc7b6 --- /dev/null +++ b/skills/scripts/e2e/workspace-compatibility-preflight.mjs @@ -0,0 +1,274 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { access, readFile, writeFile } from "node:fs/promises"; +import { delimiter, dirname, join, resolve } from "node:path"; +import { env } from "node:process"; +import { + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + resolveLangBotRepo, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "workspace-compatibility-preflight"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const startedAt = new Date(); +const detailsPath = join(paths.evidenceDir, "workspace-preflight.json"); + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + checks: [], + warnings: [], + evidence: { + workspace_preflight_json: detailsPath, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["filesystem", "api_diagnostic"], +}; + +const repositories = [ + { + id: "langbot", + directory: "LangBot", + envKey: "LANGBOT_REPO", + manifest: false, + }, + { + id: "plugin-sdk", + directory: "langbot-plugin-sdk", + envKey: "LANGBOT_PLUGIN_SDK_REPO", + manifest: false, + }, + { + id: "agent-runner", + directory: "langbot-agent-runner", + envKey: "LANGBOT_RUNNER_REPO", + manifest: false, + }, + { + id: "local-agent", + directory: "langbot-local-agent", + envKey: "LANGBOT_LOCAL_AGENT_REPO", + identity: "langbot-team/LocalAgent", + }, + { + id: "control-plane", + directory: "langbot-agent-control-plane", + envKey: "LANGBOT_AGENT_CONTROL_PLANE_REPO", + identity: "langbot/agent-control-plane", + }, + { + id: "longterm-memory", + directory: "langbot-longterm-memory", + envKey: "LANGBOT_LONGTERM_MEMORY_REPO", + identity: "langbot-team/LongTermMemory", + }, + { + id: "parser", + directory: "langbot-parser", + envKey: "LANGBOT_PARSER_PLUGIN_REPO", + identity: "langbot-team/GeneralParsers", + }, + { + id: "rag", + directory: "langbot-rag", + envKey: "LANGBOT_RAG_PLUGIN_REPO", + identity: "langbot-team/LangRAG", + }, + { + id: "skill-authoring", + directory: "langbot-skill-authoring", + envKey: "LANGBOT_SKILL_AUTHORING_REPO", + identity: "huanghuoguoguo/skill-authoring", + }, +]; + +function run(command, args, options = {}) { + return new Promise((resolvePromise) => { + const child = spawn(command, args, { + ...options, + encoding: "utf8", + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", (error) => + resolvePromise({ status: null, stdout, stderr, error }), + ); + child.on("close", (status) => + resolvePromise({ status, stdout, stderr, error: null }), + ); + }); +} + +function addCheck(name, status, detail = {}) { + result.checks.push({ name, status, ...detail }); +} + +try { + const langbotRepo = await resolveLangBotRepo(); + const workspaceRoot = resolve( + env.LANGBOT_WORKSPACE_ROOT || dirname(langbotRepo), + ); + const resolved = {}; + + for (const repository of repositories) { + const path = resolve( + env[repository.envKey] || + (repository.id === "langbot" + ? langbotRepo + : join(workspaceRoot, repository.directory)), + ); + resolved[repository.id] = path; + try { + await access(join(path, ".git")); + } catch { + addCheck(`repo:${repository.id}`, "fail", { + path, + reason: "Git checkout is missing.", + }); + continue; + } + + const branch = await run("git", ["branch", "--show-current"], { + cwd: path, + }); + const branchName = branch.stdout.trim(); + const compatible = + branch.status === 0 && /^(?:main|dev\/4\.11\.x)$/.test(branchName); + addCheck(`repo:${repository.id}`, compatible ? "pass" : "fail", { + path, + branch: branchName, + reason: compatible + ? "" + : "Expected main or dev/4.11.x compatibility branch.", + }); + + const dirty = await run("git", ["status", "--short"], { cwd: path }); + if (dirty.stdout.trim()) { + result.warnings.push({ + name: `dirty:${repository.id}`, + path, + entries: dirty.stdout.trim().split(/\r?\n/).length, + }); + } + + if (repository.identity) { + const manifestPath = join(path, "manifest.yaml"); + try { + const manifest = await readFile(manifestPath, "utf8"); + const author = manifest.match(/^\s{2}author:\s*([^\s#]+)/m)?.[1] || ""; + const name = manifest.match(/^\s{2}name:\s*([^\s#]+)/m)?.[1] || ""; + const identity = `${author}/${name}`; + addCheck( + `manifest:${repository.id}`, + identity === repository.identity ? "pass" : "fail", + { + path: manifestPath, + identity, + expected_identity: repository.identity, + }, + ); + } catch (error) { + addCheck(`manifest:${repository.id}`, "fail", { + path: manifestPath, + reason: error.message, + }); + } + } + } + + const python = join(resolved.langbot, ".venv", "bin", "python"); + try { + await access(python); + addCheck("langbot-venv", "pass", { python }); + } catch { + addCheck("langbot-venv", "fail", { + python, + reason: "LangBot virtualenv Python is missing.", + }); + } + + const sdkSrc = join(resolved["plugin-sdk"], "src"); + const importProbe = await run( + python, + [ + "-c", + [ + "import json, pathlib, langbot_plugin", + "from langbot_plugin.api.entities.builtin.runner.input import AgentInput", + "from langbot_plugin.api.entities.builtin.runner.result import RunnerResult", + "print(json.dumps({'path': str(pathlib.Path(langbot_plugin.__file__).resolve()), 'entities': [AgentInput.__name__, RunnerResult.__name__]}))", + ].join("; "), + ], + { + cwd: resolved.langbot, + env: { + ...env, + PYTHONPATH: [sdkSrc, env.PYTHONPATH].filter(Boolean).join(delimiter), + }, + }, + ); + let importDetail = {}; + try { + importDetail = JSON.parse(importProbe.stdout.trim()); + } catch { + importDetail = { stderr: importProbe.stderr.trim() }; + } + const localSdkLoaded = + importProbe.status === 0 && + resolve(importDetail.path || "").startsWith(resolve(sdkSrc)); + addCheck("local-sdk-import", localSdkLoaded ? "pass" : "fail", { + expected_root: resolve(sdkSrc), + ...importDetail, + reason: localSdkLoaded + ? "" + : "langbot_plugin did not load from the workspace SDK source tree.", + }); + + const failures = result.checks.filter((check) => check.status === "fail"); + result.status = failures.length === 0 ? "pass" : "fail"; + result.reason = + failures.length === 0 + ? `Workspace compatibility preflight passed with ${result.warnings.length} non-blocking dirty-worktree warning(s).` + : `Workspace compatibility preflight found ${failures.length} blocking check(s).`; +} catch (error) { + result.status = /missing|ENOENT|not found/i.test(error.message) + ? "env_issue" + : "fail"; + result.reason = error.message; +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeFile( + detailsPath, + `${JSON.stringify({ checks: result.checks, warnings: result.warnings }, null, 2)}\n`, + "utf8", + ); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/workspace-plugin-pages-smoke.mjs b/skills/scripts/e2e/workspace-plugin-pages-smoke.mjs new file mode 100644 index 000000000..6c238c0a4 --- /dev/null +++ b/skills/scripts/e2e/workspace-plugin-pages-smoke.mjs @@ -0,0 +1,186 @@ +#!/usr/bin/env node + +import { join } from "node:path"; +import { env } from "node:process"; +import { + apiJson, + createBrowser, + ensureAuthenticatedBrowser, + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + safeScreenshot, + scanBrowserDiagnostics, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "workspace-plugin-pages-smoke"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const startedAt = new Date(); +const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; +const backendUrl = env.LANGBOT_BACKEND_URL || ""; + +const targets = [ + { + id: "control-plane", + author: "langbot", + plugin: "agent-control-plane", + page: "control-plane", + signal: "Agent Control Plane", + endpoint: "/health", + method: "GET", + }, + { + id: "memory-console", + author: "langbot-team", + plugin: "LongTermMemory", + page: "memory_console", + signal: "Memory Console", + endpoint: "/summary", + method: "GET", + }, + { + id: "langrag-observability", + author: "langbot-team", + plugin: "LangRAG", + page: "observability", + signal: "LangRAG Observability", + endpoint: "/snapshot", + method: "GET", + }, + { + id: "skill-authoring", + author: "huanghuoguoguo", + plugin: "skill-authoring", + page: "authoring", + signal: "Skill Authoring", + endpoint: "/health", + method: "GET", + }, +]; + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + frontend_url: frontendUrl, + backend_url: backendUrl, + pages: [], + browser_diagnostics: null, + evidence: { + console_log: paths.consoleLog, + network_log: paths.networkLog, + screenshot: paths.screenshot, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["ui", "screenshot", "console", "network", "api_diagnostic"], +}; + +let browser; +try { + if (!frontendUrl || !backendUrl) throw new Error("LANGBOT_FRONTEND_URL and LANGBOT_BACKEND_URL must be configured."); + browser = await createBrowser(paths); + const { page } = browser; + await page.goto(frontendUrl, { waitUntil: "domcontentloaded" }); + const auth = await ensureAuthenticatedBrowser(page, { frontendUrl, backendUrl }); + if (auth.status !== "pass") { + result.status = auth.status; + throw new Error(auth.reason); + } + const token = await page.evaluate(() => localStorage.getItem("token") || ""); + const pluginsResponse = await apiJson(backendUrl, "/api/v1/plugins", { token }); + if (pluginsResponse.status >= 400 || pluginsResponse.json.code !== 0) { + throw new Error(pluginsResponse.json.msg || "Failed to list installed plugins."); + } + const installed = new Map((pluginsResponse.json.data?.plugins || []).map((item) => { + const metadata = item.manifest?.manifest?.metadata || item.manifest?.metadata || {}; + return [`${metadata.author || ""}/${metadata.name || ""}`, item]; + })); + const missing = targets.filter((target) => { + const plugin = installed.get(`${target.author}/${target.plugin}`); + if (!plugin) return true; + const pages = plugin.manifest?.manifest?.spec?.pages || plugin.manifest?.spec?.pages || []; + return !pages.some((pageSpec) => pageSpec.id === target.page); + }); + if (missing.length) { + result.status = "blocked"; + throw new Error(`Required plugin page fixture(s) are not installed or do not register the expected Page ID: ${missing.map((item) => item.id).join(", ")}`); + } + + for (const target of targets) { + const routeId = `${target.author}/${target.plugin}/${target.page}`; + const route = `${frontendUrl.replace(/\/$/, "")}/home/plugin-pages?id=${encodeURIComponent(routeId)}`; + const pageResult = { + id: target.id, + route, + visible_signal: target.signal, + ui_status: "fail", + api_status: "fail", + screenshot: join(paths.evidenceDir, `${target.id}.png`), + page_api: null, + }; + result.pages.push(pageResult); + + await page.goto(route, { waitUntil: "domcontentloaded" }); + await page.waitForLoadState("networkidle", { timeout: 15_000 }).catch(() => {}); + const frame = page.frameLocator("iframe").first(); + await frame.getByText(target.signal, { exact: true }).first().waitFor({ state: "visible", timeout: 30_000 }); + pageResult.ui_status = "pass"; + await safeScreenshot(page, pageResult.screenshot); + + const apiResponse = await apiJson( + backendUrl, + `/api/v1/plugins/${encodeURIComponent(target.author)}/${encodeURIComponent(target.plugin)}/page-api`, + { + method: "POST", + token, + body: { page_id: target.page, endpoint: target.endpoint, method: target.method, body: null }, + }, + ); + pageResult.page_api = { + endpoint: target.endpoint, + method: target.method, + http_status: apiResponse.status, + code: apiResponse.json.code ?? null, + has_data: apiResponse.json.data !== undefined && apiResponse.json.data !== null, + }; + if (apiResponse.status >= 400 || apiResponse.json.code !== 0 || apiResponse.json.data == null) { + throw new Error(`${target.id} Page API ${target.method} ${target.endpoint} failed.`); + } + pageResult.api_status = "pass"; + } + + await safeScreenshot(page, paths.screenshot); + result.browser_diagnostics = await scanBrowserDiagnostics(paths); + if (result.browser_diagnostics.status !== "pass") { + throw new Error(result.browser_diagnostics.reason); + } + result.status = "pass"; + result.reason = "All workspace plugin pages rendered in the LangBot WebUI and their read-only Page APIs passed."; +} catch (error) { + if (result.status === "fail" && /configured|Playwright is not installed|ECONNREFUSED|ERR_CONNECTION/i.test(error.message)) { + result.status = "env_issue"; + } + result.reason = error.message; +} finally { + if (browser?.page) await safeScreenshot(browser.page, paths.screenshot); + if (browser) await browser.close().catch(() => {}); + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/scripts/e2e/workspace-repository-contracts.mjs b/skills/scripts/e2e/workspace-repository-contracts.mjs new file mode 100644 index 000000000..ac8a64e55 --- /dev/null +++ b/skills/scripts/e2e/workspace-repository-contracts.mjs @@ -0,0 +1,136 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { access, mkdir, writeFile } from "node:fs/promises"; +import { basename, delimiter, dirname, join, resolve } from "node:path"; +import { env } from "node:process"; +import { + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + resolveLangBotRepo, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "workspace-repository-contracts"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); +const startedAt = new Date(); +const logsDir = join(paths.evidenceDir, "repository-contracts"); +await mkdir(logsDir, { recursive: true }); + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + status: "fail", + reason: "", + repositories: [], + evidence: { + contracts_dir: logsDir, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["filesystem", "metrics"], +}; + +function run(command, args, { cwd, processEnv, timeoutMs = 900_000 }) { + return new Promise((resolvePromise) => { + const started = Date.now(); + const child = spawn(command, args, { cwd, env: processEnv, stdio: ["ignore", "pipe", "pipe"] }); + let stdout = ""; + let stderr = ""; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + child.kill("SIGTERM"); + }, timeoutMs); + child.stdout.on("data", (chunk) => { stdout += chunk; }); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.on("error", (error) => { + clearTimeout(timer); + resolvePromise({ status: null, stdout, stderr, error, timedOut, durationMs: Date.now() - started }); + }); + child.on("close", (status) => { + clearTimeout(timer); + resolvePromise({ status, stdout, stderr, error: null, timedOut, durationMs: Date.now() - started }); + }); + }); +} + +try { + const langbotRepo = await resolveLangBotRepo(); + const workspaceRoot = resolve(env.LANGBOT_WORKSPACE_ROOT || dirname(langbotRepo)); + const sdkRepo = resolve(env.LANGBOT_PLUGIN_SDK_REPO || join(workspaceRoot, "langbot-plugin-sdk")); + const python = join(langbotRepo, ".venv", "bin", "python"); + await access(python); + await access(join(sdkRepo, "src", "langbot_plugin")); + const processEnv = { + ...env, + PYTHONPATH: [join(sdkRepo, "src"), env.PYTHONPATH].filter(Boolean).join(delimiter), + }; + const specs = [ + ["agent-runner", "langbot-agent-runner", ["-m", "pytest", "tests", "-q"]], + ["control-plane", "langbot-agent-control-plane", ["-m", "pytest", "tests", "-q"]], + ["longterm-memory", "langbot-longterm-memory", ["-m", "pytest", "tests", "-q"]], + ["parser", "langbot-parser", ["-m", "pytest", "tests", "-q"]], + ["rag", "langbot-rag", ["-m", "pytest", "tests", "-q"]], + ["skill-authoring", "langbot-skill-authoring", ["-m", "pytest", "tests", "-q"]], + ["local-agent", "langbot-local-agent", ["-m", "pytest", "tests", "-q"]], + ["langbot-agent-provider", "LangBot", ["-m", "pytest", "tests/unit_tests/agent", "tests/unit_tests/provider", "-q"]], + ["skills-cli", "LangBot/skills", ["--test", "test/lbs-cli.test.ts"]], + ["plugin-sdk-runtime", "langbot-plugin-sdk", ["-m", "pytest", "tests", "-q", "--ignore=tests/packaging/test_installed_cli_blackbox.py"]], + ["plugin-sdk-packaging", "langbot-plugin-sdk", ["-m", "pytest", "tests/packaging/test_installed_cli_blackbox.py", "-q"]], + ]; + + for (const [id, directory, args] of specs) { + const cwd = resolve(workspaceRoot, directory); + const command = id === "skills-cli" ? env.LANGBOT_NODE || "node" : python; + const execution = await run(command, args, { cwd, processEnv }); + const stdoutPath = join(logsDir, `${id}.stdout.log`); + const stderrPath = join(logsDir, `${id}.stderr.log`); + await writeFile(stdoutPath, execution.stdout, "utf8"); + await writeFile(stderrPath, execution.stderr, "utf8"); + const combined = `${execution.stdout}\n${execution.stderr}`; + const packagingNetworkIssue = id === "plugin-sdk-packaging" + && execution.status !== 0 + && /hatchling|PyPI|TLS|SSL|network|timed out|Temporary failure|connection|EOF/i.test(combined); + const status = execution.status === 0 ? "pass" : packagingNetworkIssue ? "env_issue" : "fail"; + result.repositories.push({ + id, + repository: basename(cwd), + status, + exit_status: execution.status, + timed_out: execution.timedOut, + duration_ms: execution.durationMs, + stdout_log: stdoutPath, + stderr_log: stderrPath, + reason: execution.error?.message || (execution.timedOut ? "Test command timed out." : packagingNetworkIssue ? "Packaging blackbox could not fetch its isolated build dependency." : ""), + }); + } + + const productFailures = result.repositories.filter((item) => item.status === "fail"); + const envIssues = result.repositories.filter((item) => item.status === "env_issue"); + result.status = productFailures.length === 0 ? "pass" : "fail"; + result.reason = productFailures.length === 0 + ? `All repository product contracts passed; ${envIssues.length} packaging environment issue(s) were classified separately.` + : `${productFailures.length} repository product contract group(s) failed.`; +} catch (error) { + result.status = "env_issue"; + result.reason = error.message; +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(exitCode(result.status)); diff --git a/skills/skills.index.json b/skills/skills.index.json index b109490b3..3e8695455 100644 --- a/skills/skills.index.json +++ b/skills/skills.index.json @@ -64,7 +64,7 @@ { "directory": "langbot-mcp-ops", "name": "langbot-mcp-ops", - "description": "Operate a LangBot instance through its built-in MCP (Model Context Protocol) server. Use when an AI agent needs to manage LangBot — list/create/update/delete bots, pipelines, models, knowledge bases, MCP servers, and skills — over MCP instead of raw HTTP. Covers the /mcp endpoint, API-key auth (web-UI lbk_ keys and the config.yaml global key), the tool surface, and client configuration. Triggers on \"langbot mcp\", \"manage langbot via mcp\", \"langbot /mcp\", \"langbot mcp server\".", + "description": "Operate a LangBot instance through its built-in MCP (Model Context Protocol) server. Use when an AI agent needs to manage LangBot — list/create/update/delete bots, agents, pipelines, models, knowledge bases, MCP servers, and skills — over MCP instead of raw HTTP. Covers the /mcp endpoint, API-key auth (web-UI lbk_ keys and the config.yaml global key), the tool surface, and client configuration. Triggers on \"langbot mcp\", \"manage langbot via mcp\", \"langbot /mcp\", \"langbot mcp server\".", "references": [], "cases": [], "case_summaries": [], @@ -134,14 +134,18 @@ "references/pipeline-debug-chat.md", "references/plugin-e2e-smoke.md", "references/sandbox-skill-authoring.md", + "references/skill-all-tool-acceptance.md", "references/troubleshooting.md", - "references/web-ui-testing.md" + "references/web-ui-testing.md", + "references/workspace-release-testing.md" ], "cases": [ "acp-agent-runner-debug-chat", + "agent-run-ledger-audit", "agent-runner-async-db-readiness", "agent-runner-behavior-matrix", "agent-runner-fixture-contract", + "agent-runner-health-visibility", "agent-runner-ledger-concurrency", "agent-runner-ledger-contention", "agent-runner-ledger-invariants", @@ -150,6 +154,10 @@ "agent-runner-qa-debug-chat", "agent-runner-release-preflight", "agent-runner-runtime-chaos", + "bot-event-routing-product-flow", + "box-mcp-heartbeat-recovery", + "claude-code-agent-debug-chat", + "codex-agent-debug-chat", "dify-agent-debug-chat", "langbot-fake-provider-debug-chat-cross-pipeline-isolation", "langbot-fake-provider-debug-chat-fault-recovery", @@ -165,14 +173,22 @@ "langrag-parser-golden-e2e", "langrag-sentinel-kb-discover", "local-agent-basic-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-complex-coding-task-debug-chat", "local-agent-context-compaction-debug-chat", "local-agent-effective-prompt-debug-chat", + "local-agent-model-fallback-before-first-chunk-debug-chat", "local-agent-multimodal-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", "local-agent-nonstreaming-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "local-agent-plugin-tool-call-debug-chat", "local-agent-rag-debug-chat", "local-agent-rag-multimodal-debug-chat", "local-agent-steering-debug-chat", + "local-agent-streaming-post-commit-failure-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", "mcp-stdio-register", "mcp-stdio-tool-call", "pipeline-debug-chat", @@ -182,12 +198,19 @@ "qa-plugin-smoke-live-install", "sandbox-skill-authoring-e2e", "sandbox-skill-authoring-edit-existing-e2e", - "webui-login-state" + "skill-discovery-via-mcp-gateway", + "webui-login-state", + "wizard-onebot-agent-runtime", + "wizard-runner-marketplace-catalog", + "wizard-scenario-routing", + "workspace-compatibility-preflight", + "workspace-plugin-pages-smoke", + "workspace-repository-contracts" ], "case_summaries": [ { "id": "acp-agent-runner-debug-chat", - "title": "ACP AgentRunner can answer through Debug Chat using real remote Claude", + "title": "ACP Runner can answer through Debug Chat using real remote Claude", "mode": "agent-browser", "area": "pipeline", "type": "regression", @@ -206,8 +229,8 @@ "node:scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env" ], "setup_provides_env": [ - "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", - "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME" + "LANGBOT_ACP_RUNNER_PIPELINE_URL", + "LANGBOT_ACP_RUNNER_PIPELINE_NAME" ], "evidence_required": [ "ui", @@ -215,9 +238,33 @@ "backend_log" ] }, + { + "id": "agent-run-ledger-audit", + "title": "Persisted Runner run ledger passes end-to-end invariants", + "mode": "probe", + "area": "agent", + "type": "regression", + "priority": "p0", + "risk": "high", + "ci_eligible": false, + "tags": [ + "agent-runner", + "ledger", + "audit", + "tools" + ], + "automation": "scripts/e2e/agent-run-ledger-audit.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "filesystem", + "metrics", + "api_diagnostic" + ] + }, { "id": "agent-runner-async-db-readiness", - "title": "AgentRunner async DB readiness probe", + "title": "Runner async DB readiness probe", "mode": "probe", "area": "release", "type": "smoke", @@ -239,7 +286,7 @@ }, { "id": "agent-runner-behavior-matrix", - "title": "AgentRunner deterministic behavior matrix probe", + "title": "Runner deterministic behavior matrix probe", "mode": "probe", "area": "release", "type": "regression", @@ -261,7 +308,7 @@ }, { "id": "agent-runner-fixture-contract", - "title": "QA AgentRunner fixture contract probe", + "title": "QA Runner fixture contract probe", "mode": "probe", "area": "release", "type": "regression", @@ -281,9 +328,34 @@ "filesystem" ] }, + { + "id": "agent-runner-health-visibility", + "title": "Agent configuration shows whether its selected runner is usable", + "mode": "agent-browser", + "area": "agent", + "type": "feature", + "priority": "p0", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "agent-runner", + "agent", + "productization", + "release-gate" + ], + "automation": "scripts/e2e/agent-runner-health-visibility.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "api_diagnostic" + ] + }, { "id": "agent-runner-ledger-concurrency", - "title": "AgentRunner run ledger concurrency and auth pytest probe", + "title": "Runner run ledger concurrency and auth pytest probe", "mode": "probe", "area": "release", "type": "regression", @@ -306,7 +378,7 @@ }, { "id": "agent-runner-ledger-contention", - "title": "AgentRunner ledger SQLite contention probe", + "title": "Runner ledger SQLite contention probe", "mode": "probe", "area": "release", "type": "regression", @@ -329,7 +401,7 @@ }, { "id": "agent-runner-ledger-invariants", - "title": "AgentRunner ledger schema and status invariants probe", + "title": "Runner ledger schema and status invariants probe", "mode": "probe", "area": "release", "type": "regression", @@ -351,7 +423,7 @@ }, { "id": "agent-runner-ledger-stress", - "title": "AgentRunner ledger lightweight stress probe", + "title": "Runner ledger lightweight stress probe", "mode": "probe", "area": "release", "type": "regression", @@ -373,7 +445,7 @@ }, { "id": "agent-runner-live-install", - "title": "QA AgentRunner package installs and registers in LangBot", + "title": "QA Runner package installs and registers in LangBot", "mode": "probe", "area": "release", "type": "regression", @@ -396,7 +468,7 @@ }, { "id": "agent-runner-qa-debug-chat", - "title": "QA AgentRunner returns deterministic output through Debug Chat", + "title": "QA Runner returns deterministic output through Debug Chat", "mode": "agent-browser", "area": "pipeline", "type": "regression", @@ -415,8 +487,8 @@ "node:scripts/e2e/ensure-qa-agent-runner-pipeline.mjs --write-env" ], "setup_provides_env": [ - "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL", - "LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME" + "LANGBOT_QA_RUNNER_PIPELINE_URL", + "LANGBOT_QA_RUNNER_PIPELINE_NAME" ], "evidence_required": [ "ui", @@ -454,7 +526,7 @@ }, { "id": "agent-runner-runtime-chaos", - "title": "AgentRunner SDK runtime chaos pytest probe", + "title": "Runner SDK runtime chaos pytest probe", "mode": "probe", "area": "release", "type": "regression", @@ -475,9 +547,117 @@ "filesystem" ] }, + { + "id": "bot-event-routing-product-flow", + "title": "Bot event routing and adapter input can be inspected from the WebUI", + "mode": "agent-browser", + "area": "bot", + "type": "feature", + "priority": "p0", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "bot", + "eba", + "event-routing", + "productization" + ], + "automation": "scripts/e2e/bot-event-routing-product-flow.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "api_diagnostic" + ] + }, + { + "id": "box-mcp-heartbeat-recovery", + "title": "Box heartbeat restores an existing stdio MCP tool session", + "mode": "probe", + "area": "reliability", + "type": "chaos", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "box", + "mcp", + "stdio", + "reliability", + "chaos", + "fault-injection" + ], + "automation": "skills/langbot-testing/probes/box-mcp-heartbeat-recovery.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "network", + "metrics", + "api_diagnostic", + "resource_log", + "filesystem" + ] + }, + { + "id": "claude-code-agent-debug-chat", + "title": "Claude Code Agent can retrieve LangBot history through MCP", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p2", + "risk": "high", + "ci_eligible": false, + "tags": [ + "agent-runner", + "claude-code", + "external-runner", + "mcp", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log" + ] + }, + { + "id": "codex-agent-debug-chat", + "title": "Codex Agent can retrieve LangBot history through MCP", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p2", + "risk": "high", + "ci_eligible": false, + "tags": [ + "agent-runner", + "codex", + "external-runner", + "mcp", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log" + ] + }, { "id": "dify-agent-debug-chat", - "title": "Dify AgentRunner returns a response through Pipeline Debug Chat", + "title": "Dify Runner returns a response through Pipeline Debug Chat", "mode": "agent-browser", "area": "pipeline", "type": "provider", @@ -823,13 +1003,18 @@ "rag" ], "automation": "scripts/e2e/langrag-kb-retrieve.mjs", - "setup_automation": [], - "setup_provides_env": [], + "setup_automation": [ + "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_RAG_KB_UUID" + ], "evidence_required": [ "ui", "screenshot", "console", - "backend_log" + "network", + "api_diagnostic" ] }, { @@ -909,6 +1094,80 @@ "backend_log" ] }, + { + "id": "local-agent-combo-rag-compaction-tool-debug-chat", + "title": "Local Agent preserves RAG, compacted history, and plugin tool result in one Debug Chat run", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "langrag", + "context", + "compaction", + "plugin", + "tools", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_RAG_KB_UUID" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, + { + "id": "local-agent-complex-coding-task-debug-chat", + "title": "Local Agent completes a multi-file coding task with iterative verification", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "agentic", + "coding", + "sandbox", + "tools", + "long-running" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/reset-complex-agent-task.mjs" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_COMPLEX_AGENT_WORKSPACE" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "network", + "api_diagnostic", + "filesystem", + "metrics" + ] + }, { "id": "local-agent-context-compaction-debug-chat", "title": "Local Agent compacts long Debug Chat history and preserves older facts", @@ -970,6 +1229,38 @@ "backend_log" ] }, + { + "id": "local-agent-model-fallback-before-first-chunk-debug-chat", + "title": "Local Agent falls back when the primary model fails before streaming starts", + "mode": "probe", + "area": "pipeline", + "type": "chaos", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "fallback", + "streaming", + "fake-provider", + "fault-injection" + ], + "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_FAKE_PROVIDER_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME" + ], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, { "id": "local-agent-multimodal-debug-chat", "title": "Local Agent Debug Chat preserves uploaded image input", @@ -1000,9 +1291,47 @@ "backend_log" ] }, + { + "id": "local-agent-multitool-rag-compaction-debug-chat", + "title": "Local Agent preserves RAG, compacted history, and two-step plugin tool loop", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "langrag", + "context", + "compaction", + "plugin", + "tools", + "tool-loop", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_RAG_KB_UUID" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, { "id": "local-agent-nonstreaming-debug-chat", - "title": "Local Agent Debug Chat returns a deterministic non-streaming response", + "title": "Local Agent Debug Chat returns a deterministic response with UI streaming disabled", "mode": "agent-browser", "area": "pipeline", "type": "regression", @@ -1028,6 +1357,44 @@ "backend_log" ] }, + { + "id": "local-agent-parallel-tools-rag-compaction-debug-chat", + "title": "Local Agent preserves RAG, compacted history, and parallel plugin tools", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "langrag", + "context", + "compaction", + "plugin", + "tools", + "parallel-tools", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_RAG_KB_UUID" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, { "id": "local-agent-plugin-tool-call-debug-chat", "title": "Local Agent can call a plugin-provided tool", @@ -1155,6 +1522,105 @@ "api_diagnostic" ] }, + { + "id": "local-agent-streaming-post-commit-failure-debug-chat", + "title": "Local Agent does not fall back after a committed stream fails", + "mode": "probe", + "area": "pipeline", + "type": "chaos", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "fallback", + "streaming", + "fake-provider", + "fault-injection" + ], + "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_FAKE_PROVIDER_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME" + ], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, + { + "id": "local-agent-tool-error-recovery-debug-chat", + "title": "Local Agent feeds plugin tool errors back to the model", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "plugin", + "tools", + "tool-error", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, + { + "id": "local-agent-tool-loop-limit-debug-chat", + "title": "Local Agent stops repeated tool calls at max-tool-iterations", + "mode": "agent-browser", + "area": "pipeline", + "type": "regression", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "local-agent", + "plugin", + "tools", + "tool-loop", + "limit", + "pipeline" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "case:qa-plugin-smoke-live-install" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "backend_log", + "api_diagnostic" + ] + }, { "id": "mcp-stdio-register", "title": "MCP stdio fixture is registered and exposes qa_mcp_echo", @@ -1196,18 +1662,21 @@ ], "automation": "scripts/e2e/pipeline-debug-chat.mjs", "setup_automation": [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env", "case:mcp-stdio-register" ], "setup_provides_env": [ - "LANGBOT_LOCAL_AGENT_PIPELINE_URL", - "LANGBOT_LOCAL_AGENT_PIPELINE_NAME" + "LANGBOT_FAKE_PROVIDER_PIPELINE_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME", + "LANGBOT_MCP_QA_STDIO_SERVER_UUID" ], "evidence_required": [ "ui", + "screenshot", "console", - "backend_log", - "api_diagnostic" + "network", + "api_diagnostic", + "metrics" ] }, { @@ -1379,8 +1848,14 @@ "edit" ], "automation": "scripts/e2e/pipeline-debug-chat.mjs", - "setup_automation": [], - "setup_provides_env": [], + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/reset-sandbox-skill.mjs --name=lb-skill-edit-regression" + ], + "setup_provides_env": [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME" + ], "evidence_required": [ "ui", "backend_log", @@ -1388,6 +1863,31 @@ "filesystem" ] }, + { + "id": "skill-discovery-via-mcp-gateway", + "title": "External harness discovers LangBot skills via langbot_list_assets (all-tool model)", + "mode": "agent-browser", + "area": "sandbox", + "type": "regression", + "priority": "p2", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "skills", + "mcp-gateway", + "acp-agent-runner", + "all-tool-model", + "tools" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "backend_log" + ] + }, { "id": "webui-login-state", "title": "Configured frontend opens with authenticated LangBot WebUI state", @@ -1409,17 +1909,169 @@ "screenshot", "console" ] + }, + { + "id": "wizard-onebot-agent-runtime", + "title": "Quick Start runs a real OneBot member event through an Agent", + "mode": "agent-browser", + "area": "wizard", + "type": "feature", + "priority": "p0", + "risk": "high", + "ci_eligible": false, + "tags": [ + "wizard", + "eba", + "onebot", + "agent-runner", + "release-gate" + ], + "automation": "scripts/e2e/wizard-onebot-agent-runtime.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "api_diagnostic" + ] + }, + { + "id": "wizard-runner-marketplace-catalog", + "title": "Quick Start installs a published Runner on a clean instance", + "mode": "agent-browser", + "area": "wizard", + "type": "feature", + "priority": "p0", + "risk": "high", + "ci_eligible": false, + "tags": [ + "wizard", + "agent-runner", + "marketplace", + "clean-install", + "productization" + ], + "automation": "scripts/e2e/wizard-runner-marketplace-catalog.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "api_diagnostic" + ] + }, + { + "id": "wizard-scenario-routing", + "title": "Quick Start filters channels by the selected bot scenario", + "mode": "agent-browser", + "area": "wizard", + "type": "feature", + "priority": "p0", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "wizard", + "eba", + "event-routing", + "productization" + ], + "automation": "scripts/e2e/wizard-scenario-routing.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "api_diagnostic" + ] + }, + { + "id": "workspace-compatibility-preflight", + "title": "LangBot workspace repositories and local SDK are mutually compatible", + "mode": "probe", + "area": "release", + "type": "smoke", + "priority": "p0", + "risk": "high", + "ci_eligible": true, + "tags": [ + "workspace", + "compatibility", + "preflight", + "sdk" + ], + "automation": "scripts/e2e/workspace-compatibility-preflight.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "filesystem", + "api_diagnostic" + ] + }, + { + "id": "workspace-plugin-pages-smoke", + "title": "Major workspace plugin pages render and answer read-only Page APIs", + "mode": "agent-browser", + "area": "plugin", + "type": "smoke", + "priority": "p0", + "risk": "high", + "ci_eligible": false, + "tags": [ + "workspace", + "plugin", + "page", + "browser" + ], + "automation": "scripts/e2e/workspace-plugin-pages-smoke.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "ui", + "screenshot", + "console", + "network", + "api_diagnostic" + ] + }, + { + "id": "workspace-repository-contracts", + "title": "Changed LangBot workspace repositories pass their contract suites", + "mode": "probe", + "area": "release", + "type": "regression", + "priority": "p0", + "risk": "high", + "ci_eligible": true, + "tags": [ + "workspace", + "contracts", + "pytest", + "sdk" + ], + "automation": "scripts/e2e/workspace-repository-contracts.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "filesystem", + "metrics" + ] } ], "suites": [ "agent-runner-release-gate", "core-smoke", + "eba-release-gate", "langbot-debug-chat-isolation-gate", "langbot-debug-chat-load-gate", "langbot-live-backend-gate", "langbot-performance-contract-gate", "langbot-performance-reliability-gate", "langbot-user-path-performance-gate", + "langbot-workspace-contract-gate", + "langbot-workspace-release-gate", "local-agent-gate" ], "suite_summaries": [ @@ -1457,6 +2109,11 @@ "local-agent-context-compaction-debug-chat", "local-agent-rag-debug-chat", "local-agent-plugin-tool-call-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "mcp-stdio-register", "mcp-stdio-tool-call", "local-agent-nonstreaming-debug-chat", @@ -1482,6 +2139,26 @@ "local-agent-basic-debug-chat" ] }, + { + "id": "eba-release-gate", + "title": "Event-based bot product release gate", + "description": "Browser and runtime gate for scenario onboarding, route diagnostics, and real platform-to-Agent execution.", + "type": "release_gate", + "priority": "p0", + "tags": [ + "eba", + "event-routing", + "wizard", + "release-gate" + ], + "cases": [ + "wizard-scenario-routing", + "wizard-runner-marketplace-catalog", + "agent-runner-health-visibility", + "bot-event-routing-product-flow", + "wizard-onebot-agent-runtime" + ] + }, { "id": "langbot-debug-chat-isolation-gate", "title": "LangBot Debug Chat isolation gate", @@ -1589,6 +2266,57 @@ "pipeline-debug-chat-performance" ] }, + { + "id": "langbot-workspace-contract-gate", + "title": "LangBot workspace contract gate", + "description": "Low-cost cross-repository gate for branch/SDK compatibility and deterministic product contracts before live workflow testing.", + "type": "regression", + "priority": "p0", + "tags": [ + "workspace", + "contracts", + "pr-gate" + ], + "cases": [ + "workspace-compatibility-preflight", + "agent-runner-fixture-contract", + "agent-runner-behavior-matrix", + "agent-runner-ledger-invariants", + "langbot-fault-taxonomy-contract", + "langbot-overhead-accounting-contract", + "workspace-repository-contracts" + ] + }, + { + "id": "langbot-workspace-release-gate", + "title": "LangBot workspace top-down release gate", + "description": "Broad release gate combining deterministic repository contracts with representative browser workflows, plugin pages, RAG/parser, EBA, external Runner, and one complex LocalAgent task.", + "type": "release_gate", + "priority": "p0", + "tags": [ + "workspace", + "release-gate", + "browser", + "agent" + ], + "cases": [ + "workspace-compatibility-preflight", + "workspace-repository-contracts", + "agent-runner-release-preflight", + "webui-login-state", + "wizard-scenario-routing", + "bot-event-routing-product-flow", + "agent-runner-health-visibility", + "agent-runner-qa-debug-chat", + "local-agent-complex-coding-task-debug-chat", + "agent-run-ledger-audit", + "langrag-kb-retrieve", + "workspace-plugin-pages-smoke", + "mcp-stdio-register", + "mcp-stdio-tool-call", + "wizard-onebot-agent-runtime" + ] + }, { "id": "local-agent-gate", "title": "Local Agent runner regression gate", @@ -1602,11 +2330,18 @@ ], "cases": [ "local-agent-basic-debug-chat", + "local-agent-model-fallback-before-first-chunk-debug-chat", + "local-agent-streaming-post-commit-failure-debug-chat", "qa-plugin-smoke-live-install", "local-agent-effective-prompt-debug-chat", "local-agent-context-compaction-debug-chat", "local-agent-rag-debug-chat", "local-agent-plugin-tool-call-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "local-agent-steering-debug-chat", "mcp-stdio-tool-call", "local-agent-nonstreaming-debug-chat", @@ -1618,7 +2353,7 @@ "fixtures": [ { "id": "qa-agent-runner-behaviors", - "title": "Deterministic AgentRunner behavior matrix", + "title": "Deterministic Runner behavior matrix", "kind": "json", "path": "fixtures/agent-runner/qa-runner-behaviors.json", "related_cases": [ @@ -1629,7 +2364,7 @@ }, { "id": "qa-agent-runner-source", - "title": "QA deterministic AgentRunner fixture source", + "title": "QA deterministic Runner fixture source", "kind": "plugin_source", "path": "fixtures/plugins/qa-agent-runner/manifest.yaml", "related_cases": [ @@ -1641,7 +2376,7 @@ }, { "id": "qa-agent-runner-package", - "title": "QA deterministic AgentRunner prebuilt package", + "title": "QA deterministic Runner prebuilt package", "kind": "plugin_package", "path": "fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg", "related_cases": [ @@ -1656,7 +2391,8 @@ "kind": "python", "path": "fixtures/mcp/qa_mcp_echo_server.py", "related_cases": [ - "mcp-stdio-tool-call" + "mcp-stdio-tool-call", + "box-mcp-heartbeat-recovery" ] }, { @@ -1666,7 +2402,10 @@ "path": "fixtures/rag/sentinel-doc.txt", "related_cases": [ "langrag-kb-retrieve", - "local-agent-rag-debug-chat" + "local-agent-rag-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat" ] }, { @@ -1698,6 +2437,11 @@ "plugin-e2e-smoke", "local-agent-effective-prompt-debug-chat", "local-agent-plugin-tool-call-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "local-agent-steering-debug-chat" ] }, @@ -1708,7 +2452,8 @@ "path": "fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg", "related_cases": [ "qa-plugin-smoke-live-install", - "plugin-e2e-smoke" + "plugin-e2e-smoke", + "local-agent-tool-error-recovery-debug-chat" ] } ], @@ -1717,10 +2462,12 @@ "aiosqlite-connect-hangs", "ambiguous-runner-default-label", "backend-not-listening", + "box-runtime-silent-action-timeout", "box-session-conflict-logical-metadata", "debug-chat-history-contaminates-automation", "dynamic-form-missing-config-id", "e2b-extra-mount-sync-missing", + "local-agent-mcp-text-result-empty-follow-up", "local-agent-model-route-unavailable", "marketplace-network-flaky", "mcp-stdio-args-not-applied", @@ -1740,7 +2487,7 @@ "troubleshooting_summaries": [ { "id": "agent-runner-actor-context-fields", - "title": "AgentRunner reads old actor.type and actor.id fields", + "title": "Runner reads old actor.type and actor.id fields", "category": "product", "related_cases": [ "dify-agent-debug-chat", @@ -1758,7 +2505,7 @@ }, { "id": "ambiguous-runner-default-label", - "title": "AgentRunner selector shows multiple Default or 默认 options", + "title": "Runner selector shows multiple Default or 默认 options", "category": "product", "related_cases": [ "dify-agent-debug-chat", @@ -1775,6 +2522,16 @@ "local-agent-basic-debug-chat" ] }, + { + "id": "box-runtime-silent-action-timeout", + "title": "Box runtime process stays alive while all action RPC calls time out", + "category": "product", + "related_cases": [ + "local-agent-complex-coding-task-debug-chat", + "agent-run-ledger-audit", + "mcp-stdio-tool-call" + ] + }, { "id": "box-session-conflict-logical-metadata", "title": "BoxSessionConflictError after a successful first exec", @@ -1810,6 +2567,16 @@ "sandbox-skill-authoring-e2e" ] }, + { + "id": "local-agent-mcp-text-result-empty-follow-up", + "title": "MCP tool succeeds but Local Agent completes without a visible reply", + "category": "product", + "related_cases": [ + "mcp-stdio-register", + "mcp-stdio-tool-call", + "agent-run-ledger-audit" + ] + }, { "id": "local-agent-model-route-unavailable", "title": "Local Agent model route is unavailable for the requested run shape", @@ -1819,7 +2586,8 @@ "local-agent-plugin-tool-call-debug-chat", "mcp-stdio-tool-call", "local-agent-multimodal-debug-chat", - "local-agent-nonstreaming-debug-chat" + "local-agent-nonstreaming-debug-chat", + "local-agent-complex-coding-task-debug-chat" ] }, { diff --git a/skills/skills/.env b/skills/skills/.env index 4e5aae69f..ca4255009 100644 --- a/skills/skills/.env +++ b/skills/skills/.env @@ -37,10 +37,10 @@ LANGBOT_NO_PROXY=localhost,127.0.0.1,::1 # LANGBOT_PIPELINE_NAME=Generic QA Pipeline # LANGBOT_LOCAL_AGENT_PIPELINE_URL=http://127.0.0.1:3000/home/pipelines?id= # LANGBOT_LOCAL_AGENT_PIPELINE_NAME=Local Agent QA Pipeline -# LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL=http://127.0.0.1:3000/home/pipelines?id= -# LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME=ACP AgentRunner QA Pipeline -# LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET=yhh@101.34.71.12 -# LANGBOT_ACP_AGENT_RUNNER_SSH_PORT=22 -# LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE= -# LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS= -# LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE=/home/yhh/langbot-e2e/acp-workspace +# LANGBOT_ACP_RUNNER_PIPELINE_URL=http://127.0.0.1:3000/home/pipelines?id= +# LANGBOT_ACP_RUNNER_PIPELINE_NAME=ACP Runner QA Pipeline +# LANGBOT_ACP_RUNNER_SSH_TARGET=yhh@101.34.71.12 +# LANGBOT_ACP_RUNNER_SSH_PORT=22 +# LANGBOT_ACP_RUNNER_SSH_IDENTITY_FILE= +# LANGBOT_ACP_RUNNER_SSH_EXTRA_OPTIONS= +# LANGBOT_ACP_RUNNER_REMOTE_WORKSPACE=/home/yhh/langbot-e2e/acp-workspace diff --git a/skills/skills/langbot-env-setup/references/service-startup.md b/skills/skills/langbot-env-setup/references/service-startup.md index b63960cdb..ddff574c5 100644 --- a/skills/skills/langbot-env-setup/references/service-startup.md +++ b/skills/skills/langbot-env-setup/references/service-startup.md @@ -24,7 +24,7 @@ Healthy startup includes: ```text Running on http://0.0.0.0: Connected to plugin runtime. -Plugin langbot/local-agent initialized +Plugin langbot-team/LocalAgent initialized ``` Quick check: diff --git a/skills/skills/langbot-mcp-ops/SKILL.md b/skills/skills/langbot-mcp-ops/SKILL.md index 3f18fdf28..c793631b3 100644 --- a/skills/skills/langbot-mcp-ops/SKILL.md +++ b/skills/skills/langbot-mcp-ops/SKILL.md @@ -1,6 +1,6 @@ --- name: langbot-mcp-ops -description: Operate a LangBot instance through its built-in MCP (Model Context Protocol) server. Use when an AI agent needs to manage LangBot — list/create/update/delete bots, pipelines, models, knowledge bases, MCP servers, and skills — over MCP instead of raw HTTP. Covers the /mcp endpoint, API-key auth (web-UI lbk_ keys and the config.yaml global key), the tool surface, and client configuration. Triggers on "langbot mcp", "manage langbot via mcp", "langbot /mcp", "langbot mcp server". +description: Operate a LangBot instance through its built-in MCP (Model Context Protocol) server. Use when an AI agent needs to manage LangBot — list/create/update/delete bots, agents, pipelines, models, knowledge bases, MCP servers, and skills — over MCP instead of raw HTTP. Covers the /mcp endpoint, API-key auth (web-UI lbk_ keys and the config.yaml global key), the tool surface, and client configuration. Triggers on "langbot mcp", "manage langbot via mcp", "langbot /mcp", "langbot mcp server". --- # LangBot MCP Operations @@ -66,6 +66,11 @@ The tools wrap the LangBot service layer. Current tools (v1): | --- | --- | | `get_system_info` | Version, edition, instance id | | `list_bots` / `get_bot` / `create_bot` / `update_bot` / `delete_bot` | Manage messaging-platform bots (secrets redacted on read) | +| `list_bot_event_route_statuses` | Inspect bot event-route runtime status | +| `list_processors` / `get_processor` / `create_processor` / `update_processor` / `delete_processor` | Manage the peer Agent, Pipeline and Event processor types | +| `get_processor_metadata` | Discover installed event-capable Runner components, schemas and supported event patterns. | +| `list_processor_runs` / `get_processor_run_events` | Read one Agent or plugin processor run history and logs; paginate with `before_id` / `after_sequence`. | +| `debug_agent` | Execute a synthetic Agent event (`processor_uuid`, `payload`); requires `runtime.operate`. Returns final text and up to 1000 execution events (thinking, text, tool arguments/results). Platform tools use Mock; other configured tools execute normally. Optional `payload.mock`: `errors`/`results` keyed by platform tool name, `unsupported_apis` lists unavailable platform APIs. | | `list_pipelines` / `get_pipeline` / `create_pipeline` / `update_pipeline` / `delete_pipeline` | Manage pipelines | | `list_llm_models` / `get_llm_model` / `list_embedding_models` / `list_model_providers` | Inspect models & providers | | `list_knowledge_bases` / `get_knowledge_base` / `retrieve_knowledge_base` | RAG knowledge bases (incl. semantic search) | @@ -142,3 +147,25 @@ state as well; it is not the same operation as disconnecting an account. - A `403` means the key is valid but lacks the permission required by the tool. - The global key is plaintext in config.yaml — only enable it on trusted/internal deployments and serve over HTTPS. + +## Event processors + +Create a processor with `kind: "event_processor"` and basic information. Without +a component it supports no events. Discover installed components with +`get_processor_metadata`, then use `update_processor` with `component_ref` and +optional `parameters`. API callers may also supply these when creating an instance. +Bind an instance by updating the bot's `plugin_processors` array with +`{"processor_uuid": "", "enabled": true}`. This replaces the full +subscription list; preserve bindings you want to keep. Do not add plugin processors +to `event_bindings`, which remains exclusive Agent/Pipeline routing. +Each enabled subscription independently receives the installed Runner's declared +events. Slow or failed subscribers do not prevent other subscribers or the primary +route from executing. Installation alone never activates a handler. Reusing an +instance shares its configuration and runtime state. Use a separate instance for +independent settings. Optional plugin behavior belongs in the Runner config schema. +`debug_agent` accepts the complete typed event in `payload.data` for this kind. +Legacy EventListener plugins remain in the Pipeline lifecycle. + +`list_processor_runs` includes `created_at_ms`, `started_at_ms`, and +`finished_at_ms`: Host lifecycle times in epoch milliseconds. Use the start and finish times for elapsed processing time; select a run and call `get_processor_run_events` for its +logs and action results. These times are not internal plugin profiling data. diff --git a/skills/skills/langbot-plugin-dev/SKILL.md b/skills/skills/langbot-plugin-dev/SKILL.md index d7858c40e..579364393 100644 --- a/skills/skills/langbot-plugin-dev/SKILL.md +++ b/skills/skills/langbot-plugin-dev/SKILL.md @@ -422,7 +422,7 @@ When a plugin doesn't work: 3. **Verify plugin loaded**: `GET /api/v1/plugins` — should list your plugin 4. **Test person mode first**: `session_type=person` always triggers pipeline, isolating trigger rule issues 5. **Check trigger rules**: Group mode requires @bot, prefix match, or random% to enter pipeline -6. **Verify model configured**: Pipeline's `config.ai.local-agent.model.primary` must point to a valid model UUID with working API keys +6. **Verify model configured**: Pipeline's `config.ai.runner_config[config.ai.runner.id].model.primary` must point to a valid model UUID with working API keys ## Publishing Plugins diff --git a/skills/skills/langbot-testing/SKILL.md b/skills/skills/langbot-testing/SKILL.md index 748ae9b81..f36498d99 100644 --- a/skills/skills/langbot-testing/SKILL.md +++ b/skills/skills/langbot-testing/SKILL.md @@ -11,17 +11,18 @@ Use this skill when an agent needs to verify LangBot behavior through the WebUI - **General WebUI testing**: read `references/web-ui-testing.md`. - **Pipeline Debug Chat**: read `references/pipeline-debug-chat.md`. -- **Dify AgentRunner**: read `references/dify-agent-runner.md`. +- **Dify Runner**: read `references/dify-agent-runner.md`. - **Model provider setup or test button**: read `references/model-provider-testing.md`. - **Plugin install/runtime/tool/page smoke**: read `references/plugin-e2e-smoke.md`. -- **Local Agent Runner**: read `references/local-agent-runner.md`. -- **Local Agent Runner path coverage**: read `references/local-agent-runner-coverage.md`. -- **Diff-aware AgentRunner QA after code changes**: read `references/agent-runner-qa-workflow.md`. -- **Agent Runner release gate**: read `references/agent-runner-release-gate.md`. +- **Local Runner**: read `references/local-agent-runner.md`. +- **Local Runner path coverage**: read `references/local-agent-runner-coverage.md`. +- **Diff-aware Runner QA after code changes**: read `references/agent-runner-qa-workflow.md`. +- **Runner release gate**: read `references/agent-runner-release-gate.md`. - **Sandbox-backed skill authoring**: read `references/sandbox-skill-authoring.md`. - **LangRAG knowledge bases**: read `references/langrag-knowledge-base.md`. - **MCP stdio tool testing**: read `references/mcp-stdio-testing.md`. - **Performance, reliability, or chaos probes**: read `references/performance-reliability-testing.md`. +- **Cross-repository workspace and release gates**: read `references/workspace-release-testing.md`. - **Drive a live instance over MCP (not raw HTTP)**: use the `langbot-mcp-ops` skill — the instance exposes an MCP server at `http://:5300/mcp` (reuses API keys). Useful for setting up bots/pipelines/models as test fixtures programmatically. - **Known failures and fixes**: read `references/troubleshooting.md`. - **Reusable test groups**: run `bin/lbs suite list` and `bin/lbs suite plan ` before manually assembling a case set. diff --git a/skills/skills/langbot-testing/cases/acp-agent-runner-debug-chat.yaml b/skills/skills/langbot-testing/cases/acp-agent-runner-debug-chat.yaml index ae10cea66..d66b7d899 100644 --- a/skills/skills/langbot-testing/cases/acp-agent-runner-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/acp-agent-runner-debug-chat.yaml @@ -1,5 +1,5 @@ id: acp-agent-runner-debug-chat -title: "ACP AgentRunner can answer through Debug Chat using real remote Claude" +title: "ACP Runner can answer through Debug Chat using real remote Claude" mode: agent-browser area: pipeline type: regression @@ -19,42 +19,42 @@ env: - LANGBOT_FRONTEND_URL - LANGBOT_BACKEND_URL env_any: - - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL|LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME + - LANGBOT_ACP_RUNNER_PIPELINE_URL|LANGBOT_ACP_RUNNER_PIPELINE_NAME automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL - LANGBOT_BACKEND_URL - LANGBOT_BROWSER_PROFILE - LANGBOT_CHROMIUM_EXECUTABLE - - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL - - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME + - LANGBOT_ACP_RUNNER_PIPELINE_URL + - LANGBOT_ACP_RUNNER_PIPELINE_NAME - LANGBOT_E2E_PROMPT - LANGBOT_E2E_EXPECTED_TEXT - LANGBOT_E2E_EXPECTED_RUNNER_ID - LANGBOT_E2E_RESPONSE_TIMEOUT_MS -automation_pipeline_url_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL -automation_pipeline_name_env: LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME -automation_expected_runner_id: "plugin:langbot/acp-agent-runner/default" -automation_prompt: "Use the injected LangBot MCP server tool langbot_get_current_event once. If the MCP call succeeds, reply with exactly ACP_AGENT_RUNNER_E2E_OK." -automation_expected_text: "ACP_AGENT_RUNNER_E2E_OK" +automation_pipeline_url_env: LANGBOT_ACP_RUNNER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_ACP_RUNNER_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/ACPRunner/default" +automation_prompt: "Do not launch any background agent, subagent, task, or worker. In this current ACP session, directly call the MCP tool named langbot_get_current_event exactly once and wait for its result. After it returns, reply exactly ACP_RUNNER_E2E_OK with no other text." +automation_expected_text: "ACP_RUNNER_E2E_OK" automation_response_timeout_ms: "300000" setup_automation: - "node:scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env" setup_provides_env: - - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL - - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME + - LANGBOT_ACP_RUNNER_PIPELINE_URL + - LANGBOT_ACP_RUNNER_PIPELINE_NAME preconditions: - "The remote machine has a working Claude Code login and can run npx -y @agentclientprotocol/claude-agent-acp." - "LangBot can non-interactively SSH to the remote machine; the runner opens the MCP reverse tunnel automatically." steps: - "Open LANGBOT_FRONTEND_URL." - - "Open the ACP AgentRunner QA pipeline." - - "Confirm the pipeline AI runner is plugin:langbot/acp-agent-runner/default." + - "Open the ACP Runner QA pipeline." + - "Confirm the pipeline AI runner is plugin:langbot-team/ACPRunner/default." - "Open Debug Chat." - - "Ask the real remote Claude ACP agent to call langbot_get_current_event and return ACP_AGENT_RUNNER_E2E_OK exactly." + - "Ask the real remote Claude ACP agent to call langbot_get_current_event and return ACP_RUNNER_E2E_OK exactly." checks: - "UI: Debug Chat shows the user prompt." - - "UI: Debug Chat shows a Bot response containing ACP_AGENT_RUNNER_E2E_OK." + - "UI: Debug Chat shows a Bot response containing ACP_RUNNER_E2E_OK." - "Logs: Backend logs include Processing request from person_websocket and Streaming completed for this run." - "Logs: No acp runner request error appears for this run." - "Console: No unexpected frontend errors appear during Debug Chat." @@ -66,13 +66,13 @@ diagnostics: - "Use scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env to create/update the pipeline." - "For remote Claude on 101, verify ssh yhh@101.34.71.12 can run without password prompts; no separate ssh -R process is required." success_patterns: - - "ACP_AGENT_RUNNER_E2E_OK" + - "ACP_RUNNER_E2E_OK" - "Processing request from person_websocket" - "Streaming completed" failure_patterns: - "acp.command_not_found" - "acp.process_exited" - - "Agent runner plugin:langbot/acp-agent-runner/default execution failed" + - "Agent runner plugin:langbot-team/ACPRunner/default execution failed" troubleshooting: - backend-not-listening - plugin-runtime-timeout diff --git a/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml b/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml new file mode 100644 index 000000000..2b24bc528 --- /dev/null +++ b/skills/skills/langbot-testing/cases/agent-run-ledger-audit.yaml @@ -0,0 +1,44 @@ +id: agent-run-ledger-audit +title: "Persisted Runner run ledger passes end-to-end invariants" +mode: probe +area: agent +type: regression +priority: p0 +risk: high +ci_eligible: false +tags: + - agent-runner + - ledger + - audit + - tools +skills: + - langbot-testing +automation: scripts/e2e/agent-run-ledger-audit.mjs +steps: + - "Set LANGBOT_AGENT_RUN_ID to audit a specific run, or leave it unset to audit the latest persisted Runner run." + - "For an external runner's own CLI tools, set LANGBOT_AGENT_TOOL_AUTHORIZATION_MODE=runner-native; keep the default strict mode for Host tool calls." + - "Read the active LangBot database configuration and inspect the selected run and its ordered events." + - "Verify completed terminal state, run.completed, paired tool.call.started/completed events, stable tool names, and monotonic ordering." + - "Compare called tools with the authorization snapshot and validate each advertised tool has owner/source, description, and parameter schema." + - "Reject malformed persisted event JSON, unrecovered invalid tool arguments, timeout, forbidden, permission-denied, or unauthorized signals." + - "Record a malformed model tool-argument payload as a recovery warning only when a later tool call succeeds and the run completes." +checks: + - "ledger-audit.json status is pass." + - "metrics has equal tool_call_started and tool_call_completed counts." + - "failures is empty and no unauthorized tool call is present." +evidence_required: + - filesystem + - metrics + - api_diagnostic +diagnostics: + - "Run this immediately after a complex UI Agent task so latest-run selection cannot drift to unrelated traffic." + - "Use LANGBOT_AGENT_RUN_ID when multiple operators share the test instance." + - "runner-native mode still checks tool pairing, stable names, ordering, terminal state, and error signals; the external runner's permission system owns its local tools." +success_patterns: + - "Agent run ledger audit passed" +failure_patterns: + - "tool_call_pairing" + - "forbidden_error_signals" + - "unauthorized_tool_calls" +troubleshooting: + - box-runtime-silent-action-timeout diff --git a/skills/skills/langbot-testing/cases/agent-runner-async-db-readiness.yaml b/skills/skills/langbot-testing/cases/agent-runner-async-db-readiness.yaml index 3324a1c8e..53a3a437e 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-async-db-readiness.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-async-db-readiness.yaml @@ -1,5 +1,5 @@ id: agent-runner-async-db-readiness -title: "AgentRunner async DB readiness probe" +title: "Runner async DB readiness probe" mode: probe area: release type: smoke diff --git a/skills/skills/langbot-testing/cases/agent-runner-behavior-matrix.yaml b/skills/skills/langbot-testing/cases/agent-runner-behavior-matrix.yaml index 944bca68a..dc765bab9 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-behavior-matrix.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-behavior-matrix.yaml @@ -1,5 +1,5 @@ id: agent-runner-behavior-matrix -title: "AgentRunner deterministic behavior matrix probe" +title: "Runner deterministic behavior matrix probe" mode: probe area: release type: regression diff --git a/skills/skills/langbot-testing/cases/agent-runner-fixture-contract.yaml b/skills/skills/langbot-testing/cases/agent-runner-fixture-contract.yaml index d8f0e04bc..bfe864098 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-fixture-contract.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-fixture-contract.yaml @@ -1,5 +1,5 @@ id: agent-runner-fixture-contract -title: "QA AgentRunner fixture contract probe" +title: "QA Runner fixture contract probe" mode: probe area: release type: regression @@ -17,19 +17,19 @@ env: automation: skills/langbot-testing/probes/agent-runner-fixture-contract.mjs steps: - "Run `rtk bin/lbs test run agent-runner-fixture-contract --dry-run` first; remove `--dry-run` after checking the planned evidence directory." - - "Automation imports the QA AgentRunner fixture source and executes normal, streaming, and controlled-failure paths with SDK entities." + - "Automation imports the QA Runner fixture source and executes normal, streaming, and controlled-failure paths with SDK entities." checks: - "automation-result.json status is pass." - - "probe-stdout.log contains QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK." - - "Normal input returns QA_AGENT_RUNNER_OK:." + - "probe-stdout.log contains QA_RUNNER_FIXTURE_CONTRACT_OK." + - "Normal input returns QA_RUNNER_OK:." - "Streaming input emits message.delta chunks and completes." - - "Failure input returns QA_AGENT_RUNNER_CONTROLLED_FAILURE." + - "Failure input returns QA_RUNNER_CONTROLLED_FAILURE." evidence_required: - filesystem diagnostics: - "This validates the deterministic fixture source contract. It does not prove the plugin package is installed in a live LangBot instance." success_patterns: - - "QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK" + - "QA_RUNNER_FIXTURE_CONTRACT_OK" failure_patterns: - "AssertionError" - "fixture contract exited" diff --git a/skills/skills/langbot-testing/cases/agent-runner-health-visibility.yaml b/skills/skills/langbot-testing/cases/agent-runner-health-visibility.yaml new file mode 100644 index 000000000..be7c826ef --- /dev/null +++ b/skills/skills/langbot-testing/cases/agent-runner-health-visibility.yaml @@ -0,0 +1,53 @@ +id: agent-runner-health-visibility +title: "Agent configuration shows whether its selected runner is usable" +mode: agent-browser +area: agent +type: feature +priority: p0 +risk: medium +ci_eligible: false +tags: + - agent-runner + - agent + - productization + - release-gate +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE +automation: scripts/e2e/agent-runner-health-visibility.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE +preconditions: + - "The plugin runtime is enabled and connected, with at least one Runner registered." + - "The target is a local test instance where a temporary Agent may be created and deleted." +steps: + - "Read the live plugin runtime status and select a registered runner from Agent metadata." + - "Create a temporary Agent using that runner and open its Runner settings in the WebUI." + - "Confirm the UI reports that the runner is ready." + - "Replace the temporary Agent binding with a syntactically valid but unregistered runner id." + - "Reload Runner settings and confirm the UI reports that the selected runner is unavailable." +checks: + - "UI: A registered runner and connected plugin runtime produce a visible ready status." + - "UI: A stale runner binding produces a visible unavailable status with recovery actions." + - "API: Plugin runtime status is connected and Agent metadata contains the registered runner." + - "Console: No unexpected frontend errors appear during the flow." + - "Cleanup: The temporary Agent is deleted after evidence is collected." +evidence_required: + - ui + - screenshot + - console + - api_diagnostic +diagnostics: + - "A disconnected plugin runtime is env_issue, not a passing unavailable-runner product check." + - "A ready status inferred only from API metadata without opening the Runner UI is not a pass." +troubleshooting: + - backend-not-listening + - plugin-runtime-timeout + - proxy-env-mismatch diff --git a/skills/skills/langbot-testing/cases/agent-runner-ledger-concurrency.yaml b/skills/skills/langbot-testing/cases/agent-runner-ledger-concurrency.yaml index 3b57c5435..d397816dc 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-ledger-concurrency.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-ledger-concurrency.yaml @@ -1,5 +1,5 @@ id: agent-runner-ledger-concurrency -title: "AgentRunner run ledger concurrency and auth pytest probe" +title: "Runner run ledger concurrency and auth pytest probe" mode: probe area: release type: regression diff --git a/skills/skills/langbot-testing/cases/agent-runner-ledger-contention.yaml b/skills/skills/langbot-testing/cases/agent-runner-ledger-contention.yaml index e90abd8b7..c8649ac17 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-ledger-contention.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-ledger-contention.yaml @@ -1,5 +1,5 @@ id: agent-runner-ledger-contention -title: "AgentRunner ledger SQLite contention probe" +title: "Runner ledger SQLite contention probe" mode: probe area: release type: regression diff --git a/skills/skills/langbot-testing/cases/agent-runner-ledger-invariants.yaml b/skills/skills/langbot-testing/cases/agent-runner-ledger-invariants.yaml index 578cd8a31..35d56caa8 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-ledger-invariants.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-ledger-invariants.yaml @@ -1,5 +1,5 @@ id: agent-runner-ledger-invariants -title: "AgentRunner ledger schema and status invariants probe" +title: "Runner ledger schema and status invariants probe" mode: probe area: release type: regression @@ -17,7 +17,7 @@ env: automation: skills/langbot-testing/probes/agent-runner-ledger-invariants.mjs steps: - "Run `rtk bin/lbs test run agent-runner-ledger-invariants --dry-run` first; remove `--dry-run` after checking the planned evidence directory." - - "Automation resolves LANGBOT_REPO, defaulting to ../LangBot, and imports the sibling SDK from LANGBOT_PLUGIN_SDK_REPO or ../langbot-plugin-sdk/src." + - "Automation resolves LANGBOT_REPO, defaulting to the parent LangBot checkout, and imports the sibling SDK from LANGBOT_PLUGIN_SDK_REPO or ../../langbot-plugin-sdk/src." - "Automation checks run status sets, terminal status validation, ledger table/index DDL, and a minimal synchronous insert/read path." checks: - "automation-result.json status is pass." diff --git a/skills/skills/langbot-testing/cases/agent-runner-ledger-stress.yaml b/skills/skills/langbot-testing/cases/agent-runner-ledger-stress.yaml index acc527943..f265d9a13 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-ledger-stress.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-ledger-stress.yaml @@ -1,5 +1,5 @@ id: agent-runner-ledger-stress -title: "AgentRunner ledger lightweight stress probe" +title: "Runner ledger lightweight stress probe" mode: probe area: release type: regression diff --git a/skills/skills/langbot-testing/cases/agent-runner-live-install.yaml b/skills/skills/langbot-testing/cases/agent-runner-live-install.yaml index f1bcaaf72..5c7e32734 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-live-install.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-live-install.yaml @@ -1,5 +1,5 @@ id: agent-runner-live-install -title: "QA AgentRunner package installs and registers in LangBot" +title: "QA Runner package installs and registers in LangBot" mode: probe area: release type: regression @@ -25,7 +25,7 @@ automation_expected_tool: "" automation_expected_runner_id: "plugin:qa/agent-runner/default" steps: - "Run `rtk bin/lbs test run agent-runner-live-install --dry-run` first; remove `--dry-run` only after readiness points at a local test LangBot instance." - - "Automation authenticates the local test user, uploads the QA AgentRunner .lbpkg package, waits for the install task, and reads pipeline metadata." + - "Automation authenticates the local test user, uploads the QA Runner .lbpkg package, waits for the install task, and reads pipeline metadata." checks: - "automation-result.json status is pass." - "/api/v1/plugins lists qa/agent-runner after install." diff --git a/skills/skills/langbot-testing/cases/agent-runner-qa-debug-chat.yaml b/skills/skills/langbot-testing/cases/agent-runner-qa-debug-chat.yaml index 1d86b5ec7..83b83f4ce 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-qa-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-qa-debug-chat.yaml @@ -1,5 +1,5 @@ id: agent-runner-qa-debug-chat -title: "QA AgentRunner returns deterministic output through Debug Chat" +title: "QA Runner returns deterministic output through Debug Chat" mode: agent-browser area: pipeline type: regression @@ -17,38 +17,39 @@ skills: env: - LANGBOT_FRONTEND_URL - LANGBOT_BACKEND_URL - - LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL - - LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME + - LANGBOT_QA_RUNNER_PIPELINE_URL + - LANGBOT_QA_RUNNER_PIPELINE_NAME automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL - LANGBOT_BACKEND_URL - LANGBOT_BROWSER_PROFILE - LANGBOT_CHROMIUM_EXECUTABLE - - LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL - - LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME -automation_pipeline_url_env: LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL -automation_pipeline_name_env: LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME + - LANGBOT_QA_RUNNER_PIPELINE_URL + - LANGBOT_QA_RUNNER_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_QA_RUNNER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_QA_RUNNER_PIPELINE_NAME automation_expected_runner_id: "plugin:qa/agent-runner/default" automation_prompt: "hello-live" -automation_expected_text: "QA_AGENT_RUNNER_OK:hello-live" +automation_expected_text: "QA_RUNNER_OK:hello-live" automation_response_timeout_ms: "120000" +automation_debug_chat_response_p95_ms: "120000" automation_reset_debug_chat: "1" setup_automation: - "case:agent-runner-live-install" - "node:scripts/e2e/ensure-qa-agent-runner-pipeline.mjs --write-env" setup_provides_env: - - LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL - - LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME + - LANGBOT_QA_RUNNER_PIPELINE_URL + - LANGBOT_QA_RUNNER_PIPELINE_NAME steps: - "Open LANGBOT_FRONTEND_URL." - - "Open the pipeline from LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL or LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME." + - "Open the pipeline from LANGBOT_QA_RUNNER_PIPELINE_URL or LANGBOT_QA_RUNNER_PIPELINE_NAME." - "Confirm the pipeline AI runner is plugin:qa/agent-runner/default." - "Open Debug Chat." - "Send: hello-live." checks: - "UI: The user message appears in Debug Chat." - - "UI: A Bot message appears and contains QA_AGENT_RUNNER_OK:hello-live." + - "UI: A Bot message appears and contains QA_RUNNER_OK:hello-live." - "API diagnostic: pipeline config uses plugin:qa/agent-runner/default." - "Console: No unexpected frontend runtime errors appear during the send/receive path." evidence_required: @@ -61,7 +62,7 @@ diagnostics: - "This is the deterministic live execution proof that sits after fixture contract and live install." - "If the runner id mismatch is reported, rerun ensure-qa-agent-runner-pipeline.mjs --write-env." success_patterns: - - "QA_AGENT_RUNNER_OK:hello-live" + - "QA_RUNNER_OK:hello-live" failure_patterns: - "plugin:qa/agent-runner/default execution failed" - "Action invoke_llm_stream call timed out" diff --git a/skills/skills/langbot-testing/cases/agent-runner-release-preflight.yaml b/skills/skills/langbot-testing/cases/agent-runner-release-preflight.yaml index b70000591..a0f93c0e0 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-release-preflight.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-release-preflight.yaml @@ -19,7 +19,7 @@ env: - LANGBOT_BACKEND_URL env_any: - LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME - - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL|LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME + - LANGBOT_ACP_RUNNER_PIPELINE_URL|LANGBOT_ACP_RUNNER_PIPELINE_NAME automation: scripts/e2e/agent-runner-release-preflight.mjs automation_env: - LANGBOT_FRONTEND_URL @@ -28,24 +28,24 @@ automation_env: - LANGBOT_CHROMIUM_EXECUTABLE automation_env_any: - LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME - - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL|LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME + - LANGBOT_ACP_RUNNER_PIPELINE_URL|LANGBOT_ACP_RUNNER_PIPELINE_NAME preconditions: - "LANGBOT_LOCAL_AGENT_PIPELINE_URL or LANGBOT_LOCAL_AGENT_PIPELINE_NAME points to the local-agent release pipeline." - - "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL or LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME points to the ACP AgentRunner release pipeline." + - "LANGBOT_ACP_RUNNER_PIPELINE_URL or LANGBOT_ACP_RUNNER_PIPELINE_NAME points to the ACP Runner release pipeline." - "The active browser profile is authenticated for the same LangBot backend." - "By default the preflight performs a cheap model test for the local-agent primary model; set LANGBOT_PREFLIGHT_TEST_MODELS=0 only when deliberately classifying model credentials outside this run." steps: - "Open LANGBOT_FRONTEND_URL with the configured browser profile." - "Use the browser token to call LangBot backend readiness APIs without printing token values." - "Check plugin runtime status, Box status, required runner plugins, qa-plugin-smoke, and qa_plugin_echo." - - "Resolve the local-agent and ACP AgentRunner QA pipelines from their case-specific env vars." + - "Resolve the local-agent and ACP Runner QA pipelines from their case-specific env vars." - "Assert each pipeline uses the expected runner id." - "Assert the external runner pipeline uses the expected runner id." - "Assert the local-agent primary model advertises func_call and vision for the full release gate." - "Run the local-agent primary model test endpoint unless LANGBOT_PREFLIGHT_TEST_MODELS=0." checks: - "API diagnostic: api-diagnostic.json has no blockers and no env_issues." - - "API diagnostic: required pipelines resolve to plugin:langbot/local-agent/default and plugin:langbot/acp-agent-runner/default." + - "API diagnostic: required pipelines resolve to plugin:langbot-team/LocalAgent/default and plugin:langbot-team/ACPRunner/default." - "API diagnostic: qa_plugin_echo is exposed by /api/v1/tools." - "API diagnostic: local-agent model check catches invalid credentials or missing func_call/vision before release E2E starts." - "Secret safety: token values, api keys, and provider secrets are not printed." diff --git a/skills/skills/langbot-testing/cases/agent-runner-runtime-chaos.yaml b/skills/skills/langbot-testing/cases/agent-runner-runtime-chaos.yaml index d7aeca392..432865c50 100644 --- a/skills/skills/langbot-testing/cases/agent-runner-runtime-chaos.yaml +++ b/skills/skills/langbot-testing/cases/agent-runner-runtime-chaos.yaml @@ -1,5 +1,5 @@ id: agent-runner-runtime-chaos -title: "AgentRunner SDK runtime chaos pytest probe" +title: "Runner SDK runtime chaos pytest probe" mode: probe area: release type: regression @@ -18,17 +18,17 @@ env: automation: skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs steps: - "Run `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run` first; remove `--dry-run` after checking the SDK repo target and evidence directory." - - "Automation resolves LANGBOT_PLUGIN_SDK_REPO, defaulting to ../langbot-plugin-sdk when the env var is unset." - - "Automation runs the existing SDK pytest files tests/runtime/plugin/test_mgr_agent_runner.py and tests/runtime/test_pull_api_handlers.py." + - "Automation resolves LANGBOT_PLUGIN_SDK_REPO, defaulting to ../../langbot-plugin-sdk when the env var is unset." + - "Automation runs the existing SDK pytest files tests/runtime/plugin/test_mgr_runner.py and tests/runtime/test_pull_api_handlers.py." checks: - "automation-result.json status is pass." - - "pytest exit status is 0 for the existing AgentRunner runtime and pull API handler tests." + - "pytest exit status is 0 for the existing Runner runtime and pull API handler tests." - "pytest-stdout.log and pytest-stderr.log are written under LBS_EVIDENCE_DIR." evidence_required: - filesystem diagnostics: - "This probe does not open the WebUI; it runs SDK pytest targets directly." - - "env_issue means LANGBOT_PLUGIN_SDK_REPO/default ../langbot-plugin-sdk did not resolve, rtk/uv was unavailable, or the expected test files are missing." + - "env_issue means LANGBOT_PLUGIN_SDK_REPO/default ../../langbot-plugin-sdk did not resolve, rtk/uv was unavailable, or the expected test files are missing." - "fail means the existing SDK runtime pytest target failed or timed out." success_patterns: - "pytest passed" diff --git a/skills/skills/langbot-testing/cases/bot-event-routing-product-flow.yaml b/skills/skills/langbot-testing/cases/bot-event-routing-product-flow.yaml new file mode 100644 index 000000000..f863b5ead --- /dev/null +++ b/skills/skills/langbot-testing/cases/bot-event-routing-product-flow.yaml @@ -0,0 +1,60 @@ +id: bot-event-routing-product-flow +title: "Bot event routing and adapter input can be inspected from the WebUI" +mode: agent-browser +area: bot +type: feature +priority: p0 +risk: medium +ci_eligible: false +tags: + - bot + - eba + - event-routing + - productization +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE +automation: scripts/e2e/bot-event-routing-product-flow.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE +preconditions: + - "The target is a local test instance where a temporary HTTP Bot may be created and deleted." +steps: + - "Open Create Bot, choose HTTP Bot, and add a message-reply behavior before the first save." + - "Create a temporary HTTP Bot with a saved message.received route to the discard processor." + - "Open the Bot configuration in the WebUI." + - "Confirm the adapter capability summary, friendly event name, target, and route status are visible." + - "Confirm overlapping routes and unmatched-event fallback behavior are explained before save." + - "Open Test event route and run a dry-run against the current form." + - "Open Platform event debugging from Adapter Configuration and send a real inbound event through the HTTP Bot adapter." + - "Confirm the normalized event, raw event code, payload, and latest discarded route status are visible." +checks: + - "UI: A user can choose a channel and add a scenario-labeled behavior during initial Bot creation." + - "UI: Event routing uses user-facing labels and does not require the raw event name in the primary route card." + - "UI: Definite route shadowing and unmatched-event fallback behavior are visible without opening raw logs." + - "UI: Dry-run visibly reports that the route matched the discard processor." + - "UI: Adapter event debugging lives under Adapter Configuration, remains separate from route preview, and starts listening only after the dialog opens." + - "UI: A real adapter event shows its friendly name, raw code, and normalized event data." + - "UI: The route card updates to discarded after the real inbound event is handled." + - "Console: No unexpected frontend errors appear during the flow." + - "Network: Bot, dry-run, route-status, log, and HTTP Bot webhook requests return without 5xx responses." + - "Cleanup: The temporary Bot is deleted after evidence is collected." +evidence_required: + - ui + - screenshot + - console + - api_diagnostic +diagnostics: + - "The fixture deliberately uses the discard processor so the product-flow test cannot invoke a model, tool, or external callback." + - "The adapter dialog observes normalized platform events; it does not simulate route matching." + - "A passing webhook call without the visible adapter event and discarded UI states is not a pass." +troubleshooting: + - backend-not-listening + - proxy-env-mismatch diff --git a/skills/skills/langbot-testing/cases/box-mcp-heartbeat-recovery.yaml b/skills/skills/langbot-testing/cases/box-mcp-heartbeat-recovery.yaml new file mode 100644 index 000000000..e2e370d4f --- /dev/null +++ b/skills/skills/langbot-testing/cases/box-mcp-heartbeat-recovery.yaml @@ -0,0 +1,85 @@ +id: box-mcp-heartbeat-recovery +title: "Box heartbeat restores an existing stdio MCP tool session" +mode: probe +area: reliability +type: chaos +priority: p1 +risk: high +ci_eligible: false +tags: + - box + - mcp + - stdio + - reliability + - chaos + - fault-injection +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_REPO + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_MCP_QA_STDIO_SERVER_UUID +automation: skills/langbot-testing/probes/box-mcp-heartbeat-recovery.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_REPO + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_MCP_QA_STDIO_SERVER_UUID +metrics_thresholds_json: '{"recovery_duration_ms":{"max":30000},"mcp_retry_count_increased":{"min":1},"debug_chat_status":{"equals":"pass"},"ledger_tool_call_status":{"equals":"pass"}}' +fault_model_json: '{"target":"local Box runtime child owned by the active LangBot checkout","fault":"SIGTERM","blast_radius":"one local development instance","expected_recovery":"heartbeat replaces Box, existing MCP session reconnects, and a fresh browser Debug Chat tool call passes without MCP registration"}' +load_profile_json: '{"requests":1,"concurrency":1,"classification":"recovery-not-throughput-benchmark"}' +preconditions: + - "Run only against a disposable local LangBot development instance; never use a shared or production-like backend." + - "The selected Local Agent pipeline uses a model route that supports qa_mcp_echo tool calls." + - "qa-local-stdio is already connected and LANGBOT_MCP_QA_STDIO_SERVER_UUID points to it." +steps: + - "Verify Box is available, the saved MCP server is connected, and qa_mcp_echo is globally visible." + - "Select exactly one Box child whose parent is main.py running from LANGBOT_REPO; abort on zero or multiple matches." + - "Send SIGTERM to that Box child and poll process, Box status, MCP runtime info, and global tools for up to 30 seconds." + - "Without running MCP setup or registration, reset Debug Chat and call qa_mcp_echo with a unique per-run value through the browser." + - "Audit the matching Runner ledger run and require the exact qa_mcp_echo arguments plus the complete tool result text." +checks: + - "The old Box PID exits and a new Box PID appears under the same LangBot parent." + - "Box returns available=true with at least one active session and managed process." + - "The MCP server returns connected, tool_count>=1, and a retry_count greater than preflight." + - "A new browser assistant message contains the unique recovery value." + - "The run ledger proves qa_mcp_echo was invoked with the exact unique value and completed with the full qa_mcp_echo result." +cleanup: + - "The probe waits for heartbeat recovery and does not spawn Box or re-register MCP. If recovery times out, restart only the disposable local LangBot instance before further tests." +evidence_required: + - ui + - screenshot + - console + - network + - metrics + - api_diagnostic + - resource_log + - filesystem +diagnostics: + - "Set LANGBOT_BOX_RUNTIME_PID only after manually verifying the exact Box process when more than one local instance is running." + - "resource-log.json records each recovery poll; api-diagnostic.json retains preflight, recovered MCP state, browser, and ledger findings even when a later assertion fails." + - "ledger-audit.json distinguishes a real tool invocation from a model merely repeating the prompt token." + - "This case intentionally has no MCP registration setup step because registration during recovery would mask the product behavior under test." +success_patterns: + - "Connected to Box runtime via stdio" + - "Box runtime reconnected, sandbox features restored" +failure_patterns: + - "Box/MCP did not recover" + - "Expected exactly one local Box child" + - "Recovered Debug Chat tool call failed" + - "Recovered Debug Chat ledger audit failed" +expected_failures: + - "Disconnected from Box runtime, trying to reconnect" + - "Box MCP transport task was cancelled unexpectedly" +troubleshooting: + - backend-not-listening + - debug-chat-history-contaminates-automation + - local-agent-model-route-unavailable diff --git a/skills/skills/langbot-testing/cases/claude-code-agent-debug-chat.yaml b/skills/skills/langbot-testing/cases/claude-code-agent-debug-chat.yaml new file mode 100644 index 000000000..d695229f9 --- /dev/null +++ b/skills/skills/langbot-testing/cases/claude-code-agent-debug-chat.yaml @@ -0,0 +1,72 @@ +id: claude-code-agent-debug-chat +title: "Claude Code Agent can retrieve LangBot history through MCP" +mode: agent-browser +area: pipeline +type: regression +priority: p2 +risk: high +ci_eligible: false +tags: + - agent-runner + - claude-code + - external-runner + - mcp + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL +env_any: + - LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_URL|LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_NAME +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_URL + - LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_CLAUDE_CODE_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/ClaudeCodeAgent/default" +automation_reset_debug_chat: "1" +automation_prompts_json: '[{"prompt":"Remember this exact passcode for the LangBot conversation: claude-history-proof-6384. Do not repeat the passcode. Reply exactly CLAUDE_HISTORY_SEEDED with no other text.","expected_text":"CLAUDE_HISTORY_SEEDED","response_timeout_ms":"600000"},{"prompt":"This CLI turn has no reused Claude session. Recover the passcode from LangBot history by calling mcp__langbot_agent__langbot_history_page exactly once. If ToolSearch is available here, first load the tool with select:mcp__langbot_agent__langbot_history_page and call it directly. If this is a coordinator session that exposes only delegation tools, delegate to exactly one worker and require that worker to perform the same ToolSearch-then-call flow, then wait for its result. Use only the returned LangBot history. If the MCP call fails, do not guess or claim success. Reply with only the recovered passcode.","expected_text":"claude-history-proof-6384","response_timeout_ms":"600000"}]' +automation_expected_text: "claude-history-proof-6384" +automation_response_timeout_ms: "600000" +preconditions: + - "Claude Code CLI is installed, authenticated, and runnable non-interactively by the LangBot plugin process." + - "The configured workspace exists and is writable by the Claude Code CLI." + - "The pipeline has reuse-session=false so the second turn cannot use Claude CLI session memory." +steps: + - "Open LANGBOT_FRONTEND_URL." + - "Open the Claude Code Agent QA pipeline." + - "Confirm the pipeline runner is plugin:langbot-team/ClaudeCodeAgent/default." + - "Open Debug Chat, seed the passcode, then ask a fresh Claude CLI turn to recover it through mcp__langbot_agent__langbot_history_page." +checks: + - "UI: The first Bot message contains CLAUDE_HISTORY_SEEDED and the second contains claude-history-proof-6384." + - "Runner evidence: The second run records a real LangBot history MCP tool call in the main session or one delegated worker, not only model text." + - "Logs: The request completes without claude_code process, MCP, approval, or runtime errors." + - "Console: No unexpected frontend errors appear during Debug Chat." +evidence_required: + - ui + - screenshot + - console + - backend_log +success_patterns: + - "CLAUDE_HISTORY_SEEDED" + - "claude-history-proof-6384" + - "Streaming completed" +failure_patterns: + - "claude_code.command_not_found" + - "claude_code.process_exited" + - "not available among the connected MCP tools" + - "not accessible in this session" + - "No such tool available" + - "Agent runner not found" + - "Agent runner plugin:langbot-team/ClaudeCodeAgent/default execution failed" +troubleshooting: + - backend-not-listening + - plugin-runtime-timeout + - proxy-env-mismatch diff --git a/skills/skills/langbot-testing/cases/codex-agent-debug-chat.yaml b/skills/skills/langbot-testing/cases/codex-agent-debug-chat.yaml new file mode 100644 index 000000000..856580062 --- /dev/null +++ b/skills/skills/langbot-testing/cases/codex-agent-debug-chat.yaml @@ -0,0 +1,71 @@ +id: codex-agent-debug-chat +title: "Codex Agent can retrieve LangBot history through MCP" +mode: agent-browser +area: pipeline +type: regression +priority: p2 +risk: high +ci_eligible: false +tags: + - agent-runner + - codex + - external-runner + - mcp + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL +env_any: + - LANGBOT_CODEX_AGENT_PIPELINE_URL|LANGBOT_CODEX_AGENT_PIPELINE_NAME +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_CODEX_AGENT_PIPELINE_URL + - LANGBOT_CODEX_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_CODEX_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_CODEX_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/CodexAgent/default" +automation_reset_debug_chat: "1" +automation_prompts_json: '[{"prompt":"Do not launch a subagent. Remember this exact passcode for the LangBot conversation: codex-history-proof-9157. Do not repeat the passcode. Reply exactly CODEX_HISTORY_SEEDED with no other text.","expected_text":"CODEX_HISTORY_SEEDED","response_timeout_ms":"600000"},{"prompt":"Do not launch a subagent. This CLI turn has no reused Codex session. Directly call the langbot_history_page tool exposed by the langbot_agent MCP server exactly once and wait for its result. Use only the returned LangBot history to recover the passcode from the previous user turn. If the tool is unavailable, do not guess or claim success. Reply with only the recovered passcode.","expected_text":"codex-history-proof-9157","response_timeout_ms":"600000"}]' +automation_expected_text: "codex-history-proof-9157" +automation_response_timeout_ms: "600000" +preconditions: + - "Codex CLI is installed, authenticated, and runnable non-interactively by the LangBot plugin process." + - "The configured workspace exists and is writable, with approval policy never and sandbox mode appropriate for the QA host." + - "The pipeline has reuse-session=false so the second turn cannot use Codex CLI thread memory." +steps: + - "Open LANGBOT_FRONTEND_URL." + - "Open the Codex Agent QA pipeline." + - "Confirm the pipeline runner is plugin:langbot-team/CodexAgent/default." + - "Open Debug Chat, seed the passcode, then ask a fresh Codex CLI turn to recover it through langbot_agent.langbot_history_page." +checks: + - "UI: The first Bot message contains CODEX_HISTORY_SEEDED and the second contains codex-history-proof-9157." + - "Runner evidence: The second run records a real LangBot history MCP tool call, not only model text." + - "Logs: The request completes without codex process, MCP, approval, sandbox, or runtime errors." + - "Console: No unexpected frontend errors appear during Debug Chat." +evidence_required: + - ui + - screenshot + - console + - backend_log +success_patterns: + - "CODEX_HISTORY_SEEDED" + - "codex-history-proof-9157" + - "Streaming completed" +failure_patterns: + - "codex.command_not_found" + - "codex.process_exited" + - "not available among the connected MCP tools" + - "not accessible in this session" + - "Agent runner not found" + - "Agent runner plugin:langbot-team/CodexAgent/default execution failed" +troubleshooting: + - backend-not-listening + - plugin-runtime-timeout + - proxy-env-mismatch diff --git a/skills/skills/langbot-testing/cases/dify-agent-debug-chat.yaml b/skills/skills/langbot-testing/cases/dify-agent-debug-chat.yaml index 42b0f62bf..5e4b37236 100644 --- a/skills/skills/langbot-testing/cases/dify-agent-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/dify-agent-debug-chat.yaml @@ -1,5 +1,5 @@ id: dify-agent-debug-chat -title: "Dify AgentRunner returns a response through Pipeline Debug Chat" +title: "Dify Runner returns a response through Pipeline Debug Chat" mode: agent-browser area: pipeline type: provider diff --git a/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml b/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml index 8ff911371..150ecaf42 100644 --- a/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml +++ b/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml @@ -16,14 +16,14 @@ skills: env: - LANGBOT_BACKEND_URL automation: skills/langbot-testing/probes/langbot-live-backend-log-health.mjs -metrics_thresholds_json: '{"fail_count":{"max":0}}' +metrics_thresholds_json: '{"scanned_line_count":{"min":1},"fail_count":{"max":0}}' load_profile_json: '{"lookback_seconds":300,"log_source":"LANGBOT_BACKEND_LOG or latest LANGBOT_REPO/data/logs/langbot-*.log"}' steps: - "Confirm the selected LangBot backend log belongs to the intended test target." - "Run `rtk bin/lbs test run langbot-live-backend-log-health --dry-run` first; remove `--dry-run` after checking evidence directory and log source." - "Automation scans the recent backend log window for fail-severity runtime findings such as Traceback, ImportError, ERROR, unclosed sessions, and unawaited coroutines." checks: - - "automation-result.json status is pass only when fail_count is 0." + - "automation-result.json status is pass only when at least one timestamped line is scanned and fail_count is 0." - "metrics_summary includes scanned_line_count, fail_count, warning_count, and finding_count." - "findings.json and scanned-backend.log are written under LBS_EVIDENCE_DIR." evidence_required: @@ -33,6 +33,7 @@ evidence_required: diagnostics: - "Set LANGBOT_BACKEND_LOG to an explicit log path when the latest log file is not the run target." - "Set LANGBOT_BACKEND_LOG_SINCE or LANGBOT_BACKEND_LOG_LOOKBACK_SECONDS to control the scan window." + - "A zero-line scan is env_issue, not pass; point LANGBOT_BACKEND_LOG at the active service output or widen the lookback." - "This probe measures runtime log health; it does not prove user-facing Debug Chat, plugin, model, or RAG behavior." success_patterns: - "Live backend log health passed" diff --git a/skills/skills/langbot-testing/cases/langrag-kb-retrieve.yaml b/skills/skills/langbot-testing/cases/langrag-kb-retrieve.yaml index d7dbd884e..c82060239 100644 --- a/skills/skills/langbot-testing/cases/langrag-kb-retrieve.yaml +++ b/skills/skills/langbot-testing/cases/langrag-kb-retrieve.yaml @@ -25,6 +25,10 @@ automation_env: automation_env_any: - LANGBOT_LOCAL_AGENT_RAG_KB_UUID|LANGBOT_RAG_KB_UUID automation_expected_text: "azalea-cobalt-7421" +setup_automation: + - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID preconditions: - "LangRAG is installed and initialized in the active LangBot instance." - "A working embedding model is available, preferably chroma-all-MiniLM-L6-v2 for local repeatability." @@ -47,7 +51,8 @@ evidence_required: - ui - screenshot - console - - backend_log + - network + - api_diagnostic diagnostics: - "If no LangRAG engine is available, check /api/v1/knowledge/engines and install langbot-team/LangRAG." - "If the embedding selector does not show a local Chroma model, confirm the model exists under embedding_models, not llm_models." diff --git a/skills/skills/langbot-testing/cases/local-agent-basic-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-basic-debug-chat.yaml index 0b1cc1352..7d4202197 100644 --- a/skills/skills/langbot-testing/cases/local-agent-basic-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-basic-debug-chat.yaml @@ -38,7 +38,7 @@ steps: - "Open LANGBOT_FRONTEND_URL." - "Navigate to Pipelines and open the target local-agent pipeline." - "Open Configuration > AI." - - "Use runner Default or the pluginized langbot/local-agent runner." + - "Use runner Default or the pluginized langbot-team/LocalAgent runner." - "Select a model that is known to answer Debug Chat in the current environment." - "Clear Knowledge Bases unless this run intentionally combines RAG with the smoke path." - "Save the pipeline." diff --git a/skills/skills/langbot-testing/cases/local-agent-combo-rag-compaction-tool-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-combo-rag-compaction-tool-debug-chat.yaml new file mode 100644 index 000000000..1c7073374 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-combo-rag-compaction-tool-debug-chat.yaml @@ -0,0 +1,103 @@ +id: local-agent-combo-rag-compaction-tool-debug-chat +title: "Local Agent preserves RAG, compacted history, and plugin tool result in one Debug Chat run" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - langrag + - context + - compaction + - plugin + - tools + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" +automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":650,"context-reserve-tokens":180,"context-keep-recent-tokens":120,"context-summary-tokens":220,"max-tool-iterations":4,"tool-execution-mode":"serial"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_expected_text: "COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent" +automation_response_timeout_ms: "180000" +automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 组合回归测试的暗号：qa_combo_compaction_sentinel_2406。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 RAG_TOOL_COMBO_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 COMBO_CONTEXT_PRESSURE_READY。填充片段 C001 combo context padding. C002 combo context padding. C003 combo context padding. C004 combo context padding. C005 combo context padding. C006 combo context padding. C007 combo context padding. C008 combo context padding. C009 combo context padding. C010 combo context padding. C011 combo context padding. C012 combo context padding. C013 combo context padding. C014 combo context padding. C015 combo context padding. C016 combo context padding. C017 combo context padding. C018 combo context padding. C019 combo context padding. C020 combo context padding. C021 combo context padding. C022 combo context padding. C023 combo context padding. C024 combo context padding. C025 combo context padding. C026 combo context padding. C027 combo context padding. C028 combo context padding. C029 combo context padding. C030 combo context padding. C031 combo context padding. C032 combo context padding. C033 combo context padding. C034 combo context padding. C035 combo context padding. C036 combo context padding. C037 combo context padding. C038 combo context padding. C039 combo context padding. C040 combo context padding.","expected_text":"COMBO_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"qa_combo final check: using the knowledge base, the compacted earlier passcode, and qa_plugin_echo with exactly combo-tool-ok-local-agent, return only COMBO_FINAL plus the passcode, the RAG sentinel, and the plugin tool result.","expected_text":"COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent","response_timeout_ms":"180000"}]' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +preconditions: + - "The selected model route supports function/tool calling and deterministic multi-turn answers, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "The LangRAG sentinel knowledge base contains azalea-cobalt-7421 and qa-plugin-smoke exposes qa_plugin_echo." +steps: + - "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available." + - "Temporarily bind the LangRAG KB, shrink the runner context budget, and allow serial plugin tool execution." + - "Temporarily bind only langbot-team/LocalAgent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the passcode memory prompt and wait for MEMORY_SET." + - "Send the long pressure prompt and wait for COMBO_CONTEXT_PRESSURE_READY." + - "Ask the runner to combine compacted passcode, RAG sentinel, and qa_plugin_echo result in one final answer." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: All three user messages appear in Debug Chat." + - "UI: The final Bot message contains COMBO_FINAL qa_combo_compaction_sentinel_2406 azalea-cobalt-7421 qa-plugin-smoke:combo-tool-ok-local-agent." + - "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, and tool-loop settings were patched." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound." + - "API diagnostic: restore diagnostics show the original runner config and extension bindings were restored." + - "Logs: Backend completes RAG retrieval, plugin tool call, and Debug Chat response without runner timeout or model setup failure." + - "Console: No unexpected frontend runtime errors appear during the run." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The ensure-langrag-sentinel-kb setup script verifies LANGBOT_E2E_RAG_EXPECTED_TEXT, defaulting to azalea-cobalt-7421, while this case's Debug Chat expected text is the full COMBO_FINAL string." + - "The fake provider intentionally returns COMBO_MISSING_* if the final model input lacks compacted memory, RAG context, or plugin tool result." + - "If the final answer lacks azalea-cobalt-7421, first rerun ensure-langrag-sentinel-kb and local-agent-rag-debug-chat." + - "If the final answer lacks qa-plugin-smoke:combo-tool-ok-local-agent, inspect /api/v1/tools and the pipeline extensions diagnostic." +success_patterns: + - "Processing request from person_websocket" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "COMBO_MISSING_" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-complex-coding-task-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-complex-coding-task-debug-chat.yaml new file mode 100644 index 000000000..7e9581131 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-complex-coding-task-debug-chat.yaml @@ -0,0 +1,96 @@ +id: local-agent-complex-coding-task-debug-chat +title: "Local Agent completes a multi-file coding task with iterative verification" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - agentic + - coding + - sandbox + - tools + - long-running +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_REPO + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_REPO + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" +automation_runner_config_patch_json: '{"knowledge-bases":[],"context-window-tokens":200000,"context-keep-recent-tokens":120000,"context-reserve-tokens":20000,"context-summary-tokens":12000,"max-tool-iterations":40,"tool-execution-mode":"serial","timeout":900}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_prompt: "Work autonomously on the project at /workspace/order-orchestrator. Read TASK.md completely, inspect the code and tests, then follow the required workflow. Run the full baseline test suite before editing and record the failing test names. Fix only production code under order_engine; do not alter TASK.md, tests, or acceptance_check.py. This is an end-to-end task: continue through failures, diagnose them, make multi-file fixes, rerun all 12 tests until green, run acceptance_check.py, and create both result.json and AGENT_REPORT.md exactly as required. Do not stop after analysis or ask me for confirmation. When and only when all verification passes, return exactly COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS." +automation_expected_text: "COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS" +automation_response_timeout_ms: "900000" +automation_stream_output: "1" +automation_filesystem_checks_json: '[{"argv":["python3","skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py","--workspace","${LANGBOT_COMPLEX_AGENT_WORKSPACE}"],"cwd":"${LANGBOT_REPO}","stdout_contains":"HOST_VERIFY_PASS tests=12 acceptance=PASS protected=PASS","exit_code":0,"timeout_ms":120000}]' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/reset-complex-agent-task.mjs" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_COMPLEX_AGENT_WORKSPACE +preconditions: + - "The active tenant Box workspace is available on the host and exposed to native tools as /workspace." + - "The selected real model supports sustained function calling across file inspection, edits, and test reruns." +steps: + - "Reset the isolated order-orchestrator workspace and confirm its baseline test suite fails." + - "Open the Local Agent pipeline in Debug Chat and send one complete autonomous coding task." + - "Observe the Agent inspect requirements and code, run the baseline tests, edit multiple production files, and rerun verification after failures." + - "Require the visible final sentinel only after all 12 tests and the acceptance check pass." + - "Run the host-side verifier to confirm protected files were not changed and required artifacts are valid." +checks: + - "UI: A new Bot message contains exactly COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS." + - "Run ledger: The real model performs a sustained sequence of native read, exec, and edit/write tool calls rather than a short answer." + - "Filesystem: Host verification reruns all 12 tests and acceptance_check.py successfully." + - "Filesystem: TASK.md, tests/test_order_engine.py, and acceptance_check.py match the immutable fixture copies." + - "Artifacts: result.json and AGENT_REPORT.md contain the required completion evidence." + - "API diagnostic: The temporary runner config uses a 200000-token context window and restores the prior config after the run." +evidence_required: + - ui + - screenshot + - console + - network + - api_diagnostic + - filesystem + - metrics +diagnostics: + - "A generic explanation or early plan without filesystem artifacts is a product-path failure, even if the model response sounds plausible." + - "If the Agent edits protected tests, the host verifier fails even when the visible test command passes." + - "If the run times out, inspect agent_run_event to distinguish model latency from a stuck tool or runtime action." +success_patterns: + - "COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS" + - "HOST_VERIFY_PASS tests=12 acceptance=PASS protected=PASS" +failure_patterns: + - "Agent runner temporarily unavailable" + - "runner.timeout" + - "Action invoke_llm_stream call timed out" + - "Tool call iteration limit reached" +troubleshooting: + - local-agent-model-route-unavailable + - sandbox-native-tools-unavailable + - box-runtime-silent-action-timeout + - plugin-runtime-timeout + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-context-compaction-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-context-compaction-debug-chat.yaml index 7c0d6a32a..637f67db8 100644 --- a/skills/skills/langbot-testing/cases/local-agent-context-compaction-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-context-compaction-debug-chat.yaml @@ -19,6 +19,8 @@ env: - LANGBOT_BACKEND_URL - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL @@ -29,9 +31,11 @@ automation_env: - LANGBOT_LOCAL_AGENT_PIPELINE_NAME automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME -automation_expected_runner_id: "plugin:langbot/local-agent/default" +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" automation_runner_config_patch_json: '{"context-window-tokens":225,"context-reserve-tokens":50,"context-keep-recent-tokens":30,"context-summary-tokens":105,"knowledge-bases":[]}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' automation_restore_runner_config: "1" +automation_restore_extensions: "1" automation_reset_debug_chat: "1" automation_debug_chat_session_type: "person" automation_expected_text: "qa_compaction_sentinel_7391" @@ -43,12 +47,13 @@ setup_provides_env: - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME preconditions: - - "The selected model route can follow short deterministic instructions across multiple Debug Chat turns." + - "The selected model route can follow short deterministic instructions across multiple Debug Chat turns, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." steps: - "Open the target local-agent pipeline through LANGBOT_FRONTEND_URL." - "Use the authenticated browser token only inside automation to GET and PUT /api/v1/pipelines/{uuid}." - - "Assert the saved runner is plugin:langbot/local-agent/default." + - "Assert the saved runner is plugin:langbot-team/LocalAgent/default." - "Temporarily set context-window-tokens, context-reserve-tokens, context-keep-recent-tokens, and context-summary-tokens to force compaction, and clear knowledge-bases so RAG does not answer the memory question." + - "Temporarily bind only the local-agent plugin, disable MCP servers, and disable skills so unrelated visible tools cannot answer or distract the memory question." - "Reset the person Debug Chat session for the target pipeline." - "Send the sentinel memory prompt and wait for MEMORY_SET." - "Send the long padding prompt and wait for CONTEXT_PRESSURE_READY." @@ -59,6 +64,8 @@ checks: - "UI: The final Bot message contains qa_compaction_sentinel_7391." - "API diagnostic: pipeline-config-diagnostic.json shows patched=true and patch_keys include the four token context compaction fields plus knowledge-bases." - "API diagnostic: pipeline-config-restore-diagnostic.json shows the original runner config was restored." + - "API diagnostic: pipeline-extensions-diagnostic.json shows enable_all_plugins=false, bound_plugins contains langbot-team/LocalAgent, enable_all_mcp_servers=false, and enable_all_skills=false." + - "API diagnostic: pipeline-extensions-restore-diagnostic.json shows the original extension bindings were restored." - "Logs: Backend completes the multi-turn Debug Chat path without runner timeout or model setup failure." - "Console: No unexpected frontend runtime errors appear during the run." evidence_required: @@ -68,7 +75,8 @@ evidence_required: - backend_log - api_diagnostic diagnostics: - - "If the final sentinel is missing, inspect whether pipeline-config-diagnostic.json targeted ai.runner_config[runnerId], cleared knowledge-bases, and whether the backend log shows the local-agent runner loading the small context settings." + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model." + - "If the final sentinel is missing, inspect whether pipeline-config-diagnostic.json targeted ai.runner_config[runnerId], cleared knowledge-bases, whether pipeline-extensions-diagnostic.json isolated unrelated tools, and whether the backend log shows the local-agent runner loading the small context settings." - "If the model ignores deterministic replies, rerun with a known-good model route before diagnosing ContextAssembler." - "If restore fails, use pipeline-config-restore-diagnostic.json and GET /api/v1/pipelines/{uuid} to confirm the current saved config before retrying." success_patterns: diff --git a/skills/skills/langbot-testing/cases/local-agent-effective-prompt-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-effective-prompt-debug-chat.yaml index d19d4a109..3c28167a9 100644 --- a/skills/skills/langbot-testing/cases/local-agent-effective-prompt-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-effective-prompt-debug-chat.yaml @@ -28,6 +28,11 @@ automation_env: - LANGBOT_LOCAL_AGENT_PIPELINE_NAME automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" automation_prompt: "qa-effective-prompt" automation_expected_text: "PROMPT_PREPROCESS_OK" automation_response_timeout_ms: "180000" @@ -44,7 +49,7 @@ steps: - "Confirm the fixture plugin is bound to the target pipeline through Extensions." - "Open the target local-agent pipeline." - "Open Configuration > AI." - - "Use runner Default or the pluginized langbot/local-agent runner." + - "Use runner Default or the pluginized langbot-team/LocalAgent runner." - "Select a model that is known to follow system prompts in Debug Chat." - "Save the pipeline." - "Open Debug Chat." diff --git a/skills/skills/langbot-testing/cases/local-agent-model-fallback-before-first-chunk-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-model-fallback-before-first-chunk-debug-chat.yaml new file mode 100644 index 000000000..2b598b797 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-model-fallback-before-first-chunk-debug-chat.yaml @@ -0,0 +1,79 @@ +id: local-agent-model-fallback-before-first-chunk-debug-chat +title: "Local Agent falls back when the primary model fails before streaming starts" +mode: probe +area: pipeline +type: chaos +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - fallback + - streaming + - fake-provider + - fault-injection +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER +automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_debug_chat_load_requests: "1" +automation_debug_chat_load_concurrency: "1" +automation_debug_chat_load_timeout_ms: "30000" +automation_debug_chat_load_response_p95_ms: "10000" +automation_debug_chat_load_max_error_rate: "0" +automation_debug_chat_load_min_ok_count: "1" +automation_debug_chat_load_min_provider_fault_count: "1" +automation_debug_chat_load_expected_prefix: "FALLBACKQA" +automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。' +automation_debug_chat_load_stream: "true" +automation_debug_chat_load_reset: "true" +automation_debug_chat_load_fail_on_final_mismatch: "true" +automation_debug_chat_load_provider_model_thresholds_json: '{"qa-fallback-primary":{"min":1},"qa-fallback-secondary":{"min":1}}' +automation_fake_provider_pipeline_name: "LangBot QA Model Fallback Debug Chat" +automation_fake_provider_model_name: "qa-fallback-primary" +automation_fake_provider_fallback_model_names: "qa-fallback-secondary" +automation_fake_provider_fail_models: "qa-fallback-primary" +automation_fake_provider_fail_after_first_chunk_models: "" +automation_fake_provider_first_token_delay_ms: "25" +automation_fake_provider_chunk_delay_ms: "10" +automation_fake_provider_fault_status: "503" +metrics_thresholds_json: '{"response_p95_ms":{"max":10000},"error_rate":{"max":0},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"min":1}}' +fault_model_json: '{"provider_fault":"HTTP 503 for every qa-fallback-primary request before the first stream chunk","expected_behavior":"Local Agent invokes qa-fallback-secondary and returns the expected Debug Chat response."}' +setup_automation: + - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" +setup_provides_env: + - LANGBOT_FAKE_PROVIDER_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +steps: + - "Create a QA pipeline with a failing primary model and a healthy fallback model on the controlled provider." + - "Send one streaming Debug Chat request with a unique response token." + - "Inspect provider request records and require both the failed primary request and the successful fallback request." +checks: + - "The user-visible request succeeds with the unique expected response." + - "The fake provider records at least one qa-fallback-primary request and at least one qa-fallback-secondary request." + - "At least one provider fault is observed without a final Debug Chat error." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +expected_failures: + - "fake_provider_fault" +failure_patterns: + - "All models failed during streaming setup" + - "Timed out after" +troubleshooting: + - backend-not-listening + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-multimodal-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-multimodal-debug-chat.yaml index 298b59dca..184381309 100644 --- a/skills/skills/langbot-testing/cases/local-agent-multimodal-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-multimodal-debug-chat.yaml @@ -42,7 +42,7 @@ steps: - "Open LANGBOT_FRONTEND_URL." - "Navigate to Pipelines and open the target local-agent pipeline." - "Open Configuration > AI." - - "Use runner Default or the pluginized langbot/local-agent runner." + - "Use runner Default or the pluginized langbot-team/LocalAgent runner." - "Select a model that supports image input in the current environment, or use a known model that at least accepts the uploaded image payload." - "Save the pipeline." - "Open Debug Chat." diff --git a/skills/skills/langbot-testing/cases/local-agent-multitool-rag-compaction-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-multitool-rag-compaction-debug-chat.yaml new file mode 100644 index 000000000..fc875d42d --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-multitool-rag-compaction-debug-chat.yaml @@ -0,0 +1,102 @@ +id: local-agent-multitool-rag-compaction-debug-chat +title: "Local Agent preserves RAG, compacted history, and two-step plugin tool loop" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - langrag + - context + - compaction + - plugin + - tools + - tool-loop + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" +automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":5,"tool-execution-mode":"serial"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_expected_text: "MULTITOOL_COMBO_FINAL qa_multitool_compaction_sentinel_6718 azalea-cobalt-7421 qa-plugin-smoke:multi-tool-a-local-agent qa-plugin-smoke:multi-tool-b-local-agent" +automation_response_timeout_ms: "180000" +automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 多工具组合回归测试的暗号：qa_multitool_compaction_sentinel_6718。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 MULTITOOL_RAG_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 MULTITOOL_CONTEXT_PRESSURE_READY。填充片段 M001 multitool context padding. M002 multitool context padding. M003 multitool context padding. M004 multitool context padding. M005 multitool context padding. M006 multitool context padding. M007 multitool context padding. M008 multitool context padding. M009 multitool context padding. M010 multitool context padding. M011 multitool context padding. M012 multitool context padding. M013 multitool context padding. M014 multitool context padding. M015 multitool context padding. M016 multitool context padding. M017 multitool context padding. M018 multitool context padding. M019 multitool context padding. M020 multitool context padding. M021 multitool context padding. M022 multitool context padding. M023 multitool context padding. M024 multitool context padding. M025 multitool context padding. M026 multitool context padding. M027 multitool context padding. M028 multitool context padding. M029 multitool context padding. M030 multitool context padding. M031 multitool context padding. M032 multitool context padding.","expected_text":"MULTITOOL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"MULTITOOL_COMBO final check: using the knowledge base, the compacted earlier passcode, and qa_plugin_echo twice with exactly multi-tool-a-local-agent then multi-tool-b-local-agent, return only MULTITOOL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"MULTITOOL_COMBO_FINAL","expected_texts":["qa_multitool_compaction_sentinel_6718","azalea-cobalt-7421","qa-plugin-smoke:multi-tool-a-local-agent","qa-plugin-smoke:multi-tool-b-local-agent"],"response_timeout_ms":"180000"}]' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +preconditions: + - "The selected model route supports function/tool calling and repeated tool loop turns, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "The LangRAG sentinel knowledge base contains azalea-cobalt-7421 and qa-plugin-smoke exposes qa_plugin_echo." +steps: + - "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available." + - "Temporarily bind the LangRAG KB, shrink the runner context budget, and set max-tool-iterations high enough for two serial plugin tool calls." + - "Temporarily bind only langbot-team/LocalAgent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the passcode memory prompt and wait for MEMORY_SET." + - "Send the long pressure prompt and wait for MULTITOOL_CONTEXT_PRESSURE_READY." + - "Ask the runner to combine compacted passcode, RAG sentinel, and two qa_plugin_echo results in one final answer." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: All three user messages appear in Debug Chat." + - "UI: The final Bot message contains MULTITOOL_COMBO_FINAL with the compacted sentinel, azalea-cobalt-7421, and both qa-plugin-smoke tool results." + - "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, and max-tool-iterations were patched." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound." + - "Logs: Backend completes RAG retrieval, two plugin tool calls, and Debug Chat response without runner timeout or model setup failure." + - "Console: No unexpected frontend runtime errors appear during the run." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The fake provider intentionally returns MULTITOOL_COMBO_MISSING_* if the final model input lacks compacted memory, RAG context, or either plugin tool result." + - "If only one tool result is present, inspect max-tool-iterations and the runner tool-loop events." + - "If the final answer lacks azalea-cobalt-7421, first rerun ensure-langrag-sentinel-kb and local-agent-rag-debug-chat." +success_patterns: + - "Processing request from person_websocket" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "MULTITOOL_COMBO_MISSING_" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-nonstreaming-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-nonstreaming-debug-chat.yaml index 910032d79..f5a143128 100644 --- a/skills/skills/langbot-testing/cases/local-agent-nonstreaming-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-nonstreaming-debug-chat.yaml @@ -1,5 +1,5 @@ id: local-agent-nonstreaming-debug-chat -title: "Local Agent Debug Chat returns a deterministic non-streaming response" +title: "Local Agent Debug Chat returns a deterministic response with UI streaming disabled" mode: agent-browser area: pipeline type: regression @@ -39,7 +39,7 @@ steps: - "Open LANGBOT_FRONTEND_URL." - "Navigate to Pipelines and open the target local-agent pipeline." - "Open Configuration > AI." - - "Use runner Default or the pluginized langbot/local-agent runner." + - "Use runner Default or the pluginized langbot-team/LocalAgent runner." - "Select a model that is known to answer Debug Chat in the current environment." - "Save the pipeline." - "Open Debug Chat." @@ -48,14 +48,16 @@ steps: checks: - "UI: The user message appears in Debug Chat." - "UI: A Bot message appears and contains NONSTREAM_OK." - - "Logs: Backend completes the request as a normal response rather than only relying on the streaming-completed path." + - "UI: The Debug Chat stream switch is disabled for this send path." + - "Logs: Backend completes the request without plugin/runtime timeout or frontend streaming UI errors." - "Console: No unexpected frontend runtime errors appear during the send/receive path." evidence_required: - ui - console - backend_log diagnostics: - - "If the UI still streams after the switch is disabled, inspect the adapter streaming capability and runner config before diagnosing the model." + - "This browser case validates the Debug Chat non-streaming UI delivery path. The runner-internal invoke_llm path is covered by langbot-local-agent component tests using runtime_metadata.streaming_supported=false." + - "If the UI still streams after the switch is disabled, inspect the Debug Chat WebSocket stream flag before diagnosing the model." - "Use GET /api/v1/pipelines/{uuid} only to confirm the saved runner and model config." troubleshooting: - local-agent-model-route-unavailable diff --git a/skills/skills/langbot-testing/cases/local-agent-parallel-tools-rag-compaction-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-parallel-tools-rag-compaction-debug-chat.yaml new file mode 100644 index 000000000..9c38751b8 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-parallel-tools-rag-compaction-debug-chat.yaml @@ -0,0 +1,103 @@ +id: local-agent-parallel-tools-rag-compaction-debug-chat +title: "Local Agent preserves RAG, compacted history, and parallel plugin tools" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - langrag + - context + - compaction + - plugin + - tools + - parallel-tools + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" +automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"],"retrieval-top-k":1,"context-window-tokens":900,"context-reserve-tokens":220,"context-keep-recent-tokens":160,"context-summary-tokens":260,"max-tool-iterations":3,"tool-execution-mode":"parallel"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_expected_text: "PARALLEL_COMBO_FINAL qa_parallel_compaction_sentinel_8142 azalea-cobalt-7421 qa-plugin-smoke:parallel-tool-a-local-agent qa-plugin-smoke:parallel-tool-b-local-agent" +automation_response_timeout_ms: "180000" +automation_prompts_json: '[{"prompt":"请记住这个用于 local-agent 并行工具组合回归测试的暗号：qa_parallel_compaction_sentinel_8142。请只回复 MEMORY_SET。","expected_text":"MEMORY_SET","response_timeout_ms":"180000"},{"prompt":"下面这轮只用于制造长历史压力，目标标记 PARALLEL_RAG_GOAL 必须被压缩保留。请忽略填充内容，最后只回复 PARALLEL_CONTEXT_PRESSURE_READY。填充片段 P001 parallel context padding. P002 parallel context padding. P003 parallel context padding. P004 parallel context padding. P005 parallel context padding. P006 parallel context padding. P007 parallel context padding. P008 parallel context padding. P009 parallel context padding. P010 parallel context padding. P011 parallel context padding. P012 parallel context padding. P013 parallel context padding. P014 parallel context padding. P015 parallel context padding. P016 parallel context padding. P017 parallel context padding. P018 parallel context padding. P019 parallel context padding. P020 parallel context padding. P021 parallel context padding. P022 parallel context padding. P023 parallel context padding. P024 parallel context padding. P025 parallel context padding. P026 parallel context padding. P027 parallel context padding. P028 parallel context padding. P029 parallel context padding. P030 parallel context padding. P031 parallel context padding. P032 parallel context padding.","expected_text":"PARALLEL_CONTEXT_PRESSURE_READY","response_timeout_ms":"180000"},{"prompt":"PARALLEL_COMBO final check: using the knowledge base, the compacted earlier passcode, and one model turn that calls qa_plugin_echo for both parallel-tool-a-local-agent and parallel-tool-b-local-agent, return only PARALLEL_COMBO_FINAL plus the passcode, the RAG sentinel, and both plugin tool results.","expected_text":"PARALLEL_COMBO_FINAL","expected_texts":["qa_parallel_compaction_sentinel_8142","azalea-cobalt-7421","qa-plugin-smoke:parallel-tool-a-local-agent","qa-plugin-smoke:parallel-tool-b-local-agent"],"response_timeout_ms":"180000"}]' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_RAG_KB_UUID +preconditions: + - "The selected model route supports function/tool calling with multiple tool calls in one turn, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "The LangRAG sentinel knowledge base contains azalea-cobalt-7421 and qa-plugin-smoke exposes qa_plugin_echo." +steps: + - "Ensure the local-agent pipeline, LangRAG sentinel KB, and qa-plugin-smoke fixture plugin are available." + - "Temporarily bind the LangRAG KB, shrink the runner context budget, and set tool-execution-mode to parallel." + - "Temporarily bind only langbot-team/LocalAgent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the passcode memory prompt and wait for MEMORY_SET." + - "Send the long pressure prompt and wait for PARALLEL_CONTEXT_PRESSURE_READY." + - "Ask the runner to combine compacted passcode, RAG sentinel, and two same-turn qa_plugin_echo results in one final answer." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: All three user messages appear in Debug Chat." + - "UI: The final Bot message contains PARALLEL_COMBO_FINAL with the compacted sentinel, azalea-cobalt-7421, and both qa-plugin-smoke tool results." + - "API diagnostic: pipeline-config-diagnostic.json shows knowledge-bases, context token settings, max-tool-iterations, and tool-execution-mode=parallel were patched." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound." + - "Logs: Backend completes RAG retrieval, both plugin tool calls, and Debug Chat response without runner timeout or model setup failure." + - "Console: No unexpected frontend runtime errors appear during the run." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The fake provider intentionally returns two qa_plugin_echo tool_calls in the same streamed model turn for this prompt." + - "The fake provider intentionally returns PARALLEL_COMBO_MISSING_* if the final model input lacks compacted memory, RAG context, or either plugin tool result." + - "If only one tool result is present, inspect the provider trace and runner parallel tool batch handling." + - "If the final answer lacks azalea-cobalt-7421, first rerun ensure-langrag-sentinel-kb and local-agent-rag-debug-chat." +success_patterns: + - "Processing request from person_websocket" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "PARALLEL_COMBO_MISSING_" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-plugin-tool-call-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-plugin-tool-call-debug-chat.yaml index 1c37e2fb1..bf241ebd9 100644 --- a/skills/skills/langbot-testing/cases/local-agent-plugin-tool-call-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-plugin-tool-call-debug-chat.yaml @@ -19,6 +19,8 @@ env: - LANGBOT_BACKEND_URL - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL @@ -28,6 +30,11 @@ automation_env: - LANGBOT_LOCAL_AGENT_PIPELINE_NAME automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" automation_prompt: "Call the qa_plugin_echo tool with exactly this text: plugin-tool-ok-local-agent. Return only the tool result." automation_expected_text: "qa-plugin-smoke:plugin-tool-ok-local-agent" automation_response_timeout_ms: "180000" @@ -38,18 +45,19 @@ setup_provides_env: - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME preconditions: - - "The selected model route supports function/tool calling." + - "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." steps: - "Install or enable the bundled qa-plugin-smoke fixture plugin." - "Confirm /api/v1/tools or the plugin detail shows qa_plugin_echo." - - "Confirm the fixture plugin is bound to the target pipeline through Extensions, or that all plugins are enabled." + - "Temporarily bind only langbot-team/LocalAgent and qa/plugin-smoke; disable MCP servers and skills to isolate the tool surface." - "Open the target local-agent pipeline." - - "Use runner Default or the pluginized langbot/local-agent runner." + - "Use runner Default or the pluginized langbot-team/LocalAgent runner." - "Select a model with function-calling ability that is known to work with tools in the current environment." - "Open Debug Chat." - "Send: Call the qa_plugin_echo tool with exactly this text: plugin-tool-ok-local-agent. Return only the tool result." checks: - "UI: Debug Chat bot response contains qa-plugin-smoke:plugin-tool-ok-local-agent." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound." - "Logs: Backend logs show the plugin tool call was executed, not only listed." - "Console: No unexpected frontend errors appear during Debug Chat." evidence_required: @@ -58,6 +66,7 @@ evidence_required: - backend_log - api_diagnostic diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model." - "If qa_plugin_echo is not listed, rebuild and reinstall the qa-plugin-smoke fixture plugin." - "If the selected model returns model_not_found or no available channel only when tools are provided, switch to a known-good function-calling model before diagnosing plugin tools or local-agent." troubleshooting: diff --git a/skills/skills/langbot-testing/cases/local-agent-rag-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-rag-debug-chat.yaml index b7e715b16..30989c100 100644 --- a/skills/skills/langbot-testing/cases/local-agent-rag-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-rag-debug-chat.yaml @@ -18,6 +18,8 @@ env: - LANGBOT_BACKEND_URL - LANGBOT_LOCAL_AGENT_PIPELINE_URL - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL @@ -29,7 +31,7 @@ automation_env: - LANGBOT_LOCAL_AGENT_RAG_KB_UUID automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME -automation_expected_runner_id: "plugin:langbot/local-agent/default" +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" automation_runner_config_patch_json: '{"knowledge-bases":["${LANGBOT_LOCAL_AGENT_RAG_KB_UUID}"]}' automation_restore_runner_config: "1" automation_reset_debug_chat: "1" @@ -43,7 +45,7 @@ setup_provides_env: - LANGBOT_LOCAL_AGENT_PIPELINE_NAME - LANGBOT_LOCAL_AGENT_RAG_KB_UUID preconditions: - - "The target pipeline already has a text-capable model route that is available for this run." + - "The target pipeline already has a text-capable model route that is available for this run, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." steps: - "Ensure case langrag-kb-retrieve has produced a knowledge base containing sentinel azalea-cobalt-7421." - "Open LANGBOT_FRONTEND_URL." @@ -68,6 +70,8 @@ evidence_required: - backend_log - api_diagnostic diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case; setup will bind the local-agent pipeline to that fake OpenAI-compatible model." + - "The ensure-langrag-sentinel-kb setup script verifies LANGBOT_E2E_RAG_EXPECTED_TEXT, defaulting to azalea-cobalt-7421, so the Debug Chat expected text does not leak into KB setup." - "Use GET /api/v1/pipelines/{uuid} only to confirm the saved runner_config contains the knowledge base uuid." - "If the bot ignores the knowledge base, rerun Retrieve Test before debugging the runner." troubleshooting: diff --git a/skills/skills/langbot-testing/cases/local-agent-steering-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-steering-debug-chat.yaml index 973822f2b..38ed5720e 100644 --- a/skills/skills/langbot-testing/cases/local-agent-steering-debug-chat.yaml +++ b/skills/skills/langbot-testing/cases/local-agent-steering-debug-chat.yaml @@ -29,7 +29,7 @@ automation_env: - LANGBOT_LOCAL_AGENT_PIPELINE_NAME automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME -automation_expected_runner_id: "plugin:langbot/local-agent/default" +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" automation_reset_debug_chat: "1" automation_stream_output: "1" automation_prompt: "You are running the LangBot steering E2E test. First call the qa_plugin_sleep tool with seconds=8 and text=steering-e2e-anchor. Do not answer before the tool result is available. After the tool returns, answer the latest user follow-up. If no follow-up was injected, reply only STEERING_NO_FOLLOWUP." @@ -45,7 +45,7 @@ preconditions: - "The selected model route supports function/tool calling and can call qa_plugin_sleep." steps: - "Open the target local-agent pipeline." - - "Assert the saved runner is plugin:langbot/local-agent/default." + - "Assert the saved runner is plugin:langbot-team/LocalAgent/default." - "Reset the person Debug Chat session for the target pipeline." - "Assert /api/v1/tools contains qa_plugin_sleep." - "Send the first prompt that instructs the model to call qa_plugin_sleep for 8 seconds." @@ -74,7 +74,6 @@ success_patterns: - "Steering" - "Streaming completed" failure_patterns: - - "STEERING_NO_FOLLOWUP" - "Action invoke_llm_stream call timed out" - "All models failed during streaming setup" - "Task exception was never retrieved" diff --git a/skills/skills/langbot-testing/cases/local-agent-streaming-post-commit-failure-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-streaming-post-commit-failure-debug-chat.yaml new file mode 100644 index 000000000..8bc471ed0 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-streaming-post-commit-failure-debug-chat.yaml @@ -0,0 +1,86 @@ +id: local-agent-streaming-post-commit-failure-debug-chat +title: "Local Agent does not fall back after a committed stream fails" +mode: probe +area: pipeline +type: chaos +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - fallback + - streaming + - fake-provider + - fault-injection +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER +automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_debug_chat_load_requests: "1" +automation_debug_chat_load_concurrency: "1" +automation_debug_chat_load_timeout_ms: "30000" +automation_debug_chat_load_max_error_rate: "1" +automation_debug_chat_load_min_error_rate: "1" +automation_debug_chat_load_min_error_count: "1" +automation_debug_chat_load_min_provider_fault_count: "1" +automation_debug_chat_load_require_success: "false" +automation_debug_chat_load_expected_prefix: "POSTCOMMITQA" +automation_debug_chat_load_prompt_template: '请只回复 "{expected}"，不要解释，不要添加其他字符。' +automation_debug_chat_load_stream: "true" +automation_debug_chat_load_reset: "true" +automation_debug_chat_load_fail_on_final_mismatch: "true" +automation_debug_chat_load_provider_model_thresholds_json: '{"qa-post-commit-primary":{"min":1},"qa-post-commit-fallback":{"max":0}}' +automation_fake_provider_pipeline_name: "LangBot QA Post Commit Failure Debug Chat" +automation_fake_provider_model_name: "qa-post-commit-primary" +automation_fake_provider_fallback_model_names: "qa-post-commit-fallback" +automation_fake_provider_fail_models: "" +automation_fake_provider_fail_after_first_chunk_models: "qa-post-commit-primary" +automation_fake_provider_fail_after_first_chunk_delay_ms: "1000" +automation_fake_provider_fail_after_first_chunk_mode: "error_event" +automation_fake_provider_first_token_delay_ms: "25" +automation_fake_provider_chunk_delay_ms: "10" +metrics_thresholds_json: '{"error_rate":{"min":1,"max":1},"fake_provider_fault_count_min":{"min":1},"primary_model_requests":{"min":1},"fallback_model_requests":{"max":0}}' +fault_model_json: '{"provider_fault":"Send a provider error event for qa-post-commit-primary one second after its first content chunk","expected_behavior":"The run terminates with a controlled error and never invokes qa-post-commit-fallback after visible output is committed."}' +setup_automation: + - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" +setup_provides_env: + - LANGBOT_FAKE_PROVIDER_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +steps: + - "Create a QA pipeline with a primary model that sends an error event after one content chunk and a healthy fallback model." + - "Send one streaming Debug Chat request and require a terminal error." + - "Inspect provider request records and prove the fallback model was never invoked." +checks: + - "The fake provider records a mid-stream error event for qa-post-commit-primary." + - "The Debug Chat request terminates as an error instead of timing out or reporting success." + - "qa-post-commit-fallback request count remains zero." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +expected_failures: + - "mid_stream_error_event" + - "failed after first chunk" + - "Agent runner execution failed." +failure_patterns: + - "Agent runner execution failed." + - "Timed out after" + - "qa-post-commit-fallback" +troubleshooting: + - backend-not-listening + - debug-chat-history-contaminates-automation +diagnostics: + - "Use error_event rather than a bare TCP disconnect for this contract: LiteLLM may normalize an abrupt EOF into a final chunk before Local Agent can observe the transport boundary." diff --git a/skills/skills/langbot-testing/cases/local-agent-tool-error-recovery-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-tool-error-recovery-debug-chat.yaml new file mode 100644 index 000000000..8b30ab8f7 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-tool-error-recovery-debug-chat.yaml @@ -0,0 +1,94 @@ +id: local-agent-tool-error-recovery-debug-chat +title: "Local Agent feeds plugin tool errors back to the model" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - plugin + - tools + - tool-error + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" +automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":3,"tool-execution-mode":"serial"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_prompt: "TOOL_ERROR_RECOVERY regression: call qa_plugin_fail with exactly tool-error-recovery-local-agent. After observing the tool error, return only TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed." +automation_expected_text: "TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed" +automation_response_timeout_ms: "180000" +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +preconditions: + - "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "qa-plugin-smoke exposes qa_plugin_fail; the setup case reinstalls the fixture if an older installed package is missing it." +steps: + - "Ensure the local-agent pipeline and qa-plugin-smoke fixture plugin are available." + - "Temporarily bind only langbot-team/LocalAgent plus qa/plugin-smoke and clear RAG knowledge bases." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the TOOL_ERROR_RECOVERY prompt." + - "Verify the failing plugin tool result is fed back into the model and the final answer contains the recovery sentinel." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: The final Bot message contains TOOL_ERROR_RECOVERY_FINAL qa-plugin-smoke forced failure observed." + - "API diagnostic: pipeline-config-diagnostic.json shows max-tool-iterations=3 and knowledge-bases cleared." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound." + - "Logs: Backend shows the request completed without runner.timeout, runner.tool_error run failure, or model setup failure." + - "Console: No unexpected frontend runtime errors appear during Debug Chat." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The fake provider calls qa_plugin_fail once, then only returns the final sentinel after it sees an Error: tool result." + - "If the run returns a generic answer, inspect whether qa_plugin_fail was exposed by /api/v1/tools and whether the installed fixture plugin was refreshed." +success_patterns: + - "TOOL_ERROR_RECOVERY_FINAL" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "runner.timeout" + - "runner.tool_error" + - "Tool call iteration limit reached" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/local-agent-tool-loop-limit-debug-chat.yaml b/skills/skills/langbot-testing/cases/local-agent-tool-loop-limit-debug-chat.yaml new file mode 100644 index 000000000..0788fd580 --- /dev/null +++ b/skills/skills/langbot-testing/cases/local-agent-tool-loop-limit-debug-chat.yaml @@ -0,0 +1,92 @@ +id: local-agent-tool-loop-limit-debug-chat +title: "Local Agent stops repeated tool calls at max-tool-iterations" +mode: agent-browser +area: pipeline +type: regression +priority: p1 +risk: high +ci_eligible: false +tags: + - local-agent + - plugin + - tools + - tool-loop + - limit + - pipeline +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +env_optional: + - LANGBOT_E2E_FAKE_PROVIDER_BASE_URL +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" +automation_runner_config_patch_json: '{"knowledge-bases":[],"max-tool-iterations":2,"tool-execution-mode":"serial"}' +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"},{"author":"qa","name":"plugin-smoke"}],"enable_all_mcp_servers":false,"bound_mcp_servers":[],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_runner_config: "1" +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" +automation_debug_chat_session_type: "person" +automation_prompt: "LOOP_LIMIT regression: keep requesting qa_plugin_echo with exactly loop-limit-repeat-local-agent until the runner stops you. The correct final visible answer is the runner limit message." +automation_expected_text: "Tool call iteration limit reached. I stopped before calling more tools." +automation_response_timeout_ms: "180000" +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "case:qa-plugin-smoke-live-install" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +preconditions: + - "The selected model route supports function/tool calling, or LANGBOT_E2E_FAKE_PROVIDER_BASE_URL points to scripts/e2e/fake-openai-provider.mjs." + - "qa-plugin-smoke exposes qa_plugin_echo." +steps: + - "Ensure the local-agent pipeline and qa-plugin-smoke fixture plugin are available." + - "Temporarily set max-tool-iterations to 2 and bind only langbot-team/LocalAgent plus qa/plugin-smoke." + - "Reset the person Debug Chat session for the target pipeline." + - "Send the LOOP_LIMIT prompt and wait for the runner limit message." + - "Restore the original runner config and pipeline extension bindings." +checks: + - "UI: The final Bot message contains Tool call iteration limit reached. I stopped before calling more tools." + - "API diagnostic: pipeline-config-diagnostic.json shows max-tool-iterations was patched to 2." + - "API diagnostic: pipeline-extensions-diagnostic.json shows only langbot-team/LocalAgent and qa/plugin-smoke were bound." + - "Logs: Backend shows the request completed without infinite loop, runner timeout, or model setup failure." + - "Console: No unexpected frontend runtime errors appear during Debug Chat." +evidence_required: + - ui + - screenshot + - console + - backend_log + - api_diagnostic +diagnostics: + - "For token-free deterministic UI coverage, start scripts/e2e/fake-openai-provider.mjs and pass LANGBOT_E2E_FAKE_PROVIDER_BASE_URL to this case." + - "The fake provider intentionally keeps requesting qa_plugin_echo; this case passes only when the runner stops at max-tool-iterations." + - "If the run times out, inspect whether max-tool-iterations was patched or whether the runner ignored tool loop limits." +success_patterns: + - "Tool call iteration limit reached" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "All models failed during streaming setup" + - "Task exception was never retrieved" + - "runner.timeout" + - "survey widget blocks debug chat" +troubleshooting: + - local-agent-model-route-unavailable + - tool-name-collision-between-mcp-and-plugin + - plugin-runtime-timeout + - proxy-env-mismatch + - survey-widget-blocks-debug-chat + - debug-chat-history-contaminates-automation diff --git a/skills/skills/langbot-testing/cases/mcp-stdio-register.yaml b/skills/skills/langbot-testing/cases/mcp-stdio-register.yaml index 6c9857ba9..3184e8ffe 100644 --- a/skills/skills/langbot-testing/cases/mcp-stdio-register.yaml +++ b/skills/skills/langbot-testing/cases/mcp-stdio-register.yaml @@ -34,8 +34,10 @@ steps: - "Poll /api/v1/tools and /api/v1/mcp/servers/qa-local-stdio until qa_mcp_echo is visible." checks: - "API diagnostic: qa-local-stdio runtime_status is connected." + - "API diagnostic: qa-local-stdio server_uuid is present for pipeline extension binding." - "API diagnostic: runtime_tool_names includes qa_mcp_echo." - "API diagnostic: /api/v1/tools includes qa_mcp_echo." + - "Automation writes LANGBOT_MCP_QA_STDIO_SERVER_UUID into skills/.env.local for downstream tool-call cases." - "Console and network logs contain no unexpected frontend/runtime failures." evidence_required: - screenshot diff --git a/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml b/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml index 593eecba3..7f2827136 100644 --- a/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml +++ b/skills/skills/langbot-testing/cases/mcp-stdio-tool-call.yaml @@ -16,26 +16,33 @@ skills: env: - LANGBOT_FRONTEND_URL - LANGBOT_BACKEND_URL - - LANGBOT_LOCAL_AGENT_PIPELINE_URL - - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME automation: scripts/e2e/pipeline-debug-chat.mjs automation_env: - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL - LANGBOT_BROWSER_PROFILE - LANGBOT_CHROMIUM_EXECUTABLE - - LANGBOT_LOCAL_AGENT_PIPELINE_URL - - LANGBOT_LOCAL_AGENT_PIPELINE_NAME -automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL -automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME + - LANGBOT_MCP_QA_STDIO_SERVER_UUID +automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_expected_runner_id: "plugin:langbot-team/LocalAgent/default" +automation_extensions_patch_json: '{"enable_all_plugins":false,"bound_plugins":[{"author":"langbot-team","name":"LocalAgent"}],"enable_all_mcp_servers":false,"bound_mcp_servers":["${LANGBOT_MCP_QA_STDIO_SERVER_UUID}"],"enable_all_skills":false,"bound_skills":[]}' +automation_restore_extensions: "1" +automation_reset_debug_chat: "1" automation_prompt: "Call the qa_mcp_echo MCP tool with exactly this text: mcp-ok-local-agent. Return only the tool result." automation_expected_text: "qa_mcp_echo:mcp-ok-local-agent" -automation_response_timeout_ms: "180000" +automation_response_timeout_ms: "60000" setup_automation: - - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" - "case:mcp-stdio-register" setup_provides_env: - - LANGBOT_LOCAL_AGENT_PIPELINE_URL - - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME + - LANGBOT_MCP_QA_STDIO_SERVER_UUID failure_patterns: - "qa-plugin-smoke:mcp-ok-local-agent" - "qa_echo:mcp-ok-local-agent" @@ -45,7 +52,7 @@ failure_patterns: - "no available channel for model" preconditions: - "box.local.allowed_mount_roots includes the bundled MCP fixture directory when LangBot runs stdio MCP servers through Box." - - "The selected model route supports function/tool calling." + - "The target is a local test instance where the QA-owned fake provider/model/pipeline may be created or updated." steps: - "Open LANGBOT_FRONTEND_URL." - "Navigate to MCP Servers." @@ -56,8 +63,8 @@ steps: - "Submit the server." - "Confirm the server detail page shows Tools: 1 and qa_mcp_echo." - "Open the target local-agent pipeline." - - "Use runner Default or the pluginized langbot/local-agent runner." - - "Select a model with function-calling ability that is known to work with tools in the current environment." + - "Use runner Default or the pluginized langbot-team/LocalAgent runner." + - "Use the QA fake-provider pipeline prepared by setup automation." - "Open Debug Chat." - "Send: Call the qa_mcp_echo tool with exactly this text: mcp-ok-local-agent. Return only the tool result." checks: @@ -65,14 +72,18 @@ checks: - "UI: The MCP detail page shows qa_mcp_echo." - "API diagnostic: /api/v1/tools contains qa_mcp_echo." - "UI: Debug Chat bot response contains qa_mcp_echo:mcp-ok-local-agent." - - "Logs: Backend logs show the MCP tool call was executed, not only listed." + - "API diagnostic: pipeline-extensions-diagnostic.json shows the qa-local-stdio server UUID is the only bound MCP server for the run." - "Console: No unexpected frontend errors appear during MCP form use or Debug Chat." evidence_required: - ui + - screenshot - console - - backend_log + - network - api_diagnostic + - metrics diagnostics: + - "The required release case uses the QA fake-provider pipeline so model instruction-following variance cannot hide or mimic an MCP bridge regression." + - "Run a separate optional live-provider tool smoke when evaluating a specific provider/model route." - "Run node scripts/e2e/mcp-stdio-fixture.mjs to verify the bundled stdio fixture can list and call qa_mcp_echo without involving a model provider." - "Run node scripts/e2e/mcp-stdio-register.mjs to upsert qa-local-stdio in LangBot and verify /api/v1/tools exposes qa_mcp_echo." - "If backend logs show host_path is outside allowed_mount_roots, add the fixture directory to box.local.allowed_mount_roots in the local LangBot data config." @@ -84,5 +95,6 @@ troubleshooting: - mcp-stdio-args-not-applied - tool-name-collision-between-mcp-and-plugin - local-agent-model-route-unavailable + - local-agent-mcp-text-result-empty-follow-up - survey-widget-blocks-debug-chat - plugin-runtime-timeout diff --git a/skills/skills/langbot-testing/cases/qa-plugin-smoke-live-install.yaml b/skills/skills/langbot-testing/cases/qa-plugin-smoke-live-install.yaml index f0ec3c336..ff01f68c0 100644 --- a/skills/skills/langbot-testing/cases/qa-plugin-smoke-live-install.yaml +++ b/skills/skills/langbot-testing/cases/qa-plugin-smoke-live-install.yaml @@ -20,14 +20,14 @@ env: automation: scripts/e2e/install-qa-plugin-smoke.mjs automation_plugin_package: "skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg" automation_expected_plugin_id: "qa/plugin-smoke" -automation_expected_tool: "qa_plugin_echo" +automation_expected_tool: "qa_plugin_echo,qa_plugin_fail" steps: - "Run `rtk bin/lbs test run qa-plugin-smoke-live-install --dry-run` first; remove `--dry-run` only after readiness points at a local test LangBot instance." - "Automation authenticates the local test user, uploads the QA plugin smoke .lbpkg package when missing, waits for install, and checks tool exposure." checks: - "automation-result.json status is pass." - "/api/v1/plugins lists qa/plugin-smoke after install." - - "/api/v1/tools lists qa_plugin_echo after install." + - "/api/v1/tools lists qa_plugin_echo and qa_plugin_fail after install." evidence_required: - api_diagnostic - filesystem @@ -39,6 +39,7 @@ success_patterns: failure_patterns: - "Plugin install task did not complete successfully" - "qa_plugin_echo is not listed" + - "qa_plugin_fail is not listed" troubleshooting: - plugin-runtime-timeout - plugin-dependency-install-offline diff --git a/skills/skills/langbot-testing/cases/sandbox-skill-authoring-edit-existing-e2e.yaml b/skills/skills/langbot-testing/cases/sandbox-skill-authoring-edit-existing-e2e.yaml index af4c383a3..34590804f 100644 --- a/skills/skills/langbot-testing/cases/sandbox-skill-authoring-edit-existing-e2e.yaml +++ b/skills/skills/langbot-testing/cases/sandbox-skill-authoring-edit-existing-e2e.yaml @@ -32,9 +32,15 @@ automation_env: - LANGBOT_LOCAL_AGENT_PIPELINE_NAME automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME -automation_expected_text: "UPDATED_MARKER_EDIT_REGRESSION" -automation_prompts_json: '[{"prompt":"Skill name is lb-skill-edit-regression. Use exactly one exec tool call with workdir=/workspace. In that single command, remove old /workspace/lb-skill-edit-regression, create SKILL.md, scripts/use.py, and data/input.json. The JSON must contain numbers [1,2,3,4], factors [2,3,4], and marker CREATE_MARKER_EDIT_REGRESSION. The Python script must read data/input.json relative to __file__ and print exactly SANDBOX_SKILL_CREATE_OK sum=10 product=24 marker=CREATE_MARKER_EDIT_REGRESSION. Run python3 /workspace/lb-skill-edit-regression/scripts/use.py in the same command. After the exec succeeds, final answer exactly CREATE_OK:lb-skill-edit-regression.","expected_text":"CREATE_MARKER_EDIT_REGRESSION","response_timeout_ms":"180000"},{"prompt":"Continue with the same skill name lb-skill-edit-regression. Strictly call register_skill with path=/workspace/lb-skill-edit-regression and name=lb-skill-edit-regression, then call activate with skill_name=lb-skill-edit-regression. Do not call exec, read, write, edit, grep, or ls in this step. After both tools succeed, final answer exactly REGISTER_ACTIVATE_OK:lb-skill-edit-regression.","expected_text":"REGISTER_ACTIVATE_OK:lb-skill-edit-regression","response_timeout_ms":"180000"},{"prompt":"Continue with the same activated skill lb-skill-edit-regression. Use exactly one exec tool call with workdir=/workspace/.skills/lb-skill-edit-regression. In that single exec command, run exactly this shell logic: set -e; overwrite SKILL.md with a heading and UPDATED_MARKER_EDIT_REGRESSION; overwrite data/input.json with numbers [5,6], factors [7,8], and marker UPDATED_MARKER_EDIT_REGRESSION; overwrite scripts/use.py so it reads data/input.json relative to __file__ and prints exactly SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION; run python3 scripts/use.py. Do not write exit(1). Do not call find, ls, read, grep as separate tools. If the single exec exits successfully and prints SANDBOX_SKILL_MODIFIED_OK, do not call more tools. Final answer exactly E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK.","expected_text":"UPDATED_MARKER_EDIT_REGRESSION","response_timeout_ms":"240000"}]' -automation_filesystem_checks_json: '[{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/SKILL.md","contains":"UPDATED_MARKER_EDIT_REGRESSION"},{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/data/input.json","contains":["UPDATED_MARKER_EDIT_REGRESSION","\"numbers\": [5, 6]","\"factors\": [7, 8]"]},{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/scripts/use.py","contains":["SANDBOX_SKILL_MODIFIED_OK"],"not_contains":["SANDBOX_SKILL_CREATE_OK","exit(1)"]},{"argv":["python3","scripts/use.py"],"cwd":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression","stdout_contains":"SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION","exit_code":0}]' +automation_expected_text: "E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK" +automation_prompts_json: '[{"prompt":"Skill name is lb-skill-edit-regression. Use exactly one exec tool call with workdir=/workspace. In that single command, remove old /workspace/lb-skill-edit-regression, create SKILL.md, scripts/use.py, and data/input.json. The JSON must contain numbers [1,2,3,4], factors [2,3,4], and marker CREATE_MARKER_EDIT_REGRESSION. The Python script must read data/input.json relative to __file__ and print exactly SANDBOX_SKILL_CREATE_OK sum=10 product=24 marker=CREATE_MARKER_EDIT_REGRESSION. Run python3 /workspace/lb-skill-edit-regression/scripts/use.py in the same command. After the exec succeeds, final answer exactly CREATE_OK:lb-skill-edit-regression.","expected_text":"CREATE_OK:lb-skill-edit-regression","response_timeout_ms":"180000"},{"prompt":"Continue with the same skill name lb-skill-edit-regression. Strictly call register_skill with path=/workspace/lb-skill-edit-regression and name=lb-skill-edit-regression, then call activate with skill_name=lb-skill-edit-regression. Do not call exec, read, write, edit, grep, or ls in this step. After both tools succeed, final answer exactly REGISTER_ACTIVATE_OK:lb-skill-edit-regression.","expected_text":"REGISTER_ACTIVATE_OK:lb-skill-edit-regression","response_timeout_ms":"180000"},{"prompt":"Continue with the same activated skill lb-skill-edit-regression. Use exactly one exec tool call with workdir=/workspace/.skills/lb-skill-edit-regression. In that single exec command, run exactly this shell logic: set -e; overwrite SKILL.md with a heading and UPDATED_MARKER_EDIT_REGRESSION; overwrite data/input.json with numbers [5,6], factors [7,8], and marker UPDATED_MARKER_EDIT_REGRESSION; overwrite scripts/use.py so it reads data/input.json relative to __file__ and prints exactly SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION; run python3 scripts/use.py. Do not write exit(1). Do not call find, ls, read, grep as separate tools. If the single exec exits successfully and prints SANDBOX_SKILL_MODIFIED_OK, do not call more tools. Final answer exactly E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK.","expected_text":"E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK","response_timeout_ms":"240000"}]' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + - "node:scripts/e2e/reset-sandbox-skill.mjs --name=lb-skill-edit-regression" +setup_provides_env: + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_filesystem_checks_json: '[{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/SKILL.md","contains":"UPDATED_MARKER_EDIT_REGRESSION"},{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/data/input.json","json_equals":{"numbers":[5,6],"factors":[7,8],"marker":"UPDATED_MARKER_EDIT_REGRESSION"}},{"path":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression/scripts/use.py","contains":["SANDBOX_SKILL_MODIFIED_OK"],"not_contains":["SANDBOX_SKILL_CREATE_OK","exit(1)"]},{"argv":["python3","scripts/use.py"],"cwd":"${LANGBOT_REPO}/data/box/skills/lb-skill-edit-regression","stdout_contains":"SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION","exit_code":0}]' automation_stream_output: "0" automation_reset_debug_chat: "1" automation_response_timeout_ms: "420000" @@ -48,9 +54,9 @@ steps: - "Run the automation prompt or the equivalent prompt pattern from references/sandbox-skill-authoring.md." - "Capture the visible final response, backend logs, and the registered skill-store files." checks: - - "UI: A Bot message, not only the user prompt, contains UPDATED_MARKER_EDIT_REGRESSION." + - "UI: A Bot message, not only the user prompt, contains E2E_OK:lb-skill-edit-regression:SANDBOX_SKILL_MODIFIED_OK." - "Logs: The model called exec, register_skill, activate, then a second exec whose workdir is /workspace/.skills/lb-skill-edit-regression." - - "Logs: The second exec stdout contains SANDBOX_SKILL_MODIFIED_OK and GREP_ALL_OK or equivalent grep success." + - "Logs: The second exec stdout contains SANDBOX_SKILL_MODIFIED_OK." - "Automation: Filesystem checks pass for the registered skill-store files under LANGBOT_REPO/data/box/skills/lb-skill-edit-regression." - "Skill store: SKILL.md and data/input.json contain UPDATED_MARKER_EDIT_REGRESSION." - "Skill store: scripts/use.py is the modified script and prints SANDBOX_SKILL_MODIFIED_OK sum=11 product=56 marker=UPDATED_MARKER_EDIT_REGRESSION." diff --git a/skills/skills/langbot-testing/cases/skill-discovery-via-mcp-gateway.yaml b/skills/skills/langbot-testing/cases/skill-discovery-via-mcp-gateway.yaml new file mode 100644 index 000000000..dfd77653d --- /dev/null +++ b/skills/skills/langbot-testing/cases/skill-discovery-via-mcp-gateway.yaml @@ -0,0 +1,58 @@ +id: skill-discovery-via-mcp-gateway +title: "External harness discovers LangBot skills via langbot_list_assets (all-tool model)" +mode: agent-browser +area: sandbox +type: regression +priority: p2 +risk: medium +ci_eligible: false +tags: + - skills + - mcp-gateway + - acp-agent-runner + - all-tool-model + - tools +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_ACP_RUNNER_PIPELINE_URL + - LANGBOT_ACP_RUNNER_PIPELINE_NAME +preconditions: + - "An external-harness runner pipeline (e.g. ACP remote claude-code) is configured with langbot-assets-enabled=true so the LangBot MCP gateway is exposed to the harness." + - "The remote harness (claude-code) is reachable and responsive (claude -p returns within the runner timeout)." + - "At least one pipeline-visible skill exists in the Box skill store (otherwise the count is 0, which is still a valid pass for the discovery surface)." +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_ACP_RUNNER_PIPELINE_URL + - LANGBOT_ACP_RUNNER_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_ACP_RUNNER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_ACP_RUNNER_PIPELINE_NAME +automation_prompt: "You have LangBot tools available via an MCP server (tools prefixed langbot_). Call langbot_list_assets with asset_types = [\"skills\",\"tools\"]. Then reply with one single line: the literal token PROBEDONE, a space, the number of skills you found, a space, and the number of tools you found." +automation_expected_text: "PROBEDONE" +automation_response_timeout_ms: "540000" +steps: + - "Open LANGBOT_FRONTEND_URL and navigate to the external-harness (ACP) pipeline." + - "Open Debug Chat with langbot-assets-enabled on the runner." + - "Send the automation_prompt asking the harness to call langbot_list_assets with asset_types [skills, tools]." + - "Capture the final reply, backend logs, and the MCP gateway call trace." +checks: + - "UI: final reply contains PROBEDONE followed by a skill count and a tool count." + - "Logs: backend shows the harness invoked langbot_list_assets and the response included a 'skills' asset class (this is the all-tool-model discovery surface added on this branch)." + - "Behavior parity: a local-agent runner reaches the same skills via use_funcs / activate; the external harness reaches them via langbot_list_assets + langbot_call_tool." +evidence_required: + - ui + - screenshot + - backend_log +expected_failures: + - "runner.timeout when the remote claude-code harness is unauthenticated or slow to start — this is an environment issue, not a discovery-surface regression." +diagnostics: + - "If runner.timeout: ssh into the harness host and confirm `claude -p 'hi'` returns quickly; the ACP runner cannot complete until the harness responds." + - "Activated-skill OPERATE on docker+shared-fs is tracked separately by issue #2271 and is out of scope for this discovery case." +troubleshooting: + - sandbox-native-tools-unavailable diff --git a/skills/skills/langbot-testing/cases/wizard-onebot-agent-runtime.yaml b/skills/skills/langbot-testing/cases/wizard-onebot-agent-runtime.yaml new file mode 100644 index 000000000..220c5b3f5 --- /dev/null +++ b/skills/skills/langbot-testing/cases/wizard-onebot-agent-runtime.yaml @@ -0,0 +1,61 @@ +id: wizard-onebot-agent-runtime +title: "Quick Start runs a real OneBot member event through an Agent" +mode: agent-browser +area: wizard +type: feature +priority: p0 +risk: high +ci_eligible: false +tags: + - wizard + - eba + - onebot + - agent-runner + - release-gate +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE +automation: scripts/e2e/wizard-onebot-agent-runtime.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE +preconditions: + - "The target is a local test instance where temporary Bots and Agents may be created and deleted." + - "Local Agent is installed so the scenario prompt can be verified in its prompt editor." + - "The QA Deterministic Runner is installed and registered as plugin:qa/agent-runner/default." + - "The LangBot virtualenv includes the websockets package used by the local OneBot probe." +steps: + - "Open Quick Start and choose Welcome new members with the OneBot v11 channel." + - "Create and enable a temporary OneBot Bot on an available local port." + - "Select Local Agent and confirm the editable welcome prompt is prefilled." + - "Choose QA Deterministic Runner and finish the wizard." + - "Connect a local OneBot reverse WebSocket and send a group_increase notice." + - "Confirm the Agent output is sent back with send_group_msg." + - "Open the Bot route editor and confirm the route status is Delivered." +checks: + - "UI: The wizard reaches All Set after creating the Agent route." + - "UI: Local Agent receives a scenario-specific editable welcome prompt." + - "Runtime: OneBot group_increase converts to group.member_joined." + - "Runtime: The saved Agent route reaches delivered status." + - "Runtime: The deterministic Agent reply invokes send_group_msg." + - "UI: Delivered is visible on the saved route card." + - "Console: No unexpected frontend errors appear during the flow." + - "Cleanup: The temporary Bot and Agent are deleted." +evidence_required: + - ui + - screenshot + - console + - api_diagnostic +diagnostics: + - "The case uses a local reverse WebSocket and deterministic runner, so it requires no platform or model credentials." + - "A passing API route status without the visible Delivered route card is not a pass." +troubleshooting: + - backend-not-listening + - plugin-runtime-timeout + - proxy-env-mismatch diff --git a/skills/skills/langbot-testing/cases/wizard-runner-marketplace-catalog.yaml b/skills/skills/langbot-testing/cases/wizard-runner-marketplace-catalog.yaml new file mode 100644 index 000000000..810dede44 --- /dev/null +++ b/skills/skills/langbot-testing/cases/wizard-runner-marketplace-catalog.yaml @@ -0,0 +1,57 @@ +id: wizard-runner-marketplace-catalog +title: "Quick Start installs a published Runner on a clean instance" +mode: agent-browser +area: wizard +type: feature +priority: p0 +risk: high +ci_eligible: false +tags: + - wizard + - agent-runner + - marketplace + - clean-install + - productization +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_REPO +automation: scripts/e2e/wizard-runner-marketplace-catalog.mjs +automation_env: + - LANGBOT_REPO + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE +preconditions: + - "The local checkout can start an isolated backend and Vite frontend and can reach LangBot Space." +steps: + - "Start an isolated first-run instance and confirm zero installed plugins and zero registered runners." + - "Resume Quick Start at the AI Engine step with a temporary disabled Bot." + - "Confirm the browser requests Marketplace plugins with component_filter=Runner." + - "Confirm langbot-team/LocalAgent is published with an installable version and the Runner Extensions link is correct." + - "Install LocalAgent and wait for plugin initialization and Runner registration." + - "Confirm Create & Deploy is disabled before installation and enabled after LocalAgent is selected." + - "Verify the layout at desktop and mobile widths." +checks: + - "API: The instance has zero installed plugins and zero registered runners." + - "API: The instance wizard status is none." + - "Network: Marketplace search uses component_filter=Runner and type_filter=plugin." + - "UI: The AI Engine step displays the published langbot-team/LocalAgent card." + - "Marketplace: LocalAgent includes latest_version so installation can proceed." + - "Runtime: LocalAgent installs and registers plugin:langbot-team/LocalAgent/default." + - "UI: Browse Runner Extensions links to the Runner-filtered market." + - "UI: Create & Deploy transitions from disabled to enabled only after Runner selection." + - "Console: No unexpected frontend errors appear during the flow." + - "Cleanup: Wizard progress and the temporary Bot are removed." +evidence_required: + - ui + - screenshot + - console + - api_diagnostic +diagnostics: + - "The automation owns a temporary data root and random ports; it must not reuse or reset the developer instance." + - "A Marketplace API response alone is not a pass; LocalAgent must install and register through the visible Wizard flow." +troubleshooting: + - backend-not-listening + - plugin-runtime-timeout + - proxy-env-mismatch diff --git a/skills/skills/langbot-testing/cases/wizard-scenario-routing.yaml b/skills/skills/langbot-testing/cases/wizard-scenario-routing.yaml new file mode 100644 index 000000000..64c9bd31c --- /dev/null +++ b/skills/skills/langbot-testing/cases/wizard-scenario-routing.yaml @@ -0,0 +1,57 @@ +id: wizard-scenario-routing +title: "Quick Start filters channels by the selected bot scenario" +mode: agent-browser +area: wizard +type: feature +priority: p0 +risk: medium +ci_eligible: false +tags: + - wizard + - eba + - event-routing + - productization +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE +automation: scripts/e2e/wizard-scenario-routing.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE +preconditions: + - "The target is a local test instance where a disabled temporary Bot and Agent may be created and deleted." +steps: + - "Open Quick Start and select Reply to messages." + - "Confirm HTTP Bot is available for the message scenario." + - "Select Welcome new members." + - "Confirm a compatible channel remains visible and HTTP Bot is removed." + - "Create a disabled temporary Bot and Agent, persist a group.member_joined route, and read it back through the API." + - "Verify the scenario and channel layout at desktop and mobile widths." +checks: + - "UI: Scenario selection appears before channel selection." + - "UI: Reply to messages includes HTTP Bot." + - "UI: Welcome new members includes a compatible event-capable channel." + - "UI: Welcome new members excludes HTTP Bot because it does not declare group.member_joined support." + - "UI: The selected non-message scenario is labeled as an Agent behavior." + - "Console: No unexpected frontend errors appear during the flow." + - "Network: Adapter discovery returns without a 5xx response." + - "API: A non-message event route to an Agent is normalized and persisted." + - "Cleanup: The temporary Bot and Agent are deleted after evidence is collected." + - "Isolation: Wizard progress writes are stubbed in the browser and do not change backend state." +evidence_required: + - ui + - screenshot + - console + - api_diagnostic +diagnostics: + - "The temporary Bot remains disabled, so the case cannot connect to an external messaging platform." + - "A passing adapter API response without the visible filtered channel list is not a pass." +troubleshooting: + - backend-not-listening + - proxy-env-mismatch diff --git a/skills/skills/langbot-testing/cases/workspace-compatibility-preflight.yaml b/skills/skills/langbot-testing/cases/workspace-compatibility-preflight.yaml new file mode 100644 index 000000000..3303cf29d --- /dev/null +++ b/skills/skills/langbot-testing/cases/workspace-compatibility-preflight.yaml @@ -0,0 +1,38 @@ +id: workspace-compatibility-preflight +title: "LangBot workspace repositories and local SDK are mutually compatible" +mode: probe +area: release +type: smoke +priority: p0 +risk: high +ci_eligible: true +tags: + - workspace + - compatibility + - preflight + - sdk +skills: + - langbot-testing +automation: scripts/e2e/workspace-compatibility-preflight.mjs +steps: + - "Discover the active LangBot checkout and sibling workspace repositories, with LANGBOT_WORKSPACE_ROOT and repository-specific env overrides available for non-default layouts." + - "Verify every checkout is on main or dev/4.11.x and record dirty worktrees as warnings only." + - "Validate plugin manifest identities for LocalAgent, Control Plane, LongTermMemory, GeneralParsers, LangRAG, and Skill Authoring." + - "Use the LangBot virtualenv to import langbot_plugin and Runner Protocol entities from the workspace SDK source tree." +checks: + - "workspace-preflight.json contains no failed checks." + - "The langbot_plugin import path is inside the discovered langbot-plugin-sdk/src directory." + - "Dirty worktrees are visible as warnings and do not hide compatibility failures." +evidence_required: + - filesystem + - api_diagnostic +diagnostics: + - "A branch failure means the checkout is outside the supported main/dev/4.11.x workspace combination." + - "A local-sdk-import failure usually means uv restored the pinned PyPI wheel or the workspace SDK path is wrong." +success_patterns: + - "Workspace compatibility preflight passed" +failure_patterns: + - "blocking check" + - "did not load from the workspace SDK" +troubleshooting: + - uv-run-resyncs-local-sdk diff --git a/skills/skills/langbot-testing/cases/workspace-plugin-pages-smoke.yaml b/skills/skills/langbot-testing/cases/workspace-plugin-pages-smoke.yaml new file mode 100644 index 000000000..1129599eb --- /dev/null +++ b/skills/skills/langbot-testing/cases/workspace-plugin-pages-smoke.yaml @@ -0,0 +1,55 @@ +id: workspace-plugin-pages-smoke +title: "Major workspace plugin pages render and answer read-only Page APIs" +mode: agent-browser +area: plugin +type: smoke +priority: p0 +risk: high +ci_eligible: false +tags: + - workspace + - plugin + - page + - browser +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL +automation: scripts/e2e/workspace-plugin-pages-smoke.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE +preconditions: + - "Control Plane, LongTermMemory, LangRAG, and Skill Authoring are installed and initialized in the selected local test instance." + - "The browser profile is authenticated for the same LangBot backend, or LANGBOT_E2E_LOGIN_USER is configured for local recovery login." +steps: + - "Open each /home/plugin-pages route through the LangBot WebUI." + - "Verify the iframe visibly renders Agent Control Plane, Memory Console, LangRAG Observability, and Skill Authoring." + - "Using the same browser login, call the read-only Page APIs /health, /summary, /snapshot, and /health respectively." + - "Capture one screenshot per page plus browser console/network diagnostics." +checks: + - "Every page has ui_status pass and api_status pass." + - "No blank iframe, Page API error, frontend uncaught exception, or HTTP 5xx is recorded." + - "No plugin is silently skipped when missing; fixture absence is blocked." +evidence_required: + - ui + - screenshot + - console + - network + - api_diagnostic +diagnostics: + - "blocked means one or more required plugins are not installed in the selected test instance." + - "A UI-only failure with a passing Page API usually points to plugin asset routing, iframe loading, or frontend page discovery." +success_patterns: + - "All workspace plugin pages rendered" +failure_patterns: + - "Required plugin page fixture" + - "Page API" + - "Browser diagnostics found" +troubleshooting: + - plugin-runtime-timeout + - backend-not-listening diff --git a/skills/skills/langbot-testing/cases/workspace-repository-contracts.yaml b/skills/skills/langbot-testing/cases/workspace-repository-contracts.yaml new file mode 100644 index 000000000..aaaf367a6 --- /dev/null +++ b/skills/skills/langbot-testing/cases/workspace-repository-contracts.yaml @@ -0,0 +1,36 @@ +id: workspace-repository-contracts +title: "Changed LangBot workspace repositories pass their contract suites" +mode: probe +area: release +type: regression +priority: p0 +risk: high +ci_eligible: true +tags: + - workspace + - contracts + - pytest + - sdk +skills: + - langbot-testing +automation: scripts/e2e/workspace-repository-contracts.mjs +steps: + - "Resolve the active LangBot virtualenv and workspace SDK source path." + - "Run tests independently for Runner, Control Plane, LongTermMemory, Parser, RAG, Skill Authoring, LocalAgent, LangBot Agent/Provider, the skills CLI, and SDK runtime contracts." + - "Run SDK packaging blackbox separately so an isolated build dependency network failure is classified as env_issue without masking product test failures." + - "Write per-repository stdout, stderr, status, and duration under repository-contracts/." +checks: + - "Every product contract group has status pass." + - "A packaging env_issue is recorded separately and does not convert a product failure into a pass." + - "Every command uses the LangBot virtualenv and local SDK PYTHONPATH." +evidence_required: + - filesystem + - metrics +diagnostics: + - "Inspect the named repository stdout/stderr logs before rerunning a broader suite." + - "Packaging dependency fetch errors belong to environment readiness; assertion/import/runtime failures belong to the product." +success_patterns: + - "All repository product contracts passed" +failure_patterns: + - "repository product contract group" + - "Test command timed out" diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py new file mode 100644 index 000000000..8ca09e680 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/verify.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +def has_initial_failure_section(report: str) -> bool: + folded = report.casefold() + if any( + marker in folded + for marker in ( + "initial fail", + "initially fail", + "baseline fail", + ) + ): + return True + + baseline_markers = ("baseline test", "before any edit", "before editing") + failure_markers = ("failing test", "failed test", "failures=", "errors=") + return any(marker in folded for marker in baseline_markers) and any( + marker in folded for marker in failure_markers + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--workspace", required=True) + args = parser.parse_args() + workspace = Path(args.workspace).resolve() + baseline = Path(__file__).resolve().parent / "workspace" + + for relative in ("TASK.md", "tests/test_order_engine.py", "acceptance_check.py"): + if (workspace / relative).read_bytes() != (baseline / relative).read_bytes(): + raise AssertionError(f"protected fixture changed: {relative}") + + tests = subprocess.run( + [sys.executable, "-m", "unittest", "discover", "-s", "tests", "-v"], + cwd=workspace, + text=True, + capture_output=True, + timeout=60, + ) + if tests.returncode != 0: + raise AssertionError(f"unit tests failed:\n{tests.stdout}\n{tests.stderr}") + + acceptance = subprocess.run( + [sys.executable, "acceptance_check.py"], + cwd=workspace, + text=True, + capture_output=True, + timeout=30, + ) + if acceptance.returncode != 0 or "ACCEPTANCE_PASS" not in acceptance.stdout: + raise AssertionError(f"acceptance failed:\n{acceptance.stdout}\n{acceptance.stderr}") + + result = json.loads((workspace / "result.json").read_text(encoding="utf-8")) + assert result["status"] == "pass" + assert result["tests_run"] == 12 + assert result["acceptance"] == "PASS" + report = (workspace / "AGENT_REPORT.md").read_text(encoding="utf-8") + folded_report = report.casefold() + assert has_initial_failure_section(report), "report missing initial failure section" + assert "root causes" in folded_report, "report missing section: root causes" + assert any( + heading in folded_report + for heading in ( + "changed files", + "files changed", + "files modified", + "modified files", + "changes made", + ) + ), "report missing section: changed files" + assert "verification" in folded_report, "report missing section: verification" + assert report.rstrip().endswith("COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS") + print("HOST_VERIFY_PASS tests=12 acceptance=PASS protected=PASS") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/TASK.md b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/TASK.md new file mode 100644 index 000000000..223c61b00 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/TASK.md @@ -0,0 +1,52 @@ +# Order Orchestrator Repair Task + +Repair the production code in `order_engine/` so the project satisfies all requirements below. + +## Required workflow + +1. Inspect the project and run the test suite before editing. +2. Record the initial failing test names in `AGENT_REPORT.md`. +3. Modify production code only. Do not edit `TASK.md`, `tests/`, or `acceptance_check.py`. +4. Re-run the full test suite after each logical fix until all 12 tests pass. +5. Run `python3 acceptance_check.py` and require `ACCEPTANCE_PASS`. +6. Write `result.json` and finish `AGENT_REPORT.md` with root causes, changed files, and verification commands. + +Use these commands from `/workspace/order-orchestrator`: + +```bash +python3 -m unittest discover -s tests -v +python3 acceptance_check.py +``` + +## Business rules + +- Monetary input must be converted through `Decimal(str(value))`. +- Every public monetary result is rounded to two decimals with `ROUND_HALF_UP`. +- Supported customer tiers are `standard` (0%), `vip` (10%), and `premium` (15%). Unknown tiers raise `ValueError`. +- Discount is calculated from the rounded subtotal. Tax is calculated from the subtotal after discount. Total is discounted subtotal plus tax. +- Every line quantity must be a positive integer. +- Duplicate SKUs are allowed, but inventory checks and decrements must use their aggregate quantity. +- Inventory reservation is atomic: no stock changes when any requested SKU is missing or insufficient. +- Processing the same successful `order_id` again is idempotent and returns the original result without decrementing stock again. +- Failed orders are not cached. A corrected retry with the same `order_id` must be processed normally. +- Batch processing continues after rejected orders and returns one result per input order. +- Reports sort orders by `order_id`, count accepted and rejected orders, and sum revenue from accepted orders only. +- Report money values are strings with exactly two decimal places. + +## Required artifacts + +`result.json` must contain at least: + +```json +{ + "status": "pass", + "tests_run": 12, + "acceptance": "PASS" +} +``` + +The final line of `AGENT_REPORT.md` must be: + +```text +COMPLEX_AGENT_TASK_OK tests=12 acceptance=PASS +``` diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/acceptance_check.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/acceptance_check.py new file mode 100644 index 000000000..da1c846b2 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/acceptance_check.py @@ -0,0 +1,26 @@ +from order_engine import Inventory, OrderLine, OrderProcessor, OrderRequest + + +def request(order_id, tier, tax, *lines): + return OrderRequest(order_id, tier, tax, tuple(lines)) + + +inventory = Inventory({"BOOK": 6, "PEN": 3}) +processor = OrderProcessor(inventory) +orders = [ + request("c-3", "vip", "0.08", OrderLine("BOOK", 2, "12.50")), + request("c-2", "standard", "0", OrderLine("PEN", 2, "1.005"), OrderLine("PEN", 2, "1.005")), + request("c-1", "premium", "0.05", OrderLine("BOOK", 1, "20.00")), +] +results, report = processor.process_batch(orders) +assert [result.status for result in results] == ["accepted", "rejected", "accepted"] +assert report == { + "accepted": 2, + "rejected": 1, + "revenue": "42.15", + "orders": ["c-1", "c-2", "c-3"], +} +assert inventory.snapshot() == {"BOOK": 3, "PEN": 3} +assert processor.process(orders[0]) == results[0] +assert inventory.snapshot() == {"BOOK": 3, "PEN": 3} +print("ACCEPTANCE_PASS") diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/__init__.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/__init__.py new file mode 100644 index 000000000..19e56acf5 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/__init__.py @@ -0,0 +1,5 @@ +from .inventory import Inventory +from .models import OrderLine, OrderRequest, OrderResult +from .service import OrderProcessor + +__all__ = ["Inventory", "OrderLine", "OrderProcessor", "OrderRequest", "OrderResult"] diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/inventory.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/inventory.py new file mode 100644 index 000000000..19e528383 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/inventory.py @@ -0,0 +1,19 @@ +from collections.abc import Mapping + +from .models import OrderLine + + +class Inventory: + def __init__(self, stock: Mapping[str, int]): + self._stock = dict(stock) + + def snapshot(self) -> dict[str, int]: + return dict(self._stock) + + def reserve(self, lines: tuple[OrderLine, ...]) -> None: + for line in lines: + if line.sku not in self._stock: + raise ValueError(f"unknown sku: {line.sku}") + if self._stock[line.sku] < line.quantity: + raise ValueError(f"insufficient stock: {line.sku}") + self._stock[line.sku] -= line.quantity diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/models.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/models.py new file mode 100644 index 000000000..e4bf2f63f --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/models.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from decimal import Decimal + + +@dataclass(frozen=True) +class OrderLine: + sku: str + quantity: int + unit_price: Decimal | str | int | float + + +@dataclass(frozen=True) +class OrderRequest: + order_id: str + customer_tier: str + tax_rate: Decimal | str | int | float + lines: tuple[OrderLine, ...] + + +@dataclass(frozen=True) +class OrderResult: + order_id: str + status: str + subtotal: str = "0.00" + discount: str = "0.00" + tax: str = "0.00" + total: str = "0.00" + error: str = "" diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/pricing.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/pricing.py new file mode 100644 index 000000000..e93aa2621 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/pricing.py @@ -0,0 +1,26 @@ +from decimal import Decimal + +from .models import OrderLine + + +CENT = Decimal("0.01") +DISCOUNT_RATES = { + "standard": Decimal("0"), + "vip": Decimal("0.10"), + "premium": Decimal("0.15"), +} + + +def money(value: Decimal) -> Decimal: + return value.quantize(CENT) + + +def calculate_price( + lines: tuple[OrderLine, ...], customer_tier: str, tax_rate: Decimal | str | int | float +) -> tuple[Decimal, Decimal, Decimal, Decimal]: + discount_rate = DISCOUNT_RATES.get(customer_tier, Decimal("0")) + subtotal = money(sum((Decimal(line.unit_price) * line.quantity for line in lines), Decimal("0"))) + tax = money(subtotal * Decimal(tax_rate)) + discount = money((subtotal + tax) * discount_rate) + total = money(subtotal + tax - discount) + return subtotal, discount, tax, total diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/report.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/report.py new file mode 100644 index 000000000..8ddab1405 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/report.py @@ -0,0 +1,15 @@ +from decimal import Decimal + +from .models import OrderResult + + +def build_report(results: list[OrderResult]) -> dict[str, object]: + accepted = [result for result in results if result.status == "accepted"] + rejected = [result for result in results if result.status == "rejected"] + revenue = sum((Decimal(result.total) for result in results), Decimal("0")) + return { + "accepted": len(accepted), + "rejected": len(rejected), + "revenue": str(revenue), + "orders": [result.order_id for result in results], + } diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/service.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/service.py new file mode 100644 index 000000000..f5afe7614 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/order_engine/service.py @@ -0,0 +1,25 @@ +from .inventory import Inventory +from .models import OrderRequest, OrderResult +from .pricing import calculate_price +from .report import build_report + + +class OrderProcessor: + def __init__(self, inventory: Inventory): + self.inventory = inventory + + def process(self, order: OrderRequest) -> OrderResult: + self.inventory.reserve(order.lines) + subtotal, discount, tax, total = calculate_price(order.lines, order.customer_tier, order.tax_rate) + return OrderResult( + order_id=order.order_id, + status="accepted", + subtotal=str(subtotal), + discount=str(discount), + tax=str(tax), + total=str(total), + ) + + def process_batch(self, orders: list[OrderRequest]) -> tuple[list[OrderResult], dict[str, object]]: + results = [self.process(order) for order in orders] + return results, build_report(results) diff --git a/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/tests/test_order_engine.py b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/tests/test_order_engine.py new file mode 100644 index 000000000..1cb3c69d1 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/complex-agent-task/workspace/tests/test_order_engine.py @@ -0,0 +1,102 @@ +import unittest + +from order_engine import Inventory, OrderLine, OrderProcessor, OrderRequest +from order_engine.pricing import calculate_price +from order_engine.report import build_report +from order_engine.models import OrderResult + + +def order(order_id, lines, tier="standard", tax="0"): + return OrderRequest(order_id, tier, tax, tuple(lines)) + + +class PricingTests(unittest.TestCase): + def test_standard_total(self): + values = calculate_price((OrderLine("A", 2, "10.00"),), "standard", "0.08") + self.assertEqual(tuple(map(str, values)), ("20.00", "0.00", "1.60", "21.60")) + + def test_vip_discount_precedes_tax(self): + values = calculate_price((OrderLine("A", 2, "25.00"),), "vip", "0.10") + self.assertEqual(tuple(map(str, values)), ("50.00", "5.00", "4.50", "49.50")) + + def test_round_half_up_and_string_conversion(self): + values = calculate_price((OrderLine("A", 1, 1.005),), "standard", "0") + self.assertEqual(str(values[0]), "1.01") + + def test_unknown_tier_is_rejected(self): + with self.assertRaisesRegex(ValueError, "unknown customer tier"): + calculate_price((OrderLine("A", 1, "1"),), "wholesale", "0") + + +class InventoryTests(unittest.TestCase): + def test_reservation_is_atomic(self): + inventory = Inventory({"A": 3, "B": 1}) + with self.assertRaisesRegex(ValueError, "insufficient stock: B"): + inventory.reserve((OrderLine("A", 2, "1"), OrderLine("B", 2, "1"))) + self.assertEqual(inventory.snapshot(), {"A": 3, "B": 1}) + + def test_duplicate_skus_are_aggregated(self): + inventory = Inventory({"A": 3}) + with self.assertRaisesRegex(ValueError, "insufficient stock: A"): + inventory.reserve((OrderLine("A", 2, "1"), OrderLine("A", 2, "1"))) + self.assertEqual(inventory.snapshot(), {"A": 3}) + + def test_successful_reservation_decrements_aggregate(self): + inventory = Inventory({"A": 5}) + inventory.reserve((OrderLine("A", 2, "1"), OrderLine("A", 1, "1"))) + self.assertEqual(inventory.snapshot(), {"A": 2}) + + def test_quantity_must_be_positive_integer(self): + for bad_quantity in (0, -1, 1.5, True): + with self.subTest(quantity=bad_quantity): + inventory = Inventory({"A": 5}) + with self.assertRaisesRegex(ValueError, "positive integer"): + inventory.reserve((OrderLine("A", bad_quantity, "1"),)) + self.assertEqual(inventory.snapshot(), {"A": 5}) + + +class ServiceTests(unittest.TestCase): + def test_success_is_idempotent(self): + inventory = Inventory({"A": 5}) + processor = OrderProcessor(inventory) + request = order("same", [OrderLine("A", 2, "4.00")]) + self.assertEqual(processor.process(request), processor.process(request)) + self.assertEqual(inventory.snapshot(), {"A": 3}) + + def test_failed_order_is_not_cached(self): + inventory = Inventory({"A": 1}) + processor = OrderProcessor(inventory) + with self.assertRaises(ValueError): + processor.process(order("retry", [OrderLine("A", 2, "1")])) + accepted = processor.process(order("retry", [OrderLine("A", 1, "1")])) + self.assertEqual(accepted.status, "accepted") + self.assertEqual(inventory.snapshot(), {"A": 0}) + + def test_batch_continues_after_rejection(self): + processor = OrderProcessor(Inventory({"A": 2})) + results, report = processor.process_batch([ + order("ok-2", [OrderLine("A", 1, "3")]), + order("bad", [OrderLine("MISSING", 1, "5")]), + order("ok-1", [OrderLine("A", 1, "7")]), + ]) + self.assertEqual([item.status for item in results], ["accepted", "rejected", "accepted"]) + self.assertIn("unknown sku", results[1].error) + self.assertEqual(report["accepted"], 2) + self.assertEqual(report["rejected"], 1) + + def test_report_is_sorted_and_accepted_revenue_only(self): + report = build_report([ + OrderResult("z-order", "accepted", total="2.50"), + OrderResult("a-rejected", "rejected", total="999.00", error="no"), + OrderResult("m-order", "accepted", total="1.25"), + ]) + self.assertEqual(report, { + "accepted": 2, + "rejected": 1, + "revenue": "3.75", + "orders": ["a-rejected", "m-order", "z-order"], + }) + + +if __name__ == "__main__": + unittest.main() diff --git a/skills/skills/langbot-testing/fixtures/fixtures.json b/skills/skills/langbot-testing/fixtures/fixtures.json index 70badac14..e6d6e3f02 100644 --- a/skills/skills/langbot-testing/fixtures/fixtures.json +++ b/skills/skills/langbot-testing/fixtures/fixtures.json @@ -1,7 +1,7 @@ [ { "id": "qa-agent-runner-behaviors", - "title": "Deterministic AgentRunner behavior matrix", + "title": "Deterministic Runner behavior matrix", "kind": "json", "path": "fixtures/agent-runner/qa-runner-behaviors.json", "related_cases": [ @@ -13,7 +13,7 @@ }, { "id": "qa-agent-runner-source", - "title": "QA deterministic AgentRunner fixture source", + "title": "QA deterministic Runner fixture source", "kind": "plugin_source", "path": "fixtures/plugins/qa-agent-runner/manifest.yaml", "related_cases": [ @@ -22,11 +22,11 @@ "agent-runner-live-install", "agent-runner-qa-debug-chat" ], - "checks": ["exists", "qa_agent_runner_source"] + "checks": ["exists", "qa_runner_source"] }, { "id": "qa-agent-runner-package", - "title": "QA deterministic AgentRunner prebuilt package", + "title": "QA deterministic Runner prebuilt package", "kind": "plugin_package", "path": "fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg", "related_cases": [ @@ -41,7 +41,7 @@ "title": "MCP stdio qa_mcp_echo server", "kind": "python", "path": "fixtures/mcp/qa_mcp_echo_server.py", - "related_cases": ["mcp-stdio-tool-call"], + "related_cases": ["mcp-stdio-tool-call", "box-mcp-heartbeat-recovery"], "checks": ["exists", "direct_mcp_script", "langbot_registration_script"] }, { @@ -49,7 +49,13 @@ "title": "LangRAG sentinel text document", "kind": "text", "path": "fixtures/rag/sentinel-doc.txt", - "related_cases": ["langrag-kb-retrieve", "local-agent-rag-debug-chat"], + "related_cases": [ + "langrag-kb-retrieve", + "local-agent-rag-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat" + ], "checks": ["exists"] }, { @@ -78,6 +84,11 @@ "plugin-e2e-smoke", "local-agent-effective-prompt-debug-chat", "local-agent-plugin-tool-call-debug-chat", + "local-agent-tool-error-recovery-debug-chat", + "local-agent-tool-loop-limit-debug-chat", + "local-agent-combo-rag-compaction-tool-debug-chat", + "local-agent-multitool-rag-compaction-debug-chat", + "local-agent-parallel-tools-rag-compaction-debug-chat", "local-agent-steering-debug-chat" ], "checks": ["exists"] @@ -87,7 +98,7 @@ "title": "QA plugin smoke prebuilt package", "kind": "plugin_package", "path": "fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg", - "related_cases": ["qa-plugin-smoke-live-install", "plugin-e2e-smoke"], + "related_cases": ["qa-plugin-smoke-live-install", "plugin-e2e-smoke", "local-agent-tool-error-recovery-debug-chat"], "checks": ["exists", "zip_package"] } ] diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/README.md b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/README.md index cbde4cc52..e35d439b7 100644 --- a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/README.md +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/README.md @@ -1,6 +1,6 @@ -# QA AgentRunner Fixture +# QA Runner Fixture -Deterministic AgentRunner plugin source used by `langbot-skills` probes and future browser release-gate cases. +Deterministic Runner plugin source used by `langbot-skills` probes and future browser release-gate cases. Runner id after installation should be: @@ -10,6 +10,6 @@ plugin:qa/agent-runner/default Expected behavior: -- normal input returns `QA_AGENT_RUNNER_OK:` +- normal input returns `QA_RUNNER_OK:` - input containing `stream` emits streaming chunks then completes -- input containing `fail` returns `QA_AGENT_RUNNER_CONTROLLED_FAILURE` +- input containing `fail` returns `QA_RUNNER_CONTROLLED_FAILURE` diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/assets/icon.svg b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/assets/icon.svg index 2a0e899bb..3544b4e35 100644 --- a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/assets/icon.svg +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/assets/icon.svg @@ -1,4 +1,4 @@ - + diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.py b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.py deleted file mode 100644 index 7d2fd37dc..000000000 --- a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.py +++ /dev/null @@ -1,39 +0,0 @@ -from __future__ import annotations - -import typing - -from langbot_plugin.api.definition.components.agent_runner.runner import AgentRunner -from langbot_plugin.api.entities.builtin.agent_runner import AgentRunContext, AgentRunResult -from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk - - -class DefaultAgentRunner(AgentRunner): - async def run( - self, - ctx: AgentRunContext, - ) -> typing.AsyncGenerator[AgentRunResult, None]: - text = (ctx.input.to_text() or "").strip() - if "fail" in text.lower(): - yield AgentRunResult.run_failed( - ctx.run_id, - error="QA_AGENT_RUNNER_CONTROLLED_FAILURE", - code="qa.controlled_failure", - retryable=False, - ) - return - - content = f"QA_AGENT_RUNNER_OK:{text or 'empty'}" - if "stream" in text.lower(): - for chunk in ("QA_", "AGENT_", f"RUNNER_OK:{text}"): - yield AgentRunResult.message_delta( - ctx.run_id, - MessageChunk(role="assistant", content=chunk), - ) - yield AgentRunResult.run_completed(ctx.run_id, finish_reason="stop") - return - - yield AgentRunResult.run_completed( - ctx.run_id, - Message(role="assistant", content=content), - finish_reason="stop", - ) diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/runner/default.py b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/runner/default.py new file mode 100644 index 000000000..038d24e86 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/runner/default.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +import typing + +from langbot_plugin.api.definition.components.runner.runner import Runner +from langbot_plugin.api.entities.builtin.runner import RunnerContext, RunnerResult +from langbot_plugin.api.entities.builtin.provider.message import Message, MessageChunk + + +class DefaultRunner(Runner): + async def run( + self, + ctx: RunnerContext, + ) -> typing.AsyncGenerator[RunnerResult, None]: + text = (ctx.input.to_text() or '').strip() + if 'fail' in text.lower(): + yield RunnerResult.run_failed( + ctx.run_id, + error='QA_RUNNER_CONTROLLED_FAILURE', + code='qa.controlled_failure', + retryable=False, + ) + return + + content = f'QA_RUNNER_OK:{text or "empty"}' + if 'stream' in text.lower(): + for chunk in ('QA_', 'AGENT_', f'RUNNER_OK:{text}'): + yield RunnerResult.message_delta( + ctx.run_id, + MessageChunk(role='assistant', content=chunk), + ) + yield RunnerResult.run_completed(ctx.run_id, finish_reason='stop') + return + + yield RunnerResult.run_completed( + ctx.run_id, + Message(role='assistant', content=content), + finish_reason='stop', + ) diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.yaml b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/runner/default.yaml similarity index 92% rename from skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.yaml rename to skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/runner/default.yaml index a2fd64d98..3e8e65057 100644 --- a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.yaml +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/runner/default.yaml @@ -1,5 +1,5 @@ apiVersion: langbot/v1 -kind: AgentRunner +kind: Runner metadata: name: default label: @@ -9,6 +9,7 @@ metadata: en_US: Deterministic runner fixture that returns stable QA sentinel output. zh_Hans: 返回稳定 QA 哨兵输出的确定性 runner 夹具。 spec: + usages: [agent] capabilities: streaming: true tool_calling: false @@ -27,4 +28,4 @@ spec: execution: python: path: default.py - attr: DefaultAgentRunner + attr: DefaultRunner diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg index aba6ead2d..fc7568d14 100644 Binary files a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg and b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg differ diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/main.py b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/main.py index 6e1c392d2..030a27e4a 100644 --- a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/main.py +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/main.py @@ -3,6 +3,6 @@ from __future__ import annotations from langbot_plugin.api.definition.plugin import BasePlugin -class QAAgentRunnerPlugin(BasePlugin): +class QARunnerPlugin(BasePlugin): async def initialize(self) -> None: - self.ready_marker = "qa-agent-runner-ready" + self.ready_marker = 'qa-agent-runner-ready' diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/manifest.yaml b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/manifest.yaml index a7fd08cac..6b82731e9 100644 --- a/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/manifest.yaml +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/manifest.yaml @@ -6,20 +6,20 @@ metadata: repository: https://example.invalid/langbot/qa-agent-runner version: 0.1.0 description: - en_US: Deterministic AgentRunner fixture for LangBot QA. - zh_Hans: LangBot QA 使用的确定性 AgentRunner 夹具。 + en_US: Deterministic Runner fixture for LangBot QA. + zh_Hans: LangBot QA 使用的确定性 Runner 夹具。 label: - en_US: QA AgentRunner - zh_Hans: QA AgentRunner + en_US: QA Runner + zh_Hans: QA Runner icon: assets/icon.svg spec: config: [] components: - AgentRunner: + Runner: fromDirs: - - path: components/agent_runner/ + - path: components/runner/ maxDepth: 1 execution: python: path: main.py - attr: QAAgentRunnerPlugin + attr: QARunnerPlugin diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/README.md b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/README.md index e276f5774..c3b7f2057 100644 --- a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/README.md +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/README.md @@ -7,3 +7,4 @@ Tools: - `qa_echo(text)` returns `qa-plugin-smoke:`. - `qa_plugin_echo(text)` returns `qa-plugin-smoke:`. - `qa_plugin_sleep(seconds, text)` waits up to 15 seconds and returns `qa-plugin-smoke:sleep::`. +- `qa_plugin_fail(text)` raises a deterministic error containing `text`. diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.py b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.py new file mode 100644 index 000000000..2c6e48f24 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from typing import Any + +from langbot_plugin.api.definition.components.tool.tool import Tool +from langbot_plugin.api.entities.builtin.provider import session as provider_session + + +class QAPluginFailTool(Tool): + async def call( + self, + params: dict[str, Any], + session: provider_session.Session, + query_id: int, + ) -> str: + text = str(params.get("text", "qa-plugin-fail")) + raise RuntimeError(f"qa-plugin-smoke forced failure: {text}") diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.yaml b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.yaml new file mode 100644 index 000000000..7bcf36bd4 --- /dev/null +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/components/tools/qa_plugin_fail.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Tool +metadata: + name: qa_plugin_fail + label: + en_US: QA Plugin Fail + zh_Hans: QA 插件失败工具 + description: + en_US: Raises a deterministic error for local-agent tool error recovery tests. + zh_Hans: 为 local-agent 工具错误恢复测试抛出确定性错误。 +spec: + parameters: + type: object + properties: + text: + type: string + description: Text included in the deterministic failure message. + required: + - text + llm_prompt: Always fail with a deterministic qa-plugin-smoke error for tool error recovery tests. +execution: + python: + path: qa_plugin_fail.py + attr: QAPluginFailTool diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg index a4a50f803..6e7cc28bb 100644 Binary files a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg and b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg differ diff --git a/skills/skills/langbot-testing/probes/agent-runner-async-db-readiness.mjs b/skills/skills/langbot-testing/probes/agent-runner-async-db-readiness.mjs index 3bf1facf0..24652f03a 100755 --- a/skills/skills/langbot-testing/probes/agent-runner-async-db-readiness.mjs +++ b/skills/skills/langbot-testing/probes/agent-runner-async-db-readiness.mjs @@ -80,13 +80,17 @@ async function main() { const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); await mkdir(evidenceDir, { recursive: true }); const startedAt = new Date(); - const langbotRepo = resolve(root, env.LANGBOT_REPO || "../LangBot"); + const langbotRepo = resolve(root, env.LANGBOT_REPO || ".."); const stdoutLog = join(evidenceDir, "probe-stdout.log"); const stderrLog = join(evidenceDir, "probe-stderr.log"); const automationResultJson = join(evidenceDir, "automation-result.json"); const resultJson = join(evidenceDir, "result.json"); const timeoutMs = Number(env.LANGBOT_ASYNC_DB_READINESS_TIMEOUT_MS || "5000"); - const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script], cwd: langbotRepo }; + const command = { + executable: "rtk", + args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script], + cwd: langbotRepo, + }; const result = { source: "automation", probe: "aiosqlite-readiness", diff --git a/skills/skills/langbot-testing/probes/agent-runner-behavior-matrix.mjs b/skills/skills/langbot-testing/probes/agent-runner-behavior-matrix.mjs index bc7ded87b..ec73bc409 100755 --- a/skills/skills/langbot-testing/probes/agent-runner-behavior-matrix.mjs +++ b/skills/skills/langbot-testing/probes/agent-runner-behavior-matrix.mjs @@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path"; import { env } from "node:process"; function timestampSlug(date = new Date()) { - return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[^0-9A-Za-z]+/g, "-") + .replace(/^-|-$/g, ""); } function localIsoWithOffset(date = new Date()) { @@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) { }); child.on("error", (error) => { clearTimeout(timeout); - resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null }); + resolveDone({ + stdout, + stderr, + error, + timedOut, + status: null, + signal: null, + }); }); child.on("close", (status, signal) => { clearTimeout(timeout); @@ -66,7 +77,7 @@ import json import sys from pathlib import Path -from langbot.pkg.agent.runner.descriptor import AgentRunnerDescriptor +from langbot.pkg.agent.runner.descriptor import RunnerDescriptor from langbot.pkg.agent.runner.errors import RunnerExecutionError, RunnerProtocolError from langbot.pkg.agent.runner.result_normalizer import AgentResultNormalizer @@ -80,10 +91,10 @@ class App: logger = Logger() def descriptor(): - return AgentRunnerDescriptor( + return RunnerDescriptor( id='plugin:qa/agent-runner/default', source='plugin', - label={'en_US': 'QA AgentRunner'}, + label={'en_US': 'QA Runner'}, plugin_author='qa', plugin_name='agent-runner', runner_name='default', @@ -139,18 +150,31 @@ async function main() { const root = resolve(env.LBS_ROOT || process.cwd()); const caseId = "agent-runner-behavior-matrix"; const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; - const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + const evidenceDir = resolve( + env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId), + ); await mkdir(evidenceDir, { recursive: true }); const startedAt = new Date(); - const langbotRepo = resolve(root, env.LANGBOT_REPO || "../LangBot"); - const sdkSrc = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk/src"); - const fixturePath = resolve(root, "skills/langbot-testing/fixtures/agent-runner/qa-runner-behaviors.json"); + const langbotRepo = resolve(root, env.LANGBOT_REPO || ".."); + const sdkRepo = resolve( + root, + env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk", + ); + const sdkSrc = resolve(sdkRepo, "src"); + const fixturePath = resolve( + root, + "skills/langbot-testing/fixtures/agent-runner/qa-runner-behaviors.json", + ); const stdoutLog = join(evidenceDir, "probe-stdout.log"); const stderrLog = join(evidenceDir, "probe-stderr.log"); const automationResultJson = join(evidenceDir, "automation-result.json"); const resultJson = join(evidenceDir, "result.json"); - const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000"); - const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script, fixturePath], cwd: langbotRepo }; + const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000"); + const command = { + executable: "rtk", + args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script, fixturePath], + cwd: langbotRepo, + }; const result = { source: "automation", probe: "agent-runner-behavior-matrix", @@ -170,7 +194,12 @@ async function main() { timeout_ms: timeoutMs, exit_status: null, signal: null, - evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, automation_result_json: automationResultJson, result_json: resultJson }, + evidence: { + stdout_log: stdoutLog, + stderr_log: stderrLog, + automation_result_json: automationResultJson, + result_json: resultJson, + }, evidence_collected: ["filesystem"], }; try { @@ -180,7 +209,9 @@ async function main() { } else { const proc = await run(command, timeoutMs, { ...process.env, - PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter), + PYTHONPATH: [sdkSrc, process.env.PYTHONPATH] + .filter(Boolean) + .join(delimiter), UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"), }); await writeFile(stdoutLog, proc.stdout, "utf8"); @@ -193,7 +224,10 @@ async function main() { } else if (proc.timedOut) { result.status = "fail"; result.reason = `behavior matrix timed out after ${timeoutMs}ms`; - } else if (proc.status === 0 && proc.stdout.includes("QA_RUNNER_BEHAVIOR_MATRIX_OK")) { + } else if ( + proc.status === 0 && + proc.stdout.includes("QA_RUNNER_BEHAVIOR_MATRIX_OK") + ) { result.status = "pass"; result.reason = "behavior matrix passed"; } else { @@ -214,7 +248,9 @@ async function main() { await writeFile(resultJson, resultText, "utf8"); console.log(JSON.stringify(result, null, 2)); } - process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); + process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, + ); } await main(); diff --git a/skills/skills/langbot-testing/probes/agent-runner-fixture-contract.mjs b/skills/skills/langbot-testing/probes/agent-runner-fixture-contract.mjs index 00c447f71..a6cc7459c 100755 --- a/skills/skills/langbot-testing/probes/agent-runner-fixture-contract.mjs +++ b/skills/skills/langbot-testing/probes/agent-runner-fixture-contract.mjs @@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path"; import { env } from "node:process"; function timestampSlug(date = new Date()) { - return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[^0-9A-Za-z]+/g, "-") + .replace(/^-|-$/g, ""); } function localIsoWithOffset(date = new Date()) { @@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) { }); child.on("error", (error) => { clearTimeout(timeout); - resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null }); + resolveDone({ + stdout, + stderr, + error, + timedOut, + status: null, + signal: null, + }); }); child.on("close", (status, signal) => { clearTimeout(timeout); @@ -66,27 +77,27 @@ import importlib.util import sys from pathlib import Path -from langbot_plugin.api.entities.builtin.agent_runner.context import AgentRunContext -from langbot_plugin.api.entities.builtin.agent_runner.delivery import DeliveryContext -from langbot_plugin.api.entities.builtin.agent_runner.event import AgentEventContext -from langbot_plugin.api.entities.builtin.agent_runner.input import AgentInput -from langbot_plugin.api.entities.builtin.agent_runner.resources import AgentResources -from langbot_plugin.api.entities.builtin.agent_runner.runtime import AgentRuntimeContext -from langbot_plugin.api.entities.builtin.agent_runner.trigger import AgentTrigger +from langbot_plugin.api.entities.builtin.runner.context import RunnerContext +from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext +from langbot_plugin.api.entities.builtin.runner.event import AgentEventContext +from langbot_plugin.api.entities.builtin.runner.input import AgentInput +from langbot_plugin.api.entities.builtin.runner.resources import AgentResources +from langbot_plugin.api.entities.builtin.runner.runtime import AgentRuntimeContext +from langbot_plugin.api.entities.builtin.runner.trigger import AgentTrigger fixture = Path(sys.argv[1]) -runner_py = fixture / "components" / "agent_runner" / "default.py" +runner_py = fixture / "components" / "runner" / "default.py" manifest = fixture / "manifest.yaml" -runner_yaml = fixture / "components" / "agent_runner" / "default.yaml" +runner_yaml = fixture / "components" / "runner" / "default.yaml" assert manifest.exists(), manifest assert runner_yaml.exists(), runner_yaml -spec = importlib.util.spec_from_file_location("qa_agent_runner_fixture", runner_py) +spec = importlib.util.spec_from_file_location("qa_runner_fixture", runner_py) module = importlib.util.module_from_spec(spec) assert spec and spec.loader spec.loader.exec_module(module) def context(run_id, text): - return AgentRunContext( + return RunnerContext( run_id=run_id, trigger=AgentTrigger(type="message.received", source="webui"), event=AgentEventContext(event_id=f"evt-{run_id}", event_type="message.received", source="webui"), @@ -97,7 +108,7 @@ def context(run_id, text): ) async def collect(text): - runner = module.DefaultAgentRunner() + runner = module.DefaultRunner() results = [] async for result in runner.run(context(f"run-{len(text)}", text)): results.append(result) @@ -107,17 +118,17 @@ async def main(): normal = await collect("hello") assert len(normal) == 1, normal assert normal[0].type.value == "run.completed" - assert normal[0].data["message"]["content"] == "QA_AGENT_RUNNER_OK:hello" + assert normal[0].data["message"]["content"] == "QA_RUNNER_OK:hello" stream = await collect("stream hello") assert [item.type.value for item in stream] == ["message.delta", "message.delta", "message.delta", "run.completed"] - assert "".join(item.data["chunk"]["content"] for item in stream[:3]) == "QA_AGENT_RUNNER_OK:stream hello" + assert "".join(item.data["chunk"]["content"] for item in stream[:3]) == "QA_RUNNER_OK:stream hello" failed = await collect("please fail") assert len(failed) == 1 assert failed[0].type.value == "run.failed" - assert failed[0].data["error"] == "QA_AGENT_RUNNER_CONTROLLED_FAILURE" - print("QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK") + assert failed[0].data["error"] == "QA_RUNNER_CONTROLLED_FAILURE" + print("QA_RUNNER_FIXTURE_CONTRACT_OK") asyncio.run(main()) `; @@ -126,18 +137,30 @@ async function main() { const root = resolve(env.LBS_ROOT || process.cwd()); const caseId = "agent-runner-fixture-contract"; const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; - const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + const evidenceDir = resolve( + env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId), + ); await mkdir(evidenceDir, { recursive: true }); const startedAt = new Date(); - const sdkRepo = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk"); + const sdkRepo = resolve( + root, + env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk", + ); const sdkSrc = resolve(sdkRepo, "src"); - const fixturePath = resolve(root, "skills/langbot-testing/fixtures/plugins/qa-agent-runner"); + const fixturePath = resolve( + root, + "skills/langbot-testing/fixtures/plugins/qa-agent-runner", + ); const stdoutLog = join(evidenceDir, "probe-stdout.log"); const stderrLog = join(evidenceDir, "probe-stderr.log"); const automationResultJson = join(evidenceDir, "automation-result.json"); const resultJson = join(evidenceDir, "result.json"); - const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000"); - const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script, fixturePath], cwd: sdkRepo }; + const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000"); + const command = { + executable: "rtk", + args: ["uv", "run", "--no-sync", "python", "-c", script, fixturePath], + cwd: sdkRepo, + }; const result = { source: "automation", probe: "agent-runner-fixture-contract", @@ -156,7 +179,12 @@ async function main() { timeout_ms: timeoutMs, exit_status: null, signal: null, - evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, automation_result_json: automationResultJson, result_json: resultJson }, + evidence: { + stdout_log: stdoutLog, + stderr_log: stderrLog, + automation_result_json: automationResultJson, + result_json: resultJson, + }, evidence_collected: ["filesystem"], }; try { @@ -166,7 +194,9 @@ async function main() { } else { const proc = await run(command, timeoutMs, { ...process.env, - PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter), + PYTHONPATH: [sdkSrc, process.env.PYTHONPATH] + .filter(Boolean) + .join(delimiter), UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"), }); await writeFile(stdoutLog, proc.stdout, "utf8"); @@ -179,9 +209,12 @@ async function main() { } else if (proc.timedOut) { result.status = "fail"; result.reason = `fixture contract probe timed out after ${timeoutMs}ms`; - } else if (proc.status === 0 && proc.stdout.includes("QA_AGENT_RUNNER_FIXTURE_CONTRACT_OK")) { + } else if ( + proc.status === 0 && + proc.stdout.includes("QA_RUNNER_FIXTURE_CONTRACT_OK") + ) { result.status = "pass"; - result.reason = "QA AgentRunner fixture contract passed"; + result.reason = "QA Runner fixture contract passed"; } else { result.status = "fail"; result.reason = `fixture contract exited with status ${proc.status}`; @@ -200,7 +233,9 @@ async function main() { await writeFile(resultJson, resultText, "utf8"); console.log(JSON.stringify(result, null, 2)); } - process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); + process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, + ); } await main(); diff --git a/skills/skills/langbot-testing/probes/agent-runner-ledger-concurrency.mjs b/skills/skills/langbot-testing/probes/agent-runner-ledger-concurrency.mjs index f6c98389a..ed50523c7 100755 --- a/skills/skills/langbot-testing/probes/agent-runner-ledger-concurrency.mjs +++ b/skills/skills/langbot-testing/probes/agent-runner-ledger-concurrency.mjs @@ -5,10 +5,11 @@ import { runPytestProbe } from "./pytest-probe.mjs"; await runPytestProbe({ caseId: "agent-runner-ledger-concurrency", repoEnvKey: "LANGBOT_REPO", - defaultRepo: "../LangBot", + defaultRepo: "..", pythonPathEnvKeys: ["LANGBOT_PLUGIN_SDK_REPO"], - defaultPythonPaths: ["../langbot-plugin-sdk/src"], - description: "LangBot AgentRunner run ledger claim, lease, authorization, and runtime-admin pytest probe.", + defaultPythonPaths: ["../../langbot-plugin-sdk/src"], + description: + "LangBot Runner run ledger claim, lease, authorization, and runtime-admin pytest probe.", testTargets: [ "tests/unit_tests/agent/test_run_ledger_store.py::test_create_queued_run_claim_renew_release", "tests/unit_tests/agent/test_run_ledger_store.py::test_expired_claim_can_be_reclaimed", diff --git a/skills/skills/langbot-testing/probes/agent-runner-ledger-contention.mjs b/skills/skills/langbot-testing/probes/agent-runner-ledger-contention.mjs index ec4670011..288489b6b 100755 --- a/skills/skills/langbot-testing/probes/agent-runner-ledger-contention.mjs +++ b/skills/skills/langbot-testing/probes/agent-runner-ledger-contention.mjs @@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path"; import { env } from "node:process"; function timestampSlug(date = new Date()) { - return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[^0-9A-Za-z]+/g, "-") + .replace(/^-|-$/g, ""); } function localIsoWithOffset(date = new Date()) { @@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) { }); child.on("error", (error) => { clearTimeout(timeout); - resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null }); + resolveDone({ + stdout, + stderr, + error, + timedOut, + status: null, + signal: null, + }); }); child.on("close", (status, signal) => { clearTimeout(timeout); @@ -149,18 +160,28 @@ async function main() { const root = resolve(env.LBS_ROOT || process.cwd()); const caseId = "agent-runner-ledger-contention"; const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; - const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + const evidenceDir = resolve( + env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId), + ); await mkdir(evidenceDir, { recursive: true }); const startedAt = new Date(); - const langbotRepo = resolve(root, env.LANGBOT_REPO || "../LangBot"); - const sdkSrc = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk/src"); + const langbotRepo = resolve(root, env.LANGBOT_REPO || ".."); + const sdkRepo = resolve( + root, + env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk", + ); + const sdkSrc = resolve(sdkRepo, "src"); const dbPath = join(evidenceDir, "ledger-contention.sqlite3"); const stdoutLog = join(evidenceDir, "probe-stdout.log"); const stderrLog = join(evidenceDir, "probe-stderr.log"); const automationResultJson = join(evidenceDir, "automation-result.json"); const resultJson = join(evidenceDir, "result.json"); - const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000"); - const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script, dbPath], cwd: langbotRepo }; + const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000"); + const command = { + executable: "rtk", + args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script, dbPath], + cwd: langbotRepo, + }; const result = { source: "automation", probe: "agent-runner-ledger-contention", @@ -180,7 +201,13 @@ async function main() { timeout_ms: timeoutMs, exit_status: null, signal: null, - evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, database: dbPath, automation_result_json: automationResultJson, result_json: resultJson }, + evidence: { + stdout_log: stdoutLog, + stderr_log: stderrLog, + database: dbPath, + automation_result_json: automationResultJson, + result_json: resultJson, + }, evidence_collected: ["filesystem"], }; try { @@ -190,7 +217,9 @@ async function main() { } else { const proc = await run(command, timeoutMs, { ...process.env, - PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter), + PYTHONPATH: [sdkSrc, process.env.PYTHONPATH] + .filter(Boolean) + .join(delimiter), UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"), }); await writeFile(stdoutLog, proc.stdout, "utf8"); @@ -203,7 +232,10 @@ async function main() { } else if (proc.timedOut) { result.status = "fail"; result.reason = `ledger contention timed out after ${timeoutMs}ms`; - } else if (proc.status === 0 && proc.stdout.includes("LEDGER_CONTENTION_OK")) { + } else if ( + proc.status === 0 && + proc.stdout.includes("LEDGER_CONTENTION_OK") + ) { result.status = "pass"; result.reason = "ledger contention probe passed"; } else { @@ -224,7 +256,9 @@ async function main() { await writeFile(resultJson, resultText, "utf8"); console.log(JSON.stringify(result, null, 2)); } - process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); + process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, + ); } await main(); diff --git a/skills/skills/langbot-testing/probes/agent-runner-ledger-invariants.mjs b/skills/skills/langbot-testing/probes/agent-runner-ledger-invariants.mjs index 1b5416d10..a8a6981cb 100755 --- a/skills/skills/langbot-testing/probes/agent-runner-ledger-invariants.mjs +++ b/skills/skills/langbot-testing/probes/agent-runner-ledger-invariants.mjs @@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path"; import { env } from "node:process"; function timestampSlug(date = new Date()) { - return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[^0-9A-Za-z]+/g, "-") + .replace(/^-|-$/g, ""); } function localIsoWithOffset(date = new Date()) { @@ -55,7 +59,14 @@ function runProcess(command, timeoutMs, childEnv) { }); child.on("error", (error) => { clearTimeout(timeout); - resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null }); + resolveDone({ + stdout, + stderr, + error, + timedOut, + status: null, + signal: null, + }); }); child.on("close", (status, signal) => { clearTimeout(timeout); @@ -124,17 +135,27 @@ async function main() { const root = resolve(env.LBS_ROOT || process.cwd()); const caseId = "agent-runner-ledger-invariants"; const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; - const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + const evidenceDir = resolve( + env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId), + ); await mkdir(evidenceDir, { recursive: true }); const startedAt = new Date(); - const langbotRepo = resolveFromRoot(root, env.LANGBOT_REPO || "../LangBot"); - const sdkSrc = resolveFromRoot(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk/src"); + const langbotRepo = resolveFromRoot(root, env.LANGBOT_REPO || ".."); + const sdkRepo = resolveFromRoot( + root, + env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk", + ); + const sdkSrc = resolve(sdkRepo, "src"); const stdoutLog = join(evidenceDir, "probe-stdout.log"); const stderrLog = join(evidenceDir, "probe-stderr.log"); const automationResultJson = join(evidenceDir, "automation-result.json"); const resultJson = join(evidenceDir, "result.json"); - const command = { executable: "rtk", args: ["uv", "run", "python", "-c", probeScript], cwd: langbotRepo }; - const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000"); + const command = { + executable: "rtk", + args: [resolve(langbotRepo, ".venv/bin/python"), "-c", probeScript], + cwd: langbotRepo, + }; + const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000"); const result = { source: "automation", probe: "python-sync", @@ -169,7 +190,9 @@ async function main() { } else { const childEnv = { ...process.env, - PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter), + PYTHONPATH: [sdkSrc, process.env.PYTHONPATH] + .filter(Boolean) + .join(delimiter), UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"), }; await mkdir(childEnv.UV_CACHE_DIR, { recursive: true }); @@ -205,7 +228,9 @@ async function main() { await writeFile(resultJson, resultText, "utf8"); console.log(JSON.stringify(result, null, 2)); } - process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); + process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, + ); } await main(); diff --git a/skills/skills/langbot-testing/probes/agent-runner-ledger-stress.mjs b/skills/skills/langbot-testing/probes/agent-runner-ledger-stress.mjs index 3575358a2..5434e257c 100755 --- a/skills/skills/langbot-testing/probes/agent-runner-ledger-stress.mjs +++ b/skills/skills/langbot-testing/probes/agent-runner-ledger-stress.mjs @@ -7,7 +7,11 @@ import { delimiter, join, resolve } from "node:path"; import { env } from "node:process"; function timestampSlug(date = new Date()) { - return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[^0-9A-Za-z]+/g, "-") + .replace(/^-|-$/g, ""); } function localIsoWithOffset(date = new Date()) { @@ -51,7 +55,14 @@ function run(command, timeoutMs, childEnv) { }); child.on("error", (error) => { clearTimeout(timeout); - resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null }); + resolveDone({ + stdout, + stderr, + error, + timedOut, + status: null, + signal: null, + }); }); child.on("close", (status, signal) => { clearTimeout(timeout); @@ -116,17 +127,27 @@ async function main() { const root = resolve(env.LBS_ROOT || process.cwd()); const caseId = "agent-runner-ledger-stress"; const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; - const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + const evidenceDir = resolve( + env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId), + ); await mkdir(evidenceDir, { recursive: true }); const startedAt = new Date(); - const langbotRepo = resolve(root, env.LANGBOT_REPO || "../LangBot"); - const sdkSrc = resolve(root, env.LANGBOT_PLUGIN_SDK_REPO || "../langbot-plugin-sdk/src"); + const langbotRepo = resolve(root, env.LANGBOT_REPO || ".."); + const sdkRepo = resolve( + root, + env.LANGBOT_PLUGIN_SDK_REPO || "../../langbot-plugin-sdk", + ); + const sdkSrc = resolve(sdkRepo, "src"); const stdoutLog = join(evidenceDir, "probe-stdout.log"); const stderrLog = join(evidenceDir, "probe-stderr.log"); const automationResultJson = join(evidenceDir, "automation-result.json"); const resultJson = join(evidenceDir, "result.json"); - const timeoutMs = Number(env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "30000"); - const command = { executable: "rtk", args: ["uv", "run", "python", "-c", script], cwd: langbotRepo }; + const timeoutMs = Number(env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "30000"); + const command = { + executable: "rtk", + args: [resolve(langbotRepo, ".venv/bin/python"), "-c", script], + cwd: langbotRepo, + }; const result = { source: "automation", probe: "agent-runner-ledger-stress", @@ -145,7 +166,12 @@ async function main() { timeout_ms: timeoutMs, exit_status: null, signal: null, - evidence: { stdout_log: stdoutLog, stderr_log: stderrLog, automation_result_json: automationResultJson, result_json: resultJson }, + evidence: { + stdout_log: stdoutLog, + stderr_log: stderrLog, + automation_result_json: automationResultJson, + result_json: resultJson, + }, evidence_collected: ["filesystem"], }; try { @@ -155,7 +181,9 @@ async function main() { } else { const proc = await run(command, timeoutMs, { ...process.env, - PYTHONPATH: [sdkSrc, process.env.PYTHONPATH].filter(Boolean).join(delimiter), + PYTHONPATH: [sdkSrc, process.env.PYTHONPATH] + .filter(Boolean) + .join(delimiter), UV_CACHE_DIR: env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"), }); await writeFile(stdoutLog, proc.stdout, "utf8"); @@ -168,7 +196,10 @@ async function main() { } else if (proc.timedOut) { result.status = "fail"; result.reason = `ledger stress timed out after ${timeoutMs}ms`; - } else if (proc.status === 0 && proc.stdout.includes("LEDGER_STRESS_OK")) { + } else if ( + proc.status === 0 && + proc.stdout.includes("LEDGER_STRESS_OK") + ) { result.status = "pass"; result.reason = "ledger stress probe passed"; } else { @@ -189,7 +220,9 @@ async function main() { await writeFile(resultJson, resultText, "utf8"); console.log(JSON.stringify(result, null, 2)); } - process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); + process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, + ); } await main(); diff --git a/skills/skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs b/skills/skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs index 07ee5f988..95127295a 100755 --- a/skills/skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs +++ b/skills/skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs @@ -5,10 +5,11 @@ import { runPytestProbe } from "./pytest-probe.mjs"; await runPytestProbe({ caseId: "agent-runner-runtime-chaos", repoEnvKey: "LANGBOT_PLUGIN_SDK_REPO", - defaultRepo: "../langbot-plugin-sdk", - description: "LangBot plugin SDK AgentRunner runtime failure, timeout, forwarding, and pull API pytest probe.", + defaultRepo: "../../langbot-plugin-sdk", + description: + "LangBot plugin SDK Runner runtime failure, timeout, forwarding, and pull API pytest probe.", testTargets: [ - "tests/runtime/plugin/test_mgr_agent_runner.py", + "tests/runtime/plugin/test_mgr_runner.py", "tests/runtime/test_pull_api_handlers.py", ], }); diff --git a/skills/skills/langbot-testing/probes/box-mcp-heartbeat-recovery.mjs b/skills/skills/langbot-testing/probes/box-mcp-heartbeat-recovery.mjs new file mode 100644 index 000000000..da3755994 --- /dev/null +++ b/skills/skills/langbot-testing/probes/box-mcp-heartbeat-recovery.mjs @@ -0,0 +1,396 @@ +#!/usr/bin/env node + +import { execFile } from "node:child_process"; +import { readFile, readlink, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { env, exit } from "node:process"; +import { promisify } from "node:util"; +import { + apiJson, + ensureEvidence, + evidencePaths, + exitCode, + loadEnvFiles, + localIsoWithOffset, + resetAndAuthLocalUser, + writeResult, +} from "../../../scripts/e2e/lib/langbot-e2e.mjs"; + +const execFileAsync = promisify(execFile); +const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; +const BOX_COMMAND_RE = /(?:^|\s)-m\s+langbot_plugin\.cli\.__init__\s+box(?:\s|$)/; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "box-mcp-heartbeat-recovery"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); + +const startedAt = new Date(); +const backendUrl = env.LANGBOT_BACKEND_URL || ""; +const user = env.LANGBOT_E2E_LOGIN_USER || ""; +const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; +const serverUuid = env.LANGBOT_MCP_QA_STDIO_SERVER_UUID || ""; +const expectedTool = env.LANGBOT_E2E_EXPECTED_TOOL || "qa_mcp_echo"; +const recoveryTimeoutMs = positiveInteger(env.LANGBOT_BOX_RECOVERY_TIMEOUT_MS, 30_000); +const pollIntervalMs = positiveInteger(env.LANGBOT_BOX_RECOVERY_POLL_INTERVAL_MS, 250); +const faultModelPath = resolve(paths.evidenceDir, "fault-model.json"); +const apiDiagnosticPath = resolve(paths.evidenceDir, "api-diagnostic.json"); +const resourceLogPath = resolve(paths.evidenceDir, "resource-log.json"); +const metricsPath = resolve(paths.evidenceDir, "metrics.json"); +const ledgerAuditPath = resolve(paths.evidenceDir, "ledger-audit.json"); +const debugChatEvidenceDir = resolve(paths.evidenceDir, "debug-chat"); +const probeValue = `box-recovery-${Date.now().toString(36)}`; + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + status: "fail", + reason: "", + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + duration_ms: 0, + evidence: { + fault_model_json: faultModelPath, + api_diagnostic_json: apiDiagnosticPath, + resource_log_json: resourceLogPath, + metrics_json: metricsPath, + ledger_audit_json: ledgerAuditPath, + debug_chat_evidence_dir: debugChatEvidenceDir, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: [ + "metrics", + "api_diagnostic", + "resource_log", + "filesystem", + ], +}; + +let resourceSamples = []; +let apiDiagnostic = { + expected_tool_call: { + tool_name: expectedTool, + parameters: { text: probeValue }, + result_text: `${expectedTool}:${probeValue}`, + }, +}; + +try { + requireEnv("LANGBOT_BACKEND_URL", backendUrl); + requireEnv("LANGBOT_E2E_LOGIN_USER", user); + requireEnv("LANGBOT_MCP_QA_STDIO_SERVER_UUID", serverUuid); + requireEnv("LANGBOT_LOCAL_AGENT_PIPELINE_URL", env.LANGBOT_LOCAL_AGENT_PIPELINE_URL || ""); + requireEnv("LANGBOT_FRONTEND_URL", env.LANGBOT_FRONTEND_URL || ""); + + const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); + const preflight = await runtimeSnapshot(auth.token); + apiDiagnostic.preflight = preflight; + assertReadySnapshot(preflight, "preflight"); + + const oldBox = await selectLocalBoxProcess(); + const faultModel = { + target: "local Box runtime child process owned by the active LangBot checkout", + injected_fault: "SIGTERM to the selected Box child process", + destructive: true, + blast_radius: "single local LangBot development instance", + old_box_pid: oldBox.pid, + parent_pid: oldBox.ppid, + recovery_timeout_ms: recoveryTimeoutMs, + abort_conditions: [ + "no exact Box command match", + "multiple eligible Box processes", + "Box parent is not main.py from LANGBOT_REPO", + ], + cleanup: "wait for the LangBot heartbeat to replace Box and restore the MCP session; do not spawn or register anything", + }; + await writeFile(faultModelPath, `${JSON.stringify(faultModel, null, 2)}\n`, "utf8"); + + process.kill(oldBox.pid, "SIGTERM"); + const faultStartedAt = performance.now(); + let observedOldProcessExit = false; + let observedBoxUnavailable = false; + let recovered = null; + const deadline = Date.now() + recoveryTimeoutMs; + + while (Date.now() < deadline) { + const processes = await boxProcesses(); + const oldAlive = processes.some((item) => item.pid === oldBox.pid); + observedOldProcessExit ||= !oldAlive; + const replacement = processes.find((item) => item.pid !== oldBox.pid && item.ppid === oldBox.ppid) || null; + const snapshot = await runtimeSnapshot(auth.token).catch((error) => ({ error: error.message })); + observedBoxUnavailable ||= snapshot.box?.available === false; + const sample = { + elapsed_ms: Math.round(performance.now() - faultStartedAt), + old_process_alive: oldAlive, + replacement_pid: replacement?.pid || null, + box_available: snapshot.box?.available ?? null, + box_active_sessions: snapshot.box?.active_sessions ?? null, + box_managed_processes: snapshot.box?.managed_processes ?? null, + mcp_status: snapshot.mcp?.status || null, + mcp_tool_count: snapshot.mcp?.tool_count ?? null, + mcp_retry_count: snapshot.mcp?.retry_count ?? null, + tools_has_expected: snapshot.tools?.includes(expectedTool) ?? false, + error: snapshot.error || "", + }; + resourceSamples.push(sample); + + if (observedOldProcessExit + && replacement + && snapshotReady(snapshot) + && (snapshot.mcp.retry_count ?? 0) > (preflight.mcp.retry_count ?? 0)) { + recovered = { replacement, snapshot, elapsedMs: sample.elapsed_ms }; + break; + } + await delay(pollIntervalMs); + } + + if (!recovered) { + throw new Error(`Box/MCP did not recover within ${recoveryTimeoutMs} ms.`); + } + apiDiagnostic.recovered = recovered.snapshot; + + const chat = await runRecoveredDebugChat(probeValue); + apiDiagnostic.debug_chat = { + status: chat.status, + reason: chat.reason, + expected_assistant_text: probeValue, + evidence_dir: debugChatEvidenceDir, + }; + result.evidence_collected.push("ui", "screenshot", "console", "network"); + const ledger = await auditRecoveredToolCall(probeValue, startedAt); + apiDiagnostic.ledger = { + status: ledger.status, + reason: ledger.reason, + run: ledger.run || null, + expected_tool_call: ledger.expected_tool_call || null, + evidence_path: ledgerAuditPath, + }; + if (chat.status !== "pass") { + throw new Error(`Recovered Debug Chat tool call failed: ${chat.reason || chat.status}`); + } + if (ledger.status !== "pass") { + throw new Error(`Recovered Debug Chat ledger audit failed: ${ledger.reason || ledger.status}`); + } + + result.status = "pass"; + result.reason = `Box was replaced in ${recovered.elapsedMs} ms, MCP reconnected without registration, and the browser plus ledger proved ${expectedTool} returned the unique recovery value.`; + result.metrics_summary = { + recovery_duration_ms: recovered.elapsedMs, + old_box_pid: oldBox.pid, + new_box_pid: recovered.replacement.pid, + observed_old_process_exit: observedOldProcessExit, + observed_box_unavailable: observedBoxUnavailable, + mcp_retry_count_before: preflight.mcp.retry_count ?? 0, + mcp_retry_count_after: recovered.snapshot.mcp.retry_count ?? 0, + debug_chat_response_ms: chat.metrics_summary?.response_p95_ms ?? null, + ledger_run_id: ledger.run?.run_id || null, + }; + result.thresholds_summary = { + recovery_duration_ms: { + actual: recovered.elapsedMs, + max: recoveryTimeoutMs, + pass: recovered.elapsedMs <= recoveryTimeoutMs, + }, + mcp_retry_count_increased: { + actual: recovered.snapshot.mcp.retry_count ?? 0, + min: (preflight.mcp.retry_count ?? 0) + 1, + pass: (recovered.snapshot.mcp.retry_count ?? 0) > (preflight.mcp.retry_count ?? 0), + }, + debug_chat_status: { actual: chat.status, expected: "pass", pass: chat.status === "pass" }, + ledger_tool_call_status: { actual: ledger.status, expected: "pass", pass: ledger.status === "pass" }, + }; +} catch (error) { + if (result.status === "fail" && /is required/.test(error.message)) result.status = "env_issue"; + result.reason = error.message; +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + result.duration_ms = finishedAt.getTime() - startedAt.getTime(); + await writeFile(resourceLogPath, `${JSON.stringify(resourceSamples, null, 2)}\n`, "utf8"); + await writeFile(apiDiagnosticPath, `${JSON.stringify(apiDiagnostic, null, 2)}\n`, "utf8"); + await writeFile(metricsPath, `${JSON.stringify({ + probe: caseId, + status: result.status, + duration_ms: result.duration_ms, + metrics_summary: result.metrics_summary || {}, + thresholds_summary: result.thresholds_summary || {}, + }, null, 2)}\n`, "utf8"); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); + exit(exitCode(result.status)); +} + +function requireEnv(name, value) { + if (!value) throw new Error(`${name} is required.`); +} + +function positiveInteger(value, fallback) { + const parsed = Number.parseInt(String(value || ""), 10); + return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback; +} + +function delay(milliseconds) { + return new Promise((resolveDelay) => setTimeout(resolveDelay, milliseconds)); +} + +function isApiFailure(response) { + return response.status >= 400 || response.json.code !== 0; +} + +async function runtimeSnapshot(token) { + const [boxResponse, serverListResponse, toolsResponse] = await Promise.all([ + apiJson(backendUrl, "/api/v1/box/status", { token }), + apiJson(backendUrl, "/api/v1/mcp/servers", { token }), + apiJson(backendUrl, "/api/v1/tools", { token }), + ]); + for (const response of [boxResponse, serverListResponse, toolsResponse]) { + if (isApiFailure(response)) throw new Error(response.json.msg || `API diagnostic failed with HTTP ${response.status}.`); + } + const servers = serverListResponse.json.data?.servers || []; + const listed = servers.find((server) => server.uuid === serverUuid); + if (!listed) throw new Error(`MCP server UUID ${serverUuid} is not registered.`); + const detailResponse = await apiJson( + backendUrl, + `/api/v1/mcp/servers/${encodeURIComponent(listed.name)}`, + { token }, + ); + if (isApiFailure(detailResponse)) { + throw new Error(detailResponse.json.msg || `MCP detail failed with HTTP ${detailResponse.status}.`); + } + const server = detailResponse.json.data?.server || listed; + const runtime = server.runtime_info || {}; + return { + box: boxResponse.json.data || {}, + mcp: { + uuid: server.uuid || listed.uuid, + name: server.name || listed.name, + status: runtime.status || null, + tool_count: runtime.tool_count ?? 0, + retry_count: runtime.retry_count ?? 0, + error_message: runtime.error_message || "", + }, + tools: (toolsResponse.json.data?.tools || []) + .map((tool) => tool.name || tool.tool_name || tool.function?.name || "") + .filter(Boolean), + }; +} + +function snapshotReady(snapshot) { + return snapshot.box?.available === true + && (snapshot.box?.active_sessions ?? 0) >= 1 + && (snapshot.box?.managed_processes ?? 0) >= 1 + && snapshot.mcp?.status === "connected" + && (snapshot.mcp?.tool_count ?? 0) >= 1 + && snapshot.tools?.includes(expectedTool); +} + +function assertReadySnapshot(snapshot, phase) { + if (!snapshotReady(snapshot)) { + throw new Error(`${phase} Box/MCP state is not ready: ${JSON.stringify(snapshot)}`); + } +} + +async function boxProcesses() { + const { stdout } = await execFileAsync("ps", ["-ww", "-eo", "pid=,ppid=,args="]); + return stdout.split(/\r?\n/).map((line) => { + const match = line.match(/^\s*(\d+)\s+(\d+)\s+(.*)$/); + return match ? { pid: Number(match[1]), ppid: Number(match[2]), args: match[3] } : null; + }).filter((item) => item && BOX_COMMAND_RE.test(item.args)); +} + +async function selectLocalBoxProcess() { + const processes = await boxProcesses(); + const configuredPid = Number.parseInt(env.LANGBOT_BOX_RUNTIME_PID || "", 10); + let candidates = Number.isFinite(configuredPid) + ? processes.filter((item) => item.pid === configuredPid) + : processes; + const repo = resolve(env.LANGBOT_REPO || ".."); + const eligible = []; + for (const candidate of candidates) { + const parentArgs = await readFile(`/proc/${candidate.ppid}/cmdline`, "utf8").catch(() => ""); + const parentCwd = await readlink(`/proc/${candidate.ppid}/cwd`).catch(() => ""); + if (/main\.py(?:\0|$)/.test(parentArgs) && resolve(parentCwd) === repo) eligible.push(candidate); + } + candidates = eligible; + if (candidates.length !== 1) { + throw new Error(`Expected exactly one local Box child for ${repo}, found ${candidates.length}. Set LANGBOT_BOX_RUNTIME_PID only after verifying the process.`); + } + return candidates[0]; +} + +async function runRecoveredDebugChat(value) { + const script = resolve("scripts/e2e/pipeline-debug-chat.mjs"); + const childEnv = { + ...env, + LBS_CASE_ID: `${caseId}-debug-chat`, + LBS_RUN_ID: paths.runId, + LBS_EVIDENCE_DIR: debugChatEvidenceDir, + LANGBOT_E2E_PIPELINE_REQUIRED: "1", + LANGBOT_E2E_PIPELINE_URL: env.LANGBOT_LOCAL_AGENT_PIPELINE_URL, + LANGBOT_E2E_PIPELINE_NAME: env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || "", + LANGBOT_E2E_RESET_DEBUG_CHAT: "1", + LANGBOT_E2E_RESPONSE_TIMEOUT_MS: env.LANGBOT_E2E_RESPONSE_TIMEOUT_MS || "60000", + LANGBOT_E2E_EXPECTED_TOOL: expectedTool, + LANGBOT_E2E_PROMPT: `Call the ${expectedTool} MCP tool with exactly this text: ${value}. Return only the tool result.`, + LANGBOT_E2E_EXPECTED_TEXT: value, + LANGBOT_E2E_EXTENSIONS_PATCH_JSON: JSON.stringify({ + enable_all_plugins: false, + bound_plugins: [{ author: "langbot-team", name: "LocalAgent" }], + enable_all_mcp_servers: false, + bound_mcp_servers: [serverUuid], + enable_all_skills: false, + bound_skills: [], + }), + LANGBOT_E2E_RESTORE_EXTENSIONS: "1", + }; + for (const key of ["ALL_PROXY", "all_proxy", "HTTP_PROXY", "HTTPS_PROXY", "http_proxy", "https_proxy"]) { + delete childEnv[key]; + } + try { + const { stdout } = await execFileAsync(process.execPath, [script], { + cwd: resolve("."), + env: childEnv, + maxBuffer: 4 * 1024 * 1024, + timeout: positiveInteger(env.LANGBOT_E2E_RESPONSE_TIMEOUT_MS, 60_000) + 30_000, + }); + return JSON.parse(stdout.trim()); + } catch (error) { + const childResultPath = resolve(debugChatEvidenceDir, "automation-result.json"); + const childResult = await readFile(childResultPath, "utf8").then(JSON.parse).catch(() => null); + if (childResult) return childResult; + throw error; + } +} + +async function auditRecoveredToolCall(value, createdAfter) { + const repo = resolve(env.LANGBOT_REPO || ".."); + const python = resolve(repo, ".venv/bin/python"); + const script = resolve(repo, "skills/scripts/e2e/agent-run-ledger-audit.py"); + const args = [ + script, + "--repo", repo, + "--output", ledgerAuditPath, + "--created-after", createdAfter.toISOString(), + "--expected-tool-name", expectedTool, + "--expected-parameters-json", JSON.stringify({ text: value }), + "--expected-result-text", `${expectedTool}:${value}`, + ]; + try { + await execFileAsync(python, args, { + cwd: repo, + env, + maxBuffer: 4 * 1024 * 1024, + timeout: 30_000, + }); + } catch (error) { + const report = await readFile(ledgerAuditPath, "utf8").then(JSON.parse).catch(() => null); + if (report) return report; + throw error; + } + return JSON.parse(await readFile(ledgerAuditPath, "utf8")); +} diff --git a/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs index af5153dbf..ec37bf6ba 100755 --- a/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +++ b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs @@ -55,6 +55,8 @@ const minErrorCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_ERROR_C const minOkCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_OK_COUNT, 0); const minProviderFaultCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_PROVIDER_FAULT_COUNT, 0); const failOnFinalMismatch = bool(env.LANGBOT_DEBUG_CHAT_LOAD_FAIL_ON_FINAL_MISMATCH, false); +const requireSuccess = bool(env.LANGBOT_DEBUG_CHAT_LOAD_REQUIRE_SUCCESS, true); +const providerModelThresholds = jsonObject(env.LANGBOT_DEBUG_CHAT_LOAD_PROVIDER_MODEL_THRESHOLDS_JSON); const failureSignals = textList(env.LANGBOT_E2E_FAILURE_SIGNALS || env.LANGBOT_DEBUG_CHAT_LOAD_FAILURE_SIGNALS || ""); const result = { @@ -80,6 +82,7 @@ const result = { stream, reset_before_run: resetBeforeRun, fail_on_final_mismatch: failOnFinalMismatch, + require_success: requireSuccess, }, evidence: { network_log: paths.networkLog, @@ -119,7 +122,7 @@ try { result.pipeline_id = pipeline.id; result.pipeline_name = pipeline.name || pipelineName; if (!result.pipeline_url && env.LANGBOT_FRONTEND_URL) { - result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.id)}`; + result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/agents?id=${encodeURIComponent(pipeline.id)}`; } if (resetBeforeRun) { @@ -190,6 +193,8 @@ try { status_counts: metrics.status_counts, fake_provider_request_count: metrics.fake_provider?.request_count ?? null, fake_provider_fault_count: metrics.fake_provider?.fault_count ?? null, + fake_provider_mid_stream_disconnect_count: metrics.fake_provider?.mid_stream_disconnect_count ?? null, + fake_provider_model_request_counts: metrics.fake_provider?.model_request_counts ?? {}, fake_provider_duration_p95_ms: metrics.provider_timing?.provider_duration_ms.p95 ?? null, langbot_overhead_estimate_p95_ms: metrics.provider_timing?.langbot_overhead_estimate_ms.p95 ?? null, send_to_provider_start_p95_ms: metrics.provider_timing?.send_to_provider_start_ms.p95 ?? null, @@ -782,12 +787,14 @@ function buildMetrics({ samples, totalRequests, concurrency, timeoutMs, loadDura function buildThresholds(metrics) { const thresholds = { error_rate: { actual: metrics.error_rate, max: maxErrorRate, pass: metrics.error_rate <= maxErrorRate }, - response_p95_ms: { + }; + if (requireSuccess) { + thresholds.response_p95_ms = { actual: metrics.response_duration_ms.p95, max: responseP95BudgetMs, pass: metrics.ok_count > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs, - }, - }; + }; + } if (minErrorRate > 0) { thresholds.error_rate_min = { actual: metrics.error_rate, @@ -824,9 +831,36 @@ function buildThresholds(metrics) { pass: metrics.ok_count > 0 && metrics.first_response_ms.p95 <= firstResponseP95BudgetMs, }; } + for (const [model, bounds] of Object.entries(providerModelThresholds)) { + const actual = metrics.fake_provider?.model_request_counts?.[model] ?? 0; + if (Number.isFinite(bounds?.min)) { + thresholds[`provider_model_${model}_min`] = { + actual, + min: bounds.min, + pass: actual >= bounds.min, + }; + } + if (Number.isFinite(bounds?.max)) { + thresholds[`provider_model_${model}_max`] = { + actual, + max: bounds.max, + pass: actual <= bounds.max, + }; + } + } return thresholds; } +function jsonObject(value) { + if (!value) return {}; + try { + const parsed = JSON.parse(String(value)); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {}; + } catch { + return {}; + } +} + function looksLikeEnvIssue(error) { const message = String(error?.message || error || ""); return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message); diff --git a/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs b/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs index 38a31c389..2fb7bfc9c 100755 --- a/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs +++ b/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs @@ -139,10 +139,15 @@ async function main() { const text = await readFile(logPath, "utf8"); scan = scanLines(text, since, startedAt.getFullYear()); const failCount = scan.findings.filter((item) => item.severity === "fail").length; - status = failCount === 0 ? "pass" : "fail"; - reason = status === "pass" - ? "Live backend log health passed; no fail-severity findings in the scanned window." - : "Live backend log health found fail-severity backend log findings."; + if (scan.scanned.length === 0) { + status = "env_issue"; + reason = "Backend log health had no timestamped lines in the selected window; no health verdict is possible."; + } else { + status = failCount === 0 ? "pass" : "fail"; + reason = status === "pass" + ? "Live backend log health passed; no fail-severity findings in the scanned window." + : "Live backend log health found fail-severity backend log findings."; + } } const warningCount = scan.findings.filter((item) => item.severity === "warning").length; @@ -158,6 +163,11 @@ async function main() { finding_count: scan.findings.length, }; const thresholds = { + scanned_line_count_min: { + actual: scan.scanned.length, + min: 1, + pass: scan.scanned.length >= 1, + }, fail_count: { actual: failCount, max: 0, pass: failCount === 0 }, }; diff --git a/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs b/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs index b383b2663..4dc21f3ee 100755 --- a/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs +++ b/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs @@ -6,8 +6,16 @@ export function summarizeFakeProviderState(state) { const faultRequests = chatRequests.filter((request) => ( request?.should_fail === true || request?.status === "http_fault" + || request?.status === "mid_stream_disconnect" + || request?.status === "mid_stream_error_event" || (Number.isFinite(request?.http_status) && request.http_status >= 400) )); + const modelRequestCounts = {}; + for (const request of chatRequests) { + const model = String(request?.model || ""); + if (!model) continue; + modelRequestCounts[model] = (modelRequestCounts[model] || 0) + 1; + } return { status: state.status || "unknown", @@ -16,6 +24,10 @@ export function summarizeFakeProviderState(state) { recent_request_count: recentRequests.length, chat_request_count: chatRequests.length, fault_count: faultRequests.length, + mid_stream_disconnect_count: chatRequests.filter( + (request) => ["mid_stream_disconnect", "mid_stream_error_event"].includes(request?.status), + ).length, + model_request_counts: modelRequestCounts, streamed_request_count: chatRequests.filter((request) => request?.stream === true).length, duration_ms: stats(chatRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)), successful_duration_ms: stats(successfulRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)), diff --git a/skills/skills/langbot-testing/probes/pytest-probe.mjs b/skills/skills/langbot-testing/probes/pytest-probe.mjs index db98361e4..e6adcee70 100755 --- a/skills/skills/langbot-testing/probes/pytest-probe.mjs +++ b/skills/skills/langbot-testing/probes/pytest-probe.mjs @@ -5,7 +5,10 @@ import { basename, delimiter, join, resolve } from "node:path"; import { env } from "node:process"; function loadEnvDefaults(root) { - for (const path of [join(root, "skills/.env"), join(root, "skills/.env.local")]) { + for (const path of [ + join(root, "skills/.env"), + join(root, "skills/.env.local"), + ]) { if (!existsSync(path)) continue; for (const rawLine of readFileSync(path, "utf8").split(/\r?\n/)) { const line = rawLine.trim(); @@ -14,13 +17,20 @@ function loadEnvDefaults(root) { if (sep === -1) continue; const key = line.slice(0, sep).trim(); if (env[key]) continue; - env[key] = line.slice(sep + 1).trim().replace(/^["']|["']$/g, ""); + env[key] = line + .slice(sep + 1) + .trim() + .replace(/^["']|["']$/g, ""); } } } function timestampSlug(date = new Date()) { - return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); + return date + .toISOString() + .replace(/\.\d{3}Z$/, "Z") + .replace(/[^0-9A-Za-z]+/g, "-") + .replace(/^-|-$/g, ""); } function localIsoWithOffset(date = new Date()) { @@ -88,7 +98,14 @@ async function runProcess(command, timeoutMs, childEnv) { }); child.on("error", (error) => { clearTimeout(timeout); - resolveDone({ stdout, stderr, error, timedOut, status: null, signal: null }); + resolveDone({ + stdout, + stderr, + error, + timedOut, + status: null, + signal: null, + }); }); child.on("close", (status, signal) => { clearTimeout(timeout); @@ -109,10 +126,14 @@ export async function runPytestProbe({ }) { const root = resolve(env.LBS_ROOT || process.cwd()); loadEnvDefaults(root); - const resolvedTimeoutMs = Number(timeoutMs || env.LANGBOT_AGENT_RUNNER_PROBE_TIMEOUT_MS || "180000"); + const resolvedTimeoutMs = Number( + timeoutMs || env.LANGBOT_RUNNER_PROBE_TIMEOUT_MS || "180000", + ); const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; - const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + const evidenceDir = resolve( + env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId), + ); await mkdir(evidenceDir, { recursive: true }); const uvCacheDir = env.UV_CACHE_DIR || join(evidenceDir, ".uv-cache"); await mkdir(uvCacheDir, { recursive: true }); @@ -129,7 +150,7 @@ export async function runPytestProbe({ const resultJson = join(evidenceDir, "result.json"); const command = { executable: "rtk", - args: ["uv", "run", "pytest", "-q", ...testTargets], + args: ["uv", "run", "--no-sync", "pytest", "-q", ...testTargets], cwd: repoPath, }; const result = { @@ -171,14 +192,21 @@ export async function runPytestProbe({ result.status = "env_issue"; result.reason = `${repoEnvKey || "repo"} did not resolve to an existing directory: ${repoPath}`; } else { - const missingTargets = testTargets.filter((target) => !existsSync(join(repoPath, target.split("::")[0]))); + const missingTargets = testTargets.filter( + (target) => !existsSync(join(repoPath, target.split("::")[0])), + ); if (missingTargets.length > 0) { result.status = "env_issue"; result.reason = `pytest target file(s) not found in ${basename(repoPath)}: ${missingTargets.join(", ")}`; } else { const childEnv = { ...process.env, UV_CACHE_DIR: uvCacheDir }; if (pythonPaths.length > 0) { - childEnv.PYTHONPATH = [pythonPaths.join(delimiter), childEnv.PYTHONPATH].filter(Boolean).join(delimiter); + childEnv.PYTHONPATH = [ + pythonPaths.join(delimiter), + childEnv.PYTHONPATH, + ] + .filter(Boolean) + .join(delimiter); } const proc = await runProcess(command, resolvedTimeoutMs, childEnv); result.exit_status = proc.status; @@ -195,7 +223,11 @@ export async function runPytestProbe({ } else if (proc.status === 0) { result.status = "pass"; result.reason = `pytest passed for ${testTargets.join(", ")}.`; - } else if (/command not found|no such file or directory|executable file not found/i.test(`${proc.stdout}\n${proc.stderr}`)) { + } else if ( + /command not found|no such file or directory|executable file not found/i.test( + `${proc.stdout}\n${proc.stderr}`, + ) + ) { result.status = "env_issue"; result.reason = `pytest command could not run in ${repoPath}. See ${stdoutLog} and ${stderrLog}.`; } else { diff --git a/skills/skills/langbot-testing/references/agent-runner-qa-workflow.md b/skills/skills/langbot-testing/references/agent-runner-qa-workflow.md index 76e62fa44..e2a3f6635 100644 --- a/skills/skills/langbot-testing/references/agent-runner-qa-workflow.md +++ b/skills/skills/langbot-testing/references/agent-runner-qa-workflow.md @@ -1,6 +1,6 @@ -# AgentRunner QA Workflow +# Runner QA Workflow -Use this workflow when an agent finishes AgentRunner-related code and enters a +Use this workflow when an agent finishes Runner-related code and enters a test phase. ## Order @@ -25,7 +25,7 @@ test phase. backend is available and installing the QA fixture is acceptable. - `rtk bin/lbs test run agent-runner-qa-debug-chat --dry-run` when WebUI live execution needs deterministic coverage without a model provider. This - case runs its setup automation first: install the QA AgentRunner fixture, + case runs its setup automation first: install the QA Runner fixture, create/update the QA pipeline, write the case-specific pipeline env, then execute Debug Chat. - `rtk bin/lbs test run agent-runner-ledger-invariants --dry-run` @@ -51,8 +51,8 @@ only to review or adjust the generated list. | --- | --- | --- | | `LangBot/src/langbot/pkg/agent/runner/*`, `tests/unit_tests/agent/test_result_normalizer.py`, protocol/result/context/resource builders | `rtk bin/lbs test run agent-runner-fixture-contract --dry-run`; `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted LangBot unit tests for touched files | Result shape, user-visible runner output, or Debug Chat delivery changed: add `pipeline-debug-chat` or `local-agent-basic-debug-chat`. | | `LangBot/src/langbot/pkg/entity/persistence/agent_run.py`, `run_journal.py`, run ledger store/API/auth tests, claim/lease/status code | `rtk bin/lbs test run agent-runner-ledger-invariants --dry-run`; `rtk bin/lbs test run agent-runner-ledger-stress --dry-run`; `rtk bin/lbs test run agent-runner-ledger-contention --dry-run`; `rtk bin/lbs test run agent-runner-async-db-readiness --dry-run` before `rtk bin/lbs test run agent-runner-ledger-concurrency --dry-run` | Debug Chat run lifecycle, resume, or visible completion changed: add `local-agent-basic-debug-chat`. | -| `langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/agent_runner/*`, `api/proxies/agent_run_api.py`, runtime pull handlers, plugin manager/runtime IO | `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted SDK pytest | Runtime delivery or tool-call surface changed: add `agent-runner-release-preflight`, then `local-agent-basic-debug-chat`. | -| `langbot-agent-runner/*/components/agent_runner/*`, external runner daemon/client code, ACP/Codex/Claude runner command wrappers | Repo-local targeted tests; `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-release-preflight --dry-run` | ACP or external coding runner behavior changed: add `acp-agent-runner-debug-chat`. | +| `langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/runner/*`, `api/proxies/agent_run_api.py`, runtime pull handlers, plugin manager/runtime IO | `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted SDK pytest | Runtime delivery or tool-call surface changed: add `agent-runner-release-preflight`, then `local-agent-basic-debug-chat`. | +| `langbot-agent-runner/*/components/runner/*`, external runner daemon/client code, ACP/Codex/Claude runner command wrappers | Repo-local targeted tests; `rtk bin/lbs test run agent-runner-runtime-chaos --dry-run`; `rtk bin/lbs test run agent-runner-release-preflight --dry-run` | ACP or external coding runner behavior changed: add `acp-agent-runner-debug-chat`. | | Prompt preprocessing, effective prompt, pipeline AI config, runner binding/default runner migration | `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted LangBot pipeline/agent tests | The runner reads host-provided prompt or saved runner config: add `local-agent-effective-prompt-debug-chat`. | | Context window, transcript, history/event state, compaction, checkpoint/steering | `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted LangBot agent state/context tests | Multi-turn memory, compaction, or steering behavior changed: add `local-agent-context-compaction-debug-chat` and, for steering-specific changes, `local-agent-steering-debug-chat`. | | Plugin tool authorization, host tool listing, MCP tool bridge, function-call conversion | `rtk bin/lbs test run agent-runner-behavior-matrix --dry-run`; targeted plugin/MCP/tool tests | Tool execution is user-visible: add `local-agent-plugin-tool-call-debug-chat`; for MCP-specific changes add `mcp-stdio-register` then `mcp-stdio-tool-call`. | diff --git a/skills/skills/langbot-testing/references/agent-runner-release-gate.md b/skills/skills/langbot-testing/references/agent-runner-release-gate.md index 89a75fcfa..ab07ae485 100644 --- a/skills/skills/langbot-testing/references/agent-runner-release-gate.md +++ b/skills/skills/langbot-testing/references/agent-runner-release-gate.md @@ -1,4 +1,4 @@ -# Agent Runner Release Gate +# Runner Release Gate Use this reference when judging whether runner externalization is release-ready. The goal is not to enumerate every possible prompt. The gate covers product abilities and trust boundaries with deterministic normal-path cases, then leaves rare negative branches to unit and contract tests. @@ -35,7 +35,7 @@ For a quick early blocker check, run: rtk bin/lbs test run agent-runner-release-preflight --dry-run ``` -For the code-level AgentRunner probes, run: +For the code-level Runner probes, run: ```bash rtk bin/lbs test run agent-runner-behavior-matrix --dry-run @@ -70,7 +70,7 @@ API integration gate, not a Debug Chat execution proof. `agent-runner-qa-debug-chat` is the deterministic live execution proof. It uses a pipeline created by `scripts/e2e/ensure-qa-agent-runner-pipeline.mjs` and -expects Debug Chat to return `QA_AGENT_RUNNER_OK:
+
{ + event.preventDefault(); + void save(); + }} + /> + {failed && ( + + + {t('agents.eventProcessor.loadError')} + + + )} +
+ + {t('agents.eventProcessor.runs')}{' '} + ({runs.length}) + + +
+ {runs.length > 0 && ( + void openRun(run)} + footer={ + cursor !== null ? ( + + ) : undefined + } + /> + )} + +
+ {!selected ? ( + + + {loading + ? t('common.loading') + : t('agents.eventProcessor.noRuns')} + + + + ) : ( + <> +
+

+ {eventPatternLabel(selected.metadata.event_type ?? '', t)} +

+

+ {new Date(selected.created_at * 1000).toLocaleString()} +

+
+ + {t(`agents.eventProcessor.status_${selected.status}`, { + defaultValue: selected.status, + })} + + + {selected.metadata.delivery != null && ( + + )} + + {selected.status === 'failed' && selected.status_reason && ( + + + {selected.status_reason} + + + )} + {eventCursor !== null && ( + + )} + + )} +
+ +
+ ); + + if (!initialLoadComplete) + return void load()} />; + + return ( + <> + + : undefined + } + titleControls={ + { + setComponentRef(value); + setActiveTab('config'); + const descriptor = components.find((item) => item.id === value); + setParameters( + Object.fromEntries( + (descriptor?.config_schema ?? []) + .filter((field) => field.default !== undefined) + .map((field) => [field.name, field.default]), + ), + ); + validate.current = null; + }} + /> + } + status={ + !loading && componentRef && !available + ? { label: t('agents.eventProcessor.unavailable'), tone: 'error' } + : undefined + } + saveLabel={t('common.save')} + saveFormId="event-processor-form" + canSave={canManage && available} + isDirty={dirty} + isSaving={saving} + headerActions={ + canManage ? ( + + ) : undefined + } + configTitle={t('agents.eventProcessor.type')} + configTabs={{ + value: activeTab, + onValueChange: setActiveTab, + items: [ + { + value: 'config', + label: t('agents.eventProcessor.configTab'), + icon: , + content: ( +
+ {!component ? ( +

+ {t('agents.eventProcessor.selectComponent')} +

+ ) : component.config_schema.length === 0 ? ( +

+ {t('agents.eventProcessor.noSettings')} +

+ ) : ( +
+ + setParameters(values as Record) + } + onValidate={(fn) => { + validate.current = fn; + }} + /> +
+ )} +
+ ), + }, + { + value: 'logs', + label: t('agents.eventProcessor.logsTab'), + icon: , + content: logsContent, + }, + ], + }} + debugTitle={canOperate ? t('agents.debugTab') : undefined} + debugDescription={t('agents.eventProcessor.debugNotice')} + debugContent={ + canOperate ? ( + !component ? ( + + + {t('agents.eventProcessor.selectToDebug')} + + + ) : ( + { + setActiveTab('logs'); + void refreshLatestRun(); + }} + supportedEventPatterns={component.supported_event_patterns} + availableEventTypes={availableEventTypes} + /> + ) + ) : undefined + } + unsavedLabel={t('pipelines.unsavedChanges')} + /> + + ); +} diff --git a/web/src/app/home/agents/components/AgentApiToolPicker.tsx b/web/src/app/home/agents/components/AgentApiToolPicker.tsx new file mode 100644 index 000000000..6b7032c53 --- /dev/null +++ b/web/src/app/home/agents/components/AgentApiToolPicker.tsx @@ -0,0 +1,383 @@ +import { useEffect, useMemo, useState } from 'react'; +import { ChevronDown, Search } from 'lucide-react'; +import { useTranslation } from 'react-i18next'; +import type { AgentPlatformTool, PluginTool } from '@/app/infra/entities/api'; +import { extractI18nObject } from '@/i18n/I18nProvider'; +import { Checkbox } from '@/components/ui/checkbox'; +import { Input } from '@/components/ui/input'; +import { cn } from '@/lib/utils'; + +interface AgentApiToolPickerProps { + platformTools: AgentPlatformTool[]; + platformValue: string[]; + onPlatformChange: (value: string[]) => void; + hostTools: PluginTool[]; + hostValue: string[]; + onHostChange: (value: string[]) => void; + platformCatalogAvailable?: boolean; + hostCatalogAvailable?: boolean; + scopes?: readonly ToolScope[]; +} + +type ToolScope = 'event' | 'platform' | 'builtin' | 'mcp' | 'plugin' | 'skill'; + +type ToolEntry = { + key: string; + kind: 'platform' | 'host'; + name: string; + scope: ToolScope; + group: string; + label: string; + description: string; + parameters: Record; + api?: string; + eventPatterns?: string[]; + risk?: AgentPlatformTool['risk']; +}; + +const PLATFORM_CATEGORY_LABELS: Record = { + message: { zh: '消息', en: 'Messages' }, + identity: { zh: '用户与身份', en: 'Users & identity' }, + group: { zh: '群组', en: 'Groups' }, + moderation: { zh: '群管理', en: 'Moderation' }, + request: { zh: '请求处理', en: 'Requests' }, +}; + +const SCOPE_ORDER: ToolScope[] = [ + 'event', + 'platform', + 'builtin', + 'mcp', + 'plugin', + 'skill', +]; + +function normalizeHostScope(tool: PluginTool): ToolScope { + if (tool.source === 'mcp' || tool.source === 'plugin') return tool.source; + if (tool.source === 'skill') return 'skill'; + return 'builtin'; +} + +export default function AgentApiToolPicker({ + platformTools, + platformValue, + onPlatformChange, + hostTools, + hostValue, + onHostChange, + platformCatalogAvailable = true, + hostCatalogAvailable = true, + scopes = SCOPE_ORDER, +}: AgentApiToolPickerProps) { + const { t, i18n } = useTranslation(); + const [query, setQuery] = useState(''); + const [activeScope, setActiveScope] = useState( + scopes[0] ?? 'event', + ); + const [expandedTool, setExpandedTool] = useState(null); + const isChinese = i18n.language.startsWith('zh'); + const selectedPlatform = useMemo( + () => new Set(platformValue), + [platformValue], + ); + const selectedHost = useMemo(() => new Set(hostValue), [hostValue]); + + const entries = useMemo( + () => [ + ...platformTools.map((tool) => ({ + key: `platform:${tool.name}`, + kind: 'platform' as const, + name: tool.name, + scope: tool.scope, + group: tool.category, + label: extractI18nObject(tool.label), + description: extractI18nObject(tool.description), + parameters: tool.parameters, + api: tool.api, + eventPatterns: tool.event_patterns, + risk: tool.risk, + })), + ...hostTools.map((tool) => ({ + key: `host:${tool.source || 'builtin'}:${tool.source_id || ''}:${tool.name}`, + kind: 'host' as const, + name: tool.name, + scope: normalizeHostScope(tool), + group: tool.source_name || t('agents.langbotBuiltIn'), + label: tool.name, + description: tool.human_desc || tool.description || tool.name, + parameters: tool.parameters as Record, + })), + ], + [hostTools, platformTools, t], + ); + + const scopeCounts = useMemo( + () => + Object.fromEntries( + SCOPE_ORDER.map((scope) => [ + scope, + entries.filter((tool) => tool.scope === scope).length, + ]), + ) as Record, + [entries], + ); + const visibleScopes = SCOPE_ORDER.filter( + (scope) => + scopes.includes(scope) && (scope !== 'skill' || scopeCounts.skill > 0), + ); + useEffect(() => { + if (!visibleScopes.includes(activeScope) && visibleScopes[0]) { + setActiveScope(visibleScopes[0]); + setExpandedTool(null); + } + }, [activeScope, visibleScopes]); + + const filteredEntries = useMemo(() => { + const needle = query.trim().toLocaleLowerCase(); + return entries.filter( + (tool) => + tool.scope === activeScope && + (!needle || + [tool.name, tool.label, tool.description, tool.api, tool.group] + .join(' ') + .toLocaleLowerCase() + .includes(needle)), + ); + }, [activeScope, entries, query]); + const groupedEntries = useMemo(() => { + const groups = new Map(); + for (const tool of filteredEntries) { + if (!groups.has(tool.group)) groups.set(tool.group, []); + groups.get(tool.group)!.push(tool); + } + return Array.from(groups.entries()); + }, [filteredEntries]); + const selectedCount = entries.filter( + (tool) => + scopes.includes(tool.scope) && + (tool.kind === 'platform' + ? selectedPlatform.has(tool.name) + : selectedHost.has(tool.name)), + ).length; + + const scopeLabel = (scope: ToolScope) => { + const keys: Record = { + event: 'agents.eventApiTools', + platform: 'agents.platformApiTools', + builtin: 'agents.sandboxTools', + mcp: 'agents.mcpTools', + plugin: 'agents.pluginTools', + skill: 'agents.skillTools', + }; + return t(keys[scope]); + }; + + const groupLabel = (group: string) => { + if (activeScope === 'event' || activeScope === 'platform') { + return isChinese + ? PLATFORM_CATEGORY_LABELS[group]?.zh || group + : PLATFORM_CATEGORY_LABELS[group]?.en || group; + } + return group; + }; + + const setTool = (tool: ToolEntry, checked: boolean) => { + if (tool.kind === 'platform') { + const next = new Set(platformValue); + if (checked) next.add(tool.name); + else next.delete(tool.name); + onPlatformChange( + platformTools + .filter((item) => next.has(item.name)) + .map((item) => item.name), + ); + return; + } + const next = new Set(hostValue); + if (checked) next.add(tool.name); + else next.delete(tool.name); + onHostChange( + hostTools.filter((item) => next.has(item.name)).map((item) => item.name), + ); + }; + + const catalogAvailable = + activeScope === 'event' || activeScope === 'platform' + ? platformCatalogAvailable + : hostCatalogAvailable; + + return ( +
+
+ {visibleScopes.length === 1 ? ( + + {scopeLabel(visibleScopes[0])} + + {scopeCounts[visibleScopes[0]]} + + + ) : ( +
+ {visibleScopes.map((scope) => ( + + ))} +
+ )} + + {t('agents.apiToolsSelected', { + count: selectedCount, + })} + +
+ +
+ + setQuery(event.target.value)} + placeholder={t('agents.apiToolsSearch')} + className="pl-9" + /> +
+ + {!catalogAvailable && ( +
+ {activeScope === 'event' || activeScope === 'platform' + ? t('agents.apiToolsCatalogUnavailable') + : t('agents.hostToolsCatalogUnavailable')} +
+ )} + {catalogAvailable && !filteredEntries.length && ( +
+ {t('agents.apiToolsNoResults')} +
+ )} + {catalogAvailable && filteredEntries.length > 0 && ( +
+ {groupedEntries.map(([group, groupTools], groupIndex) => ( +
0 && 'border-t')}> +
+ {groupLabel(group)} +
+
+ {groupTools.map((tool) => { + const checked = + tool.kind === 'platform' + ? selectedPlatform.has(tool.name) + : selectedHost.has(tool.name); + const expanded = expandedTool === tool.key; + const parameterNames = Object.keys( + (tool.parameters.properties as + | Record + | undefined) ?? {}, + ); + return ( +
+ + + {expanded && ( +
+
+ {tool.name} +
+ {tool.api &&
API: {tool.api}
} + {tool.kind === 'host' && ( +
+ {t('agents.apiToolSource')}:{' '} + {groupLabel(tool.group)} +
+ )} + {tool.eventPatterns && ( +
+ {t('agents.apiToolEvents')}:{' '} + {tool.eventPatterns.join(', ')} +
+ )} +
+ {t('agents.apiToolParameters')}:{' '} + {parameterNames.length + ? parameterNames.join(', ') + : t('agents.apiToolNoParameters')} +
+
+ )} +
+ ); + })} +
+
+ ))} +
+ )} +
+ ); +} diff --git a/web/src/app/home/agents/components/AgentCreateContent.tsx b/web/src/app/home/agents/components/AgentCreateContent.tsx new file mode 100644 index 000000000..9c5fcf852 --- /dev/null +++ b/web/src/app/home/agents/components/AgentCreateContent.tsx @@ -0,0 +1,262 @@ +import { useState } from 'react'; +import { useForm } from 'react-hook-form'; +import { zodResolver } from '@hookform/resolvers/zod'; +import { z } from 'zod'; +import { useTranslation } from 'react-i18next'; +import { toast } from 'sonner'; +import { Bot, Workflow, Puzzle } from 'lucide-react'; +import { httpClient } from '@/app/infra/http/HttpClient'; +import { AgentKind } from '@/app/infra/entities/api'; +import { Button } from '@/components/ui/button'; +import { ToggleGroup, ToggleGroupItem } from '@/components/ui/toggle-group'; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from '@/components/ui/card'; +import { + Form, + FormControl, + FormField, + FormItem, + FormLabel, + FormMessage, +} from '@/components/ui/form'; +import { Input } from '@/components/ui/input'; +import EmojiPicker from '@/components/ui/emoji-picker'; +import ProcessorTypeDiagram from './ProcessorTypeDiagram'; + +export default function AgentCreateContent({ + onCreated, +}: { + onCreated: (agentId: string) => void; +}) { + const { t } = useTranslation(); + const [kind, setKind] = useState('agent'); + const formSchema = z.object({ + name: z.string().min(1, { message: t('agents.nameRequired') }), + description: z.string().optional(), + emoji: z.string().optional(), + }); + type FormValues = z.infer; + const form = useForm({ + resolver: zodResolver(formSchema), + defaultValues: { + name: '', + description: '', + emoji: '🤖', + }, + }); + + function handleKindChange(nextKind: AgentKind) { + const previousDefaultEmoji = + kind === 'pipeline' ? '⚙️' : kind === 'event_processor' ? '🧩' : '🤖'; + const nextDefaultEmoji = + nextKind === 'pipeline' + ? '⚙️' + : nextKind === 'event_processor' + ? '🧩' + : '🤖'; + setKind(nextKind); + const currentEmoji = form.getValues('emoji'); + if (!currentEmoji || currentEmoji === previousDefaultEmoji) { + form.setValue('emoji', nextDefaultEmoji); + } + } + + async function handleSubmit(values: FormValues) { + return httpClient + .createAgent({ + kind, + name: values.name, + description: values.description ?? '', + emoji: + values.emoji || + (kind === 'pipeline' + ? '⚙️' + : kind === 'event_processor' + ? '🧩' + : '🤖'), + }) + .then((resp) => { + toast.success(t('agents.createSuccess')); + onCreated(resp.uuid); + }) + .catch((err) => { + toast.error(t('agents.createError') + err.msg); + }); + } + + const typeOptions = [ + { + kind: 'pipeline' as const, + icon: Workflow, + title: t('agents.pipelineType'), + description: t('agents.pipelineTypeDescription'), + }, + { + kind: 'agent' as const, + icon: Bot, + title: t('agents.agentType'), + description: t('agents.agentTypeDescription'), + }, + { + kind: 'event_processor' as const, + icon: Puzzle, + title: t('agents.eventProcessor.type'), + description: t('agents.eventProcessor.description'), + }, + ]; + return ( +
+
+

+ {t('agents.eventProcessor.createPageTitle')} +

+ +
+ +
+
+
+
+
+
+

+ {t('agents.chooseType')} +

+

+ {t('agents.chooseTypeDescription')} +

+
+ + { + if (value) handleKindChange(value as AgentKind); + }} + variant="outline" + spacing={3} + className="grid w-full gap-3 sm:grid-cols-2 lg:grid-cols-1" + > + {typeOptions.map((option) => { + const Icon = option.icon; + return ( + + + + + + + {option.title} + + + {option.description} + + + + ); + })} + +
+ + + + {t('agents.basicInfo')} + + {t('agents.basicInfoDescription')} + + + + + +
+ ( + + + {t('common.name')} + * + + + + + + + )} + /> + ( + + {t('common.icon')} + + + + + + )} + /> +
+ + ( + + {t('common.description')} + + + + + + )} + /> + + + + +
+ + + + + + +
+
+
+
+ ); +} diff --git a/web/src/app/home/agents/components/AgentDebugPanel.tsx b/web/src/app/home/agents/components/AgentDebugPanel.tsx new file mode 100644 index 000000000..40fd8b862 --- /dev/null +++ b/web/src/app/home/agents/components/AgentDebugPanel.tsx @@ -0,0 +1,653 @@ +import { + useCallback, + useEffect, + useLayoutEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { useTranslation } from 'react-i18next'; +import { toast } from 'sonner'; +import { + AlertCircle, + AlertTriangle, + ChevronDown, + CircleHelp, + LoaderCircle, + Play, +} from 'lucide-react'; +import { httpClient } from '@/app/infra/http/HttpClient'; +import type { AgentPlatformTool } from '@/app/infra/entities/api'; +import { extractI18nObject } from '@/i18n/I18nProvider'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { Label } from '@/components/ui/label'; +import { + Select, + SelectContent, + SelectGroup, + SelectItem, + SelectLabel, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { Textarea } from '@/components/ui/textarea'; +import { Alert, AlertDescription, AlertTitle } from '@/components/ui/alert'; +import { + Collapsible, + CollapsibleContent, + CollapsibleTrigger, +} from '@/components/ui/collapsible'; +import { + eventGroupLabel, + eventPatternDescription, + eventPatternLabel, + groupEventPatterns, +} from '@/app/home/components/event-patterns/event-pattern-groups'; +import EventSelectOptionContent from '@/app/home/components/event-patterns/EventSelectOptionContent'; +import PluginProcessorTrace from './PluginProcessorTrace'; +import AgentExecutionTrace from './AgentExecutionTrace'; +import AgentEventDataEditor from './AgentEventDataEditor'; +import { + createDebugEventData, + debugEventInputText, + invalidDebugEventField, + parseDebugEventData, + processorDebugEventTypes, +} from './debug-event-data'; +import { executionSteps, type DebugExecutionEvent } from './debug-execution'; + +interface AgentDebugPanelProps { + agentId: string; + processor?: boolean; + availableEventTypes: string[]; + platformTools?: AgentPlatformTool[]; + supportedEventPatterns?: string[]; + beforeRun?: () => Promise; + onRunFinished?: () => void; + hasUnsavedChanges?: boolean; + onOpenRunnerConfig?: () => void; +} + +interface DebugEntry { + id: string; + direction: 'input' | 'output' | 'error'; + eventType: string; + text: string; + errorCode?: string; + detail?: string; + events?: DebugExecutionEvent[]; + finished?: boolean; +} + +function createDebugSessionId(agentId: string) { + const nonce = globalThis.crypto?.randomUUID?.() ?? String(Date.now()); + return `webui:${agentId}:${nonce}`; +} + +function matchesEventPattern(pattern: string, eventType: string) { + const escaped = pattern.replace(/[.+?^${}()|[\]\\]/g, '\\$&'); + return new RegExp(`^${escaped.replaceAll('*', '.*')}$`).test(eventType); +} + +export default function AgentDebugPanel({ + agentId, + processor = false, + availableEventTypes, + platformTools = [], + supportedEventPatterns = ['*'], + beforeRun, + onRunFinished, + hasUnsavedChanges = false, + onOpenRunnerConfig, +}: AgentDebugPanelProps) { + const { t } = useTranslation(); + const toolLabels = Object.fromEntries( + platformTools.map((tool) => [tool.name, extractI18nObject(tool.label)]), + ); + const [preset, setPreset] = useState('message.received'); + const [customEventType, setCustomEventType] = useState('custom.event'); + const newEventData = useCallback( + (type: string) => + JSON.stringify( + createDebugEventData( + type, + { + user: t('agents.debugData.sampleUser'), + message: t('agents.debugData.sampleMessage'), + feedback: t('agents.debugData.sampleFeedback'), + }, + processor, + ), + null, + 2, + ), + [t, processor], + ); + const [eventDataText, setEventDataText] = useState(() => + newEventData('message.received'), + ); + const [mockOptionsText, setMockOptionsText] = useState('{}'); + const [running, setRunning] = useState(false); + const [entries, setEntries] = useState([]); + const transcriptRef = useRef(null); + const sessionIdRef = useRef(createDebugSessionId(agentId)); + const requestRef = useRef(null); + + useEffect(() => () => requestRef.current?.abort(), [agentId]); + + useLayoutEffect(() => { + const transcript = transcriptRef.current; + if (transcript) { + transcript.scrollTop = transcript.scrollHeight; + } + }, [entries]); + + const eventType = preset === 'custom' ? customEventType.trim() : preset; + const eventDataValid = parseDebugEventData(eventDataText) !== null; + const availableEvents = useMemo(() => { + const concretePatterns = supportedEventPatterns.filter( + (pattern) => pattern !== '*' && !pattern.endsWith('.*'), + ); + return Array.from( + new Set([ + ...(processor ? processorDebugEventTypes : availableEventTypes), + ...concretePatterns, + ]), + ) + .filter((candidate) => + supportedEventPatterns.some((pattern) => + matchesEventPattern(pattern, candidate), + ), + ) + .sort(); + }, [availableEventTypes, supportedEventPatterns, processor]); + const eventGroups = useMemo( + () => groupEventPatterns(availableEvents), + [availableEvents], + ); + const supportsCustomEvent = + !processor && + supportedEventPatterns.some( + (pattern) => pattern === '*' || pattern.endsWith('.*'), + ); + + const selectPreset = useCallback( + (value: string) => { + setPreset(value); + setEventDataText(newEventData(value)); + }, + [newEventData], + ); + + useEffect(() => { + if ( + availableEvents.includes(preset) || + (preset === 'custom' && supportsCustomEvent) + ) { + return; + } + selectPreset(availableEvents[0] ?? 'custom'); + }, [availableEvents, preset, supportsCustomEvent, selectPreset]); + + async function runDebugEvent() { + if (!eventType) { + toast.error(t('agents.debugEventTypeRequired')); + return; + } + if ( + !supportedEventPatterns.some((pattern) => + matchesEventPattern(pattern, eventType), + ) + ) { + toast.error(t('agents.debugUnsupportedEvent')); + return; + } + + const eventData = parseDebugEventData(eventDataText); + if (!eventData) { + toast.error(t('agents.debugInvalidPayload')); + return; + } + const invalidField = invalidDebugEventField( + eventType, + eventData, + processor, + ); + if (invalidField) { + toast.error( + t('agents.debugData.invalidField', { + field: t(`agents.debugData.${invalidField.label}`), + }), + ); + return; + } + const inputText = debugEventInputText(eventType, eventData, processor); + let mockOptions: Record; + try { + const parsed = JSON.parse(mockOptionsText || '{}'); + if (!parsed || Array.isArray(parsed) || typeof parsed !== 'object') + throw new Error('Mock options must be an object'); + mockOptions = parsed; + } catch { + toast.error(t('agents.debugInvalidMock')); + return; + } + + setRunning(true); + if (hasUnsavedChanges && beforeRun && !(await beforeRun())) { + setRunning(false); + return; + } + + const requestId = globalThis.crypto?.randomUUID?.() ?? String(Date.now()); + const controller = new AbortController(); + requestRef.current = controller; + const outputId = `execution:${requestId}`; + setEntries((current) => [ + ...current, + { + id: `input:${requestId}`, + direction: 'input', + eventType, + text: inputText.trim() || JSON.stringify(eventData, null, 2), + }, + ]); + try { + const result = await httpClient.streamDebugAgent( + agentId, + { + event_type: eventType, + text: inputText.trim(), + data: eventData, + mock: mockOptions, + conversation_id: sessionIdRef.current, + }, + (event) => { + if (controller.signal.aborted) return; + setEntries((current) => { + const existing = current.find((entry) => entry.id === outputId); + if (existing) + return current.map((entry) => + entry.id === outputId + ? { ...entry, events: [...(entry.events ?? []), event] } + : entry, + ); + return [ + ...current, + { + id: outputId, + direction: 'output', + eventType, + text: '', + events: [event], + }, + ]; + }); + }, + controller.signal, + ); + if (controller.signal.aborted) return; + setEntries((current) => + current.some((entry) => entry.id === outputId) + ? current.map((entry) => + entry.id === outputId + ? { + ...entry, + finished: true, + text: + processor || + executionSteps(entry.events ?? []).some( + (step) => + step.kind === 'tool' || step.text || step.reasoning, + ) + ? '' + : result.final_text || t('agents.debugNoTextOutput'), + } + : entry, + ) + : [ + ...current, + { + id: outputId, + direction: 'output', + eventType, + text: result.final_text || t('agents.debugNoTextOutput'), + }, + ], + ); + } catch (error) { + if (controller.signal.aborted) { + if (controller.signal.reason === 'user') { + setEntries((current) => [ + ...current.map((entry) => + entry.id === outputId ? { ...entry, finished: true } : entry, + ), + { + id: `cancel:${requestId}`, + direction: 'error', + eventType, + text: t('agents.debugCancelled'), + }, + ]); + } + return; + } + const errorCode = + typeof error === 'object' && error && 'code' in error + ? String((error as { code?: string }).code || '') + : ''; + const message = + typeof error === 'object' && error && 'msg' in error + ? String((error as { msg?: string }).msg || '') + : t('agents.debugRunFailed'); + const isConfigError = errorCode.endsWith('.config_invalid'); + const isExecutionError = errorCode === 'runner_execution_failed'; + const isTimeout = errorCode === 'runner.timeout'; + const friendlyMessage = isConfigError + ? t('agents.debugRunnerConfigInvalidDescription', { + message: + message === 'api-key is required' + ? t('agents.debugApiKeyRequired') + : message, + }) + : isExecutionError + ? t('agents.debugRunnerExecutionFailedDescription') + : isTimeout + ? t('agents.debugRunnerTimeoutDescription') + : message || t('agents.debugRunFailed'); + setEntries((current) => [ + ...current.map((entry) => + entry.id === outputId ? { ...entry, finished: true } : entry, + ), + { + id: `error:${requestId}`, + direction: 'error', + eventType, + text: friendlyMessage, + errorCode, + detail: + isExecutionError || isTimeout + ? message || t('agents.debugRunFailed') + : undefined, + }, + ]); + } finally { + if (requestRef.current === controller) { + requestRef.current = null; + setRunning(false); + onRunFinished?.(); + } + } + } + + return ( +
+
+
+

{t('agents.debugTranscript')}

+

+ {t( + processor + ? 'agents.eventProcessor.debugDescription' + : 'agents.debugTranscriptDescription', + )} +

+
+ {entries.length === 0 ? ( + !processor && ( + + + {t('agents.debugEmptyTitle')} + + {t('agents.debugEmptyTranscript')} + + + ) + ) : ( +
+ {entries + .filter( + (entry) => + entry.direction !== 'output' || + entry.text || + processor || + executionSteps(entry.events ?? []).some( + (step) => + step.kind === 'tool' || step.text || step.reasoning, + ), + ) + .map((entry) => ( + + {entry.direction === 'error' && } +
+
+ + {entry.eventType} + + + {entry.direction === 'output' + ? t( + processor + ? 'agents.eventProcessor.debugOutput' + : 'agents.debugAgentOutput', + ) + : entry.direction === 'error' + ? t('common.error') + : t('agents.debugTestInput')} + +
+ {entry.events && + (processor ? ( + + ) : ( + + ))} + {entry.text && ( + 
+                        {entry.text}
+
+ )} + {entry.detail && ( + + + + + + 
+                            {entry.detail}
+
+ + + )} + {(entry.errorCode?.endsWith('.config_invalid') || + entry.errorCode === 'runner_execution_failed' || + entry.errorCode === 'runner.timeout') && + onOpenRunnerConfig && ( + + )} +
+ + ))} +
+ )} +
+ +
+
+ {supportedEventPatterns.length === 0 ? ( + + + {t('agents.debugNoEventsTitle')} + + {t('agents.debugNoEventsDescription')} + + + ) : ( + <> +
+ + +
+ + {preset === 'custom' && ( +
+ + setCustomEventType(event.target.value)} + placeholder="custom.event" + /> +
+ )} + + + + + + + + +

+ {t('agents.debugMockOptionsHelp')} +

+