diff --git a/.gitignore b/.gitignore index d0fe6acb6..97a64ba81 100644 --- a/.gitignore +++ b/.gitignore @@ -48,6 +48,7 @@ coverage.xml .coverage src/langbot/web/ testsdk/ +.qa/ # Build artifacts /dist diff --git a/AGENTS.md b/AGENTS.md index 86eee323f..a886d2e1a 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,160 +1,105 @@ # AGENTS.md -This file guides code agents (Claude Code, GitHub Copilot, OpenAI Codex, etc.) working in the LangBot project. `CLAUDE.md` is a symlink to this file. +This file guides code agents working in the LangBot main repository. `CLAUDE.md` is a symlink to this file. -## Project Overview +Read `ARCHITECTURE.md` before non-trivial backend, frontend, runtime, plugin, Box, MCP, persistence, or cross-repo SDK changes. This file is the working checklist; `ARCHITECTURE.md` is the system map. -LangBot is an open-source, LLM-native instant-messaging bot development platform. It aims to provide an out-of-the-box IM bot development experience with Agent, RAG, MCP and other LLM application capabilities, supporting mainstream global IM platforms and exposing rich APIs for custom development. +## Quick Facts -LangBot has a comprehensive web frontend — almost every operation can be performed through it. +- Python backend: `>=3.11,<4.0`, dependencies managed by `uv`. +- Frontend: `web/` is Vite + React Router 7 + shadcn/ui + Tailwind, managed by `pnpm`. +- Backend framework: Quart served by Hypercorn on `api.port`, default `5300`. +- Frontend dev server: `web/` on `3000`, with `VITE_API_BASE_URL` pointing at the backend. +- Plugin/Box/runtime contracts live in sibling repo `langbot-plugin-sdk`, pinned as `langbot-plugin` in `pyproject.toml`. -- **Python**: `>=3.11,<4.0`, dependencies managed by `uv`. Package version is in `pyproject.toml`. -- **Frontend**: `web/` is a **Vite + React Router 7 + shadcn/ui + Tailwind CSS** SPA, managed by `pnpm`. (Note: this is NOT Next.js — the `dev` script is `vite`.) -- **Backend framework**: Quart (the async flavour of Flask). The HTTP API and the pre-built web UI are both served by the backend on `http://127.0.0.1:5300`. - -## Repository Layout - -``` -LangBot/ -├── main.py # Entrypoint shim -> langbot.__main__.main() -├── pyproject.toml # Python project + deps (uv), pins langbot-plugin== -├── src/langbot/ -│ ├── __main__.py # Real entrypoint, CLI args (--standalone-runtime, --standalone-box, --debug) -│ ├── pkg/ # Core backend package -│ │ ├── api/ # HTTP API controllers + services (Quart) -│ │ ├── core/ # App bootstrap, stages, task manager -│ │ ├── platform/ # IM platform adapters, bot managers, session managers -│ │ ├── provider/ # LLM providers, requesters, tool providers -│ │ ├── pipeline/ # Pipelines, stages, query pool -│ │ ├── plugin/ # Bridge connecting LangBot to the plugin runtime (see below) -│ │ ├── box/ # Code-sandbox subsystem (Docker / nsjail / E2B backends) -│ │ ├── skill/ # Skill subsystem -│ │ ├── rag/ , vector/ # RAG + vector store -│ │ ├── command/ # Built-in commands -│ │ ├── persistence/ # ORM models + Alembic migrations (SQLite & PostgreSQL) -│ │ ├── storage/ # Object/file storage abstractions -│ │ ├── config/, entity/, discover/, utils/, telemetry/, survey/ -│ ├── libs/ # Vendored SDKs (qq_official_api, wecom_api, etc.) -│ └── templates/ # Config/component templates (e.g. templates/config.yaml) -├── web/ # Frontend SPA (Vite + React Router 7 + shadcn + Tailwind) -└── docker/ # docker-compose deployment files -``` - -## Development Environment Setup - -Full guide lives in the wiki: **["开发配置" / Dev Config](https://docs.langbot.app/zh/develop/dev-config)**. Summary: - -### Backend - -```bash -pip install uv -uv sync --dev # uv creates a .venv/ for you; point your editor's interpreter at it -uv run main.py # serves API + web UI on http://127.0.0.1:5300 -``` - -On first run the config file is generated at `data/config.yaml`. DB is SQLite by default (zero setup); PostgreSQL is supported. Migrations run automatically on startup. - -### Frontend - -Requires Node.js + [pnpm](https://pnpm.io/installation). - -```bash -cd web -cp .env.example .env # Windows: copy .env.example .env -pnpm install -pnpm dev # http://127.0.0.1:3000 (npm install / npm run dev also work) -``` - -`pnpm dev` reads `VITE_API_BASE_URL` from `web/.env` so the dev frontend can reach the backend on port `5300`. In production the frontend is pre-built into static files served by the backend on the same origin. - -### Code formatting - -The repo runs lint + format checks in CI. Install the pre-commit hooks so the same checks run locally before each commit: +## Essential Commands ```bash +uv sync --dev +uv run main.py uv run pre-commit install + +cd web +pnpm install +pnpm dev +pnpm build ``` -## Plugin System - -LangBot's plugin system (Plugin SDK, CLI `lbp`, Plugin Runtime, and the shared entity/API definitions) lives in a **separate repository**: [`langbot-plugin-sdk`](https://github.com/langbot-app/langbot-plugin-sdk). LangBot depends on it via the pinned `langbot-plugin` package in `pyproject.toml`. - -### Architecture (what to know inside this repo) - -- Plugins run as independent processes managed by the **Plugin Runtime**. The Runtime supports two control transports: `stdio` and `websocket`. -- When LangBot is started directly by a user (not in a container), it spawns and connects to the Runtime over **stdio** (lightweight/personal use). -- When LangBot runs in a container, it connects to a standalone Runtime over **WebSocket** (production). -- The bridge code lives in `src/langbot/pkg/plugin/` (`connector.py`, `handler.py`). -- Relevant config (`data/config.yaml`): `plugin.runtime_ws_url` (e.g. `ws://langbot_plugin_runtime:5400/control/ws`). Start LangBot with `--standalone-runtime` to make it connect to an externally-launched Runtime over WebSocket instead of spawning one over stdio. - -### Debugging the Plugin Runtime / CLI / SDK - -This is documented in detail in the **SDK repo's `AGENTS.md`** and in the wiki page **["调试插件运行时、CLI、SDK" / Plugin Runtime](https://docs.langbot.app/zh/develop/plugin-runtime)**. The short version: - -- Clone `LangBot` and `langbot-plugin-sdk` as siblings under one parent dir so the editor resolves shared entities. -- Start a standalone Runtime from the SDK repo: `uv run --no-sync lbp rt` (control port `5400`, debug port `5401`). -- To make LangBot use a locally-modified SDK: from the SDK dir, with LangBot's `.venv` active, run `uv pip install .`, then launch LangBot with `uv run --no-sync main.py --standalone-runtime` (keep `--no-sync` so your local SDK isn't overwritten). - -### Debugging the Box (sandbox) runtime - -The Box subsystem (`src/langbot/pkg/box/`) is the code sandbox. It picks the first available backend among **Docker / nsjail / E2B**. The standalone Box runtime is launched via the SDK CLI: `lbp box`. Backend selection details, the `lbp box` flags, and the SDK-side architecture are documented in the SDK repo's `AGENTS.md`. - -Relevant config (`data/config.yaml`, `box:` section): `box.enabled` (master switch — disabling it also disables the native sandbox tools, skill add/edit, and stdio-mode MCP servers), `box.backend` (`'local'` = Docker/nsjail auto-pick, or `'docker'` / `'nsjail'` / `'e2b'`; also settable via `BOX__BACKEND`), and `box.runtime.endpoint` (external Box runtime base URL, e.g. `ws://127.0.0.1:5410`; empty = local auto-managed runtime). Like the plugin runtime, LangBot can connect to an externally-launched Box runtime by setting that endpoint and starting with `--standalone-box`. - -> A common false "No supported sandbox backend (Docker / nsjail / E2B) is available" comes from Docker being installed and running but the current user not being in the `docker` group → `docker info` gets `permission denied` on the socket. Fix: `sudo usermod -aG docker ` and restart the backend in a shell that has the new group. - -## Development Standards - -- LangBot is a global project: **all code comments and docstrings must be in English**, and every user-facing string must support **i18n** (`en_US` + `zh_Hans` at minimum, plus `ja_JP` where the repo already has it). -- LangBot is adopted in both toC and toB scenarios — always consider compatibility and security. -- **Commit message format**: `(): ` - - `type`: one of `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `chore`, etc. - - `scope`: the affected package/module/file/class. - - `subject`: concise description of the change. - -### Database migrations (Alembic) - -LangBot uses [Alembic](https://alembic.sqlalchemy.org/) for migrations, supporting both SQLite and PostgreSQL from a single set of scripts. Migration files live in `src/langbot/pkg/persistence/alembic/versions/`. - -If you change ORM model definitions, generate a migration: +Useful focused tests: ```bash -# Run from the project root (requires data/config.yaml to exist) -uv run python -m langbot.pkg.persistence.alembic_runner autogenerate "description of your change" +uv run pytest tests/unit_tests -q +uv run pytest tests/integration -q +uv run pytest tests/integration/persistence -q +uv run pytest tests/manual/mcp_smoke.py + +cd web +pnpm lint +pnpm test:e2e ``` -Review and edit the generated script before committing. Migrations execute automatically on startup. `autogenerate` detects schema changes (add/drop columns, tables, type changes) but **data migrations** (e.g. mutating JSON field contents) must be hand-written into the generated script. `env.py` sets `render_as_batch=True`, so SQLite's ALTER TABLE limits are handled automatically — no need to branch per database. More in the wiki ["开发配置"](https://docs.langbot.app/zh/develop/dev-config#数据库迁移). +Run the narrowest useful test first, then broader checks when confidence is needed. -When writing a migration, follow these rules: +## Where to Look -- **Revision id ≤ 32 characters.** PostgreSQL stores `alembic_version.version_num` as `varchar(32)`; a longer id raises `StringDataRightTruncationError` at runtime. Prefer short, descriptive ids like `0005_add_llm_context_length`. -- **Guard every operation against missing tables/columns.** Fresh installs build the schema via `create_all()` and then stamp the Alembic baseline, so a migration may run against a table that already has the change — or, in tests, against an empty database. Check `inspector.get_table_names()` / `inspector.get_columns(...)` before `add_column` / `drop_column`, mirroring the existing migrations. -- **Keep a single linear head.** Chain `down_revision` to the current head; do not create branches. Run the migration tests after adding one: `uv run pytest tests/integration/persistence/ -q` (the PostgreSQL test needs a running PG via `TEST_POSTGRES_URL`). +- Architecture map: `ARCHITECTURE.md`. +- Dev environment guide: https://docs.langbot.app/zh/develop/dev-config. +- Plugin runtime / CLI / SDK debugging: https://docs.langbot.app/zh/develop/plugin-runtime. +- API-key auth: `docs/API_KEY_AUTH.md`. +- Box deep-dive notes: `docs/review/box-architecture.md` and related files. +- In-repo skills: `skills/` is the single source of truth for LangBot agent skills. +- SDK repo: `../langbot-plugin-sdk/` when changing shared entities, plugin APIs, action protocol, `lbp rt`, or `lbp box`. -> **Legacy migration system (deprecated — do not extend).** The old 3.x migration system under `src/langbot/pkg/persistence/migrations/` (`DBMigration` subclasses in `dbmXXX_*.py`, run from `pkg/persistence/mgr.py`) is **frozen**. Do **not** add new `dbmXXX_*.py` files. The chain is capped at `required_database_version = 25` (`pkg/utils/constants.py`); those files only exist to upgrade pre-existing 3.x databases up to the Alembic baseline and are kept read-only. All new schema changes go through Alembic. +## Cross-Repo SDK Work -## Agent-Facing Surfaces (MCP + Skills) +When changing SDK contracts used by LangBot: -LangBot is built to be **agent-friendly**. Three surfaces let AI agents work -with LangBot, and they MUST be kept in lockstep with the HTTP API: +```bash +# from langbot-plugin-sdk, with LangBot's .venv active +uv pip install . -1. **MCP server** — `src/langbot/pkg/api/mcp/` exposes a curated subset of the - API as MCP tools at `/mcp` (API-key authenticated, including the - `api.global_api_key` from config.yaml). `server.py` defines the tools (they - call the service layer directly); `mount.py` is the ASGI dispatcher. -2. **In-repo skills** — `skills/` is the **single source of truth** for agent - skills (plugin/core/deploy/e2e/MCP-ops). Docs and the landing page link here - rather than embedding their own copies. -3. **API-key auth** — `api.global_api_key` (config.yaml) authenticates the API - and MCP without a login session; see `docs/API_KEY_AUTH.md`. +# from LangBot, preserve the locally installed SDK +uv run --no-sync main.py +``` -> **Maintenance rule (important).** When you add, remove, or change an HTTP API -> endpoint that should be agent-accessible, you MUST update **both** the matching -> MCP tool in `src/langbot/pkg/api/mcp/server.py` **and** the relevant skill under -> `skills/` (especially `skills/skills/langbot-mcp-ops`). The API, the MCP tool -> surface, and the skills are one system — drift between them is a bug. +For standalone runtime debugging: -## Some Principles +```bash +# in langbot-plugin-sdk +uv run --no-sync lbp rt +uv run --no-sync lbp box + +# in LangBot +uv run --no-sync main.py --standalone-runtime +uv run --no-sync main.py --standalone-box +``` + +Config keys to verify in `data/config.yaml` / `src/langbot/templates/config.yaml`: + +- Plugin runtime: `plugin.runtime_ws_url`, default Docker host `langbot_plugin_runtime:5400/control/ws`. +- Box runtime: `box.enabled`, `box.backend`, `box.runtime.endpoint`, Docker host `langbot_box:5410`. +- API/MCP auth: `api.global_api_key`. + +## Change Rules + +- HTTP API changes that should be agent-accessible must update the matching MCP tool in `src/langbot/pkg/api/mcp/server.py` and the relevant skill under `skills/` in the same pass. +- New schema changes use Alembic under `src/langbot/pkg/persistence/alembic/versions/`; do not add legacy `dbmXXX` migrations. +- New platform behavior belongs in platform adapters only for platform translation; pipeline/business logic belongs in `pkg/pipeline/` or services. +- User-facing strings must support i18n (`en_US`, `zh_Hans`; include `ja_JP` where the repo already does). +- Code comments and docstrings must be English. +- Keep compatibility and security in mind; LangBot is used in both self-hosted/community and toB deployments. +- Commit message format: `(): `. + +## Runtime Pitfalls + +- Local stdio Plugin Runtime disconnects do not auto-reconnect; restart LangBot if that path breaks. +- Orphan runtime processes on `5400`/`5401` commonly break plugin debugging. +- Use `uv run --no-sync` after locally installing the SDK, or `uv` may restore the pinned package. +- A false Box “no backend” often means Docker is running but the current user lacks Docker socket permission. +- Do not confuse external MCP servers LangBot connects to (`pkg/provider/tools/loaders/mcp.py`) with LangBot's own `/mcp` server (`pkg/api/mcp/`). +- `CLAUDE.md` is a symlink to this file; edit `AGENTS.md`, not the symlink. + +## Principles - Keep it simple, stupid. - Entities should not be multiplied unnecessarily. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 000000000..ded90e7ea --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,250 @@ +# Architecture + +This document is a map of LangBot's moving parts. It is intentionally more stable than a feature guide and more concrete than the README: when you need to change behavior, start here, then follow the file references into the code. + +For agent-specific working rules, see `AGENTS.md`. For plugin-runtime and Box-runtime implementation details, also read the sibling SDK repo: [`langbot-plugin-sdk`](https://github.com/langbot-app/langbot-plugin-sdk). + +## What LangBot Is + +LangBot is an open-source platform for building production IM bots backed by LLMs, agents, RAG, plugins, MCP tools, and a web management panel. + +At runtime, one LangBot process owns: + +- a Quart/Hypercorn HTTP service and the built web UI on `:5300`; +- messaging-platform adapters such as Discord, Telegram, Slack, WeChat, QQ, WeCom, Lark, DingTalk, KOOK, LINE, Satori, Matrix, and HTTP/WebSocket bots; +- a pipeline engine that turns inbound platform messages into LLM/tool/plugin work and replies; +- persistence, storage, vector database, telemetry, monitoring, and configuration managers; +- bridges to the Plugin Runtime and Box Runtime provided by `langbot-plugin-sdk`; +- an MCP server at `/mcp` exposing a curated agent-facing subset of the service layer. + +## Repository Boundary + +LangBot is not a single-repo system. + +- `LangBot/` is the main product: backend, web UI, platform adapters, pipeline engine, HTTP API, MCP server, RAG, persistence, skills integration, and the bridge code that talks to runtimes. +- `langbot-plugin-sdk/` is published as `langbot-plugin` and pinned in `LangBot/pyproject.toml`. It contains plugin developer APIs, shared entities, `lbp`, the Plugin Runtime (`lbp rt`), and the Box Runtime (`lbp box`). +- Plugins import SDK APIs from `langbot_plugin.*`; the LangBot main process imports the same package for shared entities and runtime protocols. + +This split matters. If a change modifies SDK entities, component APIs, action protocols, `lbp rt`, or `lbp box`, verify the sibling SDK repo and install the local SDK into LangBot's virtualenv when testing cross-repo behavior. + +## Startup Path + +The process entrypoint is small and layered: + +1. `main.py` delegates to `langbot.__main__.main()`. +2. `src/langbot/__main__.py` parses `--standalone-runtime`, `--standalone-box`, and `--debug`, checks dependencies, generates missing config/data files, and calls `pkg.core.boot.main()`. +3. `pkg/core/boot.py` executes startup stages in order: `LoadConfigStage`, `GenKeysStage`, `SetupLoggerStage`, `BuildAppStage`, `ShowNotesStage`. +4. `BuildAppStage` constructs the `Application` object by wiring managers, services, runtime connectors, and controllers. +5. `Application.run()` starts the platform manager, query controller, HTTP controller, telemetry/cleanup loops, and plugin initialization. + +The central runtime object is `pkg/core/app.py::Application`. It is a service locator for long-lived managers. That is not elegant, but it is the current architectural center; most subsystems receive `ap: Application` and collaborate through it. + +## Top-Level Layout + +```text +LangBot/ +├── main.py # Entrypoint shim +├── pyproject.toml # Python package, deps, pinned langbot-plugin +├── src/langbot/ +│ ├── __main__.py # CLI entrypoint and boot handoff +│ ├── pkg/ +│ │ ├── core/ # Application, boot stages, task manager +│ │ ├── api/ # HTTP API + MCP server mount +│ │ ├── platform/ # IM adapters and runtime bot manager +│ │ ├── pipeline/ # Message routing and pipeline stages +│ │ ├── provider/ # LLM runners, model manager, tools +│ │ ├── plugin/ # LangBot-side Plugin Runtime connector/handler +│ │ ├── box/ # LangBot-side Box service/connector +│ │ ├── skill/ # Skill metadata/activation integration +│ │ ├── rag/ , vector/ # Knowledge-base and vector DB integration +│ │ ├── persistence/ # SQLAlchemy/SQLModel, Alembic, legacy migrations +│ │ ├── storage/ # Local/S3 file storage abstraction +│ │ └── config/, entity/, utils/, telemetry/, survey/ +│ ├── libs/ # Vendored third-party platform SDKs +│ └── templates/ # Default config and component metadata +├── web/ # Vite + React Router + shadcn/ui + Tailwind SPA +├── docker/ # Deployment manifests +├── skills/ # In-repo agent skills, single source of truth +└── tests/ # Unit/integration/e2e/manual tests +``` + +## The Runtime Graph + +The most useful mental model is this graph: + +```text +Platform adapter + → RuntimeBot + → MessageAggregator + → QueryPool + → Controller + → RuntimePipeline + → PipelineStage chain + → RequestRunner / ToolManager / PluginRuntimeConnector / BoxService + → response via adapter +``` + +The HTTP and MCP surfaces are parallel entrypoints into the same service layer: + +```text +HTTP client / Web UI + → Quart route group + → api/http/service/* + → Application managers / persistence / runtime connectors + +MCP client + → /mcp mount + → api/mcp/server.py tools + → the same service layer directly +``` + +## Message Flow + +Inbound platform messages enter through adapter-specific SDK callbacks. The common path is: + +1. A platform adapter under `pkg/platform/sources/` converts platform-specific events into SDK message/event entities. +2. `RuntimeBot` in `pkg/platform/botmgr.py` applies pipeline routing rules and either discards the message, pushes it to webhooks, or sends it to the message aggregator. +3. `MessageAggregator` batches/normalizes messages before adding a `Query` to `QueryPool`. +4. `Controller` in `pkg/pipeline/controller.py` selects queries subject to global pipeline concurrency and per-session concurrency. +5. `RuntimePipeline` in `pkg/pipeline/pipelinemgr.py` runs configured pipeline stages using a responsibility-chain style executor that supports generator stages. +6. The chat stage emits plugin events, calls a configured `RequestRunner`, handles streaming/non-streaming responses, records telemetry, and appends conversation history. +7. Output stages send text, cards, chunks, files, or error notices back through the original platform adapter. + +Pipeline components are registered by decorators and package import side effects. When adding a new stage, loader, runner, or adapter, check the corresponding preregistration mechanism instead of inventing a second registry. + +## Platform Layer + +Platform code lives under `pkg/platform/`. + +- `botmgr.py` owns runtime bots, routing rules, event logging, webhook pushing, and adapter lifecycle. +- `sources/` contains adapter implementations. Each adapter subclasses `langbot_plugin.api.definition.abstract.platform.adapter.AbstractMessagePlatformAdapter` from the SDK. +- Platform entities such as `MessageChain`, `Image`, `At`, `Voice`, and events come from `langbot-plugin-sdk`, not from this repo. + +The platform layer should translate between external platform APIs and LangBot's shared message/event model. It should not contain LLM-provider logic or pipeline business logic. + +## Pipeline Layer + +Pipeline code lives under `pkg/pipeline/`. + +Important pieces: + +- `pool.py::QueryPool` stores pending queries and cached in-flight queries for plugin backward-compatible calls. +- `controller.py::Controller` schedules query processing and enforces concurrency. +- `pipelinemgr.py::RuntimePipeline` materializes database pipeline config into a runtime stage chain. +- `process/handlers/chat.py::ChatMessageHandler` is the main LLM conversation handler. +- Stage families include response rules, banned sessions, content filters, preprocessors, rate limits, message truncation, long text handling, response-back, command handling, and wrappers. + +Pipelines are configuration-driven. Prefer adding a stage or extending an existing stage family over hard-coding behavior in platform adapters. + +## Provider, RAG, and Tools + +Provider code lives under `pkg/provider/`. + +- `modelmgr/` manages configured model providers and requesters. +- `runners/` implements request runners such as the local agent runner and external workflow integrations. +- `tools/toolmgr.py` aggregates tools from native tools, plugin tools, external MCP servers, and skill-authoring tools. +- `tools/loaders/mcp.py` is the MCP client side: external MCP servers that LangBot connects to for agent tools. +- RAG lives across `pkg/rag/`, `pkg/vector/`, model services, and plugin KnowledgeEngine actions. + +Do not confuse LangBot's MCP client side with LangBot's own MCP server at `/mcp`; they are different surfaces. + +## Plugin System + +The plugin system crosses the repo boundary. + +In this repo: + +- `pkg/plugin/connector.py` connects LangBot to the Plugin Runtime over stdio or WebSocket. +- `pkg/plugin/handler.py` exposes LangBot actions to the runtime and calls runtime actions for plugin operations. +- `pkg/provider/tools/loaders/plugin.py` exposes plugin Tool components to LLM runners. +- Pipeline handlers emit SDK events such as normal-message events and prompt-processing events. + +In `langbot-plugin-sdk`: + +- `src/langbot_plugin/api/` defines `BasePlugin`, component base classes, message/event entities, contexts, proxies, and manifests. +- `src/langbot_plugin/runtime/` implements `lbp rt`, plugin discovery, dependency installation, process launching, and control/debug connections. +- `src/langbot_plugin/entities/io/` defines the action protocol shared by LangBot, runtime, and plugin processes. + +The Plugin Runtime supports stdio and WebSocket control transports. Direct local LangBot runs usually spawn the runtime over stdio. Containerized/standalone deployments connect over WebSocket using `plugin.runtime_ws_url` and `--standalone-runtime`. + +## Box Runtime and Skills + +Box is the sandbox subsystem used by native agent tools, stdio MCP servers, skill authoring, and managed processes. + +In this repo: + +- `pkg/box/service.py` is the application-facing facade for exec, sessions, managed processes, skill CRUD, status, reconnects, quotas, mounts, and sandbox profiles. +- `pkg/box/connector.py` connects to the Box Runtime over stdio, Windows subprocess+WebSocket, or remote WebSocket. +- `pkg/provider/tools/loaders/native.py`, `mcp_stdio.py`, and skill loaders depend on Box availability. +- `pkg/skill/manager.py` loads skills from the Box runtime, falling back to local `data/skills` when needed. + +In `langbot-plugin-sdk`: + +- `src/langbot_plugin/box/server.py` implements `lbp box` and the WebSocket endpoints on `:5410`. +- `src/langbot_plugin/box/runtime.py` owns sandbox sessions and managed processes. +- `backend.py`, `nsjail_backend.py`, and `e2b_backend.py` implement sandbox backends. +- `skill_store.py` manages skill packages from the Box side. + +Important config keys live under `box:` in `src/langbot/templates/config.yaml`: `box.enabled`, `box.backend`, `box.runtime.endpoint`, and `box.local.*`. Start LangBot with `--standalone-box` when connecting to an externally launched Box runtime. + +## HTTP API, Web UI, and MCP Server + +`pkg/api/http/controller/main.py` builds a Quart app, registers route groups, serves the built SPA, and wraps the ASGI app with the MCP dispatcher. + +- HTTP route groups live under `pkg/api/http/controller/groups/`. +- Service-layer logic lives under `pkg/api/http/service/`. +- The built web UI is served from the frontend build path with SPA fallback. +- The MCP server lives under `pkg/api/mcp/` and is mounted at `/mcp`. + +The MCP server intentionally exposes a curated subset of the API. Tools call service classes directly rather than making HTTP requests back into LangBot. + +Maintenance rule: when adding, removing, or changing an HTTP endpoint that should be agent-accessible, update the matching MCP tool and the relevant in-repo skill under `skills/` in the same pass. + +## Persistence and Configuration + +Persistence is centered on `pkg/persistence/mgr.py`. + +- SQLite is the default database; PostgreSQL is supported. +- Models live under `pkg/entity/persistence/`. +- Fresh schemas are created from metadata, then legacy migrations run up to the frozen 3.x baseline, then Alembic migrations run to head. +- New schema changes should use Alembic under `pkg/persistence/alembic/versions/`; do not extend the frozen legacy migration chain. + +Configuration starts from `src/langbot/templates/config.yaml` and is generated into `data/config.yaml` on first run. Most long-lived managers read from `ap.instance_config.data`. + +## Frontend + +The frontend lives in `web/` and is a Vite SPA using React Router 7, shadcn/ui, Tailwind CSS, and pnpm. It is not Next.js, despite some historical filenames. + +In development, `pnpm dev` serves the UI on `:3000` and reads `VITE_API_BASE_URL` to call the backend on `:5300`. In production, the built frontend is packaged into the Python distribution and served by the backend. + +Keep frontend API behavior aligned with `pkg/api/http/service/` and route groups. User-facing strings must go through the existing i18n setup. + +## Agent-Facing Surfaces + +LangBot is deliberately agent-friendly. The agent-facing surfaces are part of the architecture, not extra docs. + +- `skills/` is the single source of truth for in-repo skills. +- `pkg/api/mcp/server.py` exposes the LangBot MCP server at `/mcp`. +- `api.global_api_key` authenticates API/MCP access without a browser login. +- `AGENTS.md` and `ARCHITECTURE.md` tell coding agents how the repo works. + +When one of these changes, update the others if the behavior or contract changed. API, MCP tools, and skills are one system; drift is a bug. + +## Where to Change Things + +- New HTTP API: add/adjust a service in `pkg/api/http/service/`, a route group in `pkg/api/http/controller/groups/`, tests, and MCP/skills if agent-accessible. +- New platform adapter: add a `pkg/platform/sources/*` adapter, component metadata/templates as needed, i18n, docs, and tests/smoke coverage. +- New pipeline behavior: add or extend a pipeline stage family under `pkg/pipeline/`; avoid putting pipeline rules in adapters. +- New LLM provider/requester: work under `pkg/provider/modelmgr/` and related service/UI surfaces. +- New LLM tool source: extend `pkg/provider/tools/loaders/` and `ToolManager` intentionally. +- New plugin component/API/protocol: change `langbot-plugin-sdk` first or in lockstep, then update LangBot bridge code. +- New Box capability: change both `pkg/box/` and `langbot-plugin-sdk/src/langbot_plugin/box/`, plus config and tests. +- New database schema: add an Alembic migration, not a legacy `dbmXXX` migration. + +## Design Biases + +- Keep platform translation, pipeline orchestration, provider execution, and runtime protocols separate. +- Reuse existing registries and service layers instead of adding parallel paths. +- Prefer small, explicit agent surfaces over exposing every internal API. +- Treat cross-repo contracts with the SDK as public interfaces. +- Test behavior at the narrowest useful layer first, then add integration/e2e coverage for runtime or platform changes. diff --git a/README.md b/README.md index a6247f7fe..eb3fa5e18 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@
-LangBot - Production-grade IM bot made easy. | Product Hunt +LangBot - Easy-to-use global IM bot platform designed for the LLM era | Product Hunt

Production-grade platform for building agentic IM bots.

Quickly build, debug, and ship AI bots to Slack, Discord, Telegram, WeChat, and more.

@@ -136,7 +136,7 @@ docker compose --profile all up -d | [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | GPU Platform | ✅ | | [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | GPU Platform | ✅ | | [接口 AI](https://jiekou.ai/) | Gateway | ✅ | -| [302.AI](https://share.302.ai/SuTG99) | Gateway | ✅ | +| [302.AI](https://share.302ai.cn/SuTG99) | Gateway | ✅ | | [Qiniu](https://www.qiniu.com/ai/agent) | Gateway | ✅ | [→ View all integrations](https://link.langbot.app/en/docs/features) diff --git a/README_CN.md b/README_CN.md index 38e632247..d4544be65 100644 --- a/README_CN.md +++ b/README_CN.md @@ -136,7 +136,7 @@ docker compose --profile all up -d | [优云智算](https://www.compshare.cn/?ytag=GPU_YY-gh_langbot) | GPU 平台 | ✅ | | [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | GPU 平台 | ✅ | | [接口 AI](https://jiekou.ai/) | 聚合平台 | ✅ | -| [302.AI](https://share.302.ai/SuTG99) | 聚合平台 | ✅ | +| [302.AI](https://share.302ai.cn/SuTG99) | 聚合平台 | ✅ | | [小马算力](https://www.tokenpony.cn/453z1) | 聚合平台 | ✅ | | [百宝箱Tbox](https://www.tbox.cn/open) | 智能体平台 | ✅ | | [七牛云Qiniu](https://www.qiniu.com/ai/agent) | 聚合平台 | ✅ | diff --git a/README_ES.md b/README_ES.md index dad707921..089fceaa4 100644 --- a/README_ES.md +++ b/README_ES.md @@ -5,7 +5,7 @@
-LangBot - Production-grade IM bot made easy. | Product Hunt +LangBot - Easy-to-use global IM bot platform designed for the LLM era | Product Hunt

Plataforma de grado de producción para construir bots de mensajería instantánea con agentes de IA.

Construya, depure y despliegue bots de IA rápidamente en Slack, Discord, Telegram, WeChat y más.

@@ -135,7 +135,7 @@ docker compose --profile all up -d | [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | Plataforma GPU | ✅ | | [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | Plataforma GPU | ✅ | | [接口 AI](https://jiekou.ai/) | Pasarela | ✅ | -| [302.AI](https://share.302.ai/SuTG99) | Pasarela | ✅ | +| [302.AI](https://share.302ai.cn/SuTG99) | Pasarela | ✅ | | [Qiniu](https://www.qiniu.com/ai/agent) | Pasarela | ✅ | [→ Ver todas las integraciones](https://link.langbot.app/en/docs/features) diff --git a/README_FR.md b/README_FR.md index ac047a7e0..06afaf842 100644 --- a/README_FR.md +++ b/README_FR.md @@ -5,7 +5,7 @@
-LangBot - Production-grade IM bot made easy. | Product Hunt +LangBot - Easy-to-use global IM bot platform designed for the LLM era | Product Hunt

Plateforme de niveau production pour construire des bots de messagerie instantanée avec agents IA.

Créez, déboguez et déployez rapidement des bots IA sur Slack, Discord, Telegram, WeChat et plus.

@@ -132,7 +132,7 @@ docker compose --profile all up -d | [ModelScope](https://modelscope.cn/docs/model-service/API-Inference/intro) | Passerelle | ✅ | | [GiteeAI](https://ai.gitee.com/) | Passerelle | ✅ | | [接口 AI](https://jiekou.ai/) | Passerelle | ✅ | -| [302.AI](https://share.302.ai/SuTG99) | Passerelle | ✅ | +| [302.AI](https://share.302ai.cn/SuTG99) | Passerelle | ✅ | | [CompShare](https://www.compshare.cn/?ytag=GPU_YY-gh_langbot) | Plateforme GPU | ✅ | | [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | Plateforme GPU | ✅ | | [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | Plateforme GPU | ✅ | diff --git a/README_JP.md b/README_JP.md index 55f7b945b..5a281ff31 100644 --- a/README_JP.md +++ b/README_JP.md @@ -5,7 +5,7 @@
-LangBot - Production-grade IM bot made easy. | Product Hunt +LangBot - Easy-to-use global IM bot platform designed for the LLM era | Product Hunt

AIエージェント搭載IMボットを構築するための本番グレードプラットフォーム。

Slack、Discord、Telegram、WeChat などに AI ボットを素早く構築、デバッグ、デプロイ。

@@ -135,7 +135,7 @@ docker compose --profile all up -d | [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | GPUプラットフォーム | ✅ | | [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | GPUプラットフォーム | ✅ | | [接口 AI](https://jiekou.ai/) | ゲートウェイ | ✅ | -| [302.AI](https://share.302.ai/SuTG99) | ゲートウェイ | ✅ | +| [302.AI](https://share.302ai.cn/SuTG99) | ゲートウェイ | ✅ | | [Qiniu](https://www.qiniu.com/ai/agent) | ゲートウェイ | ✅ | [→ すべての統合を表示](https://link.langbot.app/en/docs/features) diff --git a/README_KO.md b/README_KO.md index e39ee2803..4f0fd68ad 100644 --- a/README_KO.md +++ b/README_KO.md @@ -5,7 +5,7 @@
-LangBot - Production-grade IM bot made easy. | Product Hunt +LangBot - Easy-to-use global IM bot platform designed for the LLM era | Product Hunt

AI 에이전트 IM 봇 구축을 위한 프로덕션 등급 플랫폼.

Slack, Discord, Telegram, WeChat 등에 AI 봇을 빠르게 구축, 디버그 및 배포.

@@ -135,7 +135,7 @@ docker compose --profile all up -d | [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | GPU 플랫폼 | ✅ | | [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | GPU 플랫폼 | ✅ | | [接口 AI](https://jiekou.ai/) | 게이트웨이 | ✅ | -| [302.AI](https://share.302.ai/SuTG99) | 게이트웨이 | ✅ | +| [302.AI](https://share.302ai.cn/SuTG99) | 게이트웨이 | ✅ | | [Qiniu](https://www.qiniu.com/ai/agent) | 게이트웨이 | ✅ | [→ 모든 통합 보기](https://link.langbot.app/en/docs/features) diff --git a/README_RU.md b/README_RU.md index 0bac23c9f..20ed90e08 100644 --- a/README_RU.md +++ b/README_RU.md @@ -5,7 +5,7 @@
-LangBot - Production-grade IM bot made easy. | Product Hunt +LangBot - Easy-to-use global IM bot platform designed for the LLM era | Product Hunt

Платформа производственного уровня для создания агентных IM-ботов.

Быстро создавайте, отлаживайте и развертывайте ИИ-ботов в Slack, Discord, Telegram, WeChat и других платформах.

@@ -131,7 +131,7 @@ docker compose --profile all up -d | [Volc Engine Ark](https://console.volcengine.com/ark/region:ark+cn-beijing/model?vendor=Bytedance&view=LIST_VIEW) | Шлюз | ✅ | | [ModelScope](https://modelscope.cn/docs/model-service/API-Inference/intro) | Шлюз | ✅ | | [GiteeAI](https://ai.gitee.com/) | Шлюз | ✅ | -| [302.AI](https://share.302.ai/SuTG99) | Шлюз | ✅ | +| [302.AI](https://share.302ai.cn/SuTG99) | Шлюз | ✅ | | [接口 AI](https://jiekou.ai/) | Шлюз | ✅ | | [CompShare](https://www.compshare.cn/?ytag=GPU_YY-gh_langbot) | Платформа GPU | ✅ | | [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | Платформа GPU | ✅ | diff --git a/README_TW.md b/README_TW.md index 7893486b2..c0fe2fcbb 100644 --- a/README_TW.md +++ b/README_TW.md @@ -137,7 +137,7 @@ docker compose --profile all up -d | [優雲智算](https://www.compshare.cn/?ytag=GPU_YY-gh_langbot) | GPU 平台 | ✅ | | [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | GPU 平台 | ✅ | | [接口 AI](https://jiekou.ai/) | 聚合平台 | ✅ | -| [302.AI](https://share.302.ai/SuTG99) | 聚合平台 | ✅ | +| [302.AI](https://share.302ai.cn/SuTG99) | 聚合平台 | ✅ | | [Qiniu](https://www.qiniu.com/ai/agent) | 聚合平台 | ✅ | ### TTS(語音合成) diff --git a/README_VI.md b/README_VI.md index 3d6320c3f..a958190d2 100644 --- a/README_VI.md +++ b/README_VI.md @@ -5,7 +5,7 @@
-LangBot - Production-grade IM bot made easy. | Product Hunt +LangBot - Easy-to-use global IM bot platform designed for the LLM era | Product Hunt

Nền tảng cấp sản xuất để xây dựng bot IM với AI agent.

Xây dựng, gỡ lỗi và triển khai bot AI nhanh chóng trên Slack, Discord, Telegram, WeChat và nhiều nền tảng khác.

@@ -135,7 +135,7 @@ docker compose --profile all up -d | [PPIO](https://ppinfra.com/user/register?invited_by=QJKFYD&utm_source=github_langbot) | Nền tảng GPU | ✅ | | [ShengSuanYun](https://www.shengsuanyun.com/?from=CH_KYIPP758) | Nền tảng GPU | ✅ | | [接口 AI](https://jiekou.ai/) | Cổng | ✅ | -| [302.AI](https://share.302.ai/SuTG99) | Cổng | ✅ | +| [302.AI](https://share.302ai.cn/SuTG99) | Cổng | ✅ | | [Qiniu](https://www.qiniu.com/ai/agent) | Cổng | ✅ | [→ Xem tất cả tích hợp](https://link.langbot.app/en/docs/features) diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index ee60ca7d6..bdd347021 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -62,11 +62,12 @@ services: - TZ=Asia/Shanghai # Unified env-override convention: SECTION__SUBSECTION__KEY overrides the # matching config.yaml field (see LoadConfigStage). These map onto - # box.local.* and are forwarded to the Box runtime via INIT RPC. + # box.* and are forwarded to the Box runtime via INIT RPC. - BOX__LOCAL__HOST_ROOT=${LANGBOT_BOX_ROOT:-${PWD}/data/box} - BOX__LOCAL__DEFAULT_WORKSPACE=default - BOX__LOCAL__SKILLS_ROOT=skills - BOX__LOCAL__ALLOWED_MOUNT_ROOTS=${LANGBOT_BOX_ROOT:-${PWD}/data/box} + - BOX__DOCKER__CPU_LIMIT_ENABLED=${LANGBOT_BOX_DOCKER_CPU_LIMIT_ENABLED:-true} ports: - 5300:5300 # For web ui and webhook callback - 2280-2285:2280-2285 # For platform reverse connection diff --git a/docs/review/mcp-resources-pr-2215-review.md b/docs/review/mcp-resources-pr-2215-review.md new file mode 100644 index 000000000..8e57fc5fa --- /dev/null +++ b/docs/review/mcp-resources-pr-2215-review.md @@ -0,0 +1,196 @@ +# MCP Resources PR #2215 Review + +> 更新日期: 2026-06-29 +> 分支: `mcp_resources` +> PR: langbot-app/LangBot#2215 +> 主题: MCP Resources 在 LangBot 中的产品价值、AgentRunner 集成方式与后续架构方向 + +## 结论 + +PR #2215 对 LangBot 有明确价值:它补齐了 MCP 协议中 Resources 这一重要能力,让 MCP server 不再只暴露 tools,也可以暴露文档、代码片段、配置、日志、图片等上下文资源。管理端可以发现和预览资源,Agent 也可以通过当前实现按需列出和读取资源。 + +但当前 AgentRunner 层的接入方式更接近一个可用的第一阶段方案,而不是最终架构。现在 MCP Resources 被包装成两个 synthetic tools: + +- `langbot_mcp_list_resources` +- `langbot_mcp_read_resource` + +这让模型可以通过 function calling 主动探索资源,落地成本低,也复用了已有 `ToolManager` / `LocalAgentRunner` 的工具调用链路。不过从 MCP 规范和主流实现来看,Resources 更适合作为一种一等上下文来源,而不是长期隐藏在工具列表里。 + +建议保留当前 synthetic tools 作为探索能力,同时把后续主线设计调整为:MCP Resources 是 pipeline / conversation / message 级别可选择、可固定、可审计的上下文输入。 + +## 当前实现判断 + +当前 AgentRunner 集成路径如下: + +```text +Pipeline 绑定 MCP server + -> query.variables['_pipeline_bound_mcp_servers'] + -> Preproc 为 local-agent 加载工具 + -> ToolManager.get_all_tools() + -> MCPLoader 注入 synthetic resource tools + -> LocalAgentRunner 将工具 schema 传给模型 + -> 模型发起 list/read tool call + -> ToolManager.execute_func_call() + -> MCPLoader 调 MCP session.list_resources/read_resource + -> tool result 回灌给模型 +``` + +这个路径的优点是: + +- 复用现有工具调用机制,改动范围小。 +- Agent 可以按需探索资源,不需要每轮预先读取所有资源。 +- 可以沿用 pipeline 绑定的 MCP server 范围,避免越权读取未绑定 server。 +- 对已有 MCP tools 行为影响较小。 + +主要问题是: + +- Resources 在语义上被降级成 tools,和 MCP 规范里的 resource primitive 不完全一致。 +- 模型必须先理解并主动调用 `list/read`,资源不会自然成为上下文。 +- pipeline 不能配置“默认携带某些资源”或“本轮附加某些资源”。 +- UI 资源 tab 目前是管理端预览能力,和 Agent 上下文选择没有打通。 +- 对 blob、图片、大文件、结构化资源的处理还比较粗糙。 +- 缺少 resource templates、订阅更新、缓存、chunk、token budget、trace 与审计策略。 + +## 主流项目做法 + +### MCP 官方规范 + +MCP Resources 是 server 暴露上下文数据的协议能力。规范没有要求 resources 必须以 tool call 形式给模型使用,而是把如何选择、过滤、读取和纳入上下文交给 Host application。 + +这意味着比较正统的集成方式是:LangBot 作为 Host,在 pipeline、会话或消息层决定哪些 resources 进入模型上下文。 + +参考: https://modelcontextprotocol.io/specification/2025-06-18/server/resources + +### VS Code Copilot + +VS Code 把 MCP Resources 做成 chat context 的一部分。用户可以通过 `Add Context > MCP Resources` 或命令浏览 MCP resources,并把选中的资源附加到一次 chat request。 + +这是目前最值得 LangBot 参考的产品形态:资源不是模型工具,而是用户和 Host 可控的上下文附件。 + +参考: https://code.visualstudio.com/docs/agent-customization/mcp-servers + +### Anthropic SDK + +Anthropic 的 client-side MCP helpers 提供资源读取和转换能力,例如把 MCP resource 转为 Claude message content 或 file。也就是说,应用先读取 resource,再显式放进模型消息。 + +这同样是 application-owned context injection,而不是把 resource 伪装成模型工具。 + +参考: https://platform.claude.com/docs/en/agents-and-tools/mcp-connector + +### LangChain MCP Adapters + +LangChain 把 MCP Resources 更像 data loader / document input 来处理,可以把资源加载成 `Blob`,再进入 LangChain 的文档、检索或上下文处理链路。 + +这说明 Resources 很适合作为知识源、文档源或上下文源,而不只是即时工具调用。 + +参考: https://docs.langchain.com/oss/python/langchain/mcp + +### OpenAI Agents SDK + +OpenAI Agents SDK 主路径仍偏向 MCP tools,但底层 MCP server API 已经有 `list_resources`、`list_resource_templates`、`read_resource` 等能力。当前形态说明 resources 是 client 能力,但并未默认变成 agent-visible tools。 + +参考: https://openai.github.io/openai-agents-python/mcp/ + +### Cline + +Cline 会拉取 MCP tools、resources、resourceTemplates、prompts,并通过类似 `access_mcp_resource` 的内置访问方式让模型读取资源。这个方向和 LangBot 当前 synthetic tools 比较接近。 + +这种模式适合让 Agent 自主探索,但更像 Host 自定义的模型访问协议,不应成为唯一集成路径。 + +参考: https://github.com/cline/cline/blob/main/src/services/mcp/McpHub.ts + +## 建议架构方向 + +### 1. 保留探索型工具 + +保留当前两个 synthetic tools: + +- `langbot_mcp_list_resources` +- `langbot_mcp_read_resource` + +它们适合处理“用户没有显式选择资源,但 Agent 判断需要探索 MCP server 上下文”的场景。后续可以优化工具描述、返回格式、资源大小限制和错误信息。 + +### 2. 增加一等 Resource Context + +新增一个 Host 层资源上下文概念,例如: + +```text +PipelineResourceBinding +ConversationResourceAttachment +MessageResourceAttachment +``` + +Preproc 或独立的 `ResourceContextProvider` 在模型调用前读取这些资源,按 MIME 类型、大小、token budget 转为模型可消费的上下文。 + +### 3. 打通 UI 与 Agent 上下文 + +当前 MCP 详情页的 Resources tab 可以继续作为资源发现和预览入口。建议增加操作: + +- 添加到本轮上下文 +- 固定到当前 pipeline +- 固定到当前 bot / conversation +- 查看资源读取历史和错误 + +这样 UI 资源管理能力才能真正影响 Agent 行为。 + +### 4. 支持 resource templates + +MCP resource templates 允许 server 暴露参数化资源,例如: + +```text +repo://{owner}/{repo}/file/{path} +log://{service}/{date} +``` + +LangBot 后续应支持模板发现、参数填写、实例化和绑定。否则只能使用静态 resources,覆盖面会受限。 + +### 5. 增加资源处理策略 + +建议补齐: + +- 文本资源 token budget 与截断策略。 +- 大文件 chunk 与摘要策略。 +- 图片/blob 的模型能力判断与 fallback。 +- MIME 类型白名单与安全限制。 +- 缓存与过期策略。 +- `resources/listChanged` 或订阅更新。 +- resource read trace,便于审计 Agent 读取了什么上下文。 + +## 推荐落地顺序 + +### Phase 1: 完成当前 PR 可用性 + +- 保留 synthetic tools。 +- 明确文档说明当前 Agent 集成是 tool-mediated。 +- 完善资源工具描述,降低模型误用概率。 +- 给 read/list 增加大小限制和更清晰的 MIME 处理。 +- 前端 Resources tab 与 Tools tab 分离,保持管理端清晰。 + +### Phase 2: 做 Host-owned context attachments + +- 在 pipeline 或 conversation 层新增 resource attachment 配置。 +- Preproc 读取已绑定 resources,注入模型上下文。 +- UI 支持“添加到上下文 / 固定到 pipeline”。 +- 记录每轮实际注入的 resource URI 和 token 消耗。 + +### Phase 3: 做完整 MCP Resources 能力 + +- 支持 resource templates。 +- 支持资源订阅更新。 +- 支持 chunk、summary、RAG 化接入。 +- 为 DifyAgentRunner、LocalAgentRunner 等不同 runner 定义统一资源上下文接口。 + +## 最终建议 + +PR #2215 可以作为 MCP Resources 的第一阶段实现继续推进。它让 LangBot 快速拥有“资源发现、预览、按需读取”的闭环,也给 Agent 探索资源提供了可运行路径。 + +但在正式设计上,不建议把 “Resources == Tools” 固化为长期抽象。LangBot 更应该把 MCP Resources 定位为上下文来源,与 tools、prompts、knowledge base 并列: + +```text +Tools -> Agent 可以执行的动作 +Resources -> Host/用户/Agent 可以选择的上下文数据 +Prompts -> 可复用的任务模板 +Knowledge -> 可检索、可索引的长期知识 +``` + +这样既尊重 MCP 协议语义,也能让 LangBot 在 Agent 工作流、企业知识接入和多 MCP server 管理上走得更稳。 diff --git a/skills/README.md b/skills/README.md index f45b52859..091e3d3b1 100644 --- a/skills/README.md +++ b/skills/README.md @@ -26,7 +26,7 @@ and LangBot's own Local Agent) working with the LangBot ecosystem. ## Quick start (for an AI agent) -1. Read this README, `AGENTS.md`, and `qa-agent-docs/` to understand the layout. +1. Read this README, `AGENTS.md`, and `docs/user-guide.md` to understand the layout. 2. Read `skills/.env` for shared local defaults. On a new machine, copy `skills/.env.example` to `skills/.env.local` (gitignored) and override machine-specific values there. Never commit secrets. @@ -48,6 +48,7 @@ bin/lbs env show # inspect resolved env defaults (redacted) bin/lbs env doctor # diagnose local environment readiness bin/lbs case list --ready bin/lbs test plan +bin/lbs suite plan langbot-debug-chat-load-gate ``` ## Maintenance rule diff --git a/skills/docs/user-guide.md b/skills/docs/user-guide.md new file mode 100644 index 000000000..124d3af36 --- /dev/null +++ b/skills/docs/user-guide.md @@ -0,0 +1,171 @@ +# LangBot QA Skills User Guide + +Use this guide as the first operational path after reading `README.md` and +`AGENTS.md`. + +## 1. Configure Local Inputs + +Read `skills/.env`, then create `skills/.env.local` for machine-local values. +Do not commit `.env.local`, browser profiles, reports, tokens, API keys, OAuth +state, or provider credentials. + +Minimum local fields for live browser QA: + +```bash +LANGBOT_REPO=/path/to/LangBot +LANGBOT_WEB_REPO=/path/to/LangBot/web +LANGBOT_BACKEND_URL=http://127.0.0.1:5300 +LANGBOT_FRONTEND_URL=http://127.0.0.1:3000 +LANGBOT_DEV_FRONTEND_URL=http://127.0.0.1:3000 +LANGBOT_BROWSER_PROFILE=/path/to/langbot-browser-profile +LANGBOT_CHROMIUM_EXECUTABLE=/path/to/chromium-or-playwright-chrome +LANGBOT_E2E_LOGIN_USER=qa-local@example.com +``` + +`LANGBOT_E2E_LOGIN_USER` is a local QA account. The setup automation uses the +LangBot recovery key from the active checkout to initialize or refresh that +local account and write a browser `localStorage` token. It does not need the +user's GitHub or Space credentials. + +## 2. Check Readiness + +From `skills/`: + +```bash +bin/lbs env show +bin/lbs env doctor +bin/lbs validate +bin/lbs index --check +``` + +`env doctor` should report reachable backend and frontend URLs before live +browser cases are run. Missing Space provider credentials are not a LangBot +product pass; classify them as `env_issue` and configure the local Space +provider before measuring Debug Chat performance. + +## 3. Start Services + +Start the backend from `LANGBOT_REPO`: + +```bash +cd "$LANGBOT_REPO" +uv run main.py +``` + +Start the standalone frontend from `LANGBOT_WEB_REPO` and point it at the +backend: + +```bash +cd "$LANGBOT_WEB_REPO" +VITE_API_BASE_URL="$LANGBOT_BACKEND_URL" pnpm dev --host 0.0.0.0 +``` + +If `VITE_API_BASE_URL` is missing, browser tests can load the Vite page but send +API requests to the frontend port, which produces false UI failures. + +## 4. Prepare User-Path Fixtures + +For local-agent Debug Chat cases and the user-path performance gate: + +```bash +node scripts/e2e/ensure-local-agent-pipeline.mjs --write-env +``` + +The script: + +- refreshes the local QA login and browser token; +- marks the local wizard as skipped; +- creates or updates a local QA pipeline; +- scans Space LLM models, tests candidates, and switches to the first working + Space model with tested fallback models; +- writes `LANGBOT_PIPELINE_URL`, `LANGBOT_PIPELINE_NAME`, and local-agent + pipeline/model variables into `skills/.env.local`; +- returns `env_issue` when no Space model can be scanned or tested. + +Useful model controls: + +```bash +LANGBOT_E2E_MODEL_TEST_LIMIT=8 +LANGBOT_E2E_MODEL_FALLBACK_COUNT=3 +LANGBOT_E2E_SKIP_MODEL_UUIDS=uuid-a,uuid-b +LANGBOT_E2E_SKIP_MODEL_NAMES=model-a,model-b +LANGBOT_E2E_SCAN_SPACE_MODELS=true +``` + +The setup writes a current-runtime compatibility `max-round` value into the +pipeline config because this backend still reads that field directly during +message truncation. Do not treat it as a long-term QA contract. + +## 5. Run Gates + +Fast contract gate, no live service required: + +```bash +bin/lbs suite run langbot-performance-contract-gate --run-id langbot-contract-local +``` + +Live backend gate: + +```bash +bin/lbs suite run langbot-live-backend-gate --run-id langbot-backend-local +``` + +Browser-visible user-path performance gate: + +```bash +bin/lbs suite plan langbot-user-path-performance-gate +bin/lbs suite run langbot-user-path-performance-gate --run-id langbot-user-path-local --include-manual-check +``` + +Controlled Debug Chat message-path load gate (manual/non-required; run fake-provider cases serially when they share `LANGBOT_FAKE_PROVIDER_URL`): + +```bash +bin/lbs suite plan langbot-debug-chat-load-gate +bin/lbs test run langbot-fake-provider-debug-chat-load --run-id langbot-fake-load-local +bin/lbs test run langbot-fake-provider-debug-chat-slow-load --run-id langbot-fake-slow-local +bin/lbs test run langbot-fake-provider-debug-chat-fault-recovery --run-id langbot-fake-fault-local +bin/lbs test run langbot-space-debug-chat-concurrency-smoke --run-id langbot-space-smoke-local +``` + +Cross-pipeline Debug Chat isolation is a separate manual regression gate because +current releases may fail it due to product bug #2286: + +```bash +bin/lbs suite plan langbot-debug-chat-isolation-gate +bin/lbs suite run langbot-debug-chat-isolation-gate --run-id langbot-debug-chat-isolation-local --include-manual-check +``` + +Start with `langbot-fake-provider-debug-chat-load`. It launches a local +OpenAI-compatible fake provider, creates the matching provider/model/pipeline, +then sends concurrent WebSocket Debug Chat messages through the real backend. +Use `langbot-fake-provider-debug-chat-slow-load` to measure the same path under +deterministic streaming latency. Use +`langbot-fake-provider-debug-chat-fault-recovery` to inject bounded provider +HTTP failures and confirm later Debug Chat requests recover. Use the separate +`langbot-debug-chat-isolation-gate` to verify that concurrent Debug Chat traffic +on two pipelines does not leak assistant responses across pipeline boundaries; +current releases may fail that gate because of #2286, so keep it out of the +normal load gate until the product fix lands. +Use `langbot-space-debug-chat-concurrency-smoke` only as a low-volume live +provider smoke; it includes Space/model/network latency and should be compared +against the fake-provider baseline before attributing failures to LangBot. + +`manual_check` means the agent must confirm the declared preconditions for that +run window. When setup automation is declared, run output may stop early with +`env_issue`; fix that environment input before treating the product path as +measured. + +## 6. Read Results + +Suite reports live under `skills/reports/`. Evidence lives under +`skills/reports/evidence//`. + +For performance cases, inspect: + +- `metrics.json` for p50/p95/p99, error rate, and total duration; +- `automation-result.json` for threshold decisions and artifacts; +- `console.log` and `network.log` for frontend/API failures; +- backend logs for provider, runner, WebSocket, or persistence failures. + +Do not call a user-path performance result a LangBot overhead regression until +provider/tool/network time has been separated or ruled out. diff --git a/skills/schemas/case.schema.json b/skills/schemas/case.schema.json index f6365c062..46601142a 100644 --- a/skills/schemas/case.schema.json +++ b/skills/schemas/case.schema.json @@ -48,7 +48,18 @@ }, "type": { "type": "string", - "enum": ["smoke", "regression", "feature", "provider", "exploratory"] + "enum": [ + "smoke", + "regression", + "feature", + "provider", + "exploratory", + "contract", + "performance", + "reliability", + "chaos", + "security" + ] }, "priority": { "type": "string", @@ -102,7 +113,11 @@ "backend_log", "frontend_log", "api_diagnostic", - "filesystem" + "filesystem", + "metrics", + "trace", + "profile", + "resource_log" ] }, "minItems": 1 @@ -188,9 +203,101 @@ "type": "string", "enum": ["person", "group"] }, + "automation_debug_chat_response_p95_ms": { + "type": "string" + }, + "automation_debug_chat_max_error_rate": { + "type": "string" + }, + "automation_debug_chat_load_requests": { + "type": "string" + }, + "automation_debug_chat_load_concurrency": { + "type": "string" + }, + "automation_debug_chat_load_timeout_ms": { + "type": "string" + }, + "automation_debug_chat_load_response_p95_ms": { + "type": "string" + }, + "automation_debug_chat_load_first_response_p95_ms": { + "type": "string" + }, + "automation_debug_chat_load_max_error_rate": { + "type": "string" + }, + "automation_debug_chat_load_min_error_rate": { + "type": "string" + }, + "automation_debug_chat_load_min_error_count": { + "type": "string" + }, + "automation_debug_chat_load_min_ok_count": { + "type": "string" + }, + "automation_debug_chat_load_min_provider_fault_count": { + "type": "string" + }, + "automation_debug_chat_load_expected_prefix": { + "type": "string" + }, + "automation_debug_chat_load_prompt_template": { + "type": "string" + }, + "automation_debug_chat_load_stream": { + "type": "string", + "enum": ["0", "1", "false", "true"] + }, + "automation_debug_chat_load_reset": { + "type": "string", + "enum": ["0", "1", "false", "true"] + }, + "automation_debug_chat_load_fail_on_final_mismatch": { + "type": "string", + "enum": ["0", "1", "false", "true"] + }, + "automation_fake_provider_response_text": { + "type": "string" + }, + "automation_fake_provider_first_token_delay_ms": { + "type": "string" + }, + "automation_fake_provider_chunk_delay_ms": { + "type": "string" + }, + "automation_fake_provider_chunk_count": { + "type": "string" + }, + "automation_fake_provider_fail_first_n": { + "type": "string" + }, + "automation_fake_provider_fail_every_n": { + "type": "string" + }, + "automation_fake_provider_fault_status": { + "type": "string" + }, + "automation_fake_provider_fail_after_first_chunk": { + "type": "string", + "enum": ["0", "1", "false", "true"] + }, + "automation_fake_provider_dynamic_response": { + "type": "string", + "enum": ["0", "1", "false", "true"] + }, "automation_filesystem_checks_json": { "type": "string" }, + "metrics_thresholds_json": { + "type": "string" + }, + "load_profile_json": { + "type": "string" + }, + "fault_model_json": { + "type": "string" + }, "automation_pipeline_url_env": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]*$" diff --git a/skills/schemas/suite.schema.json b/skills/schemas/suite.schema.json index 3da1a3e85..4f3fa7c7a 100644 --- a/skills/schemas/suite.schema.json +++ b/skills/schemas/suite.schema.json @@ -18,7 +18,17 @@ }, "type": { "type": "string", - "enum": ["smoke", "regression", "release_gate", "exploratory"] + "enum": [ + "smoke", + "regression", + "release_gate", + "exploratory", + "contract", + "performance", + "reliability", + "chaos", + "security" + ] }, "priority": { "type": "string", diff --git a/skills/scripts/bootstrap-lbs.mjs b/skills/scripts/bootstrap-lbs.mjs old mode 100644 new mode 100755 diff --git a/skills/scripts/e2e/agent-runner-release-preflight.mjs b/skills/scripts/e2e/agent-runner-release-preflight.mjs old mode 100644 new mode 100755 diff --git a/skills/scripts/e2e/ensure-acp-agent-runner-pipeline.mjs b/skills/scripts/e2e/ensure-acp-agent-runner-pipeline.mjs old mode 100644 new mode 100755 index e8b2e515b..462233014 --- a/skills/scripts/e2e/ensure-acp-agent-runner-pipeline.mjs +++ b/skills/scripts/e2e/ensure-acp-agent-runner-pipeline.mjs @@ -24,13 +24,19 @@ await ensureEvidence(paths); const writeEnv = process.argv.includes("--write-env"); const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; const backendUrl = env.LANGBOT_BACKEND_URL || ""; -const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME; +const pipelineName = + env.LANGBOT_E2E_CREATE_PIPELINE_NAME || + env.LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME || + DEFAULT_PIPELINE_NAME; const sshTarget = env.LANGBOT_ACP_AGENT_RUNNER_SSH_TARGET || "yhh@101.34.71.12"; -const sshConnectTimeout = env.LANGBOT_ACP_AGENT_RUNNER_SSH_CONNECT_TIMEOUT || "8"; +const sshConnectTimeout = + env.LANGBOT_ACP_AGENT_RUNNER_SSH_CONNECT_TIMEOUT || "8"; const sshPort = env.LANGBOT_ACP_AGENT_RUNNER_SSH_PORT || "22"; const sshIdentityFile = env.LANGBOT_ACP_AGENT_RUNNER_SSH_IDENTITY_FILE || ""; const sshExtraOptions = env.LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS || ""; -const remoteWorkspace = env.LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE || "/home/yhh/langbot-e2e/acp-workspace"; +const remoteWorkspace = + env.LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE || + "/home/yhh/langbot-e2e/acp-workspace"; const envLocalPath = resolve("skills/.env.local"); const result = { @@ -64,7 +70,9 @@ try { const user = env.LANGBOT_E2E_LOGIN_USER || ""; const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; if (!user) { - throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API."); + throw new Error( + "LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.", + ); } const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); @@ -114,7 +122,8 @@ try { LANGBOT_ACP_AGENT_RUNNER_SSH_EXTRA_OPTIONS: sshExtraOptions, LANGBOT_ACP_AGENT_RUNNER_REMOTE_WORKSPACE: remoteWorkspace, LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL: result.pipeline_url, - LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName, + LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME: + result.pipeline_name || pipelineName, }); result.wrote_env = true; } @@ -125,10 +134,20 @@ try { console.log(JSON.stringify(result, null, 2)); } -process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); +process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, +); -async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runnerConfig }) { - const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { token }); +async function ensurePipeline({ + backendUrl, + token, + pipelineName, + runnerId, + runnerConfig, +}) { + const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { + token, + }); if (isApiFailure(pipelineList)) { return { status: "fail", @@ -147,7 +166,8 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne token, body: { name: pipelineName, - description: "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.", + description: + "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.", emoji: "QA", }, }); @@ -159,7 +179,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }; } const pipelineId = createdResponse.json.data?.uuid || ""; - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, + { token }, + ); pipeline = loaded.json.data?.pipeline || null; created = true; } @@ -171,7 +195,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }; } - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { token }, + ); if (isApiFailure(loaded) || !loaded.json.data?.pipeline) { return { status: "fail", @@ -182,9 +210,15 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne } pipeline = loaded.json.data.pipeline; - const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; + const config = + pipeline.config && typeof pipeline.config === "object" + ? pipeline.config + : {}; const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; - const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {}; + const runnerConfigs = + ai.runner_config && typeof ai.runner_config === "object" + ? ai.runner_config + : {}; const updatedConfig = { ...config, ai: { @@ -201,16 +235,21 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }, }; - const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { - method: "PUT", - token, - body: { - name: pipelineName, - description: "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.", - emoji: "QA", - config: updatedConfig, + const updateResponse = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { + method: "PUT", + token, + body: { + name: pipelineName, + description: + "Local QA pipeline for real ACP Claude AgentRunner Debug Chat smoke tests.", + emoji: "QA", + config: updatedConfig, + }, }, - }); + ); if (isApiFailure(updateResponse)) { return { status: "fail", @@ -222,7 +261,9 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne return { status: "pass", - reason: created ? "ACP AgentRunner pipeline created and configured." : "ACP AgentRunner pipeline updated.", + reason: created + ? "ACP AgentRunner pipeline created and configured." + : "ACP AgentRunner pipeline updated.", pipeline_id: pipeline.uuid, pipeline_name: pipelineName, created, @@ -231,7 +272,12 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne } function isApiFailure(response) { - return response.status >= 400 || (response.json && response.json.code !== undefined && response.json.code !== 0); + return ( + response.status >= 400 || + (response.json && + response.json.code !== undefined && + response.json.code !== 0) + ); } async function upsertEnvLocal(path, values) { diff --git a/skills/scripts/e2e/ensure-fake-provider-cross-pipelines.mjs b/skills/scripts/e2e/ensure-fake-provider-cross-pipelines.mjs new file mode 100755 index 000000000..592a7b7f9 --- /dev/null +++ b/skills/scripts/e2e/ensure-fake-provider-cross-pipelines.mjs @@ -0,0 +1,205 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { env } from "node:process"; +import { + appendLine, + ensureEvidence, + evidencePaths, + loadEnvFiles, + redact, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +const caseId = "ensure-fake-provider-cross-pipelines"; +const DEFAULT_PIPELINE_A_NAME = "LangBot QA Fake Provider Debug Chat A"; +const DEFAULT_PIPELINE_B_NAME = "LangBot QA Fake Provider Debug Chat B"; + +await loadEnvFiles(); +const paths = evidencePaths(caseId); +await ensureEvidence(paths); + +const writeEnv = process.argv.includes("--write-env"); +const envLocalPath = resolve("skills/.env.local"); +const pipelineAName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME || DEFAULT_PIPELINE_A_NAME; +const pipelineBName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME || DEFAULT_PIPELINE_B_NAME; + +const result = { + source: "setup_automation", + case_id: caseId, + run_id: paths.runId, + status: "fail", + reason: "", + pipeline_a: { + name: pipelineAName, + id: "", + url: "", + }, + pipeline_b: { + name: pipelineBName, + id: "", + url: "", + }, + fake_provider: { + url: "", + base_url: "", + pid: null, + }, + wrote_env: false, + evidence: { + console_log: paths.consoleLog, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["api_diagnostic", "filesystem"], +}; + +try { + console.error(`[langbot-qa] configuring cross-pipeline QA fixtures: pipeline_a=\"${pipelineAName}\", pipeline_b=\"${pipelineBName}\"`); + console.error("[langbot-qa] run these fake-provider setup/probe commands serially when they share LANGBOT_FAKE_PROVIDER_URL."); + if (pipelineAName === pipelineBName) { + throw new Error("LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME and LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME must be different."); + } + + const setupA = await runPipelineSetup(pipelineAName, "A"); + const setupB = await runPipelineSetup(pipelineBName, "B"); + result.pipeline_a = { + name: setupA.pipeline_name || pipelineAName, + id: setupA.pipeline_id || "", + url: setupA.pipeline_url || "", + }; + result.pipeline_b = { + name: setupB.pipeline_name || pipelineBName, + id: setupB.pipeline_id || "", + url: setupB.pipeline_url || "", + }; + result.fake_provider = { + url: setupB.fake_provider?.url || setupA.fake_provider?.url || "", + base_url: setupB.fake_provider?.base_url || setupA.fake_provider?.base_url || "", + pid: setupB.fake_provider?.pid ?? setupA.fake_provider?.pid ?? null, + }; + + if (!result.pipeline_a.url || !result.pipeline_b.url || !result.fake_provider.url) { + throw new Error("Cross-pipeline fake provider setup did not return both pipeline URLs and provider URL."); + } + + if (writeEnv) { + await upsertEnvLocal(envLocalPath, { + LANGBOT_FAKE_PROVIDER_URL: result.fake_provider.url, + LANGBOT_FAKE_PROVIDER_BASE_URL: result.fake_provider.base_url, + LANGBOT_FAKE_PROVIDER_PID: result.fake_provider.pid ? String(result.fake_provider.pid) : "", + LANGBOT_FAKE_PROVIDER_PIPELINE_A_URL: result.pipeline_a.url, + LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME: result.pipeline_a.name, + LANGBOT_FAKE_PROVIDER_PIPELINE_B_URL: result.pipeline_b.url, + LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME: result.pipeline_b.name, + }); + result.wrote_env = true; + } + + result.status = "pass"; + result.reason = "Fake provider cross-pipeline fixtures are configured."; +} catch (error) { + result.status = looksLikeEnvIssue(error) ? "env_issue" : "fail"; + result.reason = safeReason(error.message); +} finally { + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); + +function runPipelineSetup(pipelineName, label) { + return new Promise((resolvePromise, rejectPromise) => { + const child = spawn(process.execPath, ["scripts/e2e/ensure-fake-provider-pipeline.mjs"], { + cwd: resolve("."), + env: { + ...env, + LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName, + LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS: env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS || "25", + LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS: env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS || "10", + LANGBOT_FAKE_PROVIDER_CHUNK_COUNT: env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT || "0", + LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N: "0", + LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N: "0", + LANGBOT_FAKE_PROVIDER_FAULT_STATUS: env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS || "500", + LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK: "false", + LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE: "true", + }, + stdio: ["ignore", "pipe", "pipe"], + }); + + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + const text = chunk.toString(); + stdout += text; + appendLine(paths.consoleLog, `[setup ${label} stdout] ${text.trimEnd()}`).catch(() => {}); + }); + child.stderr.on("data", (chunk) => { + const text = chunk.toString(); + stderr += text; + appendLine(paths.consoleLog, `[setup ${label} stderr] ${text.trimEnd()}`).catch(() => {}); + }); + child.on("error", rejectPromise); + child.on("close", (code) => { + const parsed = parseJsonOutput(stdout); + if (code !== 0 || parsed.status !== "pass") { + rejectPromise(new Error(parsed.reason || stderr || `Fake provider pipeline setup ${label} exited with ${code}.`)); + return; + } + resolvePromise(parsed); + }); + }); +} + +function parseJsonOutput(text) { + const trimmed = String(text || "").trim(); + if (!trimmed) return {}; + try { + return JSON.parse(trimmed); + } catch { + const start = trimmed.indexOf("{"); + const end = trimmed.lastIndexOf("}"); + if (start >= 0 && end > start) { + try { + return JSON.parse(trimmed.slice(start, end + 1)); + } catch { + return {}; + } + } + return {}; + } +} + +async function upsertEnvLocal(path, updates) { + await mkdir(dirname(path), { recursive: true }); + let text = ""; + try { + text = await readFile(path, "utf8"); + } catch { + text = ""; + } + const lines = text.split(/\r?\n/); + const seen = new Set(); + const next = lines.map((line) => { + const trimmed = line.trim(); + const match = trimmed.match(/^([A-Z][A-Z0-9_]*)=/); + if (!match || updates[match[1]] === undefined) return line; + seen.add(match[1]); + return `${match[1]}=${updates[match[1]]}`; + }); + for (const [key, value] of Object.entries(updates)) { + if (!seen.has(key)) next.push(`${key}=${value}`); + } + await writeFile(path, `${next.join("\n").replace(/\n+$/, "")}\n`, "utf8"); +} + +function looksLikeEnvIssue(error) { + const message = String(error?.message || error || ""); + return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message); +} + +function safeReason(value) { + return redact(String(value || "")).slice(0, 1000); +} diff --git a/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs new file mode 100755 index 000000000..73f2465fd --- /dev/null +++ b/skills/scripts/e2e/ensure-fake-provider-pipeline.mjs @@ -0,0 +1,635 @@ +#!/usr/bin/env node + +import { spawn } from "node:child_process"; +import { open, readFile, mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { env } from "node:process"; +import { + apiJson, + ensureEvidence, + evidencePaths, + loadEnvFiles, + redact, + resetAndAuthLocalUser, + writeResult, +} from "./lib/langbot-e2e.mjs"; + +const RUNNER_ID = "local-agent"; +const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; +const DEFAULT_PIPELINE_NAME = "LangBot QA Fake Provider Debug Chat"; +const DEFAULT_PROVIDER_NAME = "LangBot QA Fake OpenAI Provider"; +const QA_RESOURCE_DESCRIPTION = "Managed by LangBot skills QA automation for controlled fake-provider Debug Chat tests. Safe to delete when local QA fixtures are no longer needed."; +const DEFAULT_MODEL_NAME = "gpt-4o-mini"; +const DEFAULT_REQUESTER = "openai-chat-completions"; + +const caseId = "ensure-fake-provider-pipeline"; + +await loadEnvFiles(); +const paths = evidencePaths(caseId); +await ensureEvidence(paths); + +const writeEnv = process.argv.includes("--write-env"); +const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; +const backendUrl = env.LANGBOT_BACKEND_URL || ""; +const envLocalPath = resolve("skills/.env.local"); +const repoRoot = resolve(env.LANGBOT_REPO || ".."); +const fakeStateDir = resolve(env.LANGBOT_FAKE_PROVIDER_STATE_DIR || resolve(repoRoot, ".qa/fake-provider")); +const fakeStatePath = resolve(fakeStateDir, "state.json"); +const fakeStdoutPath = resolve(fakeStateDir, "fake-provider.stdout.log"); +const fakeStderrPath = resolve(fakeStateDir, "fake-provider.stderr.log"); +const pipelineName = env.LANGBOT_FAKE_PROVIDER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME; +const providerName = env.LANGBOT_FAKE_PROVIDER_NAME || DEFAULT_PROVIDER_NAME; +const requester = env.LANGBOT_FAKE_PROVIDER_REQUESTER || DEFAULT_REQUESTER; +const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || DEFAULT_MODEL_NAME; + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + status: "fail", + reason: "", + frontend_url: frontendUrl, + backend_url: backendUrl, + fake_provider: { + url: "", + base_url: "", + pid: null, + reused: false, + config: {}, + state_file: fakeStatePath, + stdout_log: fakeStdoutPath, + stderr_log: fakeStderrPath, + }, + provider: { + uuid: "", + name: providerName, + requester, + created: false, + updated: false, + }, + model: { + uuid: "", + name: modelName, + created: false, + updated: false, + test_status: "not_run", + test_reason: "", + }, + pipeline_id: "", + pipeline_name: pipelineName, + pipeline_url: "", + created: false, + updated: false, + wrote_env: false, + evidence: { + console_log: paths.consoleLog, + network_log: paths.networkLog, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["api_diagnostic", "network", "filesystem"], +}; + +try { + console.error(`[langbot-qa] configuring QA-owned fake-provider fixtures: provider=\"${providerName}\", pipeline=\"${pipelineName}\"`); + console.error("[langbot-qa] this setup may create or update local QA provider/model/pipeline resources on the selected backend."); + if (!backendUrl) { + result.status = "env_issue"; + throw new Error("LANGBOT_BACKEND_URL is not configured."); + } + if (!frontendUrl) { + result.status = "env_issue"; + throw new Error("LANGBOT_FRONTEND_URL is not configured."); + } + + const fakeProvider = await ensureFakeProvider(); + const setupConfig = await configureFakeProvider(fakeProvider.url, healthyFakeProviderConfig(), true); + result.fake_provider = { + ...result.fake_provider, + ...fakeProvider, + config: setupConfig.config || healthyFakeProviderConfig(), + }; + + const user = env.LANGBOT_E2E_LOGIN_USER || ""; + const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; + if (!user) { + result.status = "env_issue"; + throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the fake provider pipeline."); + } + + const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); + const wizard = await skipWizard({ backendUrl, token: auth.token }); + if (wizard.status !== "pass") { + result.status = "fail"; + throw new Error(wizard.reason || "Failed to mark the local QA wizard as skipped."); + } + + const provider = await ensureProvider({ + backendUrl, + token: auth.token, + name: providerName, + requester, + baseUrl: fakeProvider.base_url, + }); + result.provider = provider; + + const model = await ensureModel({ + backendUrl, + token: auth.token, + providerUuid: provider.uuid, + name: modelName, + }); + result.model = model; + + const pipeline = await ensurePipeline({ + backendUrl, + token: auth.token, + name: pipelineName, + modelUuid: model.uuid, + }); + Object.assign(result, pipeline); + result.pipeline_url = `${frontendUrl.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.pipeline_id)}`; + + const runConfig = await configureFakeProvider(fakeProvider.url, targetFakeProviderConfig(), true); + result.fake_provider.config = runConfig.config || targetFakeProviderConfig(); + + if (writeEnv) { + await upsertEnvLocal(envLocalPath, { + LANGBOT_E2E_LOGIN_USER: user, + LANGBOT_FAKE_PROVIDER_URL: fakeProvider.url, + LANGBOT_FAKE_PROVIDER_BASE_URL: fakeProvider.base_url, + LANGBOT_FAKE_PROVIDER_PID: fakeProvider.pid ? String(fakeProvider.pid) : "", + LANGBOT_FAKE_PROVIDER_PROVIDER_UUID: provider.uuid, + LANGBOT_FAKE_PROVIDER_MODEL_UUID: model.uuid, + LANGBOT_FAKE_PROVIDER_PIPELINE_URL: result.pipeline_url, + LANGBOT_FAKE_PROVIDER_PIPELINE_NAME: pipelineName, + }); + result.wrote_env = true; + } + + result.status = "pass"; + result.reason = `Fake provider pipeline is configured with ${requester}/${modelName}.`; +} catch (error) { + result.status = result.status === "env_issue" ? "env_issue" : "fail"; + result.reason = result.reason || safeReason(error.message); +} finally { + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); + +async function ensureFakeProvider() { + const envUrl = normalizeProviderRootUrl(env.LANGBOT_FAKE_PROVIDER_URL || ""); + if (envUrl && await fakeProviderHealthy(envUrl) && await fakeProviderConfigurable(envUrl)) { + return { + url: envUrl, + base_url: `${envUrl}/v1`, + pid: null, + reused: true, + }; + } + + const state = await readState(fakeStatePath); + const stateUrl = normalizeProviderRootUrl(state.url || ""); + if (stateUrl && await fakeProviderHealthy(stateUrl)) { + if (await fakeProviderConfigurable(stateUrl)) { + return { + url: stateUrl, + base_url: state.base_url || `${stateUrl}/v1`, + pid: Number.isInteger(state.pid) ? state.pid : null, + reused: true, + }; + } + if (Number.isInteger(state.pid)) await stopProcess(state.pid); + } + + await mkdir(fakeStateDir, { recursive: true }); + await writeFile(fakeStatePath, `${JSON.stringify({ status: "starting", started_at: new Date().toISOString() }, null, 2)}\n`, "utf8"); + const stdout = await open(fakeStdoutPath, "a"); + const stderr = await open(fakeStderrPath, "a"); + const scriptPath = resolve("scripts/e2e/fake-openai-provider.mjs"); + const host = env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1"; + const port = env.LANGBOT_FAKE_PROVIDER_PORT || "0"; + const child = spawn(process.execPath, [ + scriptPath, + `--host=${host}`, + `--port=${port}`, + `--state-file=${fakeStatePath}`, + ], { + cwd: resolve("."), + detached: true, + env: { + ...env, + LANGBOT_FAKE_PROVIDER_MODEL_NAME: modelName, + }, + stdio: ["ignore", stdout.fd, stderr.fd], + }); + child.unref(); + await stdout.close(); + await stderr.close(); + + const started = await waitForFakeProviderState(fakeStatePath, child.pid, 10_000); + if (!started.url || !await fakeProviderHealthy(started.url) || !await fakeProviderConfigurable(started.url)) { + throw new Error(`Fake provider did not become healthy. See ${fakeStderrPath}`); + } + + return { + url: started.url, + base_url: started.base_url || `${started.url}/v1`, + pid: child.pid ?? started.pid ?? null, + reused: false, + }; +} + +async function configureFakeProvider(rootUrl, config, resetRequestCount) { + const response = await fetch(`${normalizeProviderRootUrl(rootUrl)}/__qa/config`, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + config, + reset_request_count: resetRequestCount, + }), + signal: AbortSignal.timeout(3000), + }); + const json = await response.json().catch(() => ({})); + if (!response.ok || json.ok !== true) { + throw new Error(`Fake provider config failed with HTTP ${response.status}.`); + } + return json; +} + +async function fakeProviderHealthy(rootUrl) { + try { + const response = await fetch(`${rootUrl.replace(/\/$/, "")}/healthz`, { + signal: AbortSignal.timeout(2000), + }); + if (!response.ok) return false; + const json = await response.json().catch(() => ({})); + return json.ok === true; + } catch { + return false; + } +} + +async function fakeProviderConfigurable(rootUrl) { + try { + const response = await fetch(`${rootUrl.replace(/\/$/, "")}/__qa/config`, { + signal: AbortSignal.timeout(2000), + }); + if (!response.ok) return false; + const json = await response.json().catch(() => ({})); + return json.ok === true && json.config && typeof json.config === "object"; + } catch { + return false; + } +} + +async function stopProcess(pid) { + try { + process.kill(pid, "SIGTERM"); + } catch { + return; + } + await sleep(500); +} + +async function waitForFakeProviderState(path, expectedPid, timeoutMs) { + const startedAt = Date.now(); + let lastState = {}; + while (Date.now() - startedAt < timeoutMs) { + const state = await readState(path); + if (state.url && (!expectedPid || state.pid === expectedPid)) return state; + lastState = state; + await sleep(150); + } + return lastState; +} + +async function readState(path) { + try { + return JSON.parse(await readFile(path, "utf8")); + } catch { + return {}; + } +} + +function normalizeProviderRootUrl(value) { + const trimmed = String(value || "").trim().replace(/\/$/, ""); + return trimmed.endsWith("/v1") ? trimmed.slice(0, -3) : trimmed; +} + +function healthyFakeProviderConfig() { + return { + response_text: "OK", + first_token_delay_ms: 25, + chunk_delay_ms: 10, + chunk_count: 0, + fault_status: 500, + fail_first_n: 0, + fail_every_n: 0, + fail_after_first_chunk: false, + dynamic_response: true, + }; +} + +function targetFakeProviderConfig() { + return { + response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK", + first_token_delay_ms: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25), + chunk_delay_ms: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS, 10), + chunk_count: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT, 0), + fault_status: httpFaultStatus(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500), + fail_first_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0), + fail_every_n: nonNegativeInteger(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0), + fail_after_first_chunk: envBool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false), + dynamic_response: envBool(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE, true), + }; +} + +async function skipWizard({ backendUrl, token }) { + const response = await apiJson(backendUrl, "/api/v1/system/wizard/completed", { + method: "POST", + token, + body: { status: "skipped" }, + }); + const ok = response.status < 400 && response.json.code === 0; + return { + status: ok ? "pass" : "fail", + http_status: response.status, + code: response.json.code ?? null, + reason: ok ? "Wizard marked skipped for local QA." : response.json.msg || "Wizard status update failed.", + }; +} + +async function ensureProvider({ backendUrl, token, name, requester, baseUrl }) { + const list = await apiJson(backendUrl, "/api/v1/provider/providers", { token }); + if (isApiFailure(list)) { + throw new Error(list.json.msg || "Failed to list providers."); + } + const providers = list.json.data?.providers || []; + const existing = providers.find((provider) => ( + provider.name === name + || (provider.requester === requester && String(provider.base_url || "").replace(/\/$/, "") === baseUrl.replace(/\/$/, "")) + )); + const body = { + name, + requester, + base_url: baseUrl, + api_keys: [env.LANGBOT_FAKE_PROVIDER_API_KEY || "langbot-fake-provider-key"], + }; + + if (existing?.uuid) { + const update = await apiJson(backendUrl, `/api/v1/provider/providers/${encodeURIComponent(existing.uuid)}`, { + method: "PUT", + token, + body, + }); + if (isApiFailure(update)) { + throw new Error(update.json.msg || "Failed to update fake provider."); + } + return { + uuid: existing.uuid, + name, + requester, + created: false, + updated: true, + }; + } + + const create = await apiJson(backendUrl, "/api/v1/provider/providers", { + method: "POST", + token, + body, + }); + const uuid = create.json.data?.uuid || ""; + if (isApiFailure(create) || !uuid) { + throw new Error(create.json.msg || "Failed to create fake provider."); + } + return { + uuid, + name, + requester, + created: true, + updated: false, + }; +} + +async function ensureModel({ backendUrl, token, providerUuid, name }) { + const list = await apiJson(backendUrl, `/api/v1/provider/models/llm?provider_uuid=${encodeURIComponent(providerUuid)}`, { token }); + if (isApiFailure(list)) { + throw new Error(list.json.msg || "Failed to list fake provider models."); + } + const models = list.json.data?.models || []; + const existing = models.find((model) => model.name === name); + const body = { + name, + provider_uuid: providerUuid, + abilities: [], + context_length: positiveInteger(env.LANGBOT_FAKE_PROVIDER_CONTEXT_LENGTH, 8192), + extra_args: {}, + prefered_ranking: 0, + }; + let modelUuid = existing?.uuid || ""; + let created = false; + let updated = false; + + if (modelUuid) { + const update = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, { + method: "PUT", + token, + body, + }); + if (isApiFailure(update)) { + throw new Error(update.json.msg || "Failed to update fake provider model."); + } + updated = true; + } else { + const create = await apiJson(backendUrl, "/api/v1/provider/models/llm", { + method: "POST", + token, + body, + }); + modelUuid = create.json.data?.uuid || ""; + if (isApiFailure(create) || !modelUuid) { + throw new Error(create.json.msg || "Failed to create fake provider model."); + } + created = true; + } + + const test = await apiJson(backendUrl, `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, { + method: "POST", + token, + body: { extra_args: {} }, + }); + if (isApiFailure(test)) { + throw new Error(safeReason(test.json.msg || test.json.message || "Fake provider model test failed.")); + } + + return { + uuid: modelUuid, + name, + created, + updated, + test_status: "pass", + test_reason: "", + }; +} + +async function ensurePipeline({ backendUrl, token, name, modelUuid }) { + const list = await apiJson(backendUrl, "/api/v1/pipelines", { token }); + if (isApiFailure(list)) { + throw new Error(list.json.msg || "Failed to list pipelines."); + } + const pipelines = list.json.data?.pipelines || []; + let pipeline = pipelines.find((item) => item.name === name) || null; + let created = false; + + if (!pipeline) { + const create = await apiJson(backendUrl, "/api/v1/pipelines", { + method: "POST", + token, + body: { + name, + description: QA_RESOURCE_DESCRIPTION, + emoji: "QA", + }, + }); + const pipelineId = create.json.data?.uuid || ""; + if (isApiFailure(create) || !pipelineId) { + throw new Error(create.json.msg || "Failed to create fake provider pipeline."); + } + created = true; + pipeline = { uuid: pipelineId }; + } + + const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token }); + pipeline = loaded.json.data?.pipeline || null; + if (isApiFailure(loaded) || !pipeline?.uuid) { + throw new Error(loaded.json.msg || "Failed to load fake provider pipeline."); + } + + const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; + const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; + const existingLocalAgentConfig = ai["local-agent"] && typeof ai["local-agent"] === "object" + ? ai["local-agent"] + : {}; + const localAgentConfig = { + timeout: 60, + prompt: [{ role: "system", content: "You are a deterministic QA assistant. Reply exactly as instructed." }], + "remove-think": false, + "knowledge-bases": [], + "box-session-id-template": "{launcher_type}_{launcher_id}", + "retrieval-top-k": 5, + "rerank-model": "", + "rerank-top-k": 5, + "max-tool-iterations": 20, + "tool-execution-mode": "parallel", + "max-tool-result-chars": 20000, + "context-history-fetch-limit": 20, + "context-window-tokens": 8192, + "context-reserve-tokens": 1024, + "context-keep-recent-tokens": 2048, + "context-summary-tokens": 1024, + ...existingLocalAgentConfig, + // Current backend truncation still reads this field directly. + "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10), + model: { + primary: modelUuid, + fallbacks: [], + }, + }; + const updatedConfig = { + ...config, + ai: { + ...ai, + runner: { + ...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}), + id: RUNNER_ID, + runner: RUNNER_ID, + "expire-time": 0, + }, + "local-agent": localAgentConfig, + }, + }; + + const update = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { + method: "PUT", + token, + body: { + name, + description: QA_RESOURCE_DESCRIPTION, + emoji: "QA", + config: updatedConfig, + }, + }); + if (isApiFailure(update)) { + throw new Error(update.json.msg || "Failed to update fake provider pipeline."); + } + + return { + pipeline_id: pipeline.uuid, + pipeline_name: name, + created, + updated: true, + }; +} + +function isApiFailure(response) { + return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0); +} + +function positiveInteger(value, fallback) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function nonNegativeInteger(value, fallback) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; +} + +function httpFaultStatus(value, fallback) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed >= 400 && parsed <= 599 ? parsed : fallback; +} + +function envBool(value, fallback) { + if (value === undefined || value === "") return fallback; + if (/^(1|true|yes|on)$/i.test(String(value))) return true; + if (/^(0|false|no|off)$/i.test(String(value))) return false; + return fallback; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +function safeReason(value) { + return redact(String(value || "")).slice(0, 1000); +} + +async function upsertEnvLocal(path, updates) { + await mkdir(dirname(path), { recursive: true }); + let text = ""; + try { + text = await readFile(path, "utf8"); + } catch { + text = ""; + } + const lines = text.split(/\r?\n/); + const seen = new Set(); + const next = lines.map((line) => { + const trimmed = line.trim(); + const equals = trimmed.indexOf("="); + if (equals <= 0 || trimmed.startsWith("#")) return line; + const key = trimmed.slice(0, equals).trim(); + if (!(key in updates)) return line; + seen.add(key); + return `${key}=${updates[key]}`; + }); + for (const [key, value] of Object.entries(updates)) { + if (!seen.has(key)) next.push(`${key}=${value}`); + } + await writeFile(path, `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, "utf8"); +} diff --git a/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs b/skills/scripts/e2e/ensure-langrag-sentinel-kb.mjs old mode 100644 new mode 100755 diff --git a/skills/scripts/e2e/ensure-local-agent-pipeline.mjs b/skills/scripts/e2e/ensure-local-agent-pipeline.mjs old mode 100644 new mode 100755 index 6dffacf27..7a29cce1c --- a/skills/scripts/e2e/ensure-local-agent-pipeline.mjs +++ b/skills/scripts/e2e/ensure-local-agent-pipeline.mjs @@ -10,6 +10,7 @@ import { ensureEvidence, evidencePaths, loadEnvFiles, + redact, resetAndAuthLocalUser, safeScreenshot, setBrowserToken, @@ -17,9 +18,12 @@ import { writeResult, } from "./lib/langbot-e2e.mjs"; -const RUNNER_ID = "plugin:langbot/local-agent/default"; +const RUNNER_ID = "local-agent"; +const SPACE_PROVIDER_UUID = "00000000-0000-0000-0000-000000000000"; const DEFAULT_PIPELINE_NAME = "Agent QA Local Agent Debug Chat"; const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; +const DEFAULT_MODEL_TEST_LIMIT = 8; +const DEFAULT_MODEL_FALLBACK_COUNT = 3; const caseId = "ensure-local-agent-pipeline"; await loadEnvFiles(); @@ -27,7 +31,10 @@ const paths = evidencePaths(caseId); await ensureEvidence(paths); const writeEnv = process.argv.includes("--write-env"); -const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || DEFAULT_PIPELINE_NAME; +const pipelineName = + env.LANGBOT_E2E_CREATE_PIPELINE_NAME || + env.LANGBOT_LOCAL_AGENT_PIPELINE_NAME || + DEFAULT_PIPELINE_NAME; const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; const backendUrl = env.LANGBOT_BACKEND_URL || ""; const envLocalPath = resolve("skills/.env.local"); @@ -45,11 +52,18 @@ const result = { pipeline_url: "", runner_id: RUNNER_ID, selected_model_id: "", + selected_model_name: "", + fallback_model_ids: [], model_count: 0, + space_model_count: 0, + scanned_space_model_count: 0, + tested_model_count: 0, + model_tests: [], created: false, updated: false, wrote_env: false, auth: null, + wizard: null, browser_token_check: null, page_signal: "", evidence: { @@ -71,7 +85,10 @@ try { const user = env.LANGBOT_E2E_LOGIN_USER || ""; const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; if (!user) { - throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API."); + result.status = "env_issue"; + throw new Error( + "LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.", + ); } const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); @@ -81,6 +98,15 @@ try { backend_token_check: auth.check, }; + const wizard = await skipWizard({ backendUrl, token: auth.token }); + result.wizard = wizard; + if (wizard.status !== "pass") { + result.status = "fail"; + throw new Error( + wizard.reason || "Failed to mark the local QA wizard as skipped.", + ); + } + const prepared = await ensureLocalAgentPipeline({ backendUrl, token: auth.token, @@ -99,6 +125,12 @@ try { LANGBOT_PIPELINE_NAME: result.pipeline_name || pipelineName, LANGBOT_LOCAL_AGENT_PIPELINE_URL: result.pipeline_url, LANGBOT_LOCAL_AGENT_PIPELINE_NAME: result.pipeline_name || pipelineName, + ...(result.selected_model_id + ? { + LANGBOT_LOCAL_AGENT_MODEL_UUID: result.selected_model_id, + LANGBOT_E2E_MODEL_UUID: result.selected_model_id, + } + : {}), }); result.wrote_env = true; } @@ -109,12 +141,21 @@ try { const browserCheck = await verifyBrowserToken(page, backendUrl); result.browser_token_check = browserCheck; if (!browserCheck.authenticated) { - throw new Error(browserCheck.reason || "Browser token check failed after setup."); + throw new Error( + browserCheck.reason || "Browser token check failed after setup.", + ); } - await page.goto(result.pipeline_url || frontendUrl, { waitUntil: "domcontentloaded" }); - await page.waitForLoadState("networkidle", { timeout: 10_000 }).catch(() => {}); + await page.goto(result.pipeline_url || frontendUrl, { + waitUntil: "domcontentloaded", + }); + await page + .waitForLoadState("networkidle", { timeout: 10_000 }) + .catch(() => {}); const text = await bodyText(page); - result.page_signal = ["Pipelines", "流水线", pipelineName].find((signal) => text.includes(signal)) || ""; + result.page_signal = + ["Pipelines", "流水线", pipelineName].find((signal) => + text.includes(signal), + ) || ""; } catch (error) { result.status = result.status === "env_issue" ? "env_issue" : "fail"; result.reason = result.reason || error.message; @@ -125,9 +166,37 @@ try { console.log(JSON.stringify(result, null, 2)); } -process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); +process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, +); -async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runnerId }) { +async function skipWizard({ backendUrl, token }) { + const response = await apiJson( + backendUrl, + "/api/v1/system/wizard/completed", + { + method: "POST", + token, + body: { status: "skipped" }, + }, + ); + const ok = response.status < 400 && response.json.code === 0; + return { + status: ok ? "pass" : "fail", + http_status: response.status, + code: response.json.code ?? null, + reason: ok + ? "Wizard marked skipped for local QA." + : response.json.msg || "Wizard status update failed.", + }; +} + +async function ensureLocalAgentPipeline({ + backendUrl, + token, + pipelineName, + runnerId, +}) { const [pipelineList, modelList] = await Promise.all([ apiJson(backendUrl, "/api/v1/pipelines", { token }), apiJson(backendUrl, "/api/v1/provider/models/llm", { token }), @@ -149,7 +218,21 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne } const models = modelList.json.data?.models || []; - const selectedModel = models.find((model) => model.uuid) || null; + const skippedModelIds = new Set( + String(env.LANGBOT_E2E_SKIP_MODEL_UUIDS || "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + ); + const skippedModelNames = new Set( + String(env.LANGBOT_E2E_SKIP_MODEL_NAMES || "") + .split(",") + .map((item) => item.trim()) + .filter(Boolean), + ); + const spaceModels = models.filter( + (model) => isSpaceModel(model) && !skippedModelIds.has(model.uuid), + ); const pipelines = pipelineList.json.data?.pipelines || []; let pipeline = pipelines.find((item) => item.name === pipelineName) || null; let created = false; @@ -160,7 +243,8 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne token, body: { name: pipelineName, - description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.", + description: + "Local QA pipeline for AgentRunner Debug Chat smoke tests.", emoji: "QA", }, }); @@ -170,10 +254,15 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne reason: createdResponse.json.msg || "Failed to create pipeline.", create_status: createdResponse.status, model_count: models.length, + space_model_count: spaceModels.length, }; } const pipelineId = createdResponse.json.data?.uuid || ""; - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, + { token }, + ); pipeline = loaded.json.data?.pipeline || null; created = true; } @@ -183,10 +272,15 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne status: "fail", reason: "Pipeline was not created or resolved.", model_count: models.length, + space_model_count: spaceModels.length, }; } - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { token }, + ); if (isApiFailure(loaded) || !loaded.json.data?.pipeline) { return { status: "fail", @@ -194,27 +288,44 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne get_status: loaded.status, pipeline_id: pipeline.uuid, model_count: models.length, + space_model_count: spaceModels.length, }; } pipeline = loaded.json.data.pipeline; - const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; + const config = + pipeline.config && typeof pipeline.config === "object" + ? pipeline.config + : {}; const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; - const runnerConfig = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {}; - const rawExistingLocalAgentConfig = runnerConfig[runnerId] && typeof runnerConfig[runnerId] === "object" - ? runnerConfig[runnerId] - : {}; + const rawExistingLocalAgentConfig = + ai["local-agent"] && typeof ai["local-agent"] === "object" + ? ai["local-agent"] + : {}; const existingLocalAgentConfig = rawExistingLocalAgentConfig; - const existingModel = existingLocalAgentConfig.model && typeof existingLocalAgentConfig.model === "object" - ? existingLocalAgentConfig.model - : {}; - const requestedModelId = env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || ""; - const selectedModelId = requestedModelId || existingModel.primary || selectedModel?.uuid || ""; + const existingModel = + existingLocalAgentConfig.model && + typeof existingLocalAgentConfig.model === "object" + ? existingLocalAgentConfig.model + : {}; + const requestedModelId = + env.LANGBOT_LOCAL_AGENT_MODEL_UUID || env.LANGBOT_E2E_MODEL_UUID || ""; + const selected = await selectWorkingSpaceModel({ + backendUrl, + token, + models, + skippedModelIds, + skippedModelNames, + requestedModelId, + existingModelId: existingModel.primary || "", + }); + const selectedModelId = selected.selected_model_id || ""; const localAgentConfig = { timeout: 300, prompt: [{ role: "system", content: "You are a helpful assistant." }], "remove-think": false, "knowledge-bases": [], + "box-session-id-template": "{launcher_type}_{launcher_id}", "retrieval-top-k": 5, "rerank-model": "", "rerank-top-k": 5, @@ -227,9 +338,11 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne "context-keep-recent-tokens": 20000, "context-summary-tokens": 8000, ...existingLocalAgentConfig, + // Current backend truncation still reads this field directly. + "max-round": positiveInteger(existingLocalAgentConfig["max-round"], 10), model: { primary: selectedModelId, - fallbacks: requestedModelId ? [] : Array.isArray(existingModel.fallbacks) ? existingModel.fallbacks : [], + fallbacks: selected.fallback_model_ids || [], }, }; const updatedConfig = { @@ -239,25 +352,28 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne runner: { ...(ai.runner && typeof ai.runner === "object" ? ai.runner : {}), id: runnerId, + runner: runnerId, "expire-time": 0, }, - runner_config: { - ...runnerConfig, - [runnerId]: localAgentConfig, - }, + "local-agent": localAgentConfig, }, }; - const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { - method: "PUT", - token, - body: { - name: pipelineName, - description: "Local QA pipeline for AgentRunner Debug Chat smoke tests.", - emoji: "QA", - config: updatedConfig, + const updateResponse = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { + method: "PUT", + token, + body: { + name: pipelineName, + description: + "Local QA pipeline for AgentRunner Debug Chat smoke tests.", + emoji: "QA", + config: updatedConfig, + }, }, - }); + ); if (isApiFailure(updateResponse)) { return { status: "fail", @@ -265,26 +381,312 @@ async function ensureLocalAgentPipeline({ backendUrl, token, pipelineName, runne update_status: updateResponse.status, pipeline_id: pipeline.uuid, model_count: models.length, + space_model_count: spaceModels.length, + scanned_space_model_count: selected.scanned_space_model_count, + tested_model_count: selected.tested_model_count, + model_tests: selected.model_tests, selected_model_id: selectedModelId, + selected_model_name: selected.selected_model_name, + fallback_model_ids: selected.fallback_model_ids, }; } return { status: selectedModelId ? "pass" : "env_issue", reason: selectedModelId - ? "Local-agent pipeline is configured for Debug Chat." - : "Pipeline was created but no LLM model is configured in this LangBot instance.", + ? `Local-agent pipeline is configured for Debug Chat with Space model ${selected.selected_model_name || selectedModelId} and ${selected.fallback_model_ids.length} fallback(s).` + : selected.reason || + "No working Space LLM model is configured in this LangBot instance.", pipeline_id: pipeline.uuid, - pipeline_name: pipeline.name, + pipeline_name: pipelineName, model_count: models.length, + space_model_count: spaceModels.length, + scanned_space_model_count: selected.scanned_space_model_count, + tested_model_count: selected.tested_model_count, + model_tests: selected.model_tests, selected_model_id: selectedModelId, + selected_model_name: selected.selected_model_name, + fallback_model_ids: selected.fallback_model_ids, created, updated: true, }; } function isApiFailure(response) { - return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0); + return ( + response.status >= 400 || + (response.json.code !== undefined && response.json.code !== 0) + ); +} + +function isSpaceModel(model) { + const provider = + model?.provider && typeof model.provider === "object" ? model.provider : {}; + return ( + model?.provider_uuid === SPACE_PROVIDER_UUID || + provider.uuid === SPACE_PROVIDER_UUID || + provider.requester === "space-chat-completions" || + provider.name === "LangBot Models" + ); +} + +async function selectWorkingSpaceModel({ + backendUrl, + token, + models, + skippedModelIds, + skippedModelNames, + requestedModelId, + existingModelId, +}) { + const modelTests = []; + const testLimit = positiveInteger( + env.LANGBOT_E2E_MODEL_TEST_LIMIT, + DEFAULT_MODEL_TEST_LIMIT, + ); + const fallbackCount = positiveInteger( + env.LANGBOT_E2E_MODEL_FALLBACK_COUNT, + DEFAULT_MODEL_FALLBACK_COUNT, + ); + const workingModels = []; + const spaceModels = rankModels( + models.filter( + (model) => + model.uuid && + isSpaceModel(model) && + !skippedModelIds.has(model.uuid) && + !skippedModelNames.has(model.name), + ), + ); + const requestedModel = requestedModelId + ? spaceModels.find((model) => model.uuid === requestedModelId) || null + : null; + const existingModel = existingModelId + ? spaceModels.find((model) => model.uuid === existingModelId) || null + : null; + const candidates = uniqueCandidates([ + ...(requestedModel ? [existingCandidate(requestedModel, "requested")] : []), + ...(existingModel + ? [existingCandidate(existingModel, "existing-pipeline")] + : []), + ...spaceModels.map((model) => existingCandidate(model, "configured-space")), + ]); + + let scanResult = { status: "skipped", models: [], reason: "" }; + if (env.LANGBOT_E2E_SCAN_SPACE_MODELS !== "false") { + scanResult = await scanSpaceModels({ backendUrl, token }); + if (scanResult.status === "pass") { + const knownNames = new Set(spaceModels.map((model) => model.name)); + candidates.push( + ...scanResult.models + .filter( + (model) => + model.name && + !knownNames.has(model.name) && + !skippedModelNames.has(model.name), + ) + .map((model) => scannedCandidate(model)), + ); + } + } + + const unique = uniqueCandidates(candidates); + for (const candidate of unique.slice(0, testLimit)) { + const test = await ensureAndTestModel({ backendUrl, token, candidate }); + modelTests.push(test); + if (test.status === "pass" && test.model_uuid) { + workingModels.push(test); + if (workingModels.length >= fallbackCount + 1) break; + } + } + + if (workingModels.length > 0) { + const [primary, ...fallbacks] = workingModels; + return { + status: "pass", + reason: "", + selected_model_id: primary.model_uuid, + selected_model_name: primary.model_name, + fallback_model_ids: fallbacks.map((model) => model.model_uuid), + scanned_space_model_count: scanResult.models.length, + tested_model_count: modelTests.length, + model_tests: modelTests, + }; + } + + const baseReason = + unique.length === 0 + ? scanResult.reason || "No Space LLM model candidates are available." + : `No working Space LLM model found after testing ${modelTests.length} candidate(s).`; + return { + status: "env_issue", + reason: + requestedModelId && !requestedModel + ? `Requested Space LLM model ${requestedModelId} is missing or skipped; ${baseReason}` + : baseReason, + selected_model_id: "", + selected_model_name: "", + fallback_model_ids: [], + scanned_space_model_count: scanResult.models.length, + tested_model_count: modelTests.length, + model_tests: modelTests, + }; +} + +async function scanSpaceModels({ backendUrl, token }) { + const response = await apiJson( + backendUrl, + `/api/v1/provider/providers/${encodeURIComponent(SPACE_PROVIDER_UUID)}/scan-models?type=llm`, + { token }, + ); + if (isApiFailure(response)) { + return { + status: "env_issue", + models: [], + reason: safeReason( + response.json.msg || + response.json.message || + "Failed to scan Space LLM models.", + ), + }; + } + return { + status: "pass", + models: response.json.data?.models || [], + reason: "", + }; +} + +async function ensureAndTestModel({ backendUrl, token, candidate }) { + let modelUuid = candidate.uuid || ""; + let created = false; + if (!modelUuid) { + const create = await apiJson(backendUrl, "/api/v1/provider/models/llm", { + method: "POST", + token, + body: { + name: candidate.name, + provider_uuid: SPACE_PROVIDER_UUID, + abilities: candidate.abilities || [], + context_length: candidate.context_length ?? null, + extra_args: {}, + prefered_ranking: positiveInteger(candidate.prefered_ranking, 0), + }, + }); + modelUuid = create.json.data?.uuid || ""; + if (isApiFailure(create) || !modelUuid) { + return modelTestResult(candidate, { + status: "fail", + reason: safeReason( + create.json.msg || "Failed to create scanned Space model.", + ), + http_status: create.status, + }); + } + created = true; + } + + const test = await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}/test`, + { + method: "POST", + token, + body: { extra_args: {} }, + }, + ); + const passed = !isApiFailure(test); + if (!passed && created) { + await apiJson( + backendUrl, + `/api/v1/provider/models/llm/${encodeURIComponent(modelUuid)}`, + { + method: "DELETE", + token, + }, + ).catch(() => {}); + } + return modelTestResult(candidate, { + status: passed ? "pass" : "fail", + reason: passed + ? "" + : safeReason( + test.json.msg || test.json.message || "Space model test failed.", + ), + http_status: test.status, + model_uuid: modelUuid, + created, + }); +} + +function modelTestResult(candidate, details) { + return { + source: candidate.source, + model_uuid: details.model_uuid || candidate.uuid || "", + model_name: candidate.name, + status: details.status, + reason: details.reason || "", + http_status: details.http_status ?? null, + created: Boolean(details.created), + }; +} + +function existingCandidate(model, source) { + return { + source, + uuid: model.uuid, + name: model.name, + abilities: model.abilities || [], + context_length: model.context_length, + prefered_ranking: model.prefered_ranking, + }; +} + +function scannedCandidate(model) { + return { + source: "scanned-space", + uuid: "", + name: model.name || model.id, + abilities: model.abilities || [], + context_length: model.context_length, + prefered_ranking: model.prefered_ranking, + }; +} + +function uniqueCandidates(candidates) { + const seen = new Set(); + const result = []; + for (const candidate of candidates) { + const key = candidate.uuid + ? `uuid:${candidate.uuid}` + : `name:${candidate.name}`; + if (!candidate.name || seen.has(key)) continue; + seen.add(key); + result.push(candidate); + } + return result; +} + +function rankModels(models) { + return [...models].sort((left, right) => { + const leftRank = Number.isFinite(Number(left.prefered_ranking)) + ? Number(left.prefered_ranking) + : 9999; + const rightRank = Number.isFinite(Number(right.prefered_ranking)) + ? Number(right.prefered_ranking) + : 9999; + if (leftRank !== rightRank) return leftRank - rightRank; + return String(left.name || "").localeCompare(String(right.name || "")); + }); +} + +function positiveInteger(value, fallback) { + const parsed = Number(value); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function safeReason(value) { + return redact(String(value || "")).slice(0, 1000); } async function upsertEnvLocal(path, updates) { @@ -308,5 +710,9 @@ async function upsertEnvLocal(path, updates) { for (const [key, value] of Object.entries(updates)) { if (!seen.has(key)) next.push(`${key}=${value}`); } - await writeFile(path, `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, "utf8"); + await writeFile( + path, + `${next.filter((line, index) => line !== "" || index < next.length - 1).join("\n")}\n`, + "utf8", + ); } diff --git a/skills/scripts/e2e/ensure-qa-agent-runner-pipeline.mjs b/skills/scripts/e2e/ensure-qa-agent-runner-pipeline.mjs old mode 100644 new mode 100755 index 7346eff29..bf8c337cf --- a/skills/scripts/e2e/ensure-qa-agent-runner-pipeline.mjs +++ b/skills/scripts/e2e/ensure-qa-agent-runner-pipeline.mjs @@ -24,7 +24,10 @@ await ensureEvidence(paths); const writeEnv = process.argv.includes("--write-env"); const frontendUrl = env.LANGBOT_FRONTEND_URL || ""; const backendUrl = env.LANGBOT_BACKEND_URL || ""; -const pipelineName = env.LANGBOT_E2E_CREATE_PIPELINE_NAME || env.LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME || DEFAULT_PIPELINE_NAME; +const pipelineName = + env.LANGBOT_E2E_CREATE_PIPELINE_NAME || + env.LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME || + DEFAULT_PIPELINE_NAME; const envLocalPath = resolve("skills/.env.local"); const result = { @@ -55,7 +58,9 @@ try { const user = env.LANGBOT_E2E_LOGIN_USER || ""; const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; if (!user) { - throw new Error("LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API."); + throw new Error( + "LANGBOT_E2E_LOGIN_USER is required so this setup can create/update the pipeline via backend API.", + ); } const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); @@ -81,7 +86,8 @@ try { await upsertEnvLocal(envLocalPath, { LANGBOT_E2E_LOGIN_USER: user, LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL: result.pipeline_url, - LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME: result.pipeline_name || pipelineName, + LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME: + result.pipeline_name || pipelineName, }); result.wrote_env = true; } @@ -92,10 +98,20 @@ try { console.log(JSON.stringify(result, null, 2)); } -process.exit(result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1); +process.exit( + result.status === "pass" ? 0 : result.status === "env_issue" ? 2 : 1, +); -async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runnerConfig }) { - const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { token }); +async function ensurePipeline({ + backendUrl, + token, + pipelineName, + runnerId, + runnerConfig, +}) { + const pipelineList = await apiJson(backendUrl, "/api/v1/pipelines", { + token, + }); if (isApiFailure(pipelineList)) { return { status: "fail", @@ -114,7 +130,8 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne token, body: { name: pipelineName, - description: "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.", + description: + "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.", emoji: "QA", }, }); @@ -126,7 +143,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }; } const pipelineId = createdResponse.json.data?.uuid || ""; - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipelineId)}`, + { token }, + ); pipeline = loaded.json.data?.pipeline || null; created = true; } @@ -138,7 +159,11 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }; } - const loaded = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { token }); + const loaded = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { token }, + ); if (isApiFailure(loaded) || !loaded.json.data?.pipeline) { return { status: "fail", @@ -149,9 +174,15 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne } pipeline = loaded.json.data.pipeline; - const config = pipeline.config && typeof pipeline.config === "object" ? pipeline.config : {}; + const config = + pipeline.config && typeof pipeline.config === "object" + ? pipeline.config + : {}; const ai = config.ai && typeof config.ai === "object" ? config.ai : {}; - const runnerConfigs = ai.runner_config && typeof ai.runner_config === "object" ? ai.runner_config : {}; + const runnerConfigs = + ai.runner_config && typeof ai.runner_config === "object" + ? ai.runner_config + : {}; const updatedConfig = { ...config, ai: { @@ -168,16 +199,21 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne }, }; - const updateResponse = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, { - method: "PUT", - token, - body: { - name: pipelineName, - description: "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.", - emoji: "QA", - config: updatedConfig, + const updateResponse = await apiJson( + backendUrl, + `/api/v1/pipelines/${encodeURIComponent(pipeline.uuid)}`, + { + method: "PUT", + token, + body: { + name: pipelineName, + description: + "Local QA pipeline for deterministic QA AgentRunner Debug Chat smoke tests.", + emoji: "QA", + config: updatedConfig, + }, }, - }); + ); if (isApiFailure(updateResponse)) { return { status: "fail", @@ -189,7 +225,9 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne return { status: "pass", - reason: created ? "QA AgentRunner pipeline created and configured." : "QA AgentRunner pipeline updated.", + reason: created + ? "QA AgentRunner pipeline created and configured." + : "QA AgentRunner pipeline updated.", pipeline_id: pipeline.uuid, pipeline_name: pipelineName, created, @@ -198,7 +236,12 @@ async function ensurePipeline({ backendUrl, token, pipelineName, runnerId, runne } function isApiFailure(response) { - return response.status >= 400 || (response.json && response.json.code !== undefined && response.json.code !== 0); + return ( + response.status >= 400 || + (response.json && + response.json.code !== undefined && + response.json.code !== 0) + ); } async function upsertEnvLocal(path, values) { diff --git a/skills/scripts/e2e/fake-openai-provider.mjs b/skills/scripts/e2e/fake-openai-provider.mjs new file mode 100755 index 000000000..1cca9c46b --- /dev/null +++ b/skills/scripts/e2e/fake-openai-provider.mjs @@ -0,0 +1,496 @@ +#!/usr/bin/env node + +import { createServer } from "node:http"; +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname, resolve } from "node:path"; +import { env, exit } from "node:process"; + +const args = parseArgs(process.argv.slice(2)); +const host = args.host || env.LANGBOT_FAKE_PROVIDER_HOST || "127.0.0.1"; +const port = integer(args.port ?? env.LANGBOT_FAKE_PROVIDER_PORT, 0); +const stateFile = args["state-file"] || env.LANGBOT_FAKE_PROVIDER_STATE_FILE || ""; +const modelName = env.LANGBOT_FAKE_PROVIDER_MODEL_NAME || "gpt-4o-mini"; +const config = { + response_text: env.LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT || "OK", + first_token_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS, 25), + chunk_delay_ms: integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS, 10), + chunk_count: integer(env.LANGBOT_FAKE_PROVIDER_CHUNK_COUNT, 0), + fault_status: integer(env.LANGBOT_FAKE_PROVIDER_FAULT_STATUS, 500), + fail_first_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N, 0), + fail_every_n: integer(env.LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N, 0), + fail_after_first_chunk: bool(env.LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK, false), + dynamic_response: !/^(0|false|no|off)$/i.test(env.LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE || ""), + request_log_limit: integer(env.LANGBOT_FAKE_PROVIDER_REQUEST_LOG_LIMIT, 500), +}; + +let requestCount = 0; +const recentRequests = []; + +const server = createServer(async (request, response) => { + const startedAt = Date.now(); + const startedPerf = performance.now(); + let requestRecord = null; + const url = new URL(request.url || "/", `http://${request.headers.host || `${host}:${port}`}`); + try { + if (request.method === "GET" && url.pathname === "/healthz") { + sendJson(response, 200, { + ok: true, + model: modelName, + config, + request_count: requestCount, + recent_request_count: recentRequests.length, + }); + return; + } + + if (request.method === "GET" && url.pathname === "/__qa/config") { + sendJson(response, 200, { + ok: true, + model: modelName, + config, + request_count: requestCount, + recent_requests: recentRequests, + }); + return; + } + + if (request.method === "POST" && url.pathname === "/__qa/config") { + const body = await readJson(request); + applyConfig(body.config && typeof body.config === "object" ? body.config : body); + if (body.reset_request_count !== false) resetRequestState(); + sendJson(response, 200, { + ok: true, + model: modelName, + config, + request_count: requestCount, + }); + return; + } + + if (request.method === "POST" && url.pathname === "/__qa/reset") { + resetRequestState(); + sendJson(response, 200, { + ok: true, + model: modelName, + config, + request_count: requestCount, + }); + return; + } + + if (request.method === "GET" && ["/models", "/v1/models"].includes(url.pathname)) { + sendJson(response, 200, { + object: "list", + data: [ + { + id: modelName, + object: "model", + created: 1, + owned_by: "langbot-qa", + type: "llm", + }, + ], + }); + return; + } + + if (request.method === "POST" && ["/chat/completions", "/v1/chat/completions"].includes(url.pathname)) { + requestCount += 1; + const body = await readJson(request); + const requestId = `chatcmpl-langbot-fake-${requestCount}`; + const shouldFail = requestCount <= config.fail_first_n + || (config.fail_every_n > 0 && requestCount % config.fail_every_n === 0); + const replyText = responseTextForBody(body); + requestRecord = recordRequest({ + id: requestId, + request_number: requestCount, + path: url.pathname, + stream: Boolean(body.stream), + model: body.model || "", + message_count: Array.isArray(body.messages) ? body.messages.length : 0, + should_fail: shouldFail, + status: "running", + http_status: null, + expected_text: replyText, + response_text_preview: previewText(replyText), + started_at: new Date(startedAt).toISOString(), + started_epoch_ms: startedAt, + configured_first_token_delay_ms: config.first_token_delay_ms, + configured_chunk_delay_ms: config.chunk_delay_ms, + configured_chunk_count: config.chunk_count, + }); + + if (shouldFail) { + await sleep(config.first_token_delay_ms); + sendJson(response, config.fault_status, { + error: { + message: `LangBot fake provider injected HTTP ${config.fault_status}`, + type: "fake_provider_fault", + code: "fake_provider_fault", + }, + }); + finishRequestRecord(requestRecord, startedPerf, { + status: "http_fault", + http_status: config.fault_status, + }); + return; + } + + if (body.stream) { + await streamCompletion(response, { + requestId, + model: body.model || modelName, + content: replyText, + failAfterFirstChunk: config.fail_after_first_chunk, + requestRecord, + startedPerf, + }); + } else { + await sleep(config.first_token_delay_ms + config.chunk_delay_ms); + sendJson(response, 200, completionPayload({ + requestId, + model: body.model || modelName, + content: replyText, + })); + markRequestTiming(requestRecord, "first_chunk", startedPerf); + markRequestTiming(requestRecord, "first_content_chunk", startedPerf); + requestRecord.content_chunk_count = 1; + finishRequestRecord(requestRecord, startedPerf, { + status: "ok", + http_status: 200, + }); + } + return; + } + + sendJson(response, 404, { + error: { + message: `No fake provider route for ${request.method} ${url.pathname}`, + type: "not_found", + }, + }); + } catch (error) { + if (requestRecord) { + finishRequestRecord(requestRecord, startedPerf, { + status: "fake_provider_error", + http_status: 500, + error: error instanceof Error ? error.message : String(error), + }); + } + sendJson(response, 500, { + error: { + message: error instanceof Error ? error.message : String(error), + type: "fake_provider_error", + }, + }); + } finally { + const durationMs = Date.now() - startedAt; + if (url.pathname !== "/healthz") { + console.log(JSON.stringify({ + at: new Date().toISOString(), + method: request.method, + path: url.pathname, + duration_ms: durationMs, + })); + } + } +}); + +server.listen(port, host, async () => { + const address = server.address(); + const selectedPort = typeof address === "object" && address ? address.port : port; + const url = `http://${host}:${selectedPort}`; + const state = { + status: "ready", + pid: process.pid, + url, + base_url: `${url}/v1`, + model: modelName, + started_at: new Date().toISOString(), + }; + if (stateFile) { + const path = resolve(stateFile); + await mkdir(dirname(path), { recursive: true }); + await writeFile(path, `${JSON.stringify(state, null, 2)}\n`, "utf8"); + } + console.log(JSON.stringify(state)); +}); + +server.on("error", (error) => { + console.error(JSON.stringify({ + status: "error", + reason: error instanceof Error ? error.message : String(error), + })); + exit(1); +}); + +process.on("SIGTERM", () => { + server.close(() => exit(0)); +}); + +function parseArgs(argv) { + const result = {}; + for (const item of argv) { + const match = item.match(/^--([^=]+)(?:=(.*))?$/); + if (!match) continue; + result[match[1]] = match[2] ?? "1"; + } + return result; +} + +function integer(value, fallback) { + const parsed = Number.parseInt(String(value ?? ""), 10); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +} + +function bool(value, fallback) { + if (value === undefined || value === "") return fallback; + if (/^(1|true|yes|on)$/i.test(String(value))) return true; + if (/^(0|false|no|off)$/i.test(String(value))) return false; + return fallback; +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, Math.max(0, ms))); +} + +async function readJson(request) { + let text = ""; + for await (const chunk of request) text += chunk.toString(); + if (!text) return {}; + return JSON.parse(text); +} + +function sendJson(response, status, payload) { + const text = `${JSON.stringify(payload)}\n`; + response.writeHead(status, { + "content-type": "application/json", + "content-length": Buffer.byteLength(text), + }); + response.end(text); +} + +function completionPayload({ requestId, model, content }) { + const completionTokens = tokenEstimate(content); + return { + id: requestId, + object: "chat.completion", + created: Math.floor(Date.now() / 1000), + model, + choices: [ + { + index: 0, + message: { + role: "assistant", + content, + }, + finish_reason: "stop", + }, + ], + usage: { + prompt_tokens: 8, + completion_tokens: completionTokens, + total_tokens: 8 + completionTokens, + }, + }; +} + +async function streamCompletion(response, { + requestId, + model, + content, + failAfterFirstChunk: failMidStream, + requestRecord, + startedPerf, +}) { + response.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-cache", + "connection": "keep-alive", + }); + + await sleep(config.first_token_delay_ms); + markRequestTiming(requestRecord, "first_chunk", startedPerf); + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [{ index: 0, delta: { role: "assistant" }, finish_reason: null }], + }); + + const chunks = splitContent(content); + for (let index = 0; index < chunks.length; index += 1) { + await sleep(config.chunk_delay_ms); + if (index === 0) markRequestTiming(requestRecord, "first_content_chunk", startedPerf); + requestRecord.content_chunk_count = (requestRecord.content_chunk_count || 0) + 1; + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [{ index: 0, delta: { content: chunks[index] }, finish_reason: null }], + }); + if (failMidStream && index === 0) { + finishRequestRecord(requestRecord, startedPerf, { + status: "mid_stream_disconnect", + http_status: 200, + }); + response.destroy(new Error("LangBot fake provider injected mid-stream disconnect")); + return; + } + } + + await sleep(config.chunk_delay_ms); + const completionTokens = tokenEstimate(content); + writeSse(response, { + id: requestId, + object: "chat.completion.chunk", + created: Math.floor(Date.now() / 1000), + model, + choices: [{ index: 0, delta: {}, finish_reason: "stop" }], + usage: { + prompt_tokens: 8, + completion_tokens: completionTokens, + total_tokens: 8 + completionTokens, + }, + }); + response.write("data: [DONE]\n\n"); + response.end(); + finishRequestRecord(requestRecord, startedPerf, { + status: "ok", + http_status: 200, + }); +} + +function writeSse(response, payload) { + response.write(`data: ${JSON.stringify(payload)}\n\n`); +} + +function splitContent(content) { + const text = String(content); + const requested = config.chunk_count; + if (requested <= 1 || text.length <= 1) return [text]; + const chunkSize = Math.max(1, Math.ceil(text.length / requested)); + const chunks = []; + for (let index = 0; index < text.length; index += chunkSize) { + chunks.push(text.slice(index, index + chunkSize)); + } + return chunks; +} + +function tokenEstimate(content) { + return Math.max(1, Math.ceil(String(content || "").length / 4)); +} + +function responseTextForBody(body) { + if (!config.dynamic_response) { + return config.response_text; + } + const messages = Array.isArray(body.messages) ? body.messages : []; + const lastUser = [...messages].reverse().find((message) => message?.role === "user"); + const text = flattenContent(lastUser?.content || ""); + const quoted = text.match(/["'“”](.{1,80}?)["'“”]/); + if (quoted?.[1]) return quoted[1].trim(); + const exact = text.match(/(?:reply|回复|输出|return)\s+(?:exactly\s+)?([A-Za-z0-9_.:@-]{1,80})/i); + if (exact?.[1]) return exact[1].trim().replace(/[。.!?]+$/, ""); + const only = text.match(/只回复\s*([A-Za-z0-9_.:@-]{1,80})/); + if (only?.[1]) return only[1].trim().replace(/[。.!?]+$/, ""); + return config.response_text; +} + +function flattenContent(content) { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map((item) => { + if (typeof item === "string") return item; + if (item && typeof item === "object") return item.text || ""; + return ""; + }) + .join("\n"); + } + return ""; +} + +function recordRequest(entry) { + const item = { + ...entry, + at: new Date().toISOString(), + finished_at: null, + finished_epoch_ms: null, + duration_ms: null, + first_chunk_at: null, + first_chunk_epoch_ms: null, + first_chunk_ms: null, + first_content_chunk_at: null, + first_content_chunk_epoch_ms: null, + first_content_chunk_ms: null, + content_chunk_count: 0, + }; + recentRequests.push(item); + while (recentRequests.length > config.request_log_limit) recentRequests.shift(); + return item; +} + +function markRequestTiming(entry, key, startedPerf) { + if (!entry || entry[`${key}_at`]) return; + const now = Date.now(); + entry[`${key}_at`] = new Date(now).toISOString(); + entry[`${key}_epoch_ms`] = now; + entry[`${key}_ms`] = rounded(performance.now() - startedPerf); +} + +function finishRequestRecord(entry, startedPerf, updates = {}) { + if (!entry || entry.finished_at) return; + const now = Date.now(); + Object.assign(entry, updates); + entry.finished_at = new Date(now).toISOString(); + entry.finished_epoch_ms = now; + entry.duration_ms = rounded(performance.now() - startedPerf); +} + +function rounded(value) { + return Number(value.toFixed(3)); +} + +function previewText(value) { + return String(value || "").slice(0, 120); +} + +function resetRequestState() { + requestCount = 0; + recentRequests.length = 0; +} + +function applyConfig(updates) { + if (!updates || typeof updates !== "object") return; + assignString(updates, "response_text"); + assignNonNegativeInteger(updates, "first_token_delay_ms"); + assignNonNegativeInteger(updates, "chunk_delay_ms"); + assignNonNegativeInteger(updates, "chunk_count"); + assignNonNegativeInteger(updates, "fail_first_n"); + assignNonNegativeInteger(updates, "fail_every_n"); + assignNonNegativeInteger(updates, "request_log_limit"); + if (updates.fault_status !== undefined) { + const parsed = Number.parseInt(String(updates.fault_status), 10); + if (Number.isInteger(parsed) && parsed >= 400 && parsed <= 599) config.fault_status = parsed; + } + assignBoolean(updates, "fail_after_first_chunk"); + assignBoolean(updates, "dynamic_response"); +} + +function assignString(updates, key) { + if (updates[key] !== undefined) config[key] = String(updates[key]); +} + +function assignNonNegativeInteger(updates, key) { + if (updates[key] === undefined) return; + const parsed = Number.parseInt(String(updates[key]), 10); + if (Number.isInteger(parsed) && parsed >= 0) config[key] = parsed; +} + +function assignBoolean(updates, key) { + if (updates[key] === undefined) return; + config[key] = bool(updates[key], config[key]); +} diff --git a/skills/scripts/e2e/install-qa-plugin-smoke.mjs b/skills/scripts/e2e/install-qa-plugin-smoke.mjs old mode 100644 new mode 100755 diff --git a/skills/scripts/e2e/langrag-kb-retrieve.mjs b/skills/scripts/e2e/langrag-kb-retrieve.mjs old mode 100644 new mode 100755 diff --git a/skills/scripts/e2e/lib/debug-chat.mjs b/skills/scripts/e2e/lib/debug-chat.mjs old mode 100644 new mode 100755 diff --git a/skills/scripts/e2e/lib/langbot-e2e.mjs b/skills/scripts/e2e/lib/langbot-e2e.mjs old mode 100644 new mode 100755 index fc7a52e4f..a7584c904 --- a/skills/scripts/e2e/lib/langbot-e2e.mjs +++ b/skills/scripts/e2e/lib/langbot-e2e.mjs @@ -72,6 +72,7 @@ export async function writeResult(paths, result) { } export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) { + const processEnvKeys = new Set(Object.keys(env)); for (const path of paths) { let text = ""; try { @@ -86,7 +87,7 @@ export async function loadEnvFiles(paths = ["skills/.env", "skills/.env.local"]) if (equals <= 0) continue; const key = trimmed.slice(0, equals).trim(); const value = trimmed.slice(equals + 1).trim().replace(/^["']|["']$/g, ""); - if (!(key in env)) env[key] = value; + if (!processEnvKeys.has(key)) env[key] = value; } } } diff --git a/skills/scripts/e2e/local-agent-steering-debug-chat.mjs b/skills/scripts/e2e/local-agent-steering-debug-chat.mjs old mode 100644 new mode 100755 diff --git a/skills/scripts/e2e/mcp-stdio-register.mjs b/skills/scripts/e2e/mcp-stdio-register.mjs old mode 100644 new mode 100755 diff --git a/skills/scripts/e2e/pipeline-debug-chat.mjs b/skills/scripts/e2e/pipeline-debug-chat.mjs index 87fe9ae79..4b20f7757 100755 --- a/skills/scripts/e2e/pipeline-debug-chat.mjs +++ b/skills/scripts/e2e/pipeline-debug-chat.mjs @@ -54,6 +54,7 @@ const debugChatSessionType = env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person" const pipelineConfigDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-diagnostic.json"); const debugChatResetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json"); const pipelineConfigRestoreDiagnosticPath = resolve(paths.evidenceDir, "pipeline-config-restore-diagnostic.json"); +const metricsPath = resolve(paths.evidenceDir, "metrics.json"); const startedAt = new Date(); let browser; @@ -80,10 +81,11 @@ let result = { console_log: paths.consoleLog, network_log: paths.networkLog, screenshot: paths.screenshot, + metrics_json: metricsPath, automation_result_json: paths.automationResultJson, result_json: paths.resultJson, }, - evidence_collected: ["ui", "screenshot", "console", "network"], + evidence_collected: ["ui", "screenshot", "console", "network", "metrics"], }; function boolFromEnv(value, defaultValue) { @@ -103,6 +105,29 @@ function parseJsonEnv(key, fallback) { } } +function positiveNumberEnv(key, fallback) { + const value = Number(env[key] || ""); + return Number.isFinite(value) && value >= 0 ? value : fallback; +} + +function percentile(values, percentileValue) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return Number(sorted[index].toFixed(3)); +} + +function stats(values) { + if (values.length === 0) return { min: 0, p50: 0, p95: 0, p99: 0, max: 0 }; + return { + min: Number(Math.min(...values).toFixed(3)), + p50: percentile(values, 50), + p95: percentile(values, 95), + p99: percentile(values, 99), + max: Number(Math.max(...values).toFixed(3)), + }; +} + function promptStepsFromEnv() { const rawSteps = parseJsonEnv("LANGBOT_E2E_PROMPTS_JSON", null); if (rawSteps === null) { @@ -658,6 +683,7 @@ try { } else { for (let index = 0; index < promptSteps.length; index += 1) { const step = promptSteps[index]; + const promptStartedAt = Date.now(); const chatResult = await runDebugChatPrompt(page, { prompt: step.prompt, expectedText: step.expectedText, @@ -665,11 +691,13 @@ try { imagePath: index === 0 ? imagePath : "", failureSignals: failureSignals.length > 0 ? failureSignals : undefined, }); + const promptDurationMs = Date.now() - promptStartedAt; result.chat_results.push({ index, expected_text: step.expectedText, status: chatResult.status, reason: chatResult.reason, + response_duration_ms: promptDurationMs, min_expected_count: chatResult.min_expected_count, final_count: chatResult.final_count, before_assistant_expected_count: chatResult.before_assistant_expected_count, @@ -714,6 +742,56 @@ try { const finishedAt = new Date(); result.finished_at = finishedAt.toISOString(); result.finished_at_local = localIsoWithOffset(finishedAt); + result.duration_ms = finishedAt.getTime() - startedAt.getTime(); + const responseDurations = result.chat_results + .map((item) => item.response_duration_ms) + .filter((value) => Number.isFinite(value)); + const passedPrompts = result.chat_results.filter((item) => item.status === "pass").length; + const attemptedPrompts = result.chat_results.length; + const errorRate = attemptedPrompts === 0 ? 1 : Number(((attemptedPrompts - passedPrompts) / attemptedPrompts).toFixed(4)); + const responseStats = stats(responseDurations); + const responseP95BudgetMs = positiveNumberEnv( + "LANGBOT_E2E_DEBUG_CHAT_RESPONSE_P95_MS", + positiveNumberEnv("LANGBOT_DEBUG_CHAT_RESPONSE_P95_MS", safeResponseTimeoutMs), + ); + const maxErrorRate = positiveNumberEnv("LANGBOT_E2E_DEBUG_CHAT_MAX_ERROR_RATE", 0); + const metrics = { + probe: caseId, + url: result.url, + prompt_count: result.prompt_count, + attempted_prompt_count: attemptedPrompts, + passed_prompt_count: passedPrompts, + error_rate: errorRate, + response_duration_ms: responseStats, + total_duration_ms: result.duration_ms, + chat_results: result.chat_results, + }; + result.metrics_summary = { + prompt_count: metrics.prompt_count, + attempted_prompt_count: metrics.attempted_prompt_count, + passed_prompt_count: metrics.passed_prompt_count, + error_rate: metrics.error_rate, + response_p50_ms: metrics.response_duration_ms.p50, + response_p95_ms: metrics.response_duration_ms.p95, + total_duration_ms: metrics.total_duration_ms, + }; + result.thresholds_summary = { + response_p95_ms: { + actual: metrics.response_duration_ms.p95, + max: responseP95BudgetMs, + pass: attemptedPrompts > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs, + }, + error_rate: { + actual: metrics.error_rate, + max: maxErrorRate, + pass: metrics.error_rate <= maxErrorRate, + }, + }; + await writeFile(metricsPath, `${JSON.stringify(metrics, null, 2)}\n`, "utf8"); + if (result.status === "pass" && !Object.values(result.thresholds_summary).every((item) => item.pass)) { + result.status = "fail"; + result.reason = "Debug Chat performance breached response latency or error-rate thresholds."; + } const existingEvidence = {}; for (const [key, value] of Object.entries(result.evidence)) { if (typeof value !== "string") continue; diff --git a/skills/scripts/e2e/refresh-local-login.mjs b/skills/scripts/e2e/refresh-local-login.mjs old mode 100644 new mode 100755 diff --git a/skills/skills.index.json b/skills/skills.index.json index a1c374024..fa571e440 100644 --- a/skills/skills.index.json +++ b/skills/skills.index.json @@ -130,6 +130,7 @@ "references/local-agent-runner.md", "references/mcp-stdio-testing.md", "references/model-provider-testing.md", + "references/performance-reliability-testing.md", "references/pipeline-debug-chat.md", "references/plugin-e2e-smoke.md", "references/sandbox-skill-authoring.md", @@ -151,6 +152,16 @@ "agent-runner-release-preflight", "agent-runner-runtime-chaos", "dify-agent-debug-chat", + "langbot-fake-provider-debug-chat-cross-pipeline-isolation", + "langbot-fake-provider-debug-chat-fault-recovery", + "langbot-fake-provider-debug-chat-load", + "langbot-fake-provider-debug-chat-slow-load", + "langbot-fault-taxonomy-contract", + "langbot-live-backend-latency", + "langbot-live-backend-log-health", + "langbot-live-control-plane-api", + "langbot-overhead-accounting-contract", + "langbot-space-debug-chat-concurrency-smoke", "langrag-kb-retrieve", "langrag-parser-golden-e2e", "langrag-sentinel-kb-discover", @@ -166,6 +177,7 @@ "mcp-stdio-register", "mcp-stdio-tool-call", "pipeline-debug-chat", + "pipeline-debug-chat-performance", "plugin-e2e-smoke", "provider-deepseek", "qa-plugin-smoke-live-install", @@ -488,6 +500,316 @@ "backend_log" ] }, + { + "id": "langbot-fake-provider-debug-chat-cross-pipeline-isolation", + "title": "LangBot Debug Chat fake-provider cross-pipeline isolation probe", + "mode": "probe", + "area": "reliability", + "type": "reliability", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "reliability", + "debug-chat", + "websocket", + "fake-provider", + "isolation", + "concurrency", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-debug-chat-cross-pipeline-isolation.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-fake-provider-cross-pipelines.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_FAKE_PROVIDER_URL", + "LANGBOT_FAKE_PROVIDER_BASE_URL", + "LANGBOT_FAKE_PROVIDER_PID", + "LANGBOT_FAKE_PROVIDER_PIPELINE_A_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME", + "LANGBOT_FAKE_PROVIDER_PIPELINE_B_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME" + ], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, + { + "id": "langbot-fake-provider-debug-chat-fault-recovery", + "title": "LangBot Debug Chat fake-provider fault recovery probe", + "mode": "probe", + "area": "reliability", + "type": "chaos", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "reliability", + "chaos", + "debug-chat", + "websocket", + "fake-provider", + "fault-injection", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_FAKE_PROVIDER_URL", + "LANGBOT_FAKE_PROVIDER_BASE_URL", + "LANGBOT_FAKE_PROVIDER_PID", + "LANGBOT_FAKE_PROVIDER_PROVIDER_UUID", + "LANGBOT_FAKE_PROVIDER_MODEL_UUID", + "LANGBOT_FAKE_PROVIDER_PIPELINE_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME" + ], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, + { + "id": "langbot-fake-provider-debug-chat-load", + "title": "LangBot Debug Chat controlled fake-provider load probe", + "mode": "probe", + "area": "performance", + "type": "performance", + "priority": "p1", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "performance", + "debug-chat", + "websocket", + "fake-provider", + "load", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_FAKE_PROVIDER_URL", + "LANGBOT_FAKE_PROVIDER_BASE_URL", + "LANGBOT_FAKE_PROVIDER_PID", + "LANGBOT_FAKE_PROVIDER_PROVIDER_UUID", + "LANGBOT_FAKE_PROVIDER_MODEL_UUID", + "LANGBOT_FAKE_PROVIDER_PIPELINE_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME" + ], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, + { + "id": "langbot-fake-provider-debug-chat-slow-load", + "title": "LangBot Debug Chat slow fake-provider load probe", + "mode": "probe", + "area": "performance", + "type": "performance", + "priority": "p1", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "performance", + "debug-chat", + "websocket", + "fake-provider", + "slow-provider", + "load", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_FAKE_PROVIDER_URL", + "LANGBOT_FAKE_PROVIDER_BASE_URL", + "LANGBOT_FAKE_PROVIDER_PID", + "LANGBOT_FAKE_PROVIDER_PROVIDER_UUID", + "LANGBOT_FAKE_PROVIDER_MODEL_UUID", + "LANGBOT_FAKE_PROVIDER_PIPELINE_URL", + "LANGBOT_FAKE_PROVIDER_PIPELINE_NAME" + ], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, + { + "id": "langbot-fault-taxonomy-contract", + "title": "LangBot fault taxonomy and cleanup contract", + "mode": "probe", + "area": "reliability", + "type": "chaos", + "priority": "p1", + "risk": "medium", + "ci_eligible": true, + "tags": [ + "reliability", + "chaos", + "contract", + "synthetic" + ], + "automation": "skills/langbot-testing/probes/langbot-fault-taxonomy-contract.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "metrics", + "filesystem" + ] + }, + { + "id": "langbot-live-backend-latency", + "title": "LangBot live backend basic latency probe", + "mode": "probe", + "area": "performance", + "type": "performance", + "priority": "p1", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "performance", + "live-backend", + "latency", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-live-backend-latency.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, + { + "id": "langbot-live-backend-log-health", + "title": "LangBot live backend log health probe", + "mode": "probe", + "area": "reliability", + "type": "reliability", + "priority": "p1", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "reliability", + "live-backend", + "backend-log", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-live-backend-log-health.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "metrics", + "backend_log", + "filesystem" + ] + }, + { + "id": "langbot-live-control-plane-api", + "title": "LangBot live control-plane API probe", + "mode": "probe", + "area": "performance", + "type": "performance", + "priority": "p1", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "performance", + "reliability", + "live-backend", + "control-plane", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-live-control-plane-api.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, + { + "id": "langbot-overhead-accounting-contract", + "title": "LangBot overhead accounting metrics contract", + "mode": "probe", + "area": "performance", + "type": "performance", + "priority": "p1", + "risk": "medium", + "ci_eligible": true, + "tags": [ + "performance", + "metrics", + "contract", + "synthetic" + ], + "automation": "skills/langbot-testing/probes/langbot-overhead-accounting-contract.mjs", + "setup_automation": [], + "setup_provides_env": [], + "evidence_required": [ + "metrics", + "resource_log", + "filesystem" + ] + }, + { + "id": "langbot-space-debug-chat-concurrency-smoke", + "title": "LangBot Debug Chat real Space-provider concurrency smoke", + "mode": "probe", + "area": "performance", + "type": "performance", + "priority": "p1", + "risk": "high", + "ci_eligible": false, + "tags": [ + "performance", + "debug-chat", + "websocket", + "space", + "live-provider", + "smoke", + "metrics" + ], + "automation": "skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_PIPELINE_URL", + "LANGBOT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + "LANGBOT_LOCAL_AGENT_MODEL_UUID", + "LANGBOT_E2E_MODEL_UUID" + ], + "evidence_required": [ + "metrics", + "network", + "api_diagnostic", + "filesystem" + ] + }, { "id": "langrag-kb-retrieve", "title": "LangRAG knowledge base ingests and retrieves a sentinel document", @@ -913,6 +1235,38 @@ "backend_log" ] }, + { + "id": "pipeline-debug-chat-performance", + "title": "Pipeline Debug Chat user-path performance probe", + "mode": "agent-browser", + "area": "pipeline", + "type": "performance", + "priority": "p1", + "risk": "medium", + "ci_eligible": false, + "tags": [ + "performance", + "pipeline", + "debug-chat", + "user-path", + "metrics" + ], + "automation": "scripts/e2e/pipeline-debug-chat.mjs", + "setup_automation": [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" + ], + "setup_provides_env": [ + "LANGBOT_PIPELINE_URL", + "LANGBOT_PIPELINE_NAME" + ], + "evidence_required": [ + "ui", + "screenshot", + "console", + "network", + "metrics" + ] + }, { "id": "plugin-e2e-smoke", "title": "Plugin system installs a local plugin and exposes tool/page APIs", @@ -1086,6 +1440,12 @@ "suites": [ "agent-runner-release-gate", "core-smoke", + "langbot-debug-chat-isolation-gate", + "langbot-debug-chat-load-gate", + "langbot-live-backend-gate", + "langbot-performance-contract-gate", + "langbot-performance-reliability-gate", + "langbot-user-path-performance-gate", "local-agent-gate" ], "suite_summaries": [ @@ -1148,6 +1508,113 @@ "local-agent-basic-debug-chat" ] }, + { + "id": "langbot-debug-chat-isolation-gate", + "title": "LangBot Debug Chat isolation gate", + "description": "Manual/non-required cross-pipeline Debug Chat isolation gate. Current releases may fail this gate because of product bug #2286; use it as regression evidence after the routing fix lands.", + "type": "reliability", + "priority": "p1", + "tags": [ + "reliability", + "debug-chat", + "websocket", + "isolation", + "concurrency" + ], + "cases": [ + "langbot-fake-provider-debug-chat-cross-pipeline-isolation" + ] + }, + { + "id": "langbot-debug-chat-load-gate", + "title": "LangBot Debug Chat load gate", + "description": "Manual/non-required message-path load checks for Pipeline Debug Chat: controlled fake-provider baseline, slow-provider and fault-recovery profiles, plus optional real Space-provider smoke. Cross-pipeline isolation is split into langbot-debug-chat-isolation-gate because current releases may fail it due to product bug #2286.", + "type": "performance", + "priority": "p1", + "tags": [ + "performance", + "debug-chat", + "websocket", + "load" + ], + "cases": [ + "langbot-fake-provider-debug-chat-load", + "langbot-fake-provider-debug-chat-slow-load", + "langbot-fake-provider-debug-chat-fault-recovery", + "langbot-space-debug-chat-concurrency-smoke" + ] + }, + { + "id": "langbot-live-backend-gate", + "title": "LangBot live backend reliability gate", + "description": "Live backend control-plane responsiveness and runtime log health checks for a locally running LangBot instance.", + "type": "reliability", + "priority": "p1", + "tags": [ + "performance", + "reliability", + "live-backend", + "metrics" + ], + "cases": [ + "langbot-live-backend-latency", + "langbot-live-control-plane-api", + "langbot-live-backend-log-health" + ] + }, + { + "id": "langbot-performance-contract-gate", + "title": "LangBot performance contract gate", + "description": "Fast synthetic contract checks for performance metric accounting and non-destructive reliability fault taxonomy.", + "type": "contract", + "priority": "p1", + "tags": [ + "performance", + "reliability", + "contract", + "metrics" + ], + "cases": [ + "langbot-overhead-accounting-contract", + "langbot-fault-taxonomy-contract" + ] + }, + { + "id": "langbot-performance-reliability-gate", + "title": "LangBot performance and reliability starter gate", + "description": "Starter gate for LangBot performance accounting, live backend control-plane latency, and non-destructive fault taxonomy checks.", + "type": "reliability", + "priority": "p1", + "tags": [ + "performance", + "reliability", + "metrics", + "chaos" + ], + "cases": [ + "langbot-overhead-accounting-contract", + "langbot-fault-taxonomy-contract", + "langbot-live-backend-latency", + "langbot-live-control-plane-api", + "langbot-live-backend-log-health" + ] + }, + { + "id": "langbot-user-path-performance-gate", + "title": "LangBot user-path performance gate", + "description": "Browser-visible performance checks for user-facing LangBot paths such as Pipeline Debug Chat.", + "type": "performance", + "priority": "p1", + "tags": [ + "performance", + "browser", + "debug-chat", + "user-path" + ], + "cases": [ + "pipeline-debug-chat-performance" + ] + }, { "id": "local-agent-gate", "title": "Local Agent runner regression gate", @@ -1292,6 +1759,7 @@ "sandbox-native-tools-unavailable", "socks-proxy-without-socksio", "survey-widget-blocks-debug-chat", + "telemetry-proxy-noise", "tool-name-collision-between-mcp-and-plugin", "uv-run-resyncs-local-sdk" ], @@ -1476,6 +1944,14 @@ "mcp-stdio-tool-call" ] }, + { + "id": "telemetry-proxy-noise", + "title": "Telemetry posting fails through the proxy while the target flow succeeds", + "category": "env_issue", + "related_cases": [ + "langbot-space-debug-chat-concurrency-smoke" + ] + }, { "id": "tool-name-collision-between-mcp-and-plugin", "title": "MCP and plugin expose the same tool name", diff --git a/skills/skills/.env.example b/skills/skills/.env.example index a8f5ebf09..888c5721d 100644 --- a/skills/skills/.env.example +++ b/skills/skills/.env.example @@ -26,6 +26,23 @@ LANGBOT_NO_PROXY=localhost,127.0.0.1,::1 LANGBOT_PIPELINE_URL= LANGBOT_PIPELINE_NAME= +# Optional fake OpenAI-compatible provider controls for Debug Chat load tests. +# Leave URL empty to let setup automation start a local provider and write the +# selected URL to skills/.env.local. +LANGBOT_FAKE_PROVIDER_URL= +LANGBOT_FAKE_PROVIDER_HOST=127.0.0.1 +LANGBOT_FAKE_PROVIDER_PORT= +LANGBOT_FAKE_PROVIDER_MODEL_NAME=gpt-4o-mini +LANGBOT_FAKE_PROVIDER_RESPONSE_TEXT=OK +LANGBOT_FAKE_PROVIDER_FIRST_TOKEN_DELAY_MS=25 +LANGBOT_FAKE_PROVIDER_CHUNK_DELAY_MS=10 +LANGBOT_FAKE_PROVIDER_CHUNK_COUNT=0 +LANGBOT_FAKE_PROVIDER_FAIL_FIRST_N=0 +LANGBOT_FAKE_PROVIDER_FAIL_EVERY_N=0 +LANGBOT_FAKE_PROVIDER_FAULT_STATUS=500 +LANGBOT_FAKE_PROVIDER_FAIL_AFTER_FIRST_CHUNK=false +LANGBOT_FAKE_PROVIDER_DYNAMIC_RESPONSE=true + # Optional case-specific runner targets. Prefer these for runner-specific cases # so the automation cannot silently test the wrong runner. LANGBOT_LOCAL_AGENT_PIPELINE_URL= diff --git a/skills/skills/langbot-env-setup/references/service-startup.md b/skills/skills/langbot-env-setup/references/service-startup.md index 4f7b3ec27..b63960cdb 100644 --- a/skills/skills/langbot-env-setup/references/service-startup.md +++ b/skills/skills/langbot-env-setup/references/service-startup.md @@ -53,7 +53,7 @@ Start the new frontend from the web repo: ```bash cd "$LANGBOT_WEB_REPO" -npm run dev +VITE_API_BASE_URL="$LANGBOT_BACKEND_URL" pnpm dev --host 0.0.0.0 ``` Healthy startup includes: @@ -68,6 +68,10 @@ Quick check: curl -I --max-time 3 "$LANGBOT_FRONTEND_URL" ``` +If `VITE_API_BASE_URL` is missing, Vite still serves the page but frontend API +calls may go to the frontend port instead of the backend port. That produces +false browser failures in login, wizard, pipeline, and Debug Chat cases. + ## Completion Signal Environment setup is not complete until the required frontend/backend URLs are reachable and the chosen browser-control path can open the WebUI. diff --git a/skills/skills/langbot-testing/SKILL.md b/skills/skills/langbot-testing/SKILL.md index e9db1980f..748ae9b81 100644 --- a/skills/skills/langbot-testing/SKILL.md +++ b/skills/skills/langbot-testing/SKILL.md @@ -21,6 +21,7 @@ Use this skill when an agent needs to verify LangBot behavior through the WebUI - **Sandbox-backed skill authoring**: read `references/sandbox-skill-authoring.md`. - **LangRAG knowledge bases**: read `references/langrag-knowledge-base.md`. - **MCP stdio tool testing**: read `references/mcp-stdio-testing.md`. +- **Performance, reliability, or chaos probes**: read `references/performance-reliability-testing.md`. - **Drive a live instance over MCP (not raw HTTP)**: use the `langbot-mcp-ops` skill — the instance exposes an MCP server at `http://:5300/mcp` (reuses API keys). Useful for setting up bots/pipelines/models as test fixtures programmatically. - **Known failures and fixes**: read `references/troubleshooting.md`. - **Reusable test groups**: run `bin/lbs suite list` and `bin/lbs suite plan ` before manually assembling a case set. @@ -36,6 +37,8 @@ Use this skill when an agent needs to verify LangBot behavior through the WebUI - Use an authenticated browser profile prepared by `langbot-env-setup`. - Do not expose API keys, OAuth secrets, tokens, or localStorage token values in output. - A WebUI test is not complete until the visible UI result is checked against backend logs or network behavior. +- A performance result is not complete without `metrics` evidence and a clear split between LangBot overhead and external provider/tool/network time. +- A chaos or reliability result is not complete until the fault scope, cleanup, and recovery checks are recorded. - For a suite, use `bin/lbs suite start ` to create the suite evidence root, per-case directories, and `suite-start.json`/`suite-start.md` handoff files; use `bin/lbs test result ` to write final per-case `result.json`, then run `bin/lbs suite report --evidence-dir `. - Do not mark a case `pass` until `test result --evidence` covers every value in the case's `evidence_required`. - For runner-specific Debug Chat cases, use the case-specific pipeline env declared by `automation_pipeline_url_env` / `automation_pipeline_name_env`; do not silently reuse a generic `LANGBOT_PIPELINE_URL`. diff --git a/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-cross-pipeline-isolation.yaml b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-cross-pipeline-isolation.yaml new file mode 100644 index 000000000..9e8e09af0 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-cross-pipeline-isolation.yaml @@ -0,0 +1,84 @@ +id: langbot-fake-provider-debug-chat-cross-pipeline-isolation +title: "LangBot Debug Chat fake-provider cross-pipeline isolation probe" +mode: probe +area: reliability +type: reliability +priority: p1 +risk: high +ci_eligible: false +tags: + - reliability + - debug-chat + - websocket + - fake-provider + - isolation + - concurrency + - metrics +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER +automation: skills/langbot-testing/probes/langbot-debug-chat-cross-pipeline-isolation.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_FAKE_PROVIDER_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_A_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME + - LANGBOT_FAKE_PROVIDER_PIPELINE_B_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME +automation_debug_chat_load_requests: "6" +automation_debug_chat_load_concurrency: "4" +automation_debug_chat_load_timeout_ms: "30000" +automation_debug_chat_load_response_p95_ms: "5000" +automation_debug_chat_load_max_error_rate: "0" +automation_debug_chat_load_prompt_template: '请只回复 "{expected}",不要解释,不要添加其他字符。' +automation_debug_chat_load_stream: "true" +automation_debug_chat_load_reset: "true" +metrics_thresholds_json: '{"cross_pipeline_leak_count":{"max":0},"response_p95_ms":{"max":5000},"error_rate":{"max":0}}' +load_profile_json: '{"requests_per_pipeline":6,"pipelines":2,"concurrency":4,"path":"Pipeline Debug Chat WebSocket","provider":"controlled fake OpenAI-compatible provider","metric":"cross-pipeline response isolation and send-to-final-assistant-response"}' +setup_automation: + - "node:scripts/e2e/ensure-fake-provider-cross-pipelines.mjs --write-env" +setup_provides_env: + - LANGBOT_FAKE_PROVIDER_URL + - LANGBOT_FAKE_PROVIDER_BASE_URL + - LANGBOT_FAKE_PROVIDER_PID + - LANGBOT_FAKE_PROVIDER_PIPELINE_A_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME + - LANGBOT_FAKE_PROVIDER_PIPELINE_B_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME +steps: + - "Start or reuse the local fake OpenAI-compatible provider." + - "Create or update two local-agent pipelines that both point at the controlled fake provider." + - "Reset both Debug Chat sessions and the fake-provider request log." + - "Open concurrent WebSocket Debug Chat connections to both pipelines and send unique pipeline-scoped response tokens." +checks: + - "automation-result.json status is pass only when every request receives its own expected token and cross_pipeline_leak_count is zero." + - "metrics_summary includes by_pipeline status counts, fake-provider request count, and LangBot/provider timing estimates." + - "samples.json contains per-request pipeline labels so any leak can be attributed to the receiving pipeline." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +diagnostics: + - "This probe targets Debug Chat isolation under concurrent traffic from two pipelines." + - "It is designed to expose regressions where global pipeline state causes one pipeline's assistant response to be delivered to another pipeline's Debug Chat session." + - "Same-pipeline foreign responses are tolerated because Debug Chat intentionally broadcasts within the same pipeline/session; cross-pipeline tokens are never tolerated." + - "Known product bug: current releases may fail this probe because Debug Chat replies can read singleton WebSocket proxy pipeline state after another pipeline overwrites it. See https://github.com/langbot-app/LangBot/issues/2286." +expected_failures: + - "https://github.com/langbot-app/LangBot/issues/2286" +success_patterns: + - "Debug Chat cross-pipeline isolation probe passed" +failure_patterns: + - "cross_pipeline_leak" + - "Timed out after" + - "WebSocket connection error" + - "Final assistant response did not include" +troubleshooting: + - backend-not-listening + - debug-chat-history-contaminates-automation + - local-agent-model-route-unavailable diff --git a/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-fault-recovery.yaml b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-fault-recovery.yaml new file mode 100644 index 000000000..7dfa45c91 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-fault-recovery.yaml @@ -0,0 +1,95 @@ +id: langbot-fake-provider-debug-chat-fault-recovery +title: "LangBot Debug Chat fake-provider fault recovery probe" +mode: probe +area: reliability +type: chaos +priority: p1 +risk: high +ci_eligible: false +tags: + - reliability + - chaos + - debug-chat + - websocket + - fake-provider + - fault-injection + - metrics +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER +automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_debug_chat_load_requests: "6" +automation_debug_chat_load_concurrency: "1" +automation_debug_chat_load_timeout_ms: "15000" +automation_debug_chat_load_response_p95_ms: "5000" +automation_debug_chat_load_max_error_rate: "0" +automation_debug_chat_load_min_ok_count: "6" +automation_debug_chat_load_min_provider_fault_count: "2" +automation_debug_chat_load_expected_prefix: "FAULTQA" +automation_debug_chat_load_prompt_template: '请只回复 "{expected}",不要解释,不要添加其他字符。' +automation_debug_chat_load_stream: "true" +automation_debug_chat_load_reset: "true" +automation_debug_chat_load_fail_on_final_mismatch: "true" +automation_fake_provider_first_token_delay_ms: "25" +automation_fake_provider_chunk_delay_ms: "10" +automation_fake_provider_chunk_count: "0" +automation_fake_provider_fail_first_n: "2" +automation_fake_provider_fail_every_n: "0" +automation_fake_provider_fault_status: "503" +metrics_thresholds_json: '{"response_p95_ms":{"max":5000},"error_rate":{"max":0},"ok_count_min":{"min":6},"fake_provider_fault_count_min":{"min":2}}' +fault_model_json: '{"provider_fault":"HTTP 503 for first 2 fake-provider chat completions after reset","expected_behavior":"LangBot retries or otherwise recovers from bounded provider failures so every Debug Chat request receives its expected response without backend crash."}' +load_profile_json: '{"requests":6,"concurrency":1,"path":"Pipeline Debug Chat WebSocket","provider":"controlled fake OpenAI-compatible provider","classification":"fault-recovery-not-throughput-benchmark"}' +setup_automation: + - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" +setup_provides_env: + - LANGBOT_FAKE_PROVIDER_URL + - LANGBOT_FAKE_PROVIDER_BASE_URL + - LANGBOT_FAKE_PROVIDER_PID + - LANGBOT_FAKE_PROVIDER_PROVIDER_UUID + - LANGBOT_FAKE_PROVIDER_MODEL_UUID + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +steps: + - "Configure the local fake provider to return HTTP 503 for the first two chat completions after reset." + - "Create or update the LangBot provider, model, and local-agent pipeline that points at the fake provider." + - "Reset the target Debug Chat session and fake-provider request counter." + - "Send a sequential Debug Chat batch and verify later requests recover after the injected provider faults." +checks: + - "automation-result.json status is pass when the fake provider records at least two injected faults, every Debug Chat request succeeds, and total user-visible error rate stays at zero." + - "metrics_summary includes fake_provider_fault_count and status_counts for the same run window." + - "backend logs show request handling for the same run window without unexpected Traceback or task-leak findings." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +diagnostics: + - "This is a fault-recovery probe, not a throughput benchmark." + - "Provider faults may be retried inside the provider/requester path; judge this case by fake_provider_fault_count plus user-visible success/error metrics." + - "The profile uses concurrency 1 because Debug Chat broadcasts assistant responses to every connection in a session, and failed responses do not carry the unique success token needed for concurrent attribution." +success_patterns: + - "Debug Chat WebSocket concurrency probe passed" + - "Streaming completed" +failure_patterns: + - "fake_provider_fault" + - "HTTP 503" + - "Timed out after" + - "All models failed during streaming setup" +expected_failures: + - "fake_provider_fault" + - "HTTP 503" +troubleshooting: + - backend-not-listening + - debug-chat-history-contaminates-automation + - local-agent-model-route-unavailable diff --git a/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-load.yaml b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-load.yaml new file mode 100644 index 000000000..8a71c3558 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-load.yaml @@ -0,0 +1,81 @@ +id: langbot-fake-provider-debug-chat-load +title: "LangBot Debug Chat controlled fake-provider load probe" +mode: probe +area: performance +type: performance +priority: p1 +risk: medium +ci_eligible: false +tags: + - performance + - debug-chat + - websocket + - fake-provider + - load + - metrics +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER +automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_debug_chat_load_requests: "12" +automation_debug_chat_load_concurrency: "4" +automation_debug_chat_load_timeout_ms: "30000" +automation_debug_chat_load_response_p95_ms: "5000" +automation_debug_chat_load_first_response_p95_ms: "3000" +automation_debug_chat_load_max_error_rate: "0" +automation_debug_chat_load_expected_prefix: "FAKEQA" +automation_debug_chat_load_prompt_template: '请只回复 "{expected}",不要解释,不要添加其他字符。' +automation_debug_chat_load_stream: "true" +automation_debug_chat_load_reset: "true" +metrics_thresholds_json: '{"response_p95_ms":{"max":5000},"first_response_p95_ms":{"max":3000},"error_rate":{"max":0}}' +load_profile_json: '{"requests":12,"concurrency":4,"path":"Pipeline Debug Chat WebSocket","provider":"controlled fake OpenAI-compatible provider","metric":"send-to-final-assistant-response"}' +setup_automation: + - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" +setup_provides_env: + - LANGBOT_FAKE_PROVIDER_URL + - LANGBOT_FAKE_PROVIDER_BASE_URL + - LANGBOT_FAKE_PROVIDER_PID + - LANGBOT_FAKE_PROVIDER_PROVIDER_UUID + - LANGBOT_FAKE_PROVIDER_MODEL_UUID + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +steps: + - "Start or reuse the local fake OpenAI-compatible provider." + - "Create or update the LangBot provider, model, and local-agent pipeline that points at the fake provider." + - "Reset the target Debug Chat session." + - "Open concurrent WebSocket Debug Chat connections and send unique deterministic prompts through the real backend pipeline." +checks: + - "automation-result.json status is pass when every request receives its own expected assistant response." + - "metrics_summary includes request count, concurrency, p50/p95 response latency, first response latency, throughput, and error rate." + - "thresholds_summary shows response_p95_ms, first_response_p95_ms, and error_rate pass." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +diagnostics: + - "This probe removes external model latency from the measurement; it still exercises the live LangBot backend, provider requester, local-agent runner, pipeline, and Debug Chat WebSocket adapter." + - "Use this as the repeatable message-path baseline before comparing against Space or another real provider." +success_patterns: + - "Debug Chat WebSocket concurrency probe passed" + - "Streaming completed" +failure_patterns: + - "WebSocket connection error" + - "Timed out after" + - "Final assistant response did not include" + - "All models failed during streaming setup" +troubleshooting: + - backend-not-listening + - debug-chat-history-contaminates-automation + - local-agent-model-route-unavailable diff --git a/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-slow-load.yaml b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-slow-load.yaml new file mode 100644 index 000000000..afa7de154 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-fake-provider-debug-chat-slow-load.yaml @@ -0,0 +1,88 @@ +id: langbot-fake-provider-debug-chat-slow-load +title: "LangBot Debug Chat slow fake-provider load probe" +mode: probe +area: performance +type: performance +priority: p1 +risk: medium +ci_eligible: false +tags: + - performance + - debug-chat + - websocket + - fake-provider + - slow-provider + - load + - metrics +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER +automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_FAKE_PROVIDER_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +automation_debug_chat_load_requests: "8" +automation_debug_chat_load_concurrency: "4" +automation_debug_chat_load_timeout_ms: "45000" +automation_debug_chat_load_response_p95_ms: "10000" +automation_debug_chat_load_first_response_p95_ms: "7000" +automation_debug_chat_load_max_error_rate: "0" +automation_debug_chat_load_expected_prefix: "SLOWQA" +automation_debug_chat_load_prompt_template: '请只回复 "{expected}",不要解释,不要添加其他字符。' +automation_debug_chat_load_stream: "true" +automation_debug_chat_load_reset: "true" +automation_fake_provider_first_token_delay_ms: "1000" +automation_fake_provider_chunk_delay_ms: "250" +automation_fake_provider_chunk_count: "4" +automation_fake_provider_fail_first_n: "0" +automation_fake_provider_fail_every_n: "0" +automation_fake_provider_fault_status: "500" +metrics_thresholds_json: '{"response_p95_ms":{"max":10000},"first_response_p95_ms":{"max":7000},"error_rate":{"max":0}}' +load_profile_json: '{"requests":8,"concurrency":4,"path":"Pipeline Debug Chat WebSocket","provider":"controlled slow fake OpenAI-compatible provider","metric":"send-to-final-assistant-response","provider_profile":{"first_token_delay_ms":1000,"chunk_delay_ms":250,"chunk_count":4}}' +setup_automation: + - "node:scripts/e2e/ensure-fake-provider-pipeline.mjs --write-env" +setup_provides_env: + - LANGBOT_FAKE_PROVIDER_URL + - LANGBOT_FAKE_PROVIDER_BASE_URL + - LANGBOT_FAKE_PROVIDER_PID + - LANGBOT_FAKE_PROVIDER_PROVIDER_UUID + - LANGBOT_FAKE_PROVIDER_MODEL_UUID + - LANGBOT_FAKE_PROVIDER_PIPELINE_URL + - LANGBOT_FAKE_PROVIDER_PIPELINE_NAME +steps: + - "Configure the local fake provider with deterministic slow streaming latency." + - "Create or update the LangBot provider, model, and local-agent pipeline that points at the fake provider." + - "Reset the target Debug Chat session." + - "Open concurrent WebSocket Debug Chat connections and send unique deterministic prompts through the real backend pipeline." +checks: + - "automation-result.json status is pass when every request receives its own expected assistant response." + - "metrics_summary shows zero errors under the slow-provider profile." + - "thresholds_summary shows response_p95_ms, first_response_p95_ms, and error_rate pass." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +diagnostics: + - "This probe keeps the model deterministic while injecting provider latency, so it catches backend timeout, streaming, and WebSocket backpressure issues without Space variability." + - "Compare with langbot-fake-provider-debug-chat-load to separate fixed LangBot overhead from provider-latency amplification." +success_patterns: + - "Debug Chat WebSocket concurrency probe passed" + - "Streaming completed" +failure_patterns: + - "WebSocket connection error" + - "Timed out after" + - "Final assistant response did not include" + - "All models failed during streaming setup" +troubleshooting: + - backend-not-listening + - debug-chat-history-contaminates-automation + - local-agent-model-route-unavailable diff --git a/skills/skills/langbot-testing/cases/langbot-fault-taxonomy-contract.yaml b/skills/skills/langbot-testing/cases/langbot-fault-taxonomy-contract.yaml new file mode 100644 index 000000000..2b990f837 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-fault-taxonomy-contract.yaml @@ -0,0 +1,35 @@ +id: langbot-fault-taxonomy-contract +title: "LangBot fault taxonomy and cleanup contract" +mode: probe +area: reliability +type: chaos +priority: p1 +risk: medium +ci_eligible: true +tags: + - reliability + - chaos + - contract + - synthetic +skills: + - langbot-testing +automation: skills/langbot-testing/probes/langbot-fault-taxonomy-contract.mjs +fault_model_json: '{"kind":"taxonomy-contract","destructive":false,"scenarios":["provider-timeout","plugin-runtime-disconnect","mcp-stdio-server-exit","operator-missing-login","transient-marketplace-timeout"]}' +steps: + - "Run `rtk bin/lbs test run langbot-fault-taxonomy-contract --dry-run` first; remove `--dry-run` after checking the evidence directory." + - "Automation validates that representative fault scenarios declare target, injected fault, expected status, recovery check, and cleanup." + - "Review metrics.json, fault-model.json, and automation-result.json under LBS_EVIDENCE_DIR." +checks: + - "automation-result.json status is pass." + - "Every scenario has an expected status in pass, fail, blocked, env_issue, or flaky." + - "Every scenario declares a cleanup action and recovery check." +evidence_required: + - metrics + - filesystem +diagnostics: + - "This is a non-destructive taxonomy contract probe; it does not inject real runtime faults." + - "Use it as a gate before adding live chaos cases that kill runtimes, route traffic through a proxy, or disrupt a backend dependency." +success_patterns: + - "Fault taxonomy contract declares status" +failure_patterns: + - "missing required scenario fields" diff --git a/skills/skills/langbot-testing/cases/langbot-live-backend-latency.yaml b/skills/skills/langbot-testing/cases/langbot-live-backend-latency.yaml new file mode 100644 index 000000000..1922d06f0 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-live-backend-latency.yaml @@ -0,0 +1,42 @@ +id: langbot-live-backend-latency +title: "LangBot live backend basic latency probe" +mode: probe +area: performance +type: performance +priority: p1 +risk: medium +ci_eligible: false +tags: + - performance + - live-backend + - latency + - metrics +skills: + - langbot-testing +env: + - LANGBOT_BACKEND_URL +automation: skills/langbot-testing/probes/langbot-live-backend-latency.mjs +metrics_thresholds_json: '{"backend_p95_ms":{"max":1000},"error_rate":{"max":0}}' +load_profile_json: '{"requests":12,"concurrency":2,"endpoints":["/healthz"]}' +steps: + - "Confirm the selected LangBot backend is the intended test target." + - "Run `rtk bin/lbs test run langbot-live-backend-latency --dry-run` first; remove `--dry-run` after checking LANGBOT_BACKEND_URL and evidence directory." + - "Automation sends a small request batch to LANGBOT_BACKEND_URL/healthz and records latency, status counts, and network errors." +checks: + - "automation-result.json status is pass when the backend responds and p95/error-rate thresholds pass." + - "automation-result.json status is env_issue when the backend is not reachable." + - "metrics.json and network.log are written under LBS_EVIDENCE_DIR." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +diagnostics: + - "This probe measures backend health endpoint reachability latency only; it does not cover model/provider, browser, Debug Chat, RAG, or plugin runtime latency." +success_patterns: + - "Live backend latency probe passed" +failure_patterns: + - "Backend did not respond" + - "breached latency or error-rate thresholds" +troubleshooting: + - socks-proxy-without-socksio diff --git a/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml b/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml new file mode 100644 index 000000000..8ff911371 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-live-backend-log-health.yaml @@ -0,0 +1,45 @@ +id: langbot-live-backend-log-health +title: "LangBot live backend log health probe" +mode: probe +area: reliability +type: reliability +priority: p1 +risk: medium +ci_eligible: false +tags: + - reliability + - live-backend + - backend-log + - metrics +skills: + - langbot-testing +env: + - LANGBOT_BACKEND_URL +automation: skills/langbot-testing/probes/langbot-live-backend-log-health.mjs +metrics_thresholds_json: '{"fail_count":{"max":0}}' +load_profile_json: '{"lookback_seconds":300,"log_source":"LANGBOT_BACKEND_LOG or latest LANGBOT_REPO/data/logs/langbot-*.log"}' +steps: + - "Confirm the selected LangBot backend log belongs to the intended test target." + - "Run `rtk bin/lbs test run langbot-live-backend-log-health --dry-run` first; remove `--dry-run` after checking evidence directory and log source." + - "Automation scans the recent backend log window for fail-severity runtime findings such as Traceback, ImportError, ERROR, unclosed sessions, and unawaited coroutines." +checks: + - "automation-result.json status is pass only when fail_count is 0." + - "metrics_summary includes scanned_line_count, fail_count, warning_count, and finding_count." + - "findings.json and scanned-backend.log are written under LBS_EVIDENCE_DIR." +evidence_required: + - metrics + - backend_log + - filesystem +diagnostics: + - "Set LANGBOT_BACKEND_LOG to an explicit log path when the latest log file is not the run target." + - "Set LANGBOT_BACKEND_LOG_SINCE or LANGBOT_BACKEND_LOG_LOOKBACK_SECONDS to control the scan window." + - "This probe measures runtime log health; it does not prove user-facing Debug Chat, plugin, model, or RAG behavior." +success_patterns: + - "Live backend log health passed" +failure_patterns: + - "Traceback" + - "ImportError" + - "ERROR" + - "unclosed" +troubleshooting: + - socks-proxy-without-socksio diff --git a/skills/skills/langbot-testing/cases/langbot-live-control-plane-api.yaml b/skills/skills/langbot-testing/cases/langbot-live-control-plane-api.yaml new file mode 100644 index 000000000..2cd8ee2c7 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-live-control-plane-api.yaml @@ -0,0 +1,44 @@ +id: langbot-live-control-plane-api +title: "LangBot live control-plane API probe" +mode: probe +area: performance +type: performance +priority: p1 +risk: medium +ci_eligible: false +tags: + - performance + - reliability + - live-backend + - control-plane + - metrics +skills: + - langbot-testing +env: + - LANGBOT_BACKEND_URL +automation: skills/langbot-testing/probes/langbot-live-control-plane-api.mjs +metrics_thresholds_json: '{"error_rate":{"max":0},"response_shape_failures":{"max":0},"healthz_p95_ms":{"max":500},"system_info_p95_ms":{"max":1000}}' +load_profile_json: '{"requests":20,"concurrency":4,"endpoints":["/healthz","/api/v1/system/info"],"auth_required":false}' +steps: + - "Confirm the selected LangBot backend is the intended test target." + - "Run `rtk bin/lbs test run langbot-live-control-plane-api --dry-run` first; remove `--dry-run` after checking LANGBOT_BACKEND_URL and evidence directory." + - "Automation sends a small request batch to /healthz and /api/v1/system/info, then validates status code, JSON shape, and latency budgets." +checks: + - "automation-result.json status is pass when every control-plane request returns HTTP 200, JSON code 0, and required response fields." + - "metrics_summary includes per-endpoint p50/p95 latency, error rate, status counts, and response_shape_failures." + - "thresholds_summary shows error_rate, response_shape_failures, healthz_p95_ms, and system_info_p95_ms all pass." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +diagnostics: + - "This probe measures unauthenticated backend control-plane readiness; it does not cover authenticated UI flows, Debug Chat, model calls, plugins, or RAG." + - "A system_info shape failure usually means the API contract or startup state changed and should be investigated before treating latency as healthy." +success_patterns: + - "Live control-plane API probe passed" +failure_patterns: + - "Backend did not respond" + - "breached shape, latency, or error-rate thresholds" +troubleshooting: + - socks-proxy-without-socksio diff --git a/skills/skills/langbot-testing/cases/langbot-overhead-accounting-contract.yaml b/skills/skills/langbot-testing/cases/langbot-overhead-accounting-contract.yaml new file mode 100644 index 000000000..650dfe7d9 --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-overhead-accounting-contract.yaml @@ -0,0 +1,37 @@ +id: langbot-overhead-accounting-contract +title: "LangBot overhead accounting metrics contract" +mode: probe +area: performance +type: performance +priority: p1 +risk: medium +ci_eligible: true +tags: + - performance + - metrics + - contract + - synthetic +skills: + - langbot-testing +automation: skills/langbot-testing/probes/langbot-overhead-accounting-contract.mjs +metrics_thresholds_json: '{"sample_count":{"min":50},"langbot_overhead_p95_ms":{"max":25},"accounting_gap_max_ms":{"max":0.001}}' +load_profile_json: '{"kind":"synthetic-overhead-accounting","samples":80,"external_latency_segments":["provider","external_tool","network"]}' +steps: + - "Run `rtk bin/lbs test run langbot-overhead-accounting-contract --dry-run` first; remove `--dry-run` after checking the evidence directory." + - "Automation generates deterministic message-path latency samples and separates LangBot overhead from provider/tool/network latency." + - "Review metrics.json, thresholds.json, resource-log.json, and automation-result.json under LBS_EVIDENCE_DIR." +checks: + - "automation-result.json status is pass." + - "metrics_summary includes sample_count, langbot_overhead_p95_ms, e2e_latency_p95_ms, external_latency_p95_ms, and accounting_gap_max_ms." + - "thresholds_summary shows sample_count, langbot_overhead_p95_ms, and accounting_gap_max_ms all pass." +evidence_required: + - metrics + - resource_log + - filesystem +diagnostics: + - "This is a synthetic contract probe for the QA harness; it is not live product performance." + - "Use it to verify that reports can carry overhead accounting metrics before running live backend or browser performance probes." +success_patterns: + - "Overhead accounting contract passed" +failure_patterns: + - "breached one or more thresholds" diff --git a/skills/skills/langbot-testing/cases/langbot-space-debug-chat-concurrency-smoke.yaml b/skills/skills/langbot-testing/cases/langbot-space-debug-chat-concurrency-smoke.yaml new file mode 100644 index 000000000..4f9fc779b --- /dev/null +++ b/skills/skills/langbot-testing/cases/langbot-space-debug-chat-concurrency-smoke.yaml @@ -0,0 +1,84 @@ +id: langbot-space-debug-chat-concurrency-smoke +title: "LangBot Debug Chat real Space-provider concurrency smoke" +mode: probe +area: performance +type: performance +priority: p1 +risk: high +ci_eligible: false +tags: + - performance + - debug-chat + - websocket + - space + - live-provider + - smoke + - metrics +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_BACKEND_URL + - LANGBOT_FRONTEND_URL + - LANGBOT_E2E_LOGIN_USER +automation: skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs +automation_env: + - LANGBOT_BACKEND_URL + - LANGBOT_E2E_LOGIN_USER + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_pipeline_url_env: LANGBOT_LOCAL_AGENT_PIPELINE_URL +automation_pipeline_name_env: LANGBOT_LOCAL_AGENT_PIPELINE_NAME +automation_debug_chat_load_requests: "3" +automation_debug_chat_load_concurrency: "2" +automation_debug_chat_load_timeout_ms: "120000" +automation_debug_chat_load_response_p95_ms: "120000" +automation_debug_chat_load_max_error_rate: "0" +automation_debug_chat_load_expected_prefix: "SPACEQA" +automation_debug_chat_load_prompt_template: '请只回复 "{expected}",不要解释,不要添加其他字符。' +automation_debug_chat_load_stream: "true" +automation_debug_chat_load_reset: "true" +metrics_thresholds_json: '{"response_p95_ms":{"max":120000},"error_rate":{"max":0}}' +load_profile_json: '{"requests":3,"concurrency":2,"path":"Pipeline Debug Chat WebSocket","provider":"LangBot Space model route","metric":"send-to-final-assistant-response","classification":"smoke-not-benchmark"}' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" +setup_provides_env: + - LANGBOT_PIPELINE_URL + - LANGBOT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_PIPELINE_URL + - LANGBOT_LOCAL_AGENT_PIPELINE_NAME + - LANGBOT_LOCAL_AGENT_MODEL_UUID + - LANGBOT_E2E_MODEL_UUID +preconditions: + - "The selected local LangBot instance is safe for a low-volume real Space model smoke run." + - "Treat Space/provider/network failures as environment or dependency findings until fake-provider baseline evidence separates LangBot overhead." +steps: + - "Prepare a local-agent pipeline with a tested Space model and fallback models." + - "Reset the target Debug Chat session." + - "Open a small number of concurrent WebSocket Debug Chat connections and send unique deterministic prompts through the live Space provider path." +checks: + - "automation-result.json status is pass when every request receives its own expected assistant response." + - "metrics_summary includes request count, concurrency, p95 response latency, throughput, and error rate." + - "The report classifies the result as a live-provider smoke, not a stable LangBot overhead benchmark." +evidence_required: + - metrics + - network + - api_diagnostic + - filesystem +diagnostics: + - "This probe measures real user-path latency through Space and includes provider latency, model behavior, and network effects." + - "Compare with langbot-fake-provider-debug-chat-load before attributing slow or failed runs to LangBot itself." +success_patterns: + - "Debug Chat WebSocket concurrency probe passed" + - "Streaming completed" +failure_patterns: + - "invalid api key" + - "WebSocket connection error" + - "Timed out after" + - "Final assistant response did not include" + - "All models failed during streaming setup" +troubleshooting: + - local-agent-model-route-unavailable + - marketplace-network-flaky + - proxy-env-mismatch + - telemetry-proxy-noise diff --git a/skills/skills/langbot-testing/cases/pipeline-debug-chat-performance.yaml b/skills/skills/langbot-testing/cases/pipeline-debug-chat-performance.yaml new file mode 100644 index 000000000..266cbb57d --- /dev/null +++ b/skills/skills/langbot-testing/cases/pipeline-debug-chat-performance.yaml @@ -0,0 +1,80 @@ +id: pipeline-debug-chat-performance +title: "Pipeline Debug Chat user-path performance probe" +mode: agent-browser +area: pipeline +type: performance +priority: p1 +risk: medium +ci_eligible: false +tags: + - performance + - pipeline + - debug-chat + - user-path + - metrics +skills: + - langbot-env-setup + - langbot-testing +env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL +env_any: + - LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME +automation: scripts/e2e/pipeline-debug-chat.mjs +automation_env: + - LANGBOT_FRONTEND_URL + - LANGBOT_BACKEND_URL + - LANGBOT_BROWSER_PROFILE + - LANGBOT_CHROMIUM_EXECUTABLE + - LANGBOT_E2E_PROMPT + - LANGBOT_E2E_EXPECTED_TEXT + - LANGBOT_E2E_RESPONSE_TIMEOUT_MS +automation_env_any: + - LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME +automation_prompt: "请只回复 OK,用于性能测试。" +automation_expected_text: "OK" +automation_response_timeout_ms: "120000" +automation_reset_debug_chat: "true" +automation_debug_chat_response_p95_ms: "120000" +automation_debug_chat_max_error_rate: "0" +metrics_thresholds_json: '{"response_p95_ms":{"max":120000},"error_rate":{"max":0}}' +load_profile_json: '{"prompts":1,"browser":true,"path":"Pipeline Debug Chat","metric":"send-to-visible-completion"}' +setup_automation: + - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env" +setup_provides_env: + - LANGBOT_PIPELINE_URL + - LANGBOT_PIPELINE_NAME +preconditions: + - "LANGBOT_PIPELINE_URL or LANGBOT_PIPELINE_NAME points to the pipeline intended for this Debug Chat performance run." + - "The target pipeline is safe to reset Debug Chat history for this run." + - "The target pipeline has a known-good runner/model; provider latency should be interpreted separately from LangBot overhead." +steps: + - "Open LANGBOT_FRONTEND_URL with the prepared browser profile." + - "Open the target pipeline and select Debug Chat." + - "Reset Debug Chat history through the backend API when configured." + - "Send the deterministic prompt and wait for the expected assistant response." +checks: + - "automation-result.json status is pass when the expected assistant response appears." + - "metrics_summary includes response_p50_ms, response_p95_ms, error_rate, and total_duration_ms." + - "thresholds_summary shows response_p95_ms and error_rate pass." +evidence_required: + - ui + - screenshot + - console + - network + - metrics +diagnostics: + - "This case measures browser-visible send-to-completion latency; it does not split provider latency from LangBot overhead." + - "Use backend logs and provider diagnostics to explain slow runs before calling them LangBot regressions." +success_patterns: + - "Processing request from person_websocket" + - "Streaming completed" +failure_patterns: + - "Action invoke_llm_stream call timed out" + - "Task exception was never retrieved" + - "All models failed during streaming setup" +troubleshooting: + - debug-chat-history-contaminates-automation + - local-agent-model-route-unavailable + - plugin-runtime-timeout + - proxy-env-mismatch diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/.gitignore b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/.gitignore index 849ddff3b..89d8e500c 100644 --- a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/.gitignore +++ b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/.gitignore @@ -1 +1,3 @@ -dist/ +dist/* +!dist/ +!dist/qa-plugin-smoke-0.1.0.lbpkg diff --git a/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg new file mode 100644 index 000000000..a4a50f803 Binary files /dev/null and b/skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/dist/qa-plugin-smoke-0.1.0.lbpkg differ diff --git a/skills/skills/langbot-testing/probes/agent-runner-async-db-readiness.mjs b/skills/skills/langbot-testing/probes/agent-runner-async-db-readiness.mjs old mode 100644 new mode 100755 diff --git a/skills/skills/langbot-testing/probes/agent-runner-behavior-matrix.mjs b/skills/skills/langbot-testing/probes/agent-runner-behavior-matrix.mjs old mode 100644 new mode 100755 diff --git a/skills/skills/langbot-testing/probes/agent-runner-fixture-contract.mjs b/skills/skills/langbot-testing/probes/agent-runner-fixture-contract.mjs old mode 100644 new mode 100755 diff --git a/skills/skills/langbot-testing/probes/agent-runner-ledger-concurrency.mjs b/skills/skills/langbot-testing/probes/agent-runner-ledger-concurrency.mjs old mode 100644 new mode 100755 diff --git a/skills/skills/langbot-testing/probes/agent-runner-ledger-contention.mjs b/skills/skills/langbot-testing/probes/agent-runner-ledger-contention.mjs old mode 100644 new mode 100755 diff --git a/skills/skills/langbot-testing/probes/agent-runner-ledger-invariants.mjs b/skills/skills/langbot-testing/probes/agent-runner-ledger-invariants.mjs old mode 100644 new mode 100755 diff --git a/skills/skills/langbot-testing/probes/agent-runner-ledger-stress.mjs b/skills/skills/langbot-testing/probes/agent-runner-ledger-stress.mjs old mode 100644 new mode 100755 diff --git a/skills/skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs b/skills/skills/langbot-testing/probes/agent-runner-runtime-chaos.mjs old mode 100644 new mode 100755 diff --git a/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs new file mode 100755 index 000000000..af5153dbf --- /dev/null +++ b/skills/skills/langbot-testing/probes/langbot-debug-chat-concurrency.mjs @@ -0,0 +1,837 @@ +#!/usr/bin/env node + +import crypto from "node:crypto"; +import net from "node:net"; +import tls from "node:tls"; +import { mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { env, exit } from "node:process"; +import { + apiJson, + appendLine, + ensureEvidence, + evidencePaths, + loadEnvFiles, + localIsoWithOffset, + redact, + resetAndAuthLocalUser, + writeResult, +} from "../../../scripts/e2e/lib/langbot-e2e.mjs"; +import { + buildProviderTimingMetrics, + summarizeFakeProviderState, +} from "./lib/fake-provider-timing.mjs"; + +const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "langbot-debug-chat-concurrency"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); + +const startedAt = new Date(); +const metricsPath = resolve(paths.evidenceDir, "metrics.json"); +const samplesPath = resolve(paths.evidenceDir, "samples.json"); +const fakeProviderStatePath = resolve(paths.evidenceDir, "fake-provider-state.json"); +const resetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json"); +const backendUrl = env.LANGBOT_BACKEND_URL || ""; +const fakeProviderUrl = env.LANGBOT_FAKE_PROVIDER_URL || ""; +const pipelineUrl = env.LANGBOT_E2E_PIPELINE_URL || env.LANGBOT_PIPELINE_URL || ""; +const pipelineName = env.LANGBOT_E2E_PIPELINE_NAME || env.LANGBOT_PIPELINE_NAME || ""; +const sessionType = env.LANGBOT_DEBUG_CHAT_LOAD_SESSION_TYPE || env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person"; +const totalRequests = positiveInteger(env.LANGBOT_DEBUG_CHAT_LOAD_REQUESTS, defaultRequests(caseId)); +const concurrency = Math.min(totalRequests, positiveInteger(env.LANGBOT_DEBUG_CHAT_LOAD_CONCURRENCY, defaultConcurrency(caseId))); +const timeoutMs = positiveInteger(env.LANGBOT_DEBUG_CHAT_LOAD_TIMEOUT_MS, defaultTimeout(caseId)); +const expectedPrefix = env.LANGBOT_DEBUG_CHAT_LOAD_EXPECTED_PREFIX || "LBQA"; +const promptTemplate = env.LANGBOT_DEBUG_CHAT_LOAD_PROMPT_TEMPLATE + || "请只回复 \"{expected}\",不要解释,不要添加其他字符。"; +const stream = bool(env.LANGBOT_DEBUG_CHAT_LOAD_STREAM, true); +const resetBeforeRun = bool(env.LANGBOT_DEBUG_CHAT_LOAD_RESET, true); +const responseP95BudgetMs = positiveNumber(env.LANGBOT_DEBUG_CHAT_LOAD_RESPONSE_P95_MS, defaultP95Budget(caseId)); +const firstResponseP95BudgetMs = positiveNumber(env.LANGBOT_DEBUG_CHAT_LOAD_FIRST_RESPONSE_P95_MS, 0); +const maxErrorRate = positiveNumber(env.LANGBOT_DEBUG_CHAT_LOAD_MAX_ERROR_RATE, 0); +const minErrorRate = positiveNumber(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_ERROR_RATE, 0); +const minErrorCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_ERROR_COUNT, 0); +const minOkCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_OK_COUNT, 0); +const minProviderFaultCount = nonNegativeInteger(env.LANGBOT_DEBUG_CHAT_LOAD_MIN_PROVIDER_FAULT_COUNT, 0); +const failOnFinalMismatch = bool(env.LANGBOT_DEBUG_CHAT_LOAD_FAIL_ON_FINAL_MISMATCH, false); +const failureSignals = textList(env.LANGBOT_E2E_FAILURE_SIGNALS || env.LANGBOT_DEBUG_CHAT_LOAD_FAILURE_SIGNALS || ""); + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + status: "fail", + reason: "", + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + duration_ms: 0, + backend_url: backendUrl, + pipeline_url: pipelineUrl, + pipeline_name: pipelineName, + pipeline_id: "", + session_type: sessionType, + load_profile: { + requests: totalRequests, + concurrency, + timeout_ms: timeoutMs, + stream, + reset_before_run: resetBeforeRun, + fail_on_final_mismatch: failOnFinalMismatch, + }, + evidence: { + network_log: paths.networkLog, + metrics_json: metricsPath, + samples_json: samplesPath, + fake_provider_state_json: fakeProviderStatePath, + debug_chat_reset_diagnostic_json: resetDiagnosticPath, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["metrics", "network", "api_diagnostic", "filesystem"], +}; + +try { + if (!backendUrl) { + result.status = "env_issue"; + throw new Error("LANGBOT_BACKEND_URL is not configured."); + } + if (!["person", "group"].includes(sessionType)) { + throw new Error(`LANGBOT_DEBUG_CHAT_LOAD_SESSION_TYPE must be person or group, got ${sessionType}.`); + } + const backendReady = await backendReachable(backendUrl); + if (!backendReady) { + result.status = "env_issue"; + throw new Error(`Backend did not respond at ${backendUrl}.`); + } + + const user = env.LANGBOT_E2E_LOGIN_USER || ""; + const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; + if (!user) { + result.status = "env_issue"; + throw new Error("LANGBOT_E2E_LOGIN_USER is required so this probe can resolve/reset the Debug Chat session."); + } + const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); + + const pipeline = await resolvePipeline({ backendUrl, token: auth.token, pipelineUrl, pipelineName }); + result.pipeline_id = pipeline.id; + result.pipeline_name = pipeline.name || pipelineName; + if (!result.pipeline_url && env.LANGBOT_FRONTEND_URL) { + result.pipeline_url = `${env.LANGBOT_FRONTEND_URL.replace(/\/$/, "")}/home/pipelines?id=${encodeURIComponent(pipeline.id)}`; + } + + if (resetBeforeRun) { + const reset = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.id)}/ws/reset/${encodeURIComponent(sessionType)}`, { + method: "POST", + token: auth.token, + }); + const resetDiagnostic = { + status: isApiFailure(reset) ? "fail" : "ready", + http_status: reset.status, + code: reset.json.code ?? null, + reason: isApiFailure(reset) ? reset.json.msg || "Debug Chat reset failed." : "Debug Chat session reset.", + }; + await writeFile(resetDiagnosticPath, `${JSON.stringify(resetDiagnostic, null, 2)}\n`, "utf8"); + if (resetDiagnostic.status === "fail") { + throw new Error(resetDiagnostic.reason); + } + } + + const wsUrl = websocketUrl(backendUrl, pipeline.id, sessionType); + const loadStartedAt = performance.now(); + const samples = await runLoad({ + wsUrl, + totalRequests, + concurrency, + timeoutMs, + promptTemplate, + expectedPrefix, + stream, + failOnFinalMismatch, + failureSignals, + }); + const loadDurationMs = performance.now() - loadStartedAt; + const fakeProviderState = await readFakeProviderState(fakeProviderUrl); + if (fakeProviderState) { + await writeFile(fakeProviderStatePath, `${JSON.stringify(fakeProviderState, null, 2)}\n`, "utf8"); + } + const metrics = buildMetrics({ + samples, + totalRequests, + concurrency, + timeoutMs, + loadDurationMs, + backendUrl, + pipelineId: pipeline.id, + sessionType, + fakeProviderState, + }); + const thresholds = buildThresholds(metrics); + const passed = Object.values(thresholds).every((item) => item.pass); + result.status = passed ? "pass" : "fail"; + result.reason = passed + ? "Debug Chat WebSocket concurrency probe passed all thresholds." + : "Debug Chat WebSocket concurrency probe breached latency or error-rate thresholds."; + result.metrics_summary = { + requests: metrics.total_requests, + concurrency: metrics.concurrency, + ok_count: metrics.ok_count, + error_count: metrics.error_count, + timeout_count: metrics.timeout_count, + error_rate: metrics.error_rate, + response_p50_ms: metrics.response_duration_ms.p50, + response_p95_ms: metrics.response_duration_ms.p95, + first_assistant_event_p95_ms: metrics.first_assistant_event_ms.p95, + first_assistant_content_p95_ms: metrics.first_assistant_content_ms.p95, + first_response_p95_ms: metrics.first_response_ms.p95, + throughput_rps: metrics.throughput_rps, + status_counts: metrics.status_counts, + fake_provider_request_count: metrics.fake_provider?.request_count ?? null, + fake_provider_fault_count: metrics.fake_provider?.fault_count ?? null, + fake_provider_duration_p95_ms: metrics.provider_timing?.provider_duration_ms.p95 ?? null, + langbot_overhead_estimate_p95_ms: metrics.provider_timing?.langbot_overhead_estimate_ms.p95 ?? null, + send_to_provider_start_p95_ms: metrics.provider_timing?.send_to_provider_start_ms.p95 ?? null, + provider_finish_to_ws_final_p95_ms: metrics.provider_timing?.provider_finish_to_ws_final_ms.p95 ?? null, + provider_timing_matched_request_count: metrics.provider_timing?.matched_request_count ?? null, + }; + result.thresholds_summary = thresholds; + result.artifacts = { + metrics_json: metricsPath, + samples_json: samplesPath, + fake_provider_state_json: fakeProviderState ? fakeProviderStatePath : "", + network_log: paths.networkLog, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }; + + await writeFile(metricsPath, `${JSON.stringify({ ...metrics, thresholds }, null, 2)}\n`, "utf8"); + await writeFile(samplesPath, `${JSON.stringify(samples, null, 2)}\n`, "utf8"); +} catch (error) { + if (!["env_issue", "blocked"].includes(result.status)) { + result.status = looksLikeEnvIssue(error) ? "env_issue" : "fail"; + } + result.reason = result.reason || safeReason(error.message); +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + result.duration_ms = finishedAt.getTime() - startedAt.getTime(); + await mkdir(paths.evidenceDir, { recursive: true }); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +exit(result.status === "pass" ? 0 : result.status === "env_issue" || result.status === "blocked" ? 2 : 1); + +function defaultRequests(id) { + return id.includes("space") ? 3 : 12; +} + +function defaultConcurrency(id) { + return id.includes("space") ? 1 : 4; +} + +function defaultTimeout(id) { + return id.includes("space") ? 120_000 : 30_000; +} + +function defaultP95Budget(id) { + return id.includes("space") ? 120_000 : 5_000; +} + +function positiveInteger(value, fallback) { + const parsed = Number.parseInt(String(value || ""), 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function nonNegativeInteger(value, fallback) { + const parsed = Number.parseInt(String(value ?? ""), 10); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; +} + +function positiveNumber(value, fallback) { + const parsed = Number(value || ""); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +} + +function bool(value, fallback) { + if (value === undefined || value === "") return fallback; + if (/^(1|true|yes|on)$/i.test(String(value))) return true; + if (/^(0|false|no|off)$/i.test(String(value))) return false; + return fallback; +} + +function textList(value) { + return String(value || "") + .split(/\r?\n|,/) + .map((item) => item.trim()) + .filter(Boolean); +} + +async function backendReachable(baseUrl) { + try { + const response = await fetch(`${baseUrl.replace(/\/$/, "")}/healthz`, { + signal: AbortSignal.timeout(3000), + }); + return response.status < 500; + } catch { + return false; + } +} + +async function readFakeProviderState(rootUrl) { + if (!rootUrl) return null; + try { + const response = await fetch(`${normalizeProviderRootUrl(rootUrl)}/__qa/config`, { + signal: AbortSignal.timeout(3000), + }); + const json = await response.json().catch(() => ({})); + return { + status: response.ok && json.ok === true ? "loaded" : "unavailable", + url: normalizeProviderRootUrl(rootUrl), + http_status: response.status, + model: json.model || "", + config: json.config || {}, + request_count: Number.isFinite(json.request_count) ? json.request_count : null, + recent_requests: Array.isArray(json.recent_requests) ? json.recent_requests : [], + }; + } catch (error) { + return { + status: "unavailable", + url: normalizeProviderRootUrl(rootUrl), + reason: safeReason(error.message), + request_count: null, + recent_requests: [], + }; + } +} + +function normalizeProviderRootUrl(value) { + const trimmed = String(value || "").trim().replace(/\/$/, ""); + return trimmed.endsWith("/v1") ? trimmed.slice(0, -3) : trimmed; +} + +function pipelineIdFromUrl(url) { + if (!url) return ""; + try { + const parsed = new URL(url); + return parsed.searchParams.get("id") || ""; + } catch { + return ""; + } +} + +async function resolvePipeline({ backendUrl, token, pipelineUrl, pipelineName }) { + const idFromUrl = pipelineIdFromUrl(pipelineUrl); + if (idFromUrl) { + const response = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(idFromUrl)}`, { token }); + const pipeline = response.json.data?.pipeline; + if (isApiFailure(response) || !pipeline?.uuid) { + throw new Error(response.json.msg || `Could not load pipeline ${idFromUrl}.`); + } + return { id: pipeline.uuid, name: pipeline.name || "" }; + } + if (!pipelineName) { + throw new Error("Set LANGBOT_E2E_PIPELINE_URL or LANGBOT_E2E_PIPELINE_NAME before running this probe."); + } + const response = await apiJson(backendUrl, "/api/v1/pipelines", { token }); + if (isApiFailure(response)) { + throw new Error(response.json.msg || "Failed to list pipelines."); + } + const pipeline = (response.json.data?.pipelines || []).find((item) => item.name === pipelineName); + if (!pipeline?.uuid) { + throw new Error(`Could not find pipeline named ${pipelineName}.`); + } + return { id: pipeline.uuid, name: pipeline.name || pipelineName }; +} + +function isApiFailure(response) { + return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0); +} + +function websocketUrl(baseUrl, pipelineId, sessionType) { + const parsed = new URL(baseUrl); + parsed.protocol = parsed.protocol === "https:" ? "wss:" : "ws:"; + parsed.pathname = `/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/connect`; + parsed.search = `?session_type=${encodeURIComponent(sessionType)}`; + return parsed.toString(); +} + +async function runLoad(options) { + const samples = []; + let nextIndex = 0; + const workers = Array.from({ length: options.concurrency }, async () => { + while (nextIndex < options.totalRequests) { + const index = nextIndex; + nextIndex += 1; + const sample = await runSingleRequest({ ...options, index }); + samples.push(sample); + } + }); + await Promise.all(workers); + return samples.sort((left, right) => left.index - right.index); +} + +function expectedForIndex(prefix, index) { + return `${prefix}-${String(index + 1).padStart(4, "0")}`; +} + +function promptForIndex(template, expected) { + return template.replaceAll("{expected}", expected); +} + +function runSingleRequest({ + wsUrl, + index, + timeoutMs, + promptTemplate, + expectedPrefix, + stream, + failOnFinalMismatch, + failureSignals, +}) { + return new Promise((resolve) => { + const expected = expectedForIndex(expectedPrefix, index); + const prompt = promptForIndex(promptTemplate, expected); + const sample = { + index, + status: "running", + ok: false, + expected_text: expected, + prompt, + response_text: "", + started_at: new Date().toISOString(), + started_epoch_ms: Date.now(), + connected_at: null, + connected_epoch_ms: null, + sent_at: null, + sent_epoch_ms: null, + first_assistant_event_at: null, + first_assistant_event_epoch_ms: null, + first_assistant_event_ms: null, + first_assistant_content_at: null, + first_assistant_content_epoch_ms: null, + first_assistant_content_ms: null, + first_response_at: null, + first_response_epoch_ms: null, + connected_ms: null, + first_response_ms: null, + response_duration_ms: null, + finished_at: null, + finished_epoch_ms: null, + event_count: 0, + foreign_response_count: 0, + last_foreign_response_text: "", + error: "", + close_code: null, + close_reason: "", + }; + let closed = false; + let connectedAt = 0; + let sentAt = 0; + const startedAt = performance.now(); + let client = null; + const timer = setTimeout(() => { + finish("timeout", `Timed out after ${timeoutMs} ms.`); + }, timeoutMs); + + client = openRawWebSocket(wsUrl, { + onOpen() { + connectedAt = performance.now(); + const now = Date.now(); + sample.connected_at = new Date(now).toISOString(); + sample.connected_epoch_ms = now; + sample.connected_ms = rounded(connectedAt - startedAt); + }, + onMessage(text) { + sample.event_count += 1; + let data; + try { + data = JSON.parse(String(text || "")); + } catch (error) { + finish("error", `Invalid WebSocket JSON: ${error.message}`); + return; + } + appendLine(paths.networkLog, JSON.stringify({ + request_index: index, + type: data.type, + session_type: data.session_type || "", + role: data.data?.role || "", + is_final: data.data?.is_final ?? null, + content_preview: redact(String(data.data?.content || data.message || "").slice(0, 200)), + })).catch(() => {}); + + if (data.type === "connected") { + sentAt = performance.now(); + const now = Date.now(); + sample.sent_at = new Date(now).toISOString(); + sample.sent_epoch_ms = now; + client.send(JSON.stringify({ + type: "message", + message: [{ type: "Plain", text: prompt }], + stream, + })); + return; + } + if (data.type === "error") { + finish("error", data.message || "WebSocket error message."); + return; + } + if (data.type !== "response" || data.data?.role !== "assistant") return; + + const content = String(data.data.content || ""); + markFirstAssistantEvent(sample, sentAt); + if (content) sample.response_text = content; + if (content) markFirstAssistantContent(sample, sentAt); + if (content.includes(expected) && sample.first_response_ms === null && sentAt > 0) { + const now = Date.now(); + sample.first_response_at = new Date(now).toISOString(); + sample.first_response_epoch_ms = now; + sample.first_response_ms = rounded(performance.now() - sentAt); + } + if (data.data.is_final === true) { + const ok = sample.response_text.includes(expected); + if (ok) { + if (sample.first_response_ms === null && sentAt > 0) { + sample.first_response_ms = rounded(performance.now() - sentAt); + } + finish("pass", ""); + } else if (matchesFailureSignal(sample.response_text, failureSignals)) { + finish("app_error", `Assistant final response matched a failure signal: ${sample.response_text}`); + } else if (failOnFinalMismatch && !containsLoadToken(sample.response_text, expectedPrefix)) { + finish("mismatch", `Final assistant response did not include ${expected}: ${sample.response_text}`); + } else { + sample.foreign_response_count += 1; + sample.last_foreign_response_text = sample.response_text; + } + } + }, + onError(error) { + finish("connection_error", `WebSocket connection error: ${error.message}`); + }, + onClose(event) { + sample.close_code = event.code; + sample.close_reason = event.reason || ""; + if (!closed) finish("closed", `WebSocket closed before final assistant response: ${event.code}`); + }, + }); + + function finish(status, reason) { + if (closed) return; + closed = true; + clearTimeout(timer); + sample.status = status; + sample.ok = status === "pass"; + sample.error = status === "timeout" && sample.foreign_response_count > 0 + ? `${reason || ""} Saw ${sample.foreign_response_count} foreign assistant response(s); last=${sample.last_foreign_response_text}` + : reason || ""; + if (sentAt > 0) sample.response_duration_ms = rounded(performance.now() - sentAt); + else sample.response_duration_ms = rounded(performance.now() - startedAt); + const now = Date.now(); + sample.finished_at = new Date(now).toISOString(); + sample.finished_epoch_ms = now; + try { + client?.close(); + } catch { + // Closing a failed socket should not hide the sample result. + } + resolve(sample); + } + }); +} + +function markFirstAssistantEvent(sample, sentAt) { + if (sample.first_assistant_event_ms !== null || sentAt <= 0) return; + const now = Date.now(); + sample.first_assistant_event_at = new Date(now).toISOString(); + sample.first_assistant_event_epoch_ms = now; + sample.first_assistant_event_ms = rounded(performance.now() - sentAt); +} + +function markFirstAssistantContent(sample, sentAt) { + if (sample.first_assistant_content_ms !== null || sentAt <= 0) return; + const now = Date.now(); + sample.first_assistant_content_at = new Date(now).toISOString(); + sample.first_assistant_content_epoch_ms = now; + sample.first_assistant_content_ms = rounded(performance.now() - sentAt); +} + +function containsLoadToken(text, prefix) { + const escaped = String(prefix).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`${escaped}-\\d{4}`).test(String(text || "")); +} + +function matchesFailureSignal(text, signals) { + const lower = String(text || "").toLowerCase(); + return signals.some((signal) => lower.includes(signal.toLowerCase())); +} + +function openRawWebSocket(wsUrl, handlers) { + const parsed = new URL(wsUrl); + const secure = parsed.protocol === "wss:"; + const port = Number(parsed.port || (secure ? 443 : 80)); + const host = parsed.hostname; + const path = `${parsed.pathname}${parsed.search}`; + const key = crypto.randomBytes(16).toString("base64"); + const socket = secure + ? tls.connect({ host, port, servername: host }) + : net.connect({ host, port }); + let opened = false; + let closed = false; + let buffer = Buffer.alloc(0); + + socket.setNoDelay(true); + socket.on("connect", () => { + const originProtocol = secure ? "https" : "http"; + const request = [ + `GET ${path} HTTP/1.1`, + `Host: ${parsed.host}`, + "Upgrade: websocket", + "Connection: Upgrade", + `Sec-WebSocket-Key: ${key}`, + "Sec-WebSocket-Version: 13", + `Origin: ${originProtocol}://${parsed.host}`, + "", + "", + ].join("\r\n"); + socket.write(request); + }); + socket.on("data", (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + if (!opened) { + const headerEnd = buffer.indexOf("\r\n\r\n"); + if (headerEnd === -1) return; + const headerText = buffer.slice(0, headerEnd).toString("utf8"); + buffer = buffer.slice(headerEnd + 4); + if (!/^HTTP\/1\.1 101\b/i.test(headerText)) { + handlers.onError(new Error(`Handshake failed: ${headerText.split("\r\n")[0] || "missing status"}`)); + socket.destroy(); + return; + } + opened = true; + handlers.onOpen(); + } + processFrames(); + }); + socket.on("error", (error) => { + if (!closed) handlers.onError(error); + }); + socket.on("close", () => { + if (closed) return; + closed = true; + handlers.onClose({ code: null, reason: "" }); + }); + + function processFrames() { + while (true) { + const frame = readFrame(buffer); + if (!frame) return; + buffer = buffer.slice(frame.consumed); + if (frame.opcode === 0x1) { + handlers.onMessage(frame.payload.toString("utf8")); + } else if (frame.opcode === 0x8) { + const code = frame.payload.length >= 2 ? frame.payload.readUInt16BE(0) : null; + const reason = frame.payload.length > 2 ? frame.payload.slice(2).toString("utf8") : ""; + closed = true; + handlers.onClose({ code, reason }); + socket.end(); + return; + } else if (frame.opcode === 0x9) { + writeFrame(socket, 0xA, frame.payload); + } + } + } + + return { + send(text) { + if (closed || !opened) return; + writeFrame(socket, 0x1, Buffer.from(text, "utf8")); + }, + close() { + if (closed) return; + closed = true; + if (!socket.destroyed) { + if (opened) writeFrame(socket, 0x8, Buffer.alloc(0)); + setTimeout(() => socket.end(), 50).unref(); + } + }, + }; +} + +function readFrame(buffer) { + if (buffer.length < 2) return null; + const first = buffer[0]; + const second = buffer[1]; + const opcode = first & 0x0f; + const masked = Boolean(second & 0x80); + let length = second & 0x7f; + let offset = 2; + if (length === 126) { + if (buffer.length < offset + 2) return null; + length = buffer.readUInt16BE(offset); + offset += 2; + } else if (length === 127) { + if (buffer.length < offset + 8) return null; + const high = buffer.readUInt32BE(offset); + const low = buffer.readUInt32BE(offset + 4); + length = high * 2 ** 32 + low; + offset += 8; + } + let mask = null; + if (masked) { + if (buffer.length < offset + 4) return null; + mask = buffer.slice(offset, offset + 4); + offset += 4; + } + if (buffer.length < offset + length) return null; + let payload = buffer.slice(offset, offset + length); + if (mask) { + payload = Buffer.from(payload); + for (let index = 0; index < payload.length; index += 1) { + payload[index] ^= mask[index % 4]; + } + } + return { + opcode, + payload, + consumed: offset + length, + }; +} + +function writeFrame(socket, opcode, payload) { + const body = Buffer.isBuffer(payload) ? payload : Buffer.from(payload || ""); + const mask = crypto.randomBytes(4); + const headerLength = body.length < 126 ? 2 : body.length <= 0xffff ? 4 : 10; + const header = Buffer.alloc(headerLength); + header[0] = 0x80 | opcode; + if (body.length < 126) { + header[1] = 0x80 | body.length; + } else if (body.length <= 0xffff) { + header[1] = 0x80 | 126; + header.writeUInt16BE(body.length, 2); + } else { + header[1] = 0x80 | 127; + header.writeUInt32BE(Math.floor(body.length / 2 ** 32), 2); + header.writeUInt32BE(body.length >>> 0, 6); + } + const masked = Buffer.from(body); + for (let index = 0; index < masked.length; index += 1) { + masked[index] ^= mask[index % 4]; + } + socket.write(Buffer.concat([header, mask, masked])); +} + +function rounded(value) { + return Number(value.toFixed(3)); +} + +function percentile(values, percentileValue) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return rounded(sorted[index]); +} + +function stats(values) { + if (values.length === 0) return { min: 0, p50: 0, p95: 0, p99: 0, max: 0 }; + return { + min: rounded(Math.min(...values)), + p50: percentile(values, 50), + p95: percentile(values, 95), + p99: percentile(values, 99), + max: rounded(Math.max(...values)), + }; +} + +function buildMetrics({ samples, totalRequests, concurrency, timeoutMs, loadDurationMs, backendUrl, pipelineId, sessionType, fakeProviderState }) { + const okSamples = samples.filter((sample) => sample.ok); + const statusCounts = {}; + for (const sample of samples) { + statusCounts[sample.status] = (statusCounts[sample.status] || 0) + 1; + } + const errorCount = samples.length - okSamples.length; + return { + probe: caseId, + backend_url: backendUrl, + pipeline_id: pipelineId, + session_type: sessionType, + total_requests: totalRequests, + completed_requests: samples.length, + concurrency, + timeout_ms: timeoutMs, + ok_count: okSamples.length, + error_count: errorCount, + timeout_count: samples.filter((sample) => sample.status === "timeout").length, + error_rate: samples.length === 0 ? 1 : rounded(errorCount / samples.length), + load_duration_ms: rounded(loadDurationMs), + throughput_rps: loadDurationMs <= 0 ? 0 : rounded(okSamples.length / (loadDurationMs / 1000)), + status_counts: statusCounts, + connected_ms: stats(samples.map((sample) => sample.connected_ms).filter(Number.isFinite)), + first_assistant_event_ms: stats(samples.map((sample) => sample.first_assistant_event_ms).filter(Number.isFinite)), + first_assistant_content_ms: stats(samples.map((sample) => sample.first_assistant_content_ms).filter(Number.isFinite)), + first_response_ms: stats(okSamples.map((sample) => sample.first_response_ms).filter(Number.isFinite)), + response_duration_ms: stats(okSamples.map((sample) => sample.response_duration_ms).filter(Number.isFinite)), + fake_provider: summarizeFakeProviderState(fakeProviderState), + provider_timing: buildProviderTimingMetrics(samples, fakeProviderState), + samples, + }; +} + +function buildThresholds(metrics) { + const thresholds = { + error_rate: { actual: metrics.error_rate, max: maxErrorRate, pass: metrics.error_rate <= maxErrorRate }, + response_p95_ms: { + actual: metrics.response_duration_ms.p95, + max: responseP95BudgetMs, + pass: metrics.ok_count > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs, + }, + }; + if (minErrorRate > 0) { + thresholds.error_rate_min = { + actual: metrics.error_rate, + min: minErrorRate, + pass: metrics.error_rate >= minErrorRate, + }; + } + if (minErrorCount > 0) { + thresholds.error_count_min = { + actual: metrics.error_count, + min: minErrorCount, + pass: metrics.error_count >= minErrorCount, + }; + } + if (minOkCount > 0) { + thresholds.ok_count_min = { + actual: metrics.ok_count, + min: minOkCount, + pass: metrics.ok_count >= minOkCount, + }; + } + if (minProviderFaultCount > 0) { + const actual = metrics.fake_provider?.fault_count ?? 0; + thresholds.fake_provider_fault_count_min = { + actual, + min: minProviderFaultCount, + pass: actual >= minProviderFaultCount, + }; + } + if (firstResponseP95BudgetMs > 0) { + thresholds.first_response_p95_ms = { + actual: metrics.first_response_ms.p95, + max: firstResponseP95BudgetMs, + pass: metrics.ok_count > 0 && metrics.first_response_ms.p95 <= firstResponseP95BudgetMs, + }; + } + return thresholds; +} + +function looksLikeEnvIssue(error) { + const message = String(error?.message || error || ""); + return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message); +} + +function safeReason(value) { + return redact(String(value || "")).slice(0, 1000); +} diff --git a/skills/skills/langbot-testing/probes/langbot-debug-chat-cross-pipeline-isolation.mjs b/skills/skills/langbot-testing/probes/langbot-debug-chat-cross-pipeline-isolation.mjs new file mode 100755 index 000000000..b83f6161d --- /dev/null +++ b/skills/skills/langbot-testing/probes/langbot-debug-chat-cross-pipeline-isolation.mjs @@ -0,0 +1,861 @@ +#!/usr/bin/env node + +import crypto from "node:crypto"; +import net from "node:net"; +import tls from "node:tls"; +import { mkdir, writeFile } from "node:fs/promises"; +import { resolve } from "node:path"; +import { env, exit } from "node:process"; +import { + apiJson, + appendLine, + ensureEvidence, + evidencePaths, + loadEnvFiles, + localIsoWithOffset, + redact, + resetAndAuthLocalUser, + writeResult, +} from "../../../scripts/e2e/lib/langbot-e2e.mjs"; +import { + buildProviderTimingMetrics, + summarizeFakeProviderState, +} from "./lib/fake-provider-timing.mjs"; + +const DEFAULT_LOCAL_PASSWORD = "LangBotE2ELocalPass!2026"; + +await loadEnvFiles(); +const caseId = env.LBS_CASE_ID || "langbot-debug-chat-cross-pipeline-isolation"; +const paths = evidencePaths(caseId); +await ensureEvidence(paths); + +const startedAt = new Date(); +const metricsPath = resolve(paths.evidenceDir, "metrics.json"); +const samplesPath = resolve(paths.evidenceDir, "samples.json"); +const fakeProviderStatePath = resolve(paths.evidenceDir, "fake-provider-state.json"); +const resetDiagnosticPath = resolve(paths.evidenceDir, "debug-chat-reset-diagnostic.json"); +const backendUrl = env.LANGBOT_BACKEND_URL || ""; +const fakeProviderUrl = env.LANGBOT_FAKE_PROVIDER_URL || ""; +const sessionType = env.LANGBOT_DEBUG_CHAT_LOAD_SESSION_TYPE || env.LANGBOT_E2E_DEBUG_CHAT_SESSION_TYPE || "person"; +const requestsPerPipeline = positiveInteger(env.LANGBOT_DEBUG_CHAT_LOAD_REQUESTS, 6); +const concurrency = Math.min(requestsPerPipeline * 2, positiveInteger(env.LANGBOT_DEBUG_CHAT_LOAD_CONCURRENCY, 4)); +const timeoutMs = positiveInteger(env.LANGBOT_DEBUG_CHAT_LOAD_TIMEOUT_MS, 30_000); +const stream = bool(env.LANGBOT_DEBUG_CHAT_LOAD_STREAM, true); +const resetBeforeRun = bool(env.LANGBOT_DEBUG_CHAT_LOAD_RESET, true); +const responseP95BudgetMs = positiveNumber(env.LANGBOT_DEBUG_CHAT_LOAD_RESPONSE_P95_MS, 5_000); +const maxErrorRate = positiveNumber(env.LANGBOT_DEBUG_CHAT_LOAD_MAX_ERROR_RATE, 0); +const promptTemplate = env.LANGBOT_DEBUG_CHAT_LOAD_PROMPT_TEMPLATE + || "请只回复 \"{expected}\",不要解释,不要添加其他字符。"; +const failureSignals = textList(env.LANGBOT_E2E_FAILURE_SIGNALS || env.LANGBOT_DEBUG_CHAT_LOAD_FAILURE_SIGNALS || ""); + +const pipelineTargets = [ + { + label: "A", + expectedPrefix: "PIPEA", + otherPrefix: "PIPEB", + url: env.LANGBOT_FAKE_PROVIDER_PIPELINE_A_URL || "", + name: env.LANGBOT_FAKE_PROVIDER_PIPELINE_A_NAME || "", + }, + { + label: "B", + expectedPrefix: "PIPEB", + otherPrefix: "PIPEA", + url: env.LANGBOT_FAKE_PROVIDER_PIPELINE_B_URL || "", + name: env.LANGBOT_FAKE_PROVIDER_PIPELINE_B_NAME || "", + }, +]; + +const result = { + source: "automation", + case_id: caseId, + run_id: paths.runId, + status: "fail", + reason: "", + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: "", + finished_at_local: "", + duration_ms: 0, + backend_url: backendUrl, + session_type: sessionType, + pipelines: [], + load_profile: { + requests_per_pipeline: requestsPerPipeline, + total_requests: requestsPerPipeline * 2, + concurrency, + timeout_ms: timeoutMs, + stream, + reset_before_run: resetBeforeRun, + }, + evidence: { + network_log: paths.networkLog, + metrics_json: metricsPath, + samples_json: samplesPath, + fake_provider_state_json: fakeProviderStatePath, + debug_chat_reset_diagnostic_json: resetDiagnosticPath, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }, + evidence_collected: ["metrics", "network", "api_diagnostic", "filesystem"], +}; + +try { + if (!backendUrl) { + result.status = "env_issue"; + throw new Error("LANGBOT_BACKEND_URL is not configured."); + } + if (!["person", "group"].includes(sessionType)) { + throw new Error(`LANGBOT_DEBUG_CHAT_LOAD_SESSION_TYPE must be person or group, got ${sessionType}.`); + } + for (const target of pipelineTargets) { + if (!target.url && !target.name) { + result.status = "env_issue"; + throw new Error(`Set LANGBOT_FAKE_PROVIDER_PIPELINE_${target.label}_URL or LANGBOT_FAKE_PROVIDER_PIPELINE_${target.label}_NAME.`); + } + } + + const backendReady = await backendReachable(backendUrl); + if (!backendReady) { + result.status = "env_issue"; + throw new Error(`Backend did not respond at ${backendUrl}.`); + } + + const user = env.LANGBOT_E2E_LOGIN_USER || ""; + const password = env.LANGBOT_E2E_LOGIN_PASSWORD || DEFAULT_LOCAL_PASSWORD; + if (!user) { + result.status = "env_issue"; + throw new Error("LANGBOT_E2E_LOGIN_USER is required so this probe can resolve/reset Debug Chat sessions."); + } + const auth = await resetAndAuthLocalUser({ backendUrl, user, password }); + const pipelines = []; + for (const target of pipelineTargets) { + const pipeline = await resolvePipeline({ + backendUrl, + token: auth.token, + pipelineUrl: target.url, + pipelineName: target.name, + }); + pipelines.push({ + ...target, + id: pipeline.id, + name: pipeline.name || target.name, + wsUrl: websocketUrl(backendUrl, pipeline.id, sessionType), + }); + } + result.pipelines = pipelines.map((pipeline) => ({ + label: pipeline.label, + id: pipeline.id, + name: pipeline.name, + url: pipeline.url, + })); + + if (resetBeforeRun) { + const resetDiagnostics = []; + for (const pipeline of pipelines) { + const reset = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(pipeline.id)}/ws/reset/${encodeURIComponent(sessionType)}`, { + method: "POST", + token: auth.token, + }); + resetDiagnostics.push({ + pipeline_label: pipeline.label, + pipeline_id: pipeline.id, + status: isApiFailure(reset) ? "fail" : "ready", + http_status: reset.status, + code: reset.json.code ?? null, + reason: isApiFailure(reset) ? reset.json.msg || "Debug Chat reset failed." : "Debug Chat session reset.", + }); + } + await writeFile(resetDiagnosticPath, `${JSON.stringify(resetDiagnostics, null, 2)}\n`, "utf8"); + const failedReset = resetDiagnostics.find((item) => item.status === "fail"); + if (failedReset) throw new Error(failedReset.reason); + } + await resetFakeProvider(fakeProviderUrl); + + const jobs = []; + for (let index = 0; index < requestsPerPipeline; index += 1) { + for (const pipeline of pipelines) { + jobs.push({ ...pipeline, index }); + } + } + + const loadStartedAt = performance.now(); + const samples = await runLoad({ + jobs, + concurrency, + timeoutMs, + promptTemplate, + stream, + failureSignals, + }); + const loadDurationMs = performance.now() - loadStartedAt; + const fakeProviderState = await readFakeProviderState(fakeProviderUrl); + if (fakeProviderState) { + await writeFile(fakeProviderStatePath, `${JSON.stringify(fakeProviderState, null, 2)}\n`, "utf8"); + } + const metrics = buildMetrics({ + samples, + requestsPerPipeline, + concurrency, + timeoutMs, + loadDurationMs, + backendUrl, + sessionType, + fakeProviderState, + }); + const thresholds = buildThresholds(metrics); + const passed = Object.values(thresholds).every((item) => item.pass); + result.status = passed ? "pass" : "fail"; + result.reason = passed + ? "Debug Chat cross-pipeline isolation probe passed all thresholds." + : "Debug Chat cross-pipeline isolation probe found leaks, errors, or latency threshold breaches."; + result.metrics_summary = { + requests_per_pipeline: metrics.requests_per_pipeline, + total_requests: metrics.total_requests, + concurrency: metrics.concurrency, + ok_count: metrics.ok_count, + error_count: metrics.error_count, + cross_pipeline_leak_count: metrics.cross_pipeline_leak_count, + timeout_count: metrics.timeout_count, + error_rate: metrics.error_rate, + response_p95_ms: metrics.response_duration_ms.p95, + first_response_p95_ms: metrics.first_response_ms.p95, + throughput_rps: metrics.throughput_rps, + status_counts: metrics.status_counts, + by_pipeline: metrics.by_pipeline, + fake_provider_request_count: metrics.fake_provider?.request_count ?? null, + fake_provider_duration_p95_ms: metrics.provider_timing?.provider_duration_ms.p95 ?? null, + langbot_overhead_estimate_p95_ms: metrics.provider_timing?.langbot_overhead_estimate_ms.p95 ?? null, + send_to_provider_start_p95_ms: metrics.provider_timing?.send_to_provider_start_ms.p95 ?? null, + provider_finish_to_ws_final_p95_ms: metrics.provider_timing?.provider_finish_to_ws_final_ms.p95 ?? null, + }; + result.thresholds_summary = thresholds; + result.artifacts = { + metrics_json: metricsPath, + samples_json: samplesPath, + fake_provider_state_json: fakeProviderState ? fakeProviderStatePath : "", + network_log: paths.networkLog, + automation_result_json: paths.automationResultJson, + result_json: paths.resultJson, + }; + + await writeFile(metricsPath, `${JSON.stringify({ ...metrics, thresholds }, null, 2)}\n`, "utf8"); + await writeFile(samplesPath, `${JSON.stringify(samples, null, 2)}\n`, "utf8"); +} catch (error) { + if (!["env_issue", "blocked"].includes(result.status)) { + result.status = looksLikeEnvIssue(error) ? "env_issue" : "fail"; + } + result.reason = result.reason || safeReason(error.message); +} finally { + const finishedAt = new Date(); + result.finished_at = finishedAt.toISOString(); + result.finished_at_local = localIsoWithOffset(finishedAt); + result.duration_ms = finishedAt.getTime() - startedAt.getTime(); + await mkdir(paths.evidenceDir, { recursive: true }); + await writeResult(paths, result); + console.log(JSON.stringify(result, null, 2)); +} + +exit(result.status === "pass" ? 0 : result.status === "env_issue" || result.status === "blocked" ? 2 : 1); + +async function backendReachable(baseUrl) { + try { + const response = await fetch(`${baseUrl.replace(/\/$/, "")}/healthz`, { + signal: AbortSignal.timeout(3000), + }); + return response.status < 500; + } catch { + return false; + } +} + +async function resetFakeProvider(rootUrl) { + if (!rootUrl) return; + try { + await fetch(`${normalizeProviderRootUrl(rootUrl)}/__qa/reset`, { + method: "POST", + signal: AbortSignal.timeout(3000), + }); + } catch { + // Missing fake-provider diagnostics should not hide the isolation result. + } +} + +async function readFakeProviderState(rootUrl) { + if (!rootUrl) return null; + try { + const response = await fetch(`${normalizeProviderRootUrl(rootUrl)}/__qa/config`, { + signal: AbortSignal.timeout(3000), + }); + const json = await response.json().catch(() => ({})); + return { + status: response.ok && json.ok === true ? "loaded" : "unavailable", + url: normalizeProviderRootUrl(rootUrl), + http_status: response.status, + model: json.model || "", + config: json.config || {}, + request_count: Number.isFinite(json.request_count) ? json.request_count : null, + recent_requests: Array.isArray(json.recent_requests) ? json.recent_requests : [], + }; + } catch (error) { + return { + status: "unavailable", + url: normalizeProviderRootUrl(rootUrl), + reason: safeReason(error.message), + request_count: null, + recent_requests: [], + }; + } +} + +function normalizeProviderRootUrl(value) { + const trimmed = String(value || "").trim().replace(/\/$/, ""); + return trimmed.endsWith("/v1") ? trimmed.slice(0, -3) : trimmed; +} + +function pipelineIdFromUrl(url) { + if (!url) return ""; + try { + const parsed = new URL(url); + return parsed.searchParams.get("id") || ""; + } catch { + return ""; + } +} + +async function resolvePipeline({ backendUrl, token, pipelineUrl, pipelineName }) { + const idFromUrl = pipelineIdFromUrl(pipelineUrl); + if (idFromUrl) { + const response = await apiJson(backendUrl, `/api/v1/pipelines/${encodeURIComponent(idFromUrl)}`, { token }); + const pipeline = response.json.data?.pipeline; + if (isApiFailure(response) || !pipeline?.uuid) { + throw new Error(response.json.msg || `Could not load pipeline ${idFromUrl}.`); + } + return { id: pipeline.uuid, name: pipeline.name || "" }; + } + if (!pipelineName) { + throw new Error("Set pipeline URL or name before running this probe."); + } + const response = await apiJson(backendUrl, "/api/v1/pipelines", { token }); + if (isApiFailure(response)) { + throw new Error(response.json.msg || "Failed to list pipelines."); + } + const pipeline = (response.json.data?.pipelines || []).find((item) => item.name === pipelineName); + if (!pipeline?.uuid) { + throw new Error(`Could not find pipeline named ${pipelineName}.`); + } + return { id: pipeline.uuid, name: pipeline.name || pipelineName }; +} + +function isApiFailure(response) { + return response.status >= 400 || (response.json.code !== undefined && response.json.code !== 0); +} + +function websocketUrl(baseUrl, pipelineId, sessionTypeValue) { + const parsed = new URL(baseUrl); + parsed.protocol = parsed.protocol === "https:" ? "wss:" : "ws:"; + parsed.pathname = `/api/v1/pipelines/${encodeURIComponent(pipelineId)}/ws/connect`; + parsed.search = `?session_type=${encodeURIComponent(sessionTypeValue)}`; + return parsed.toString(); +} + +async function runLoad(options) { + const samples = []; + const queue = [...options.jobs]; + const workers = Array.from({ length: options.concurrency }, async () => { + while (queue.length > 0) { + const job = queue.shift(); + if (!job) continue; + const sample = await runSingleRequest({ ...options, job }); + samples.push(sample); + } + }); + await Promise.all(workers); + return samples.sort((left, right) => ( + left.pipeline_label.localeCompare(right.pipeline_label) || left.index - right.index + )); +} + +function expectedForIndex(prefix, index) { + return `${prefix}-${String(index + 1).padStart(4, "0")}`; +} + +function promptForIndex(template, expected) { + return template.replaceAll("{expected}", expected); +} + +function runSingleRequest({ + job, + timeoutMs, + promptTemplate, + stream, + failureSignals, +}) { + return new Promise((resolvePromise) => { + const expected = expectedForIndex(job.expectedPrefix, job.index); + const prompt = promptForIndex(promptTemplate, expected); + const sample = { + index: job.index, + pipeline_label: job.label, + pipeline_id: job.id, + pipeline_name: job.name, + status: "running", + ok: false, + expected_text: expected, + expected_prefix: job.expectedPrefix, + other_prefix: job.otherPrefix, + prompt, + response_text: "", + started_at: new Date().toISOString(), + started_epoch_ms: Date.now(), + connected_at: null, + connected_epoch_ms: null, + sent_at: null, + sent_epoch_ms: null, + first_assistant_event_at: null, + first_assistant_event_epoch_ms: null, + first_assistant_event_ms: null, + first_assistant_content_at: null, + first_assistant_content_epoch_ms: null, + first_assistant_content_ms: null, + first_response_at: null, + first_response_epoch_ms: null, + connected_ms: null, + first_response_ms: null, + response_duration_ms: null, + finished_at: null, + finished_epoch_ms: null, + event_count: 0, + same_pipeline_foreign_response_count: 0, + cross_pipeline_leak_count: 0, + last_foreign_response_text: "", + error: "", + close_code: null, + close_reason: "", + }; + let closed = false; + let connectedAt = 0; + let sentAt = 0; + const startedPerf = performance.now(); + let client = null; + const timer = setTimeout(() => { + finish("timeout", `Timed out after ${timeoutMs} ms.`); + }, timeoutMs); + + client = openRawWebSocket(job.wsUrl, { + onOpen() { + connectedAt = performance.now(); + const now = Date.now(); + sample.connected_at = new Date(now).toISOString(); + sample.connected_epoch_ms = now; + sample.connected_ms = rounded(connectedAt - startedPerf); + }, + onMessage(text) { + sample.event_count += 1; + let data; + try { + data = JSON.parse(String(text || "")); + } catch (error) { + finish("error", `Invalid WebSocket JSON: ${error.message}`); + return; + } + appendLine(paths.networkLog, JSON.stringify({ + pipeline_label: job.label, + request_index: job.index, + type: data.type, + session_type: data.session_type || "", + role: data.data?.role || "", + is_final: data.data?.is_final ?? null, + content_preview: redact(String(data.data?.content || data.message || "").slice(0, 200)), + })).catch(() => {}); + + if (data.type === "connected") { + sentAt = performance.now(); + const now = Date.now(); + sample.sent_at = new Date(now).toISOString(); + sample.sent_epoch_ms = now; + client.send(JSON.stringify({ + type: "message", + message: [{ type: "Plain", text: prompt }], + stream, + })); + return; + } + if (data.type === "error") { + finish("error", data.message || "WebSocket error message."); + return; + } + if (data.type !== "response" || data.data?.role !== "assistant") return; + + const content = String(data.data.content || ""); + markFirstAssistantEvent(sample, sentAt); + if (content) sample.response_text = content; + if (content) markFirstAssistantContent(sample, sentAt); + if (containsPipelineToken(content, job.otherPrefix)) { + sample.cross_pipeline_leak_count += 1; + finish("cross_pipeline_leak", `Pipeline ${job.label} received response from ${job.otherPrefix}: ${content}`); + return; + } + if (content.includes(expected) && sample.first_response_ms === null && sentAt > 0) { + const now = Date.now(); + sample.first_response_at = new Date(now).toISOString(); + sample.first_response_epoch_ms = now; + sample.first_response_ms = rounded(performance.now() - sentAt); + } + if (data.data.is_final === true) { + const ok = sample.response_text.includes(expected); + if (ok) { + if (sample.first_response_ms === null && sentAt > 0) { + const now = Date.now(); + sample.first_response_at = new Date(now).toISOString(); + sample.first_response_epoch_ms = now; + sample.first_response_ms = rounded(performance.now() - sentAt); + } + finish("pass", ""); + } else if (matchesFailureSignal(sample.response_text, failureSignals)) { + finish("app_error", `Assistant final response matched a failure signal: ${sample.response_text}`); + } else if (containsPipelineToken(sample.response_text, job.expectedPrefix)) { + sample.same_pipeline_foreign_response_count += 1; + sample.last_foreign_response_text = sample.response_text; + } else { + finish("mismatch", `Final assistant response did not include ${expected}: ${sample.response_text}`); + } + } + }, + onError(error) { + finish("connection_error", `WebSocket connection error: ${error.message}`); + }, + onClose(event) { + sample.close_code = event.code; + sample.close_reason = event.reason || ""; + if (!closed) finish("closed", `WebSocket closed before final assistant response: ${event.code}`); + }, + }); + + function finish(status, reason) { + if (closed) return; + closed = true; + clearTimeout(timer); + sample.status = status; + sample.ok = status === "pass"; + sample.error = status === "timeout" && sample.same_pipeline_foreign_response_count > 0 + ? `${reason || ""} Saw ${sample.same_pipeline_foreign_response_count} same-pipeline foreign assistant response(s); last=${sample.last_foreign_response_text}` + : reason || ""; + if (sentAt > 0) sample.response_duration_ms = rounded(performance.now() - sentAt); + else sample.response_duration_ms = rounded(performance.now() - startedPerf); + const now = Date.now(); + sample.finished_at = new Date(now).toISOString(); + sample.finished_epoch_ms = now; + try { + client?.close(); + } catch { + // Closing a failed socket should not hide the sample result. + } + resolvePromise(sample); + } + }); +} + +function markFirstAssistantEvent(sample, sentAt) { + if (sample.first_assistant_event_ms !== null || sentAt <= 0) return; + const now = Date.now(); + sample.first_assistant_event_at = new Date(now).toISOString(); + sample.first_assistant_event_epoch_ms = now; + sample.first_assistant_event_ms = rounded(performance.now() - sentAt); +} + +function markFirstAssistantContent(sample, sentAt) { + if (sample.first_assistant_content_ms !== null || sentAt <= 0) return; + const now = Date.now(); + sample.first_assistant_content_at = new Date(now).toISOString(); + sample.first_assistant_content_epoch_ms = now; + sample.first_assistant_content_ms = rounded(performance.now() - sentAt); +} + +function containsPipelineToken(text, prefix) { + const escaped = String(prefix).replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + return new RegExp(`${escaped}-\\d{4}`).test(String(text || "")); +} + +function matchesFailureSignal(text, signals) { + const lower = String(text || "").toLowerCase(); + return signals.some((signal) => lower.includes(signal.toLowerCase())); +} + +function openRawWebSocket(wsUrl, handlers) { + const parsed = new URL(wsUrl); + const secure = parsed.protocol === "wss:"; + const port = Number(parsed.port || (secure ? 443 : 80)); + const host = parsed.hostname; + const path = `${parsed.pathname}${parsed.search}`; + const key = crypto.randomBytes(16).toString("base64"); + const socket = secure + ? tls.connect({ host, port, servername: host }) + : net.connect({ host, port }); + let opened = false; + let closed = false; + let buffer = Buffer.alloc(0); + + socket.setNoDelay(true); + socket.on("connect", () => { + const originProtocol = secure ? "https" : "http"; + const request = [ + `GET ${path} HTTP/1.1`, + `Host: ${parsed.host}`, + "Upgrade: websocket", + "Connection: Upgrade", + `Sec-WebSocket-Key: ${key}`, + "Sec-WebSocket-Version: 13", + `Origin: ${originProtocol}://${parsed.host}`, + "", + "", + ].join("\r\n"); + socket.write(request); + }); + socket.on("data", (chunk) => { + buffer = Buffer.concat([buffer, chunk]); + if (!opened) { + const headerEnd = buffer.indexOf("\r\n\r\n"); + if (headerEnd === -1) return; + const headerText = buffer.slice(0, headerEnd).toString("utf8"); + buffer = buffer.slice(headerEnd + 4); + if (!/^HTTP\/1\.1 101\b/i.test(headerText)) { + handlers.onError(new Error(`Handshake failed: ${headerText.split("\r\n")[0] || "missing status"}`)); + socket.destroy(); + return; + } + opened = true; + handlers.onOpen(); + } + processFrames(); + }); + socket.on("error", (error) => { + if (!closed) handlers.onError(error); + }); + socket.on("close", () => { + if (closed) return; + closed = true; + handlers.onClose({ code: null, reason: "" }); + }); + + function processFrames() { + while (true) { + const frame = readFrame(buffer); + if (!frame) return; + buffer = buffer.slice(frame.consumed); + if (frame.opcode === 0x1) { + handlers.onMessage(frame.payload.toString("utf8")); + } else if (frame.opcode === 0x8) { + const code = frame.payload.length >= 2 ? frame.payload.readUInt16BE(0) : null; + const reason = frame.payload.length > 2 ? frame.payload.slice(2).toString("utf8") : ""; + closed = true; + handlers.onClose({ code, reason }); + socket.end(); + return; + } else if (frame.opcode === 0x9) { + writeFrame(socket, 0xA, frame.payload); + } + } + } + + return { + send(text) { + if (closed || !opened) return; + writeFrame(socket, 0x1, Buffer.from(text, "utf8")); + }, + close() { + if (closed) return; + closed = true; + if (!socket.destroyed) { + if (opened) writeFrame(socket, 0x8, Buffer.alloc(0)); + setTimeout(() => socket.end(), 50).unref(); + } + }, + }; +} + +function readFrame(buffer) { + if (buffer.length < 2) return null; + const first = buffer[0]; + const second = buffer[1]; + const opcode = first & 0x0f; + const masked = Boolean(second & 0x80); + let length = second & 0x7f; + let offset = 2; + if (length === 126) { + if (buffer.length < offset + 2) return null; + length = buffer.readUInt16BE(offset); + offset += 2; + } else if (length === 127) { + if (buffer.length < offset + 8) return null; + const high = buffer.readUInt32BE(offset); + const low = buffer.readUInt32BE(offset + 4); + length = high * 2 ** 32 + low; + offset += 8; + } + let mask = null; + if (masked) { + if (buffer.length < offset + 4) return null; + mask = buffer.slice(offset, offset + 4); + offset += 4; + } + if (buffer.length < offset + length) return null; + let payload = buffer.slice(offset, offset + length); + if (mask) { + payload = Buffer.from(payload); + for (let index = 0; index < payload.length; index += 1) { + payload[index] ^= mask[index % 4]; + } + } + return { + opcode, + payload, + consumed: offset + length, + }; +} + +function writeFrame(socket, opcode, payload) { + const body = Buffer.isBuffer(payload) ? payload : Buffer.from(payload || ""); + const mask = crypto.randomBytes(4); + const headerLength = body.length < 126 ? 2 : body.length <= 0xffff ? 4 : 10; + const header = Buffer.alloc(headerLength); + header[0] = 0x80 | opcode; + if (body.length < 126) { + header[1] = 0x80 | body.length; + } else if (body.length <= 0xffff) { + header[1] = 0x80 | 126; + header.writeUInt16BE(body.length, 2); + } else { + header[1] = 0x80 | 127; + header.writeUInt32BE(Math.floor(body.length / 2 ** 32), 2); + header.writeUInt32BE(body.length >>> 0, 6); + } + const masked = Buffer.from(body); + for (let index = 0; index < masked.length; index += 1) { + masked[index] ^= mask[index % 4]; + } + socket.write(Buffer.concat([header, mask, masked])); +} + +function buildMetrics({ samples, requestsPerPipeline, concurrency, timeoutMs, loadDurationMs, backendUrl, sessionType, fakeProviderState }) { + const okSamples = samples.filter((sample) => sample.ok); + const statusCounts = {}; + const byPipeline = {}; + for (const sample of samples) { + statusCounts[sample.status] = (statusCounts[sample.status] || 0) + 1; + if (!byPipeline[sample.pipeline_label]) { + byPipeline[sample.pipeline_label] = { + ok_count: 0, + error_count: 0, + cross_pipeline_leak_count: 0, + timeout_count: 0, + }; + } + if (sample.ok) byPipeline[sample.pipeline_label].ok_count += 1; + else byPipeline[sample.pipeline_label].error_count += 1; + byPipeline[sample.pipeline_label].cross_pipeline_leak_count += sample.cross_pipeline_leak_count || 0; + if (sample.status === "timeout") byPipeline[sample.pipeline_label].timeout_count += 1; + } + const errorCount = samples.length - okSamples.length; + return { + probe: caseId, + backend_url: backendUrl, + session_type: sessionType, + requests_per_pipeline: requestsPerPipeline, + total_requests: requestsPerPipeline * 2, + completed_requests: samples.length, + concurrency, + timeout_ms: timeoutMs, + ok_count: okSamples.length, + error_count: errorCount, + timeout_count: samples.filter((sample) => sample.status === "timeout").length, + cross_pipeline_leak_count: samples.reduce((count, sample) => count + (sample.cross_pipeline_leak_count || 0), 0), + error_rate: samples.length === 0 ? 1 : rounded(errorCount / samples.length), + load_duration_ms: rounded(loadDurationMs), + throughput_rps: loadDurationMs <= 0 ? 0 : rounded(okSamples.length / (loadDurationMs / 1000)), + status_counts: statusCounts, + by_pipeline: byPipeline, + connected_ms: stats(samples.map((sample) => sample.connected_ms).filter(Number.isFinite)), + first_assistant_event_ms: stats(samples.map((sample) => sample.first_assistant_event_ms).filter(Number.isFinite)), + first_assistant_content_ms: stats(samples.map((sample) => sample.first_assistant_content_ms).filter(Number.isFinite)), + first_response_ms: stats(okSamples.map((sample) => sample.first_response_ms).filter(Number.isFinite)), + response_duration_ms: stats(okSamples.map((sample) => sample.response_duration_ms).filter(Number.isFinite)), + fake_provider: summarizeFakeProviderState(fakeProviderState), + provider_timing: buildProviderTimingMetrics(samples, fakeProviderState), + samples, + }; +} + +function buildThresholds(metrics) { + return { + cross_pipeline_leak_count: { + actual: metrics.cross_pipeline_leak_count, + max: 0, + pass: metrics.cross_pipeline_leak_count === 0, + }, + error_rate: { + actual: metrics.error_rate, + max: maxErrorRate, + pass: metrics.error_rate <= maxErrorRate, + }, + response_p95_ms: { + actual: metrics.response_duration_ms.p95, + max: responseP95BudgetMs, + pass: metrics.ok_count > 0 && metrics.response_duration_ms.p95 <= responseP95BudgetMs, + }, + }; +} + +function positiveInteger(value, fallback) { + const parsed = Number.parseInt(String(value || ""), 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; +} + +function positiveNumber(value, fallback) { + const parsed = Number(value || ""); + return Number.isFinite(parsed) && parsed >= 0 ? parsed : fallback; +} + +function bool(value, fallback) { + if (value === undefined || value === "") return fallback; + if (/^(1|true|yes|on)$/i.test(String(value))) return true; + if (/^(0|false|no|off)$/i.test(String(value))) return false; + return fallback; +} + +function textList(value) { + return String(value || "") + .split(/\r?\n|,/) + .map((item) => item.trim()) + .filter(Boolean); +} + +function rounded(value) { + return Number(value.toFixed(3)); +} + +function percentile(values, percentileValue) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return rounded(sorted[index]); +} + +function stats(values) { + if (values.length === 0) return { min: 0, p50: 0, p95: 0, p99: 0, max: 0 }; + return { + min: rounded(Math.min(...values)), + p50: percentile(values, 50), + p95: percentile(values, 95), + p99: percentile(values, 99), + max: rounded(Math.max(...values)), + }; +} + +function looksLikeEnvIssue(error) { + const message = String(error?.message || error || ""); + return /fetch failed|ECONNREFUSED|ENOTFOUND|LANGBOT_.*not configured|Could not read recovery_key|Backend did not respond/i.test(message); +} + +function safeReason(value) { + return redact(String(value || "")).slice(0, 1000); +} diff --git a/skills/skills/langbot-testing/probes/langbot-fault-taxonomy-contract.mjs b/skills/skills/langbot-testing/probes/langbot-fault-taxonomy-contract.mjs new file mode 100755 index 000000000..8c9628e58 --- /dev/null +++ b/skills/skills/langbot-testing/probes/langbot-fault-taxonomy-contract.mjs @@ -0,0 +1,159 @@ +#!/usr/bin/env node + +import { mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { env, exit } from "node:process"; + +function pad(value, size = 2) { + return String(value).padStart(size, "0"); +} + +function localIsoWithOffset(date = new Date()) { + const offsetMinutes = -date.getTimezoneOffset(); + const sign = offsetMinutes >= 0 ? "+" : "-"; + const absolute = Math.abs(offsetMinutes); + return [ + `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`, + `T${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.${pad(date.getMilliseconds(), 3)}`, + `${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`, + ].join(""); +} + +function timestampSlug(date = new Date()) { + return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); +} + +const scenarios = [ + { + id: "provider-timeout", + target: "provider", + injected_fault: "fake provider request exceeds the configured timeout", + expected_status: "env_issue", + recovery_check: "provider route is reachable or the case remains outside product pass/fail", + cleanup: "stop fake provider or reset proxy route", + }, + { + id: "plugin-runtime-disconnect", + target: "plugin-runtime", + injected_fault: "runtime control channel disconnects during an action", + expected_status: "fail", + recovery_check: "runtime reconnects and a deterministic plugin action succeeds", + cleanup: "restart the local plugin runtime process", + }, + { + id: "mcp-stdio-server-exit", + target: "mcp", + injected_fault: "stdio server exits mid-call", + expected_status: "fail", + recovery_check: "server can be registered again and exposes the expected tool", + cleanup: "remove temporary MCP server registration", + }, + { + id: "operator-missing-login", + target: "webui", + injected_fault: "browser profile is not authenticated", + expected_status: "blocked", + recovery_check: "authenticated profile can open the same WebUI origin", + cleanup: "no product cleanup; refresh local login state", + }, + { + id: "transient-marketplace-timeout", + target: "marketplace", + injected_fault: "marketplace request times out once and then succeeds", + expected_status: "flaky", + recovery_check: "rerun passes with the same product revision and no code change", + cleanup: "clear retry-only evidence and keep the run classified as flaky", + }, +]; + +function validateScenario(scenario) { + const missing = ["id", "target", "injected_fault", "expected_status", "recovery_check", "cleanup"] + .filter((key) => !scenario[key]); + const allowedStatuses = new Set(["pass", "fail", "blocked", "env_issue", "flaky"]); + return { + id: scenario.id, + pass: missing.length === 0 && allowedStatuses.has(scenario.expected_status), + missing, + expected_status: scenario.expected_status, + }; +} + +async function main() { + const root = resolve(env.LBS_ROOT || process.cwd()); + const caseId = "langbot-fault-taxonomy-contract"; + const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; + const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + await mkdir(evidenceDir, { recursive: true }); + + const startedAt = new Date(); + const validations = scenarios.map(validateScenario); + const statusCounts = {}; + for (const scenario of scenarios) { + statusCounts[scenario.expected_status] = (statusCounts[scenario.expected_status] || 0) + 1; + } + const metrics = { + probe: caseId, + scenario_count: scenarios.length, + status_counts: statusCounts, + scenarios, + validations, + }; + const thresholds = { + scenario_count: { actual: scenarios.length, min: 5, pass: scenarios.length >= 5 }, + invalid_scenario_count: { + actual: validations.filter((item) => !item.pass).length, + max: 0, + pass: validations.every((item) => item.pass), + }, + cleanup_declared_count: { + actual: scenarios.filter((item) => item.cleanup).length, + min: scenarios.length, + pass: scenarios.every((item) => item.cleanup), + }, + }; + const status = Object.values(thresholds).every((item) => item.pass) ? "pass" : "fail"; + const metricsPath = join(evidenceDir, "metrics.json"); + const faultModelPath = join(evidenceDir, "fault-model.json"); + const automationResultPath = join(evidenceDir, "automation-result.json"); + const resultPath = join(evidenceDir, "result.json"); + + await writeFile(metricsPath, `${JSON.stringify(metrics, null, 2)}\n`, "utf8"); + await writeFile(faultModelPath, `${JSON.stringify({ scenarios }, null, 2)}\n`, "utf8"); + + const finishedAt = new Date(); + const result = { + source: "automation", + case_id: caseId, + run_id: runId, + status, + reason: status === "pass" + ? "Fault taxonomy contract declares status, recovery, and cleanup for every scenario." + : "Fault taxonomy contract is missing required scenario fields.", + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: finishedAt.toISOString(), + finished_at_local: localIsoWithOffset(finishedAt), + duration_ms: finishedAt.getTime() - startedAt.getTime(), + metrics_summary: { + scenario_count: metrics.scenario_count, + status_counts: metrics.status_counts, + invalid_scenario_count: thresholds.invalid_scenario_count.actual, + }, + thresholds_summary: thresholds, + artifacts: { + metrics_json: metricsPath, + fault_model_json: faultModelPath, + automation_result_json: automationResultPath, + result_json: resultPath, + }, + evidence_collected: ["metrics", "filesystem"], + }; + + const resultText = `${JSON.stringify(result, null, 2)}\n`; + await writeFile(automationResultPath, resultText, "utf8"); + await writeFile(resultPath, resultText, "utf8"); + console.log(JSON.stringify(result, null, 2)); + exit(status === "pass" ? 0 : 1); +} + +await main(); diff --git a/skills/skills/langbot-testing/probes/langbot-live-backend-latency.mjs b/skills/skills/langbot-testing/probes/langbot-live-backend-latency.mjs new file mode 100755 index 000000000..747c84c6a --- /dev/null +++ b/skills/skills/langbot-testing/probes/langbot-live-backend-latency.mjs @@ -0,0 +1,212 @@ +#!/usr/bin/env node + +import { mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { env, exit } from "node:process"; + +function pad(value, size = 2) { + return String(value).padStart(size, "0"); +} + +function localIsoWithOffset(date = new Date()) { + const offsetMinutes = -date.getTimezoneOffset(); + const sign = offsetMinutes >= 0 ? "+" : "-"; + const absolute = Math.abs(offsetMinutes); + return [ + `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`, + `T${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.${pad(date.getMilliseconds(), 3)}`, + `${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`, + ].join(""); +} + +function timestampSlug(date = new Date()) { + return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); +} + +function percentile(values, percentileValue) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return Number(sorted[index].toFixed(3)); +} + +function stats(values) { + if (values.length === 0) return { min: 0, p50: 0, p95: 0, p99: 0, max: 0 }; + return { + min: Number(Math.min(...values).toFixed(3)), + p50: percentile(values, 50), + p95: percentile(values, 95), + p99: percentile(values, 99), + max: Number(Math.max(...values).toFixed(3)), + }; +} + +function parseJsonList(value, fallback) { + if (!value) return fallback; + try { + const parsed = JSON.parse(value); + return Array.isArray(parsed) && parsed.every((item) => typeof item === "string") ? parsed : fallback; + } catch { + return fallback; + } +} + +function joinUrl(baseUrl, path) { + const base = baseUrl.replace(/\/+$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; +} + +async function fetchOnce(url, timeoutMs) { + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const started = performance.now(); + try { + const response = await fetch(url, { method: "GET", signal: controller.signal }); + await response.arrayBuffer(); + const latencyMs = performance.now() - started; + return { + url, + ok: response.status < 500, + status: response.status, + latency_ms: Number(latencyMs.toFixed(3)), + error: "", + }; + } catch (error) { + const latencyMs = performance.now() - started; + return { + url, + ok: false, + status: 0, + latency_ms: Number(latencyMs.toFixed(3)), + error: error instanceof Error ? error.message : String(error), + }; + } finally { + clearTimeout(timeout); + } +} + +async function runBatches(urls, totalRequests, concurrency, timeoutMs) { + const queue = Array.from({ length: totalRequests }, (_, index) => urls[index % urls.length]); + const results = []; + while (queue.length > 0) { + const batch = queue.splice(0, concurrency); + results.push(...await Promise.all(batch.map((url) => fetchOnce(url, timeoutMs)))); + } + return results; +} + +async function main() { + const root = resolve(env.LBS_ROOT || process.cwd()); + const caseId = "langbot-live-backend-latency"; + const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; + const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + await mkdir(evidenceDir, { recursive: true }); + + const startedAt = new Date(); + const backendUrl = env.LANGBOT_BACKEND_URL || ""; + const endpoints = parseJsonList(env.LANGBOT_PERF_ENDPOINTS_JSON, ["/healthz"]); + const totalRequests = Number(env.LANGBOT_PERF_REQUESTS || "12"); + const concurrency = Number(env.LANGBOT_PERF_CONCURRENCY || "2"); + const timeoutMs = Number(env.LANGBOT_PERF_TIMEOUT_MS || "5000"); + const p95BudgetMs = Number(env.LANGBOT_PERF_BACKEND_P95_MS || "1000"); + const maxErrorRate = Number(env.LANGBOT_PERF_MAX_ERROR_RATE || "0"); + const metricsPath = join(evidenceDir, "metrics.json"); + const networkLogPath = join(evidenceDir, "network.log"); + const automationResultPath = join(evidenceDir, "automation-result.json"); + const resultPath = join(evidenceDir, "result.json"); + + let status = "fail"; + let reason = ""; + let results = []; + if (!backendUrl) { + status = "env_issue"; + reason = "LANGBOT_BACKEND_URL is not configured."; + } else { + const urls = endpoints.map((path) => joinUrl(backendUrl, path)); + results = await runBatches(urls, totalRequests, concurrency, timeoutMs); + const okCount = results.filter((item) => item.ok).length; + const errorCount = results.length - okCount; + const errorRate = results.length === 0 ? 1 : errorCount / results.length; + const latencies = results.filter((item) => item.ok).map((item) => item.latency_ms); + const latencyStats = stats(latencies); + const allConnectionFailures = results.length > 0 && results.every((item) => item.status === 0); + if (allConnectionFailures) { + status = "env_issue"; + reason = `Backend did not respond at ${backendUrl}.`; + } else if (latencyStats.p95 <= p95BudgetMs && errorRate <= maxErrorRate) { + status = "pass"; + reason = "Live backend latency probe passed all thresholds."; + } else { + status = "fail"; + reason = "Live backend latency probe breached latency or error-rate thresholds."; + } + } + + const statusCounts = {}; + for (const item of results) { + const key = item.status === 0 ? "network_error" : String(item.status); + statusCounts[key] = (statusCounts[key] || 0) + 1; + } + const okResults = results.filter((item) => item.ok); + const metrics = { + probe: caseId, + backend_url: backendUrl, + endpoints, + total_requests: totalRequests, + concurrency, + timeout_ms: timeoutMs, + ok_count: okResults.length, + error_count: results.length - okResults.length, + error_rate: results.length === 0 ? 1 : Number(((results.length - okResults.length) / results.length).toFixed(4)), + latency_ms: stats(okResults.map((item) => item.latency_ms)), + status_counts: statusCounts, + }; + const thresholds = { + backend_p95_ms: { actual: metrics.latency_ms.p95, max: p95BudgetMs, pass: metrics.latency_ms.p95 <= p95BudgetMs }, + error_rate: { actual: metrics.error_rate, max: maxErrorRate, pass: metrics.error_rate <= maxErrorRate }, + }; + + await writeFile(metricsPath, `${JSON.stringify({ ...metrics, samples: results }, null, 2)}\n`, "utf8"); + await writeFile(networkLogPath, results.map((item) => JSON.stringify(item)).join("\n") + (results.length > 0 ? "\n" : ""), "utf8"); + + const finishedAt = new Date(); + const result = { + source: "automation", + case_id: caseId, + run_id: runId, + status, + reason, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: finishedAt.toISOString(), + finished_at_local: localIsoWithOffset(finishedAt), + duration_ms: finishedAt.getTime() - startedAt.getTime(), + url: backendUrl, + metrics_summary: { + requests: metrics.total_requests, + concurrency: metrics.concurrency, + ok_count: metrics.ok_count, + error_rate: metrics.error_rate, + latency_p50_ms: metrics.latency_ms.p50, + latency_p95_ms: metrics.latency_ms.p95, + status_counts: metrics.status_counts, + }, + thresholds_summary: thresholds, + artifacts: { + metrics_json: metricsPath, + network_log: networkLogPath, + automation_result_json: automationResultPath, + result_json: resultPath, + }, + evidence_collected: ["metrics", "network", "api_diagnostic", "filesystem"], + }; + + const resultText = `${JSON.stringify(result, null, 2)}\n`; + await writeFile(automationResultPath, resultText, "utf8"); + await writeFile(resultPath, resultText, "utf8"); + console.log(JSON.stringify(result, null, 2)); + exit(status === "pass" ? 0 : status === "env_issue" ? 2 : 1); +} + +await main(); diff --git a/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs b/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs new file mode 100755 index 000000000..38a31c389 --- /dev/null +++ b/skills/skills/langbot-testing/probes/langbot-live-backend-log-health.mjs @@ -0,0 +1,205 @@ +#!/usr/bin/env node + +import { existsSync, readdirSync, statSync } from "node:fs"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { env, exit } from "node:process"; + +function pad(value, size = 2) { + return String(value).padStart(size, "0"); +} + +function localIsoWithOffset(date = new Date()) { + const offsetMinutes = -date.getTimezoneOffset(); + const sign = offsetMinutes >= 0 ? "+" : "-"; + const absolute = Math.abs(offsetMinutes); + return [ + `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`, + `T${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.${pad(date.getMilliseconds(), 3)}`, + `${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`, + ].join(""); +} + +function timestampSlug(date = new Date()) { + return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); +} + +function repoRootFromEnv(root) { + return env.LANGBOT_REPO ? resolve(env.LANGBOT_REPO) : resolve(root, ".."); +} + +function latestBackendLog(root) { + const explicit = env.LANGBOT_BACKEND_LOG; + if (explicit) return resolve(explicit); + + const logsDir = join(repoRootFromEnv(root), "data", "logs"); + if (!existsSync(logsDir)) return ""; + const candidates = readdirSync(logsDir) + .filter((name) => /^langbot-.*\.log$/.test(name)) + .map((name) => join(logsDir, name)) + .filter((path) => { + try { + return statSync(path).isFile(); + } catch { + return false; + } + }) + .sort((left, right) => statSync(right).mtimeMs - statSync(left).mtimeMs); + return candidates[0] || ""; +} + +function parseSince(startedAt) { + if (env.LANGBOT_BACKEND_LOG_SINCE) return new Date(env.LANGBOT_BACKEND_LOG_SINCE); + const lookbackSeconds = Number(env.LANGBOT_BACKEND_LOG_LOOKBACK_SECONDS || "300"); + return new Date(startedAt.getTime() - lookbackSeconds * 1000); +} + +function parseTimestamp(line, year) { + const localMatch = line.match(/^\[(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2})\.(\d{3})\]/); + if (localMatch) { + const [, month, day, hour, minute, second, millisecond] = localMatch; + return new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.${millisecond}+08:00`); + } + + const accessMatch = line.match(/^\[(\d{4})-(\d{2})-(\d{2}) (\d{2}):(\d{2}):(\d{2}) ([+-]\d{4})\]/); + if (accessMatch) { + const [, fullYear, month, day, hour, minute, second, offset] = accessMatch; + const normalizedOffset = `${offset.slice(0, 3)}:${offset.slice(3)}`; + return new Date(`${fullYear}-${month}-${day}T${hour}:${minute}:${second}${normalizedOffset}`); + } + + return null; +} + +function findingForLine(line, number) { + const rules = [ + { severity: "fail", kind: "python_traceback", pattern: /\bTraceback(?: \(most recent call last\))?/i }, + { severity: "fail", kind: "unretrieved_task_exception", pattern: /Task exception was never retrieved/i }, + { severity: "fail", kind: "unawaited_coroutine", pattern: /RuntimeWarning:\s+coroutine .* was never awaited/i }, + { severity: "fail", kind: "unclosed_client_session", pattern: /Unclosed client session/i }, + { severity: "fail", kind: "unclosed_connector", pattern: /Unclosed connector/i }, + { severity: "fail", kind: "import_error", pattern: /\bImportError\b/i }, + { severity: "fail", kind: "error_log", pattern: /\b(?:ERROR|CRITICAL)\b/ }, + { severity: "warning", kind: "warning_log", pattern: /\bWARNING\b/ }, + ]; + + for (const rule of rules) { + if (rule.pattern.test(line)) { + return { + severity: rule.severity, + kind: rule.kind, + line: number, + excerpt: line, + }; + } + } + return null; +} + +function scanLines(text, since, year) { + const findings = []; + const scanned = []; + let includeContinuation = false; + const lines = text.split(/\r?\n/); + for (const [index, line] of lines.entries()) { + const number = index + 1; + const timestamp = parseTimestamp(line, year); + if (timestamp) includeContinuation = timestamp >= since; + if (!includeContinuation) continue; + scanned.push({ number, text: line }); + const finding = findingForLine(line, number); + if (finding) findings.push(finding); + } + return { findings, scanned, total_lines: lines.length }; +} + +async function main() { + const root = resolve(env.LBS_ROOT || process.cwd()); + const caseId = "langbot-live-backend-log-health"; + const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; + const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + await mkdir(evidenceDir, { recursive: true }); + + const startedAt = new Date(); + const since = parseSince(startedAt); + const logPath = latestBackendLog(root); + const metricsPath = join(evidenceDir, "metrics.json"); + const findingsPath = join(evidenceDir, "findings.json"); + const scannedLogPath = join(evidenceDir, "scanned-backend.log"); + const automationResultPath = join(evidenceDir, "automation-result.json"); + const resultPath = join(evidenceDir, "result.json"); + + let status = "fail"; + let reason = ""; + let scan = { findings: [], scanned: [], total_lines: 0 }; + if (!logPath || !existsSync(logPath)) { + status = "env_issue"; + reason = "No LangBot backend log file was found. Set LANGBOT_BACKEND_LOG or LANGBOT_REPO."; + } else { + const text = await readFile(logPath, "utf8"); + scan = scanLines(text, since, startedAt.getFullYear()); + const failCount = scan.findings.filter((item) => item.severity === "fail").length; + status = failCount === 0 ? "pass" : "fail"; + reason = status === "pass" + ? "Live backend log health passed; no fail-severity findings in the scanned window." + : "Live backend log health found fail-severity backend log findings."; + } + + const warningCount = scan.findings.filter((item) => item.severity === "warning").length; + const failCount = scan.findings.filter((item) => item.severity === "fail").length; + const metrics = { + probe: caseId, + backend_log: logPath, + since: since.toISOString(), + scanned_line_count: scan.scanned.length, + total_line_count: scan.total_lines, + fail_count: failCount, + warning_count: warningCount, + finding_count: scan.findings.length, + }; + const thresholds = { + fail_count: { actual: failCount, max: 0, pass: failCount === 0 }, + }; + + await writeFile(metricsPath, `${JSON.stringify(metrics, null, 2)}\n`, "utf8"); + await writeFile(findingsPath, `${JSON.stringify(scan.findings, null, 2)}\n`, "utf8"); + await writeFile(scannedLogPath, scan.scanned.map((item) => `${item.number}: ${item.text}`).join("\n") + (scan.scanned.length > 0 ? "\n" : ""), "utf8"); + + const finishedAt = new Date(); + const result = { + source: "automation", + case_id: caseId, + run_id: runId, + status, + reason, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: finishedAt.toISOString(), + finished_at_local: localIsoWithOffset(finishedAt), + duration_ms: finishedAt.getTime() - startedAt.getTime(), + url: logPath, + metrics_summary: { + scanned_line_count: metrics.scanned_line_count, + fail_count: metrics.fail_count, + warning_count: metrics.warning_count, + finding_count: metrics.finding_count, + }, + thresholds_summary: thresholds, + artifacts: { + metrics_json: metricsPath, + findings_json: findingsPath, + scanned_backend_log: scannedLogPath, + automation_result_json: automationResultPath, + result_json: resultPath, + }, + evidence_collected: ["metrics", "backend_log", "filesystem"], + }; + + const resultText = `${JSON.stringify(result, null, 2)}\n`; + await writeFile(automationResultPath, resultText, "utf8"); + await writeFile(resultPath, resultText, "utf8"); + console.log(JSON.stringify(result, null, 2)); + exit(status === "pass" ? 0 : status === "env_issue" ? 2 : 1); +} + +await main(); diff --git a/skills/skills/langbot-testing/probes/langbot-live-control-plane-api.mjs b/skills/skills/langbot-testing/probes/langbot-live-control-plane-api.mjs new file mode 100755 index 000000000..8232d1fc3 --- /dev/null +++ b/skills/skills/langbot-testing/probes/langbot-live-control-plane-api.mjs @@ -0,0 +1,311 @@ +#!/usr/bin/env node + +import { mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { env, exit } from "node:process"; + +function pad(value, size = 2) { + return String(value).padStart(size, "0"); +} + +function localIsoWithOffset(date = new Date()) { + const offsetMinutes = -date.getTimezoneOffset(); + const sign = offsetMinutes >= 0 ? "+" : "-"; + const absolute = Math.abs(offsetMinutes); + return [ + `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`, + `T${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.${pad(date.getMilliseconds(), 3)}`, + `${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`, + ].join(""); +} + +function timestampSlug(date = new Date()) { + return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); +} + +function percentile(values, percentileValue) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return Number(sorted[index].toFixed(3)); +} + +function stats(values) { + if (values.length === 0) return { min: 0, p50: 0, p95: 0, p99: 0, max: 0 }; + return { + min: Number(Math.min(...values).toFixed(3)), + p50: percentile(values, 50), + p95: percentile(values, 95), + p99: percentile(values, 99), + max: Number(Math.max(...values).toFixed(3)), + }; +} + +function joinUrl(baseUrl, path) { + const base = baseUrl.replace(/\/+$/, ""); + const suffix = path.startsWith("/") ? path : `/${path}`; + return `${base}${suffix}`; +} + +function parseJsonObject(value, fallback) { + if (!value) return fallback; + try { + const parsed = JSON.parse(value); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : fallback; + } catch { + return fallback; + } +} + +function controlPlaneEndpoints() { + return [ + { + id: "healthz", + path: "/healthz", + expected_status: 200, + expected_code: 0, + p95_budget_ms: Number(env.LANGBOT_PERF_HEALTHZ_P95_MS || "500"), + required_data_fields: [], + }, + { + id: "system_info", + path: "/api/v1/system/info", + expected_status: 200, + expected_code: 0, + p95_budget_ms: Number(env.LANGBOT_PERF_SYSTEM_INFO_P95_MS || "1000"), + required_data_fields: ["version", "edition", "enable_marketplace"], + }, + ]; +} + +async function fetchEndpoint(backendUrl, endpoint, timeoutMs) { + const url = joinUrl(backendUrl, endpoint.path); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + const started = performance.now(); + let bodyText = ""; + let json = null; + let jsonValid = false; + let error = ""; + + try { + const response = await fetch(url, { + method: "GET", + headers: { "accept": "application/json" }, + signal: controller.signal, + }); + bodyText = await response.text(); + try { + json = bodyText ? JSON.parse(bodyText) : null; + jsonValid = json !== null; + } catch (parseError) { + error = parseError instanceof Error ? parseError.message : String(parseError); + } + + const data = json && typeof json === "object" && json.data && typeof json.data === "object" ? json.data : {}; + const missingFields = endpoint.required_data_fields.filter((field) => !(field in data)); + const statusOk = response.status === endpoint.expected_status; + const codeOk = !json || typeof json !== "object" ? false : json.code === endpoint.expected_code; + const shapeOk = jsonValid && missingFields.length === 0; + const latencyMs = performance.now() - started; + return { + endpoint_id: endpoint.id, + path: endpoint.path, + url, + status: response.status, + ok: statusOk && codeOk && shapeOk, + status_ok: statusOk, + code_ok: codeOk, + json_valid: jsonValid, + missing_fields: missingFields, + response_code: json && typeof json === "object" ? json.code : null, + latency_ms: Number(latencyMs.toFixed(3)), + error, + }; + } catch (fetchError) { + const latencyMs = performance.now() - started; + return { + endpoint_id: endpoint.id, + path: endpoint.path, + url, + status: 0, + ok: false, + status_ok: false, + code_ok: false, + json_valid: false, + missing_fields: endpoint.required_data_fields, + response_code: null, + latency_ms: Number(latencyMs.toFixed(3)), + error: fetchError instanceof Error ? fetchError.message : String(fetchError), + }; + } finally { + clearTimeout(timeout); + } +} + +async function runBatches(backendUrl, endpoints, totalRequests, concurrency, timeoutMs) { + const queue = Array.from({ length: totalRequests }, (_, index) => endpoints[index % endpoints.length]); + const results = []; + while (queue.length > 0) { + const batch = queue.splice(0, concurrency); + results.push(...await Promise.all(batch.map((endpoint) => fetchEndpoint(backendUrl, endpoint, timeoutMs)))); + } + return results; +} + +function endpointMetrics(endpoints, results) { + return Object.fromEntries(endpoints.map((endpoint) => { + const samples = results.filter((item) => item.endpoint_id === endpoint.id); + const okSamples = samples.filter((item) => item.ok); + return [ + endpoint.id, + { + path: endpoint.path, + requests: samples.length, + ok_count: okSamples.length, + error_rate: samples.length === 0 ? 1 : Number(((samples.length - okSamples.length) / samples.length).toFixed(4)), + latency_ms: stats(okSamples.map((item) => item.latency_ms)), + p95_budget_ms: endpoint.p95_budget_ms, + }, + ]; + })); +} + +async function main() { + const root = resolve(env.LBS_ROOT || process.cwd()); + const caseId = "langbot-live-control-plane-api"; + const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; + const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + await mkdir(evidenceDir, { recursive: true }); + + const startedAt = new Date(); + const backendUrl = env.LANGBOT_BACKEND_URL || ""; + const endpoints = controlPlaneEndpoints(); + const configuredBudgets = parseJsonObject(env.LANGBOT_CONTROL_PLANE_P95_BUDGETS_JSON, {}); + for (const endpoint of endpoints) { + const budget = configuredBudgets[endpoint.id]; + if (typeof budget === "number" && Number.isFinite(budget)) endpoint.p95_budget_ms = budget; + } + const totalRequests = Number(env.LANGBOT_CONTROL_PLANE_REQUESTS || "20"); + const concurrency = Number(env.LANGBOT_CONTROL_PLANE_CONCURRENCY || "4"); + const timeoutMs = Number(env.LANGBOT_CONTROL_PLANE_TIMEOUT_MS || "5000"); + const maxErrorRate = Number(env.LANGBOT_CONTROL_PLANE_MAX_ERROR_RATE || "0"); + const metricsPath = join(evidenceDir, "metrics.json"); + const endpointsPath = join(evidenceDir, "endpoints.json"); + const networkLogPath = join(evidenceDir, "network.log"); + const automationResultPath = join(evidenceDir, "automation-result.json"); + const resultPath = join(evidenceDir, "result.json"); + + let status = "fail"; + let reason = ""; + let results = []; + if (!backendUrl) { + status = "env_issue"; + reason = "LANGBOT_BACKEND_URL is not configured."; + } else { + results = await runBatches(backendUrl, endpoints, totalRequests, concurrency, timeoutMs); + const allConnectionFailures = results.length > 0 && results.every((item) => item.status === 0); + if (allConnectionFailures) { + status = "env_issue"; + reason = `Backend did not respond at ${backendUrl}.`; + } + } + + const okResults = results.filter((item) => item.ok); + const statusCounts = {}; + for (const item of results) { + const key = item.status === 0 ? "network_error" : String(item.status); + statusCounts[key] = (statusCounts[key] || 0) + 1; + } + const perEndpoint = endpointMetrics(endpoints, results); + const responseShapeFailures = results.filter((item) => !item.json_valid || item.missing_fields.length > 0 || !item.code_ok).length; + const errorRate = results.length === 0 ? 1 : Number(((results.length - okResults.length) / results.length).toFixed(4)); + const thresholds = { + error_rate: { actual: errorRate, max: maxErrorRate, pass: errorRate <= maxErrorRate }, + response_shape_failures: { actual: responseShapeFailures, max: 0, pass: responseShapeFailures === 0 }, + }; + for (const endpoint of endpoints) { + const actual = perEndpoint[endpoint.id].latency_ms.p95; + thresholds[`${endpoint.id}_p95_ms`] = { + actual, + max: endpoint.p95_budget_ms, + pass: actual <= endpoint.p95_budget_ms, + }; + } + + if (status !== "env_issue") { + const passed = Object.values(thresholds).every((item) => item.pass); + status = passed ? "pass" : "fail"; + reason = passed + ? "Live control-plane API probe passed all thresholds." + : "Live control-plane API probe breached shape, latency, or error-rate thresholds."; + } + + const metrics = { + probe: caseId, + backend_url: backendUrl, + total_requests: totalRequests, + concurrency, + timeout_ms: timeoutMs, + ok_count: okResults.length, + error_count: results.length - okResults.length, + error_rate: errorRate, + status_counts: statusCounts, + response_shape_failures: responseShapeFailures, + endpoints: perEndpoint, + }; + + await writeFile(metricsPath, `${JSON.stringify({ ...metrics, samples: results }, null, 2)}\n`, "utf8"); + await writeFile(endpointsPath, `${JSON.stringify(endpoints, null, 2)}\n`, "utf8"); + await writeFile(networkLogPath, results.map((item) => JSON.stringify(item)).join("\n") + (results.length > 0 ? "\n" : ""), "utf8"); + + const finishedAt = new Date(); + const result = { + source: "automation", + case_id: caseId, + run_id: runId, + status, + reason, + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: finishedAt.toISOString(), + finished_at_local: localIsoWithOffset(finishedAt), + duration_ms: finishedAt.getTime() - startedAt.getTime(), + url: backendUrl, + metrics_summary: { + requests: metrics.total_requests, + concurrency: metrics.concurrency, + ok_count: metrics.ok_count, + error_rate: metrics.error_rate, + response_shape_failures: metrics.response_shape_failures, + endpoints: Object.fromEntries(Object.entries(metrics.endpoints).map(([id, value]) => [ + id, + { + path: value.path, + ok_count: value.ok_count, + error_rate: value.error_rate, + latency_p50_ms: value.latency_ms.p50, + latency_p95_ms: value.latency_ms.p95, + }, + ])), + status_counts: metrics.status_counts, + }, + thresholds_summary: thresholds, + artifacts: { + metrics_json: metricsPath, + endpoints_json: endpointsPath, + network_log: networkLogPath, + automation_result_json: automationResultPath, + result_json: resultPath, + }, + evidence_collected: ["metrics", "network", "api_diagnostic", "filesystem"], + }; + + const resultText = `${JSON.stringify(result, null, 2)}\n`; + await writeFile(automationResultPath, resultText, "utf8"); + await writeFile(resultPath, resultText, "utf8"); + console.log(JSON.stringify(result, null, 2)); + exit(status === "pass" ? 0 : status === "env_issue" ? 2 : 1); +} + +await main(); diff --git a/skills/skills/langbot-testing/probes/langbot-overhead-accounting-contract.mjs b/skills/skills/langbot-testing/probes/langbot-overhead-accounting-contract.mjs new file mode 100755 index 000000000..5338df003 --- /dev/null +++ b/skills/skills/langbot-testing/probes/langbot-overhead-accounting-contract.mjs @@ -0,0 +1,162 @@ +#!/usr/bin/env node + +import { mkdir, writeFile } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { env, exit } from "node:process"; + +function pad(value, size = 2) { + return String(value).padStart(size, "0"); +} + +function localIsoWithOffset(date = new Date()) { + const offsetMinutes = -date.getTimezoneOffset(); + const sign = offsetMinutes >= 0 ? "+" : "-"; + const absolute = Math.abs(offsetMinutes); + return [ + `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`, + `T${pad(date.getHours())}:${pad(date.getMinutes())}:${pad(date.getSeconds())}.${pad(date.getMilliseconds(), 3)}`, + `${sign}${pad(Math.floor(absolute / 60))}:${pad(absolute % 60)}`, + ].join(""); +} + +function timestampSlug(date = new Date()) { + return date.toISOString().replace(/\.\d{3}Z$/, "Z").replace(/[^0-9A-Za-z]+/g, "-").replace(/^-|-$/g, ""); +} + +function percentile(values, percentileValue) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return Number(sorted[index].toFixed(3)); +} + +function stats(values) { + return { + min: Number(Math.min(...values).toFixed(3)), + p50: percentile(values, 50), + p95: percentile(values, 95), + p99: percentile(values, 99), + max: Number(Math.max(...values).toFixed(3)), + }; +} + +function threshold(actual, limit, operator) { + const pass = operator === "<=" ? actual <= limit : actual >= limit; + return { actual, [operator === "<=" ? "max" : "min"]: limit, pass }; +} + +function makeSample(index) { + const ingress = 1 + (index % 5) * 0.22; + const pipeline = 2.8 + (index % 7) * 0.31; + const persistence = 1.1 + (index % 4) * 0.2; + const pluginIpc = 1.9 + (index % 6) * 0.27; + const rag = index % 3 === 0 ? 4.4 : 0.8 + (index % 5) * 0.18; + const streaming = 1.5 + (index % 8) * 0.24; + const provider = 80 + (index % 13) * 11; + const externalTool = index % 4 === 0 ? 25 + (index % 9) * 3 : 0; + const network = 8 + (index % 10) * 1.7; + const overhead = ingress + pipeline + persistence + pluginIpc + rag + streaming; + const external = provider + externalTool + network; + const total = overhead + external; + return { + index, + segments_ms: { + ingress, + pipeline, + persistence, + plugin_ipc: pluginIpc, + rag, + streaming, + provider, + external_tool: externalTool, + network, + }, + langbot_overhead_ms: Number(overhead.toFixed(3)), + external_latency_ms: Number(external.toFixed(3)), + e2e_latency_ms: Number(total.toFixed(3)), + accounting_gap_ms: Number((total - external - overhead).toFixed(6)), + }; +} + +async function main() { + const root = resolve(env.LBS_ROOT || process.cwd()); + const caseId = "langbot-overhead-accounting-contract"; + const runId = env.LBS_RUN_ID || `${timestampSlug()}-${caseId}`; + const evidenceDir = resolve(env.LBS_EVIDENCE_DIR || join(root, "reports", "evidence", runId)); + await mkdir(evidenceDir, { recursive: true }); + + const startedAt = new Date(); + const sampleCount = Number(env.LANGBOT_PERF_CONTRACT_SAMPLES || "80"); + const overheadP95BudgetMs = Number(env.LANGBOT_PERF_OVERHEAD_P95_MS || "25"); + const samples = Array.from({ length: sampleCount }, (_, index) => makeSample(index)); + const overheads = samples.map((sample) => sample.langbot_overhead_ms); + const e2e = samples.map((sample) => sample.e2e_latency_ms); + const external = samples.map((sample) => sample.external_latency_ms); + const gaps = samples.map((sample) => Math.abs(sample.accounting_gap_ms)); + const memory = process.memoryUsage(); + + const metrics = { + probe: caseId, + sample_count: sampleCount, + langbot_overhead_ms: stats(overheads), + e2e_latency_ms: stats(e2e), + external_latency_ms: stats(external), + accounting_gap_max_ms: Number(Math.max(...gaps).toFixed(6)), + samples, + }; + const thresholds = { + sample_count: threshold(sampleCount, 50, ">="), + langbot_overhead_p95_ms: threshold(metrics.langbot_overhead_ms.p95, overheadP95BudgetMs, "<="), + accounting_gap_max_ms: threshold(metrics.accounting_gap_max_ms, 0.001, "<="), + }; + const status = Object.values(thresholds).every((item) => item.pass) ? "pass" : "fail"; + const metricsPath = join(evidenceDir, "metrics.json"); + const thresholdsPath = join(evidenceDir, "thresholds.json"); + const resourceLogPath = join(evidenceDir, "resource-log.json"); + const automationResultPath = join(evidenceDir, "automation-result.json"); + const resultPath = join(evidenceDir, "result.json"); + + await writeFile(metricsPath, `${JSON.stringify(metrics, null, 2)}\n`, "utf8"); + await writeFile(thresholdsPath, `${JSON.stringify(thresholds, null, 2)}\n`, "utf8"); + await writeFile(resourceLogPath, `${JSON.stringify({ memory, pid: process.pid }, null, 2)}\n`, "utf8"); + + const finishedAt = new Date(); + const result = { + source: "automation", + case_id: caseId, + run_id: runId, + status, + reason: status === "pass" + ? "Overhead accounting contract passed all thresholds." + : "Overhead accounting contract breached one or more thresholds.", + started_at: startedAt.toISOString(), + started_at_local: localIsoWithOffset(startedAt), + finished_at: finishedAt.toISOString(), + finished_at_local: localIsoWithOffset(finishedAt), + duration_ms: finishedAt.getTime() - startedAt.getTime(), + metrics_summary: { + sample_count: metrics.sample_count, + langbot_overhead_p95_ms: metrics.langbot_overhead_ms.p95, + e2e_latency_p95_ms: metrics.e2e_latency_ms.p95, + external_latency_p95_ms: metrics.external_latency_ms.p95, + accounting_gap_max_ms: metrics.accounting_gap_max_ms, + }, + thresholds_summary: thresholds, + artifacts: { + metrics_json: metricsPath, + thresholds_json: thresholdsPath, + resource_log_json: resourceLogPath, + automation_result_json: automationResultPath, + result_json: resultPath, + }, + evidence_collected: ["metrics", "resource_log", "filesystem"], + }; + + const resultText = `${JSON.stringify(result, null, 2)}\n`; + await writeFile(automationResultPath, resultText, "utf8"); + await writeFile(resultPath, resultText, "utf8"); + console.log(JSON.stringify(result, null, 2)); + exit(status === "pass" ? 0 : 1); +} + +await main(); diff --git a/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs b/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs new file mode 100755 index 000000000..b383b2663 --- /dev/null +++ b/skills/skills/langbot-testing/probes/lib/fake-provider-timing.mjs @@ -0,0 +1,134 @@ +export function summarizeFakeProviderState(state) { + if (!state) return null; + const recentRequests = Array.isArray(state.recent_requests) ? state.recent_requests : []; + const chatRequests = recentRequests.filter((request) => String(request?.path || "").includes("/chat/completions")); + const successfulRequests = chatRequests.filter((request) => request?.status === "ok"); + const faultRequests = chatRequests.filter((request) => ( + request?.should_fail === true + || request?.status === "http_fault" + || (Number.isFinite(request?.http_status) && request.http_status >= 400) + )); + + return { + status: state.status || "unknown", + url: state.url || "", + request_count: Number.isFinite(state.request_count) ? state.request_count : recentRequests.length, + recent_request_count: recentRequests.length, + chat_request_count: chatRequests.length, + fault_count: faultRequests.length, + streamed_request_count: chatRequests.filter((request) => request?.stream === true).length, + duration_ms: stats(chatRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)), + successful_duration_ms: stats(successfulRequests.map((request) => numberOrNull(request?.duration_ms)).filter(Number.isFinite)), + first_chunk_ms: stats(successfulRequests.map((request) => numberOrNull(request?.first_chunk_ms)).filter(Number.isFinite)), + first_content_chunk_ms: stats(successfulRequests.map((request) => numberOrNull(request?.first_content_chunk_ms)).filter(Number.isFinite)), + content_chunk_count: stats(successfulRequests.map((request) => numberOrNull(request?.content_chunk_count)).filter(Number.isFinite)), + config: state.config || {}, + }; +} + +export function buildProviderTimingMetrics(samples, state) { + const recentRequests = Array.isArray(state?.recent_requests) ? state.recent_requests : []; + const byExpectedText = new Map(); + for (const request of recentRequests) { + const expected = String(request?.expected_text || ""); + if (!expected) continue; + if (!byExpectedText.has(expected)) byExpectedText.set(expected, []); + byExpectedText.get(expected).push(request); + } + + const segments = []; + const missingExpectedText = []; + for (const sample of samples) { + const expected = String(sample?.expected_text || ""); + if (!expected) continue; + const request = (byExpectedText.get(expected) || []).shift(); + if (!request) { + missingExpectedText.push(expected); + continue; + } + const segment = buildTimingSegment(sample, request); + if (segment) segments.push(segment); + } + + const values = (key) => segments.map((segment) => numberOrNull(segment[key])).filter(Number.isFinite); + return { + matched_request_count: segments.length, + missing_provider_match_count: missingExpectedText.length, + missing_expected_text: missingExpectedText.slice(0, 20), + send_to_provider_start_ms: stats(values("send_to_provider_start_ms")), + provider_duration_ms: stats(values("provider_duration_ms")), + provider_finish_to_ws_final_ms: stats(values("provider_finish_to_ws_final_ms")), + langbot_overhead_estimate_ms: stats(values("langbot_overhead_estimate_ms")), + e2e_minus_provider_ms: stats(values("e2e_minus_provider_ms")), + provider_first_content_to_ws_first_content_ms: stats(values("provider_first_content_to_ws_first_content_ms")), + segments, + }; +} + +function buildTimingSegment(sample, request) { + const sentEpochMs = numberOrNull(sample.sent_epoch_ms); + const finishedEpochMs = numberOrNull(sample.finished_epoch_ms); + const providerStartedEpochMs = numberOrNull(request.started_epoch_ms); + const providerFinishedEpochMs = numberOrNull(request.finished_epoch_ms); + const providerFirstContentEpochMs = numberOrNull(request.first_content_chunk_epoch_ms); + const wsFirstContentEpochMs = numberOrNull(sample.first_assistant_content_epoch_ms); + const responseDurationMs = numberOrNull(sample.response_duration_ms); + const providerDurationMs = numberOrNull(request.duration_ms); + + const sendToProviderStartMs = finiteDelta(providerStartedEpochMs, sentEpochMs); + const providerFinishToWsFinalMs = finiteDelta(finishedEpochMs, providerFinishedEpochMs); + const e2eMinusProviderMs = Number.isFinite(responseDurationMs) && Number.isFinite(providerDurationMs) + ? rounded(responseDurationMs - providerDurationMs) + : null; + const overheadEstimateMs = Number.isFinite(sendToProviderStartMs) && Number.isFinite(providerFinishToWsFinalMs) + ? rounded(sendToProviderStartMs + providerFinishToWsFinalMs) + : e2eMinusProviderMs; + + return { + sample_index: sample.index, + pipeline_label: sample.pipeline_label || "", + expected_text: sample.expected_text || "", + provider_request_id: request.id || "", + provider_request_number: request.request_number ?? null, + response_duration_ms: responseDurationMs, + provider_duration_ms: providerDurationMs, + send_to_provider_start_ms: sendToProviderStartMs, + provider_finish_to_ws_final_ms: providerFinishToWsFinalMs, + langbot_overhead_estimate_ms: overheadEstimateMs, + e2e_minus_provider_ms: e2eMinusProviderMs, + provider_first_content_to_ws_first_content_ms: finiteDelta(wsFirstContentEpochMs, providerFirstContentEpochMs), + provider_status: request.status || "", + provider_http_status: request.http_status ?? null, + }; +} + +function finiteDelta(left, right) { + return Number.isFinite(left) && Number.isFinite(right) ? rounded(left - right) : null; +} + +export function stats(values) { + if (values.length === 0) return { min: 0, p50: 0, p95: 0, p99: 0, max: 0 }; + return { + min: rounded(Math.min(...values)), + p50: percentile(values, 50), + p95: percentile(values, 95), + p99: percentile(values, 99), + max: rounded(Math.max(...values)), + }; +} + +export function percentile(values, percentileValue) { + if (values.length === 0) return 0; + const sorted = [...values].sort((a, b) => a - b); + const index = Math.min(sorted.length - 1, Math.ceil((percentileValue / 100) * sorted.length) - 1); + return rounded(sorted[index]); +} + +export function rounded(value) { + return Number(value.toFixed(3)); +} + +function numberOrNull(value) { + const number = Number(value); + return Number.isFinite(number) ? number : null; +} diff --git a/skills/skills/langbot-testing/probes/pytest-probe.mjs b/skills/skills/langbot-testing/probes/pytest-probe.mjs old mode 100644 new mode 100755 diff --git a/skills/skills/langbot-testing/references/performance-reliability-testing.md b/skills/skills/langbot-testing/references/performance-reliability-testing.md new file mode 100644 index 000000000..42aaa0467 --- /dev/null +++ b/skills/skills/langbot-testing/references/performance-reliability-testing.md @@ -0,0 +1,285 @@ +# Performance And Reliability Testing + +Use this reference when a QA request asks whether LangBot is fast enough, +stable under load, or resilient to controlled faults. + +These probes are manual/non-required QA gates unless a case or suite explicitly +states otherwise. They depend on a live local backend, mutable QA fixtures, and +operator-selected environment variables, so do not promote them to required CI +checks until fake-provider isolation, ownership markers, and cleanup are in +place. + +## Scope + +Treat `skills/` as the QA control plane: + +- Cases define intent, readiness, thresholds, and required evidence. +- Probe scripts collect metrics, traces, resource logs, and artifacts. +- Reports classify the same run as `pass`, `fail`, `blocked`, + `env_issue`, or `flaky`. + +Do not turn `skills/` into a load generator or chaos engine. Call a focused +tool from a `mode: probe` case when the test needs one, for example k6, +Locust, pytest-benchmark, Playwright trace collection, Toxiproxy, Docker, or a +Kubernetes disruption tool. + +## LangBot Performance Model + +For LangBot, performance is the cost LangBot adds around external systems: + +```text +LangBot overhead = end-to-end latency - provider latency - external tool latency - network/fault injection latency +``` + +Measure user experience and internal composition separately: + +- WebUI load and interaction latency. +- Debug Chat send-to-first-visible-token and send-to-completion latency. +- Pipeline, RAG, plugin runtime, MCP, AgentRunner, and persistence segment + latency. +- Queue wait time, concurrency, throughput, timeout rate, and p95/p99 latency. +- Startup, plugin install, knowledge-base ingestion, migration, and recovery + time. + +Do not report a single message round-trip time as "LangBot performance" unless +the report also explains external provider/tool/network time. + +## Evidence Contract + +Performance and reliability cases should declare the evidence they need: + +- `metrics`: machine-readable latency, throughput, error-rate, or recovery + metrics, usually `metrics.json`. +- `resource_log`: CPU, memory, process, connection, queue, or file descriptor + samples. +- `trace`: browser, HTTP, database, or runtime trace artifacts. +- `profile`: CPU, memory, or flamegraph profile artifacts. +- `backend_log`, `network`, `api_diagnostic`, and `filesystem` as supporting + evidence when relevant. + +Automation should write `automation-result.json` with these fields when +available: + +```json +{ + "status": "pass", + "reason": "Probe passed all thresholds.", + "metrics_summary": { + "langbot_overhead_p95_ms": 12.4, + "error_rate": 0 + }, + "thresholds_summary": { + "langbot_overhead_p95_ms": { "actual": 12.4, "max": 50, "pass": true } + }, + "artifacts": { + "metrics_json": "/path/to/metrics.json" + }, + "evidence_collected": ["metrics", "filesystem"] +} +``` + +Synthetic contract probes are useful for checking the QA harness, but they are +not live product performance results. Label them as contract probes in the case +title, checks, and report. + +## Chaos And Reliability Rules + +Chaos tests must be narrow and reversible: + +- Declare the fault model in `fault_model_json`. +- Record blast radius, target component, injection method, duration, and abort + conditions. +- Capture recovery checks and cleanup steps in the case. +- Classify unavailable dependencies as `env_issue` unless the target behavior + is LangBot's handling of that dependency failure. +- Do not run destructive fault injection against a shared or production-like + instance without explicit operator approval. + +Recommended first fault models: + +- Provider timeout or HTTP 429 from a fake provider endpoint. +- Plugin runtime disconnect/reconnect in a local instance. +- MCP stdio server exits mid-call. +- RAG parser fixture fails once and recovers on retry. +- Backend API endpoint returns 5xx from a controlled local proxy. + +## Starter Live Probes + +The starter gate separates QA-harness contracts from live product checks: + +- `langbot-overhead-accounting-contract` verifies that reports can carry + overhead accounting metrics. It uses deterministic synthetic samples and is + not live product performance. +- `langbot-fault-taxonomy-contract` verifies that fault scenarios declare + expected status, recovery, and cleanup before destructive chaos tests are + added. +- `langbot-live-backend-latency` checks the unauthenticated `/healthz` + endpoint for basic backend responsiveness. +- `langbot-live-control-plane-api` checks `/healthz` and + `/api/v1/system/info` for HTTP 200, JSON `code: 0`, response shape, and + per-endpoint p95 latency. +- `langbot-live-backend-log-health` scans the recent backend log window for + fail-severity runtime findings. It is the reliability guard that should fail + the gate when HTTP probes pass but backend logs contain Traceback, ImportError, + ERROR, unclosed sessions, or unawaited coroutine signals. + +Do not treat these starter live probes as Debug Chat or model-provider +performance. They are control-plane readiness checks; user-facing performance +needs browser/WebSocket/message-path measurements. + +## Debug Chat Load And Fake Provider Baseline + +Use `langbot-fake-provider-debug-chat-load` before real-provider load checks. +The setup automation starts a local OpenAI-compatible fake provider, registers +it as a normal LangBot provider/model, configures a local-agent pipeline, resets +Debug Chat, and then drives concurrent WebSocket messages through the live +backend. + +This is not a mocked backend test. It still exercises: + +- provider/model persistence and runtime reload; +- LiteLLM OpenAI-compatible requester path; +- local-agent runner selection and pipeline execution; +- Debug Chat WebSocket adapter and broadcast behavior; +- backend concurrency, timeout, and error-rate accounting. + +The fake provider is deterministic and can inject controlled latency or faults +with `LANGBOT_FAKE_PROVIDER_*` variables, so it is the baseline for LangBot +message-path overhead. A fake-provider process keeps process-global config, +request counters, and recent request history; run fake-provider probes serially +or give each run its own provider instance. Concurrent probes against the same +fake-provider URL can reset or reconfigure each other's metrics. + +The probe uses unique expected response tokens per +request because Debug Chat broadcasts messages to every connection in the same +session; unique tokens prevent one connection from counting another +connection's response as its own. + +When the fake provider is used, reports also include provider-side timing in +`metrics.json`: + +- `fake_provider.duration_ms` and `fake_provider.first_content_chunk_ms` + measure the controlled provider itself. +- `provider_timing.send_to_provider_start_ms` estimates WebSocket ingress, + pipeline dispatch, runner setup, and requester time before the provider + receives the request. +- `provider_timing.provider_finish_to_ws_final_ms` estimates the path from + provider completion back to the final Debug Chat WebSocket response. +- `provider_timing.langbot_overhead_estimate_ms` is the sum of those two + LangBot-side segments when wall-clock timestamps can be matched by the + unique expected response token. + +After the baseline passes, run `langbot-fake-provider-debug-chat-slow-load` to +keep the same live backend path while injecting deterministic streaming latency. +Run `langbot-fake-provider-debug-chat-fault-recovery` to inject bounded HTTP +provider failures and require both observed failures and later successful +requests. The fault-recovery case is deliberately sequential because failed +Debug Chat responses do not carry a unique success token that can be attributed +to one concurrent connection. + +Run `langbot-fake-provider-debug-chat-cross-pipeline-isolation` separately via +`langbot-debug-chat-isolation-gate`. Current LangBot releases may fail it because +of product bug [#2286](https://github.com/langbot-app/LangBot/issues/2286), where +Debug Chat replies can read singleton WebSocket proxy pipeline state after a +later message overwrites it. Treat that failure as regression evidence for the +product fix rather than as a fake-provider latency finding. + +Use `langbot-space-debug-chat-concurrency-smoke` after the fake-provider +baseline. It runs a deliberately small real Space-provider batch and reports +user-visible latency, not pure LangBot overhead. Space/model/network failures +are dependency findings until the fake baseline shows the same symptom. +If a Space smoke passes but log guard finds telemetry posting Tracebacks, +classify that separately as `telemetry-proxy-noise` instead of clearing the +proxy or treating the Debug Chat path as failed. + +Useful commands: + +```bash +rtk bin/lbs test run langbot-fake-provider-debug-chat-load --run-id langbot-fake-load-local +rtk bin/lbs test run langbot-fake-provider-debug-chat-slow-load --run-id langbot-fake-slow-local +rtk bin/lbs test run langbot-fake-provider-debug-chat-fault-recovery --run-id langbot-fake-fault-local +rtk bin/lbs suite run langbot-debug-chat-isolation-gate --run-id langbot-debug-chat-isolation-local --include-manual-check +rtk bin/lbs test run langbot-space-debug-chat-concurrency-smoke --run-id langbot-space-smoke-local +rtk bin/lbs suite run langbot-debug-chat-load-gate --run-id langbot-debug-chat-load-local --include-manual-check +``` + +## Gate Layers + +Use the smallest gate that answers the quality question: + +- `langbot-performance-contract-gate`: fast synthetic checks for report shape, + threshold accounting, and fault taxonomy. Good for PR feedback when no live + service is running. +- `langbot-live-backend-gate`: live backend `/healthz`, + `/api/v1/system/info`, and backend log health. Good after starting a local + LangBot backend. +- `langbot-user-path-performance-gate`: browser-visible user path performance, + starting with Pipeline Debug Chat send-to-visible-completion latency. Run it + only when the browser profile and target pipeline are ready. +- `langbot-debug-chat-load-gate`: manual WebSocket Debug Chat load checks, + starting with controlled fake-provider baseline, slow-provider, and + fault-recovery profiles, plus an optional low-volume real Space-provider + smoke. Run fake-provider cases serially when they share a provider URL. +- `langbot-debug-chat-isolation-gate`: manual cross-pipeline Debug Chat + isolation regression gate. Current releases may fail because of #2286; keep it + separate from the normal load gate until that product fix lands. +- `langbot-performance-reliability-gate`: combined starter gate for synthetic + contracts plus live backend checks. + +Keep environment diagnostics separate from product regressions. For example, a +SOCKS proxy without Python `socksio` support should be fixed or clearly +classified by `bin/lbs env doctor`; do not hide the resulting backend +Traceback in reports. + +## Debug Chat Performance + +`pipeline-debug-chat-performance` reuses the browser Debug Chat automation and +adds `metrics.json`, `metrics_summary`, and `thresholds_summary` to +`automation-result.json`. + +Current metric: + +```text +response_duration_ms = prompt send -> expected assistant response visible and stable +``` + +This is a user-path metric, not pure LangBot overhead. If it regresses, inspect +provider latency, model route health, plugin/runtime logs, WebSocket behavior, +and browser console/network evidence before attributing the whole duration to +LangBot. + +### User-Path Gate Runbook + +1. Start the backend and frontend. The frontend must be launched with + `VITE_API_BASE_URL="$LANGBOT_BACKEND_URL"` so browser API calls reach the + backend. +2. Run `node scripts/e2e/ensure-local-agent-pipeline.mjs --write-env`. The + setup refreshes the local QA login, skips the wizard, prepares a Debug Chat + pipeline, scans Space models, tests candidates, writes tested fallback + models, and writes the selected pipeline/model env values to + `skills/.env.local`. +3. If setup returns `env_issue`, read `model_tests` and provider errors first. + A missing Space key, failed Space scan, or unavailable model route is not a + LangBot performance regression. +4. Run + `bin/lbs suite run langbot-user-path-performance-gate --include-manual-check`. +5. Interpret `response_p95_ms` as browser-visible send-to-completion time. It + includes provider latency; use backend logs and model test evidence to + separate LangBot overhead from the external model route. + +The setup keeps a `max-round` value in the generated pipeline config only +because the current backend truncator still reads that field directly. Do not +use it as a quality requirement for future local-agent behavior. + +## Running The First Gate + +Start with the reusable suite: + +```bash +rtk bin/lbs suite plan langbot-performance-reliability-gate +rtk bin/lbs suite start langbot-performance-reliability-gate --run-id langbot-perf-rel-local +``` + +Run synthetic contract probes first. Run live probes only after the selected +backend/frontend instance is reachable and the run owner accepts any fault +scope. diff --git a/skills/skills/langbot-testing/suites/langbot-debug-chat-isolation-gate.yaml b/skills/skills/langbot-testing/suites/langbot-debug-chat-isolation-gate.yaml new file mode 100644 index 000000000..d2b31dd32 --- /dev/null +++ b/skills/skills/langbot-testing/suites/langbot-debug-chat-isolation-gate.yaml @@ -0,0 +1,13 @@ +id: langbot-debug-chat-isolation-gate +title: "LangBot Debug Chat isolation gate" +description: "Manual/non-required cross-pipeline Debug Chat isolation gate. Current releases may fail this gate because of product bug #2286; use it as regression evidence after the routing fix lands." +type: reliability +priority: p1 +tags: + - reliability + - debug-chat + - websocket + - isolation + - concurrency +cases: + - langbot-fake-provider-debug-chat-cross-pipeline-isolation diff --git a/skills/skills/langbot-testing/suites/langbot-debug-chat-load-gate.yaml b/skills/skills/langbot-testing/suites/langbot-debug-chat-load-gate.yaml new file mode 100644 index 000000000..5b4950f16 --- /dev/null +++ b/skills/skills/langbot-testing/suites/langbot-debug-chat-load-gate.yaml @@ -0,0 +1,15 @@ +id: langbot-debug-chat-load-gate +title: "LangBot Debug Chat load gate" +description: "Manual/non-required message-path load checks for Pipeline Debug Chat: controlled fake-provider baseline, slow-provider and fault-recovery profiles, plus optional real Space-provider smoke. Cross-pipeline isolation is split into langbot-debug-chat-isolation-gate because current releases may fail it due to product bug #2286." +type: performance +priority: p1 +tags: + - performance + - debug-chat + - websocket + - load +cases: + - langbot-fake-provider-debug-chat-load + - langbot-fake-provider-debug-chat-slow-load + - langbot-fake-provider-debug-chat-fault-recovery + - langbot-space-debug-chat-concurrency-smoke diff --git a/skills/skills/langbot-testing/suites/langbot-live-backend-gate.yaml b/skills/skills/langbot-testing/suites/langbot-live-backend-gate.yaml new file mode 100644 index 000000000..58a978527 --- /dev/null +++ b/skills/skills/langbot-testing/suites/langbot-live-backend-gate.yaml @@ -0,0 +1,14 @@ +id: langbot-live-backend-gate +title: "LangBot live backend reliability gate" +description: "Live backend control-plane responsiveness and runtime log health checks for a locally running LangBot instance." +type: reliability +priority: p1 +tags: + - performance + - reliability + - live-backend + - metrics +cases: + - langbot-live-backend-latency + - langbot-live-control-plane-api + - langbot-live-backend-log-health diff --git a/skills/skills/langbot-testing/suites/langbot-performance-contract-gate.yaml b/skills/skills/langbot-testing/suites/langbot-performance-contract-gate.yaml new file mode 100644 index 000000000..b5a9eb47f --- /dev/null +++ b/skills/skills/langbot-testing/suites/langbot-performance-contract-gate.yaml @@ -0,0 +1,13 @@ +id: langbot-performance-contract-gate +title: "LangBot performance contract gate" +description: "Fast synthetic contract checks for performance metric accounting and non-destructive reliability fault taxonomy." +type: contract +priority: p1 +tags: + - performance + - reliability + - contract + - metrics +cases: + - langbot-overhead-accounting-contract + - langbot-fault-taxonomy-contract diff --git a/skills/skills/langbot-testing/suites/langbot-performance-reliability-gate.yaml b/skills/skills/langbot-testing/suites/langbot-performance-reliability-gate.yaml new file mode 100644 index 000000000..1e0d58d26 --- /dev/null +++ b/skills/skills/langbot-testing/suites/langbot-performance-reliability-gate.yaml @@ -0,0 +1,16 @@ +id: langbot-performance-reliability-gate +title: "LangBot performance and reliability starter gate" +description: "Starter gate for LangBot performance accounting, live backend control-plane latency, and non-destructive fault taxonomy checks." +type: reliability +priority: p1 +tags: + - performance + - reliability + - metrics + - chaos +cases: + - langbot-overhead-accounting-contract + - langbot-fault-taxonomy-contract + - langbot-live-backend-latency + - langbot-live-control-plane-api + - langbot-live-backend-log-health diff --git a/skills/skills/langbot-testing/suites/langbot-user-path-performance-gate.yaml b/skills/skills/langbot-testing/suites/langbot-user-path-performance-gate.yaml new file mode 100644 index 000000000..a6a138ec0 --- /dev/null +++ b/skills/skills/langbot-testing/suites/langbot-user-path-performance-gate.yaml @@ -0,0 +1,12 @@ +id: langbot-user-path-performance-gate +title: "LangBot user-path performance gate" +description: "Browser-visible performance checks for user-facing LangBot paths such as Pipeline Debug Chat." +type: performance +priority: p1 +tags: + - performance + - browser + - debug-chat + - user-path +cases: + - pipeline-debug-chat-performance diff --git a/skills/skills/langbot-testing/troubleshooting/telemetry-proxy-noise.yaml b/skills/skills/langbot-testing/troubleshooting/telemetry-proxy-noise.yaml new file mode 100644 index 000000000..945109029 --- /dev/null +++ b/skills/skills/langbot-testing/troubleshooting/telemetry-proxy-noise.yaml @@ -0,0 +1,23 @@ +id: telemetry-proxy-noise +title: "Telemetry posting fails through the proxy while the target flow succeeds" +date: 2026-06-25 +category: env_issue +symptoms: + - "The target Debug Chat or provider smoke request completes successfully." + - "The same log window contains a Traceback for telemetry posting." + - "The traceback references the Space telemetry endpoint." +patterns: + - "Failed to post telemetry" + - "https://space.langbot.app/api/v1/telemetry" + - "httpx.ConnectError" +likely_causes: + - "The backend process inherited proxy settings that are required for model/provider access but unreliable for telemetry posting." + - "The telemetry endpoint is temporarily unreachable through the local proxy route." + - "TLS or proxy negotiation failed for the non-critical telemetry request." +fix_steps: + - "Keep the proxy configuration needed for model/provider access; do not clear it only to hide telemetry noise." + - "Check that uppercase and lowercase proxy variables are consistent before rerunning a live Space smoke." + - "Classify the target flow and log-health result separately: a successful Debug Chat run can still have an environment log-health finding." +verification: "A rerun shows the target case success patterns and no telemetry Traceback in the scanned log window, or the report explicitly records the telemetry issue as environment noise." +related_cases: + - langbot-space-debug-chat-concurrency-smoke diff --git a/skills/src/commands/env.ts b/skills/src/commands/env.ts index d5d1eeaf4..76ef33aec 100644 --- a/skills/src/commands/env.ts +++ b/skills/src/commands/env.ts @@ -1,5 +1,7 @@ import { existsSync } from "node:fs"; +import { spawnSync } from "node:child_process"; import { Socket } from "node:net"; +import { join } from "node:path"; import type { CommandContext } from "../types.ts"; import { parseOptions } from "../cli.ts"; import { loadEnv } from "../fs.ts"; @@ -88,6 +90,37 @@ function compareProxyPair(env: Record, upper: string, lower: str return null; } +function envValue(env: Record, key: string): string { + return process.env[key] ?? env[key] ?? ""; +} + +function activeSocksProxy(env: Record): { key: string; value: string } | null { + for (const key of ["ALL_PROXY", "all_proxy", "HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"]) { + const value = envValue(env, key); + if (/^socks/i.test(value)) return { key, value }; + } + return null; +} + +function checkSocksio(env: Record): string | null { + const proxy = activeSocksProxy(env); + if (!proxy) return null; + + const repo = env.LANGBOT_REPO; + const python = repo ? join(repo, ".venv", "bin", "python") : ""; + if (!python || !existsSync(python)) { + return `SOCKS proxy ${proxy.key} is configured (${redactEnvValue(proxy.key, proxy.value)}), but LangBot venv python was not found; after creating the venv, verify it can import socksio.`; + } + + const result = spawnSync(python, ["-c", "import socksio"], { + encoding: "utf8", + timeout: 5000, + }); + if (result.status === 0) return null; + + return `SOCKS proxy ${proxy.key} is configured (${redactEnvValue(proxy.key, proxy.value)}), but ${python} cannot import socksio; run \`${python} -m pip install socksio\` or start LangBot without SOCKS proxy env.`; +} + export async function commandEnvDoctor(ctx: CommandContext): Promise { const env = loadEnv(ctx.root); const failures: string[] = []; @@ -117,6 +150,8 @@ export async function commandEnvDoctor(ctx: CommandContext): Promise { ]) { if (mismatch) failures.push(mismatch); } + const socksioFailure = checkSocksio(env); + if (socksioFailure) failures.push(socksioFailure); for (const [label, result] of await Promise.all([ checkUrl("LANGBOT_BACKEND_URL", env.LANGBOT_BACKEND_URL).then((result) => ["LANGBOT_BACKEND_URL", result] as const), diff --git a/skills/src/commands/suite.ts b/skills/src/commands/suite.ts index 403156100..7ab556c5b 100644 --- a/skills/src/commands/suite.ts +++ b/skills/src/commands/suite.ts @@ -465,6 +465,41 @@ function outputTail(value: string | Buffer | null | undefined): string { return String(value ?? "").trim().slice(-4000); } +function exitStatusFromResultStatus(status: string): number { + if (status === "pass") return 0; + if (status === "blocked" || status === "env_issue" || status === "flaky") return 2; + return 1; +} + +function executionStatusFromExitStatus(status: number): string { + if (status === 0) return "ok"; + if (status === 2) return "classified"; + return "nonzero"; +} + +function executionFromCaseResultFile(caseItem: Record): Record | null { + const resultPath = join(String(caseItem.evidence_dir), "result.json"); + if (!existsSync(resultPath)) return null; + try { + const parsed = JSON.parse(readFileSync(resultPath, "utf8")) as Record; + if ( + parsed.case_id !== caseItem.id || + parsed.run_id !== caseItem.run_id || + typeof parsed.status !== "string" + ) return null; + const exitStatus = exitStatusFromResultStatus(parsed.status); + return { + status: executionStatusFromExitStatus(exitStatus), + exit_status: exitStatus, + reason: typeof parsed.reason === "string" ? parsed.reason : "result.json completed", + result_status: parsed.status, + result_json: resultPath, + }; + } catch { + return null; + } +} + function executionProblemStatus(executions: Array>): string { const statuses = executions.map((item) => String(item.status)); if (statuses.includes("nonzero")) return "fail"; @@ -523,12 +558,18 @@ export function commandSuiteRun(ctx: CommandContext): number { encoding: "utf8", stdio: options.json === true ? "pipe" : "inherit", }); - const status = result.error ? 1 : result.status ?? 1; + const fileExecution = result.error ? executionFromCaseResultFile(caseItem) : null; + const status = typeof fileExecution?.exit_status === "number" + ? fileExecution.exit_status + : result.error ? 1 : result.status ?? 1; executions.push({ id: caseItem.id, - status: status === 0 ? "ok" : "nonzero", + status: fileExecution?.status ?? executionStatusFromExitStatus(status), exit_status: status, - reason: result.error?.message || "", + reason: fileExecution?.reason ?? result.error?.message ?? "", + result_status: fileExecution?.result_status, + result_json: fileExecution?.result_json, + spawn_error: fileExecution && result.error ? result.error.message : undefined, stdout: outputTail(result.stdout), stderr: outputTail(result.stderr), }); diff --git a/skills/src/commands/test.ts b/skills/src/commands/test.ts index 2cce7a1e5..67ddc3122 100644 --- a/skills/src/commands/test.ts +++ b/skills/src/commands/test.ts @@ -271,7 +271,7 @@ function reportTemplate(mode: string): Record { target_tested: "Probe target, endpoint, file, command, or service actually checked", execution_path: "automation script | shell command | direct API | other", probe_result: "What the probe observed", - logs_or_artifacts: "Log, filesystem, API, or other artifact paths collected", + metrics_or_artifacts: "Metrics, logs, filesystem artifacts, traces, or profiles collected", diagnostics: "Extra diagnostics used, if any", matched_troubleshooting: "Troubleshooting ids matched, if any", assets_to_update: "New case/reference/troubleshooting entries to add", @@ -320,7 +320,7 @@ function manualEvidenceTemplate(mode: string): ManualEvidenceTemplate { target_tested: "TODO: probe target, endpoint, file, command, or service actually checked", execution_path: "TODO: automation script | shell command | direct API | other", probe_result: "TODO: observed probe result", - logs_or_artifacts: "TODO: evidence paths or skipped reason", + metrics_or_artifacts: "TODO: metrics, logs, filesystem artifacts, traces, or profiles collected", diagnostics: "TODO: additional diagnostics used, if any", matched_troubleshooting: "TODO: troubleshooting ids matched, if any", assets_to_update: "TODO: case/reference/troubleshooting updates to make", @@ -1099,6 +1099,41 @@ function executionTail(value: string | Buffer | null | undefined): string { return String(value ?? "").trim().slice(-4000); } +function exitStatusFromResultStatus(status: string): number { + if (status === "pass") return 0; + if (status === "blocked" || status === "env_issue" || status === "flaky") return 2; + return 1; +} + +function executionStatusFromExitStatus(status: number): string { + if (status === 0) return "ok"; + if (status === 2) return "classified"; + return "nonzero"; +} + +function executionFromAutomationResultFile( + evidenceDir: string, + caseId: string, + runId: string, +): { status: string; exit_status: number; reason: string; result_status: string; path: string } | null { + const resultPath = join(evidenceDir, "automation-result.json"); + if (!existsSync(resultPath)) return null; + try { + const parsed = JSON.parse(readFileSync(resultPath, "utf8")) as Record; + if (parsed.case_id !== caseId || parsed.run_id !== runId || typeof parsed.status !== "string") return null; + const exitStatus = exitStatusFromResultStatus(parsed.status); + return { + status: executionStatusFromExitStatus(exitStatus), + exit_status: exitStatus, + reason: typeof parsed.reason === "string" ? parsed.reason : "automation-result.json completed", + result_status: parsed.status, + path: resultPath, + }; + } catch { + return null; + } +} + function runSetupAutomation( ctx: CommandContext, item: StructuredItem, @@ -1224,6 +1259,30 @@ export function commandTestRun(ctx: CommandContext): number { }); if (result.error) { + const fileExecution = executionFromAutomationResultFile( + run.automation.evidence_dir, + String(run.case.id), + run.run_id, + ); + if (fileExecution) { + if (options.json !== true) { + console.error(`WARN: automation spawn reported an error, but ${fileExecution.path} completed: ${result.error.message}`); + } + if (options.json === true) { + console.log(JSON.stringify({ + run, + setup_executions: setupExecutions, + automation_execution: { + ...fileExecution, + spawn_error: result.error.message, + stdout: executionTail(result.stdout), + stderr: executionTail(result.stderr), + }, + exit_status: fileExecution.exit_status, + }, null, 2)); + } + return fileExecution.exit_status; + } if (options.json !== true) console.error(`ERROR: failed to run automation: ${result.error.message}`); if (options.json === true) { console.log(JSON.stringify({ @@ -1247,7 +1306,7 @@ export function commandTestRun(ctx: CommandContext): number { run, setup_executions: setupExecutions, automation_execution: { - status: status === 0 ? "ok" : "nonzero", + status: executionStatusFromExitStatus(status), exit_status: status, stdout: executionTail(result.stdout), stderr: executionTail(result.stderr), @@ -1311,6 +1370,7 @@ function renderMarkdownReport(report: TestReport): string { const environment = report.environment; const logGuard = report.log_guard; const troubleshooting = report.troubleshooting; + const automation = report.automation_result; const lines: string[] = []; lines.push(`# Test Report: ${reportCase.id}`); @@ -1323,20 +1383,41 @@ function renderMarkdownReport(report: TestReport): string { lines.push(`Type: ${reportCase.type}`); lines.push(""); lines.push("## Result"); - lines.push(`- result: ${evidence.result}`); - for (const [key, value] of Object.entries(evidence)) { - if (key !== "result") lines.push(`- ${key}: ${value}`); + if (automation.status === "loaded" && automation.result) { + lines.push(`- result: ${automation.result}`); + if (automation.reason) lines.push(`- reason: ${automation.reason}`); + if (automation.url) lines.push(`- target_tested: ${automation.url}`); + if (automation.path) lines.push(`- automation_result: ${automation.path}`); + if (automation.artifacts) lines.push(`- artifacts: ${JSON.stringify(automation.artifacts)}`); + } else { + lines.push(`- result: ${evidence.result}`); + for (const [key, value] of Object.entries(evidence)) { + if (key !== "result") lines.push(`- ${key}: ${value}`); + } } lines.push(""); lines.push("## Automation Result"); - lines.push(`- status: ${report.automation_result.status}`); - if (report.automation_result.path) lines.push(`- path: ${report.automation_result.path}`); - if (report.automation_result.result) lines.push(`- result: ${report.automation_result.result}`); - if (report.automation_result.reason) lines.push(`- reason: ${report.automation_result.reason}`); - if (report.automation_result.started_at_local) lines.push(`- started_at_local: ${report.automation_result.started_at_local}`); - if (report.automation_result.finished_at_local) lines.push(`- finished_at_local: ${report.automation_result.finished_at_local}`); - if (report.automation_result.url) lines.push(`- url: ${report.automation_result.url}`); - if (report.automation_result.expected_text) lines.push(`- expected_text: ${report.automation_result.expected_text}`); + lines.push(`- status: ${automation.status}`); + if (automation.path) lines.push(`- path: ${automation.path}`); + if (automation.result) lines.push(`- result: ${automation.result}`); + if (automation.reason) lines.push(`- reason: ${automation.reason}`); + if (automation.duration_ms !== undefined) lines.push(`- duration_ms: ${automation.duration_ms}`); + if (automation.started_at_local) lines.push(`- started_at_local: ${automation.started_at_local}`); + if (automation.finished_at_local) lines.push(`- finished_at_local: ${automation.finished_at_local}`); + if (automation.url) lines.push(`- url: ${automation.url}`); + if (automation.expected_text) lines.push(`- expected_text: ${automation.expected_text}`); + if (automation.metrics_summary) { + lines.push("- metrics_summary:"); + lines.push(` ${JSON.stringify(automation.metrics_summary)}`); + } + if (automation.thresholds_summary) { + lines.push("- thresholds_summary:"); + lines.push(` ${JSON.stringify(automation.thresholds_summary)}`); + } + if (automation.artifacts) { + lines.push("- artifacts:"); + lines.push(` ${JSON.stringify(automation.artifacts)}`); + } lines.push(""); lines.push("## Environment"); for (const [key, value] of Object.entries(environment)) lines.push(`- ${key}=${value}`); diff --git a/skills/src/commands/validate.ts b/skills/src/commands/validate.ts index 8b15d6344..590032ef8 100644 --- a/skills/src/commands/validate.ts +++ b/skills/src/commands/validate.ts @@ -126,6 +126,9 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set ( validateSetupAutomationEntry(root, entry, caseIds).map((error) => `${item.path}: ${error}`) )), @@ -183,10 +186,62 @@ function validateCaseItem(root: string, item: StructuredItem, skillNames: Set 599) { + errors.push(`${item.path}: 'automation_fake_provider_fault_status' must be an HTTP 4xx or 5xx status string`); + } + } const streamOutput = scalar(item.fields, "automation_stream_output"); if (streamOutput && !["0", "1", "false", "true"].includes(streamOutput)) { errors.push(`${item.path}: 'automation_stream_output' must be one of 0, 1, false, or true`); } + for (const key of [ + "automation_debug_chat_load_stream", + "automation_debug_chat_load_reset", + "automation_debug_chat_load_fail_on_final_mismatch", + "automation_fake_provider_fail_after_first_chunk", + "automation_fake_provider_dynamic_response", + ]) { + const value = scalar(item.fields, key); + if (value && !["0", "1", "false", "true"].includes(value)) { + errors.push(`${item.path}: '${key}' must be one of 0, 1, false, or true`); + } + } const imageBase64Fixture = scalar(item.fields, "automation_image_base64_fixture"); if (imageBase64Fixture && !existsSync(join(root, imageBase64Fixture))) { errors.push(`${item.path}: automation image fixture does not exist: ${imageBase64Fixture}`); diff --git a/skills/src/constants.ts b/skills/src/constants.ts index 015a9bd39..5cfe37f8a 100644 --- a/skills/src/constants.ts +++ b/skills/src/constants.ts @@ -9,7 +9,18 @@ export const requiredEnvKeys = [ ]; export const caseModeValues = ["agent-browser", "probe"]; -export const caseTypeValues = ["smoke", "regression", "feature", "provider", "exploratory"]; +export const caseTypeValues = [ + "smoke", + "regression", + "feature", + "provider", + "exploratory", + "contract", + "performance", + "reliability", + "chaos", + "security", +]; export const casePriorityValues = ["p0", "p1", "p2"]; export const caseRiskValues = ["low", "medium", "high"]; export const caseEvidenceValues = [ @@ -21,10 +32,24 @@ export const caseEvidenceValues = [ "frontend_log", "api_diagnostic", "filesystem", + "metrics", + "trace", + "profile", + "resource_log", ]; export const testResultStatusValues = ["pass", "fail", "blocked", "env_issue", "flaky"]; export const troubleshootingCategoryValues = ["product", "env_issue", "external_dependency", "blocked", "flaky"]; -export const suiteTypeValues = ["smoke", "regression", "release_gate", "exploratory"]; +export const suiteTypeValues = [ + "smoke", + "regression", + "release_gate", + "exploratory", + "contract", + "performance", + "reliability", + "chaos", + "security", +]; export const suiteRequiredStrings = ["id", "title", "description", "type", "priority"]; export const suiteRequiredLists = ["tags", "cases"]; diff --git a/skills/src/log-guard.ts b/skills/src/log-guard.ts index 253cb229e..6f7f541a7 100644 --- a/skills/src/log-guard.ts +++ b/skills/src/log-guard.ts @@ -91,6 +91,7 @@ export type AutomationResultEvidence = { path?: string; result?: string; reason?: string; + duration_ms?: number; started_at?: string; started_at_local?: string; finished_at?: string; @@ -98,6 +99,9 @@ export type AutomationResultEvidence = { url?: string; prompt?: string; expected_text?: string; + metrics_summary?: Record; + thresholds_summary?: Record; + artifacts?: Record; }; type MutableScanState = { @@ -594,6 +598,18 @@ function stringField(data: Record, key: string): string | undef return typeof value === "string" && value.trim() ? value : undefined; } +function numberField(data: Record, key: string): number | undefined { + const value = data[key]; + return typeof value === "number" && Number.isFinite(value) ? value : undefined; +} + +function objectField(data: Record, key: string): Record | undefined { + const value = data[key]; + return value && typeof value === "object" && !Array.isArray(value) + ? value as Record + : undefined; +} + function evidenceDirFromOptions(options: Record): string | undefined { const explicit = typeof options["evidence-dir"] === "string" ? options["evidence-dir"] : undefined; if (explicit) return resolve(explicit); @@ -628,6 +644,7 @@ export function readAutomationResultEvidence(options: Record number): { code: number; output: string } { } } -function captureAll(fn: () => number): { code: number; output: string; error: string } { +function captureAll(fn: () => number): { + code: number; + output: string; + error: string; +} { const originalLog = console.log; const originalWrite = process.stderr.write; const lines: string[] = []; @@ -76,7 +115,12 @@ function captureAll(fn: () => number): { code: number; output: string; error: st } } -function suiteResult(caseId: string, runId: string, status = "pass", evidence = ["ui", "screenshot", "console", "backend_log"]): string { +function suiteResult( + caseId: string, + runId: string, + status = "pass", + evidence = ["ui", "screenshot", "console", "backend_log"], +): string { return JSON.stringify({ source: "final", case_id: caseId, @@ -90,7 +134,9 @@ function suiteResult(caseId: string, runId: string, status = "pass", evidence = } function withEnv(values: Record, fn: () => T): T { - const previous = new Map(Object.keys(values).map((key) => [key, process.env[key]])); + const previous = new Map( + Object.keys(values).map((key) => [key, process.env[key]]), + ); try { for (const [key, value] of Object.entries(values)) process.env[key] = value; return fn(); @@ -102,7 +148,9 @@ function withEnv(values: Record, fn: () => T): T { } } -async function captureAsync(fn: () => Promise): Promise<{ code: number; output: string }> { +async function captureAsync( + fn: () => Promise, +): Promise<{ code: number; output: string }> { const originalLog = console.log; const lines: string[] = []; console.log = (...args: unknown[]) => { @@ -130,10 +178,18 @@ test("validate allows blank shared env values but requires declared keys", () => const testingDir = join(skillsDir, "langbot-testing"); mkdirSync(schemasDir, { recursive: true }); mkdirSync(testingDir, { recursive: true }); - for (const schemaName of ["case.schema.json", "suite.schema.json", "troubleshooting.schema.json", "skill-index.schema.json"]) { + for (const schemaName of [ + "case.schema.json", + "suite.schema.json", + "troubleshooting.schema.json", + "skill-index.schema.json", + ]) { writeFileSync(join(schemasDir, schemaName), "{}"); } - writeFileSync(join(testingDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(testingDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); const envText = [ "LANGBOT_FRONTEND_URL=http://127.0.0.1:3000", "LANGBOT_BACKEND_URL=http://127.0.0.1:5300", @@ -158,23 +214,48 @@ test("validate allows blank shared env values but requires declared keys", () => test("index includes case summaries for agent discovery", () => { const result = capture(() => commandIndex({ root, args: ["index"] })); assert.equal(result.code, 0); - const index = JSON.parse(readFileSync(join(root, "skills.index.json"), "utf8")); - const testing = index.skills.find((skill: { name: string }) => skill.name === "langbot-testing"); + const index = JSON.parse( + readFileSync(join(root, "skills.index.json"), "utf8"), + ); + const testing = index.skills.find( + (skill: { name: string }) => skill.name === "langbot-testing", + ); assert.ok(testing); - assert.ok(testing.case_summaries.some((item: { id: string; priority: string; evidence_required: string[] }) => ( - item.id === "pipeline-debug-chat" && item.priority === "p0" && item.evidence_required.includes("backend_log") - ))); - assert.ok(testing.case_summaries.some((item: { id: string; setup_automation: string[]; setup_provides_env: string[] }) => ( - item.id === "agent-runner-qa-debug-chat" && - item.setup_automation.includes("case:agent-runner-live-install") && - item.setup_provides_env.includes("LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL") - ))); - assert.ok(testing.suite_summaries.some((item: { id: string; cases: string[] }) => ( - item.id === "core-smoke" && item.cases.includes("pipeline-debug-chat") - ))); - assert.ok(testing.fixtures.some((item: { id: string; related_cases: string[] }) => ( - item.id === "mcp-stdio-echo-server" && item.related_cases.includes("mcp-stdio-tool-call") - ))); + assert.ok( + testing.case_summaries.some( + (item: { id: string; priority: string; evidence_required: string[] }) => + item.id === "pipeline-debug-chat" && + item.priority === "p0" && + item.evidence_required.includes("backend_log"), + ), + ); + assert.ok( + testing.case_summaries.some( + (item: { + id: string; + setup_automation: string[]; + setup_provides_env: string[]; + }) => + item.id === "agent-runner-qa-debug-chat" && + item.setup_automation.includes("case:agent-runner-live-install") && + item.setup_provides_env.includes( + "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL", + ), + ), + ); + assert.ok( + testing.suite_summaries.some( + (item: { id: string; cases: string[] }) => + item.id === "core-smoke" && item.cases.includes("pipeline-debug-chat"), + ), + ); + assert.ok( + testing.fixtures.some( + (item: { id: string; related_cases: string[] }) => + item.id === "mcp-stdio-echo-server" && + item.related_cases.includes("mcp-stdio-tool-call"), + ), + ); }); test("index check detects stale index without writing", () => { @@ -184,12 +265,16 @@ test("index check detects stale index without writing", () => { const fresh = readFileSync(path, "utf8"); try { - const ok = capture(() => commandIndex({ root, args: ["index", "--check"] })); + const ok = capture(() => + commandIndex({ root, args: ["index", "--check"] }), + ); assert.equal(ok.code, 0); assert.match(ok.output, /^OK /); writeFileSync(path, "{}\n"); - const stale = captureAll(() => commandIndex({ root, args: ["index", "--check"] })); + const stale = captureAll(() => + commandIndex({ root, args: ["index", "--check"] }), + ); assert.equal(stale.code, 1); assert.match(stale.error, /index is stale/); assert.equal(readFileSync(path, "utf8"), "{}\n"); @@ -207,22 +292,24 @@ test("case list exposes seeded QA cases", () => { }); test("case list JSON filters by reusable agent-selection metadata", () => { - const result = capture(() => commandCaseList(ctx([ - "case", - "list", - "--json", - "--priority", - "p0", - "--automation", - ]))); + const result = capture(() => + commandCaseList( + ctx(["case", "list", "--json", "--priority", "p0", "--automation"]), + ), + ); assert.equal(result.code, 0); const rows = JSON.parse(result.output); assert.ok(rows.length >= 2); assert.ok(rows.every((row: { priority: string }) => row.priority === "p0")); assert.ok(rows.every((row: { automation: string }) => row.automation)); - assert.ok(rows.some((row: { id: string; evidence_required: string[]; readiness: string }) => ( - row.id === "pipeline-debug-chat" && row.evidence_required.includes("backend_log") && row.readiness - ))); + assert.ok( + rows.some( + (row: { id: string; evidence_required: string[]; readiness: string }) => + row.id === "pipeline-debug-chat" && + row.evidence_required.includes("backend_log") && + row.readiness, + ), + ); }); test("case list distinguishes machine readiness from manual precondition checks", () => { @@ -234,7 +321,10 @@ test("case list distinguishes machine readiness from manual precondition checks" join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", ); - writeFileSync(join(tmp, "skills", ".env"), "LANGBOT_FRONTEND_URL=http://127.0.0.1:3000\n"); + writeFileSync( + join(tmp, "skills", ".env"), + "LANGBOT_FRONTEND_URL=http://127.0.0.1:3000\n", + ); writeFileSync( join(skillDir, "cases", "manual-case.yaml"), [ @@ -263,12 +353,16 @@ test("case list distinguishes machine readiness from manual precondition checks" ].join("\n"), ); - const machineReady = capture(() => commandCaseList({ root: tmp, args: ["case", "list", "--machine-ready"] })); + const machineReady = capture(() => + commandCaseList({ root: tmp, args: ["case", "list", "--machine-ready"] }), + ); assert.equal(machineReady.code, 0); assert.match(machineReady.output, /manual-case/); assert.match(machineReady.output, /manual-check/); - const ready = capture(() => commandCaseList({ root: tmp, args: ["case", "list", "--ready"] })); + const ready = capture(() => + commandCaseList({ root: tmp, args: ["case", "list", "--ready"] }), + ); assert.equal(ready.code, 0); assert.doesNotMatch(ready.output, /manual-case/); } finally { @@ -277,7 +371,9 @@ test("case list distinguishes machine readiness from manual precondition checks" }); test("case show prints structured agent-browser case", () => { - const result = capture(() => commandCaseShow(ctx(["case", "show", "pipeline-debug-chat"]))); + const result = capture(() => + commandCaseShow(ctx(["case", "show", "pipeline-debug-chat"])), + ); assert.equal(result.code, 0); assert.match(result.output, /^id: pipeline-debug-chat/m); assert.match(result.output, /^mode: agent-browser/m); @@ -294,10 +390,12 @@ test("case new writes required selection metadata", () => { "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", ); - const result = capture(() => commandCaseNew({ - root: tmp, - args: ["case", "new", "new-case", "--title", "New Case"], - })); + const result = capture(() => + commandCaseNew({ + root: tmp, + args: ["case", "new", "new-case", "--title", "New Case"], + }), + ); assert.equal(result.code, 0); const text = readFileSync(join(skillDir, "cases", "new-case.yaml"), "utf8"); @@ -311,34 +409,59 @@ test("case new writes required selection metadata", () => { }); test("suite list and plan expose reusable case groups", () => { - const list = capture(() => commandSuiteList(ctx(["suite", "list", "--json", "--priority", "p0"]))); + const list = capture(() => + commandSuiteList(ctx(["suite", "list", "--json", "--priority", "p0"])), + ); assert.equal(list.code, 0); const suites = JSON.parse(list.output); - assert.ok(suites.some((suite: { id: string; cases: string[] }) => ( - suite.id === "core-smoke" && suite.cases.includes("webui-login-state") - ))); + assert.ok( + suites.some( + (suite: { id: string; cases: string[] }) => + suite.id === "core-smoke" && suite.cases.includes("webui-login-state"), + ), + ); - const plan = capture(() => commandSuitePlan(ctx(["suite", "plan", "core-smoke", "--json"]))); + const plan = capture(() => + commandSuitePlan(ctx(["suite", "plan", "core-smoke", "--json"])), + ); assert.equal(plan.code, 0); const suitePlan = JSON.parse(plan.output); assert.equal(suitePlan.id, "core-smoke"); - assert.ok(suitePlan.cases.some((item: { id: string; evidence_required: string[] }) => ( - item.id === "pipeline-debug-chat" && item.evidence_required.includes("backend_log") - ))); - assert.ok(suitePlan.commands.some((item: { id: string; automation: string }) => ( - item.id === "pipeline-debug-chat" && item.automation.includes("test run") - ))); + assert.ok( + suitePlan.cases.some( + (item: { id: string; evidence_required: string[] }) => + item.id === "pipeline-debug-chat" && + item.evidence_required.includes("backend_log"), + ), + ); + assert.ok( + suitePlan.commands.some( + (item: { id: string; automation: string }) => + item.id === "pipeline-debug-chat" && + item.automation.includes("test run"), + ), + ); - const localAgent = capture(() => commandSuitePlan(ctx(["suite", "plan", "local-agent-gate", "--json"]))); + const localAgent = capture(() => + commandSuitePlan(ctx(["suite", "plan", "local-agent-gate", "--json"])), + ); assert.equal(localAgent.code, 0); const localAgentPlan = JSON.parse(localAgent.output); - assert.ok(["ready", "missing", "manual_check"].includes(localAgentPlan.readiness.status)); - const basic = localAgentPlan.cases.find((item: { id: string }) => item.id === "local-agent-basic-debug-chat"); + assert.ok( + ["ready", "missing", "manual_check"].includes( + localAgentPlan.readiness.status, + ), + ); + const basic = localAgentPlan.cases.find( + (item: { id: string }) => item.id === "local-agent-basic-debug-chat", + ); assert.equal(basic.automation_readiness.pipeline_env_required, true); }); test("suite show prints structured suite YAML", () => { - const result = capture(() => commandSuiteShow(ctx(["suite", "show", "local-agent-gate"]))); + const result = capture(() => + commandSuiteShow(ctx(["suite", "show", "local-agent-gate"])), + ); assert.equal(result.code, 0); assert.match(result.output, /^id: local-agent-gate/m); assert.match(result.output, /^cases:/m); @@ -349,16 +472,20 @@ test("suite start creates a run handoff with per-case evidence commands", () => const tmp = mkdtempSync(join(tmpdir(), "lbs-suite-start-")); try { const evidenceRoot = join(tmp, "evidence"); - const result = capture(() => commandSuiteStart(ctx([ - "suite", - "start", - "core-smoke", - "--run-id", - "core-smoke-local", - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteStart( + ctx([ + "suite", + "start", + "core-smoke", + "--run-id", + "core-smoke-local", + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const start = JSON.parse(result.output); assert.equal(start.suite.id, "core-smoke"); @@ -369,12 +496,23 @@ test("suite start creates a run handoff with per-case evidence commands", () => assert.match(start.report_command, /bin\/lbs suite report core-smoke/); assert.ok(existsSync(join(evidenceRoot, "suite-start.json"))); assert.ok(existsSync(join(evidenceRoot, "suite-start.md"))); - const pipeline = start.cases.find((item: { id: string }) => item.id === "pipeline-debug-chat"); + const pipeline = start.cases.find( + (item: { id: string }) => item.id === "pipeline-debug-chat", + ); assert.ok(pipeline); assert.ok(existsSync(join(evidenceRoot, "pipeline-debug-chat"))); - assert.match(pipeline.automation_command, /bin\/lbs test run pipeline-debug-chat/); - assert.match(pipeline.report_command, /--evidence-dir .+pipeline-debug-chat/); - assert.match(pipeline.result_command_template, /bin\/lbs test result pipeline-debug-chat/); + assert.match( + pipeline.automation_command, + /bin\/lbs test run pipeline-debug-chat/, + ); + assert.match( + pipeline.report_command, + /--evidence-dir .+pipeline-debug-chat/, + ); + assert.match( + pipeline.result_command_template, + /bin\/lbs test result pipeline-debug-chat/, + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -398,25 +536,33 @@ test("suite report aggregates case result JSON files", () => { ); } - const result = capture(() => commandSuiteReport(ctx([ - "suite", - "report", - "core-smoke", - "--run-id", - runId, - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteReport( + ctx([ + "suite", + "report", + "core-smoke", + "--run-id", + runId, + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "env_issue"); assert.equal(report.counts.pass, 2); assert.equal(report.counts.env_issue, 1); - assert.ok(report.cases.some((item: { id: string; result: { status: string } }) => ( - item.id === "local-agent-basic-debug-chat" && item.result.status === "env_issue" - ))); + assert.ok( + report.cases.some( + (item: { id: string; result: { status: string } }) => + item.id === "local-agent-basic-debug-chat" && + item.result.status === "env_issue", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -427,7 +573,11 @@ test("suite report treats pass without required evidence as incomplete", () => { try { const evidenceRoot = join(tmp, "suite-evidence"); const runId = "suite-report-evidence"; - for (const caseId of ["webui-login-state", "pipeline-debug-chat", "local-agent-basic-debug-chat"]) { + for (const caseId of [ + "webui-login-state", + "pipeline-debug-chat", + "local-agent-basic-debug-chat", + ]) { const dir = join(evidenceRoot, caseId); mkdirSync(dir, { recursive: true }); writeFileSync( @@ -436,23 +586,31 @@ test("suite report treats pass without required evidence as incomplete", () => { ); } - const result = capture(() => commandSuiteReport(ctx([ - "suite", - "report", - "core-smoke", - "--run-id", - runId, - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteReport( + ctx([ + "suite", + "report", + "core-smoke", + "--run-id", + runId, + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "incomplete"); - assert.ok(report.cases.some((item: { id: string; result: { evidence_missing: string[] } }) => ( - item.id === "pipeline-debug-chat" && item.result.evidence_missing.includes("backend_log") - ))); + assert.ok( + report.cases.some( + (item: { id: string; result: { evidence_missing: string[] } }) => + item.id === "pipeline-debug-chat" && + item.result.evidence_missing.includes("backend_log"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -464,25 +622,35 @@ test("suite report marks missing case evidence as incomplete", () => { const evidenceRoot = join(tmp, "suite-evidence"); const runId = "suite-report-missing"; mkdirSync(join(evidenceRoot, "webui-login-state"), { recursive: true }); - writeFileSync(join(evidenceRoot, "webui-login-state", "result.json"), suiteResult("webui-login-state", runId, "pass")); + writeFileSync( + join(evidenceRoot, "webui-login-state", "result.json"), + suiteResult("webui-login-state", runId, "pass"), + ); - const result = capture(() => commandSuiteReport(ctx([ - "suite", - "report", - "core-smoke", - "--run-id", - runId, - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteReport( + ctx([ + "suite", + "report", + "core-smoke", + "--run-id", + runId, + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "incomplete"); - assert.ok(report.cases.some((item: { id: string; result: { status: string } }) => ( - item.id === "pipeline-debug-chat" && item.result.status === "missing" - ))); + assert.ok( + report.cases.some( + (item: { id: string; result: { status: string } }) => + item.id === "pipeline-debug-chat" && item.result.status === "missing", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -493,34 +661,61 @@ test("suite report rejects result files from the wrong case or run", () => { try { const evidenceRoot = join(tmp, "suite-evidence"); const runId = "suite-report-mismatch"; - for (const caseId of ["webui-login-state", "pipeline-debug-chat", "local-agent-basic-debug-chat"]) { + for (const caseId of [ + "webui-login-state", + "pipeline-debug-chat", + "local-agent-basic-debug-chat", + ]) { const dir = join(evidenceRoot, caseId); mkdirSync(dir, { recursive: true }); - writeFileSync(join(dir, "result.json"), suiteResult(caseId, runId, "pass")); + writeFileSync( + join(dir, "result.json"), + suiteResult(caseId, runId, "pass"), + ); } - writeFileSync(join(evidenceRoot, "pipeline-debug-chat", "result.json"), suiteResult("webui-login-state", runId, "pass")); - writeFileSync(join(evidenceRoot, "local-agent-basic-debug-chat", "result.json"), suiteResult("local-agent-basic-debug-chat", "old-run", "pass")); + writeFileSync( + join(evidenceRoot, "pipeline-debug-chat", "result.json"), + suiteResult("webui-login-state", runId, "pass"), + ); + writeFileSync( + join(evidenceRoot, "local-agent-basic-debug-chat", "result.json"), + suiteResult("local-agent-basic-debug-chat", "old-run", "pass"), + ); - const result = capture(() => commandSuiteReport(ctx([ - "suite", - "report", - "core-smoke", - "--run-id", - runId, - "--evidence-dir", - evidenceRoot, - "--json", - ]))); + const result = capture(() => + commandSuiteReport( + ctx([ + "suite", + "report", + "core-smoke", + "--run-id", + runId, + "--evidence-dir", + evidenceRoot, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "fail"); - assert.ok(report.cases.some((item: { id: string; result: { status: string; reason: string } }) => ( - item.id === "pipeline-debug-chat" && item.result.status === "invalid" && item.result.reason.includes("case_id mismatch") - ))); - assert.ok(report.cases.some((item: { id: string; result: { status: string; reason: string } }) => ( - item.id === "local-agent-basic-debug-chat" && item.result.status === "invalid" && item.result.reason.includes("run_id mismatch") - ))); + assert.ok( + report.cases.some( + (item: { id: string; result: { status: string; reason: string } }) => + item.id === "pipeline-debug-chat" && + item.result.status === "invalid" && + item.result.reason.includes("case_id mismatch"), + ), + ); + assert.ok( + report.cases.some( + (item: { id: string; result: { status: string; reason: string } }) => + item.id === "local-agent-basic-debug-chat" && + item.result.status === "invalid" && + item.result.reason.includes("run_id mismatch"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -536,7 +731,10 @@ test("suite run executes automated cases and aggregates a verdict", () => { mkdirSync(casesDir, { recursive: true }); mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "one.yaml"), @@ -602,16 +800,30 @@ test("suite run executes automated cases and aggregates a verdict", () => { ].join("\n"), ); - const result = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "mini", "--run-id", "mini-run", "--evidence-dir", join(tmp, "evidence"), "--json"], - })); + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "mini", + "--run-id", + "mini-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); assert.equal(result.code, 0); const payload = JSON.parse(result.output); assert.equal(payload.report.status, "pass"); assert.equal(payload.report.counts.pass, 2); - assert.deepEqual(payload.executions.map((item: { status: string }) => item.status), ["ok", "ok"]); + assert.deepEqual( + payload.executions.map((item: { status: string }) => item.status), + ["ok", "ok"], + ); assert.ok(existsSync(join(tmp, "evidence", "one", "result.json"))); assert.ok(existsSync(join(tmp, "evidence", "two", "result.json"))); } finally { @@ -629,7 +841,10 @@ test("suite run JSON captures failed case output", () => { mkdirSync(casesDir, { recursive: true }); mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "fail-case.yaml"), @@ -659,12 +874,26 @@ test("suite run JSON captures failed case output", () => { " - fail-case", ].join("\n"), ); - writeFileSync(join(scriptsDir, "fail.mjs"), "console.error('child failure detail'); process.exit(1);\n"); + writeFileSync( + join(scriptsDir, "fail.mjs"), + "console.error('child failure detail'); process.exit(1);\n", + ); - const result = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "mini", "--run-id", "mini-run", "--evidence-dir", join(tmp, "evidence"), "--json"], - })); + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "mini", + "--run-id", + "mini-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); assert.equal(result.code, 1); const payload = JSON.parse(result.output); @@ -676,6 +905,96 @@ test("suite run JSON captures failed case output", () => { } }); +test("suite run preserves classified env_issue automation results", () => { + const tmp = mkdtempSync(join(tmpdir(), "lbs-suite-run-env-issue-")); + try { + const skillDir = join(tmp, "skills", "langbot-testing"); + const casesDir = join(skillDir, "cases"); + const suitesDir = join(skillDir, "suites"); + const scriptsDir = join(tmp, "scripts"); + mkdirSync(casesDir, { recursive: true }); + mkdirSync(suitesDir, { recursive: true }); + mkdirSync(scriptsDir, { recursive: true }); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); + writeFileSync(join(tmp, "skills", ".env"), ""); + writeFileSync( + join(casesDir, "env-case.yaml"), + [ + "id: env-case", + "title: Env Case", + "mode: probe", + "area: qa", + "type: smoke", + "priority: p2", + "risk: low", + "ci_eligible: true", + "automation: scripts/env-issue.mjs", + "evidence_required:", + " - filesystem", + ].join("\n"), + ); + writeFileSync( + join(suitesDir, "mini.yaml"), + [ + "id: mini", + "title: Mini", + "description: Mini suite.", + "type: smoke", + "priority: p2", + "tags:", + " - qa", + "cases:", + " - env-case", + ].join("\n"), + ); + writeFileSync( + join(scriptsDir, "env-issue.mjs"), + [ + "import { mkdirSync, writeFileSync } from 'node:fs';", + "import { join } from 'node:path';", + "mkdirSync(process.env.LBS_EVIDENCE_DIR, { recursive: true });", + "const result = {", + " case_id: process.env.LBS_CASE_ID,", + " run_id: process.env.LBS_RUN_ID,", + " status: 'env_issue',", + " reason: 'backend not reachable',", + " evidence_collected: ['filesystem']", + "};", + "writeFileSync(join(process.env.LBS_EVIDENCE_DIR, 'result.json'), JSON.stringify(result));", + "writeFileSync(join(process.env.LBS_EVIDENCE_DIR, 'automation-result.json'), JSON.stringify({ ...result, source: 'automation' }));", + "process.exit(2);", + ].join("\n"), + ); + + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "mini", + "--run-id", + "mini-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); + + assert.equal(result.code, 2); + const payload = JSON.parse(result.output); + assert.equal(payload.executions[0].status, "classified"); + assert.equal(payload.report.status, "env_issue"); + assert.equal(payload.report.execution_status, "ok"); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); + test("suite run failure cannot be masked by stale pass result", () => { const tmp = mkdtempSync(join(tmpdir(), "lbs-suite-run-stale-pass-")); try { @@ -688,7 +1007,10 @@ test("suite run failure cannot be masked by stale pass result", () => { mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); mkdirSync(join(evidenceDir, "fail-case"), { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "fail-case.yaml"), @@ -721,17 +1043,31 @@ test("suite run failure cannot be masked by stale pass result", () => { ].join("\n"), ); writeFileSync(join(scriptsDir, "fail.mjs"), "process.exit(1);\n"); - writeFileSync(join(evidenceDir, "fail-case", "result.json"), JSON.stringify({ - case_id: "fail-case", - run_id: "stale-run-fail-case", - status: "pass", - evidence_collected: ["filesystem"], - })); + writeFileSync( + join(evidenceDir, "fail-case", "result.json"), + JSON.stringify({ + case_id: "fail-case", + run_id: "stale-run-fail-case", + status: "pass", + evidence_collected: ["filesystem"], + }), + ); - const result = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "mini", "--run-id", "stale-run", "--evidence-dir", evidenceDir, "--json"], - })); + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "mini", + "--run-id", + "stale-run", + "--evidence-dir", + evidenceDir, + "--json", + ], + }), + ); assert.equal(result.code, 1); const payload = JSON.parse(result.output); @@ -753,7 +1089,10 @@ test("suite run dry-run plans automation without creating evidence", () => { mkdirSync(casesDir, { recursive: true }); mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "dry-case.yaml"), @@ -786,10 +1125,22 @@ test("suite run dry-run plans automation without creating evidence", () => { writeFileSync(join(scriptsDir, "fail-if-run.mjs"), "process.exit(9);\n"); const evidenceDir = join(tmp, "evidence"); - const result = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "dry-suite", "--run-id", "dry-run", "--evidence-dir", evidenceDir, "--dry-run", "--json"], - })); + const result = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "dry-suite", + "--run-id", + "dry-run", + "--evidence-dir", + evidenceDir, + "--dry-run", + "--json", + ], + }), + ); assert.equal(result.code, 0); const payload = JSON.parse(result.output); @@ -799,13 +1150,27 @@ test("suite run dry-run plans automation without creating evidence", () => { assert.equal(existsSync(evidenceDir), false); assert.equal(existsSync(join(tmp, "reports", "dry-run.md")), false); - const markdown = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "dry-suite", "--run-id", "dry-run-markdown", "--evidence-dir", join(tmp, "evidence-md"), "--dry-run"], - })); + const markdown = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "dry-suite", + "--run-id", + "dry-run-markdown", + "--evidence-dir", + join(tmp, "evidence-md"), + "--dry-run", + ], + }), + ); assert.equal(markdown.code, 0); assert.match(markdown.output, /# Suite Report: dry-suite/); - assert.equal(existsSync(join(tmp, "reports", "dry-run-markdown.md")), false); + assert.equal( + existsSync(join(tmp, "reports", "dry-run-markdown.md")), + false, + ); assert.equal(existsSync(join(tmp, "evidence-md")), false); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -822,7 +1187,10 @@ test("suite run skips manual-check cases unless explicitly included", () => { mkdirSync(casesDir, { recursive: true }); mkdirSync(suitesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "manual-case.yaml"), @@ -866,25 +1234,53 @@ test("suite run skips manual-check cases unless explicitly included", () => { ].join("\n"), ); - const skipped = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "manual-suite", "--run-id", "manual-run", "--evidence-dir", join(tmp, "evidence"), "--json"], - })); + const skipped = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "manual-suite", + "--run-id", + "manual-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); assert.equal(skipped.code, 1); const skippedPayload = JSON.parse(skipped.output); assert.equal(skippedPayload.executions[0].status, "skipped"); assert.match(skippedPayload.executions[0].reason, /manual_check/); - assert.equal(existsSync(join(tmp, "evidence", "manual-case", "result.json")), false); + assert.equal( + existsSync(join(tmp, "evidence", "manual-case", "result.json")), + false, + ); - const included = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "manual-suite", "--run-id", "manual-run-included", "--evidence-dir", join(tmp, "evidence-included"), "--include-manual-check", "--json"], - })); + const included = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "manual-suite", + "--run-id", + "manual-run-included", + "--evidence-dir", + join(tmp, "evidence-included"), + "--include-manual-check", + "--json", + ], + }), + ); assert.equal(included.code, 0); const includedPayload = JSON.parse(included.output); assert.equal(includedPayload.executions[0].status, "ok"); assert.equal(includedPayload.report.status, "pass"); - assert.ok(existsSync(join(tmp, "evidence-included", "manual-case", "result.json"))); + assert.ok( + existsSync(join(tmp, "evidence-included", "manual-case", "result.json")), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -902,7 +1298,10 @@ test("suite run skips cases with missing machine readiness unless explicitly inc mkdirSync(suitesDir, { recursive: true }); mkdirSync(fixturesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "not-ready-case.yaml"), @@ -926,14 +1325,20 @@ test("suite run skips cases with missing machine readiness unless explicitly inc ); writeFileSync( join(fixturesDir, "fixtures.json"), - `${JSON.stringify([{ - id: "missing-fixture", - title: "Missing fixture", - kind: "file", - path: "fixtures/missing.txt", - related_cases: ["not-ready-case"], - checks: ["exists"], - }], null, 2)}\n`, + `${JSON.stringify( + [ + { + id: "missing-fixture", + title: "Missing fixture", + kind: "file", + path: "fixtures/missing.txt", + related_cases: ["not-ready-case"], + checks: ["exists"], + }, + ], + null, + 2, + )}\n`, ); writeFileSync( join(suitesDir, "readiness-suite.yaml"), @@ -959,28 +1364,64 @@ test("suite run skips cases with missing machine readiness unless explicitly inc ].join("\n"), ); - const skipped = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "readiness-suite", "--run-id", "readiness-run", "--evidence-dir", join(tmp, "evidence"), "--json"], - })); + const skipped = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "readiness-suite", + "--run-id", + "readiness-run", + "--evidence-dir", + join(tmp, "evidence"), + "--json", + ], + }), + ); assert.equal(skipped.code, 1); const skippedPayload = JSON.parse(skipped.output); assert.equal(skippedPayload.executions[0].status, "skipped"); assert.match(skippedPayload.executions[0].reason, /readiness missing/); - assert.match(skippedPayload.executions[0].reason, /LBS_TEST_SUITE_RUN_MISSING_ENV/); - assert.match(skippedPayload.executions[0].reason, /LBS_TEST_SUITE_RUN_MISSING_AUTOMATION_ENV/); + assert.match( + skippedPayload.executions[0].reason, + /LBS_TEST_SUITE_RUN_MISSING_ENV/, + ); + assert.match( + skippedPayload.executions[0].reason, + /LBS_TEST_SUITE_RUN_MISSING_AUTOMATION_ENV/, + ); assert.match(skippedPayload.executions[0].reason, /missing-fixture/); - assert.equal(existsSync(join(tmp, "evidence", "not-ready-case", "result.json")), false); + assert.equal( + existsSync(join(tmp, "evidence", "not-ready-case", "result.json")), + false, + ); - const included = capture(() => commandSuiteRun({ - root: tmp, - args: ["suite", "run", "readiness-suite", "--run-id", "readiness-run-included", "--evidence-dir", join(tmp, "evidence-included"), "--include-not-ready", "--json"], - })); + const included = capture(() => + commandSuiteRun({ + root: tmp, + args: [ + "suite", + "run", + "readiness-suite", + "--run-id", + "readiness-run-included", + "--evidence-dir", + join(tmp, "evidence-included"), + "--include-not-ready", + "--json", + ], + }), + ); assert.equal(included.code, 0); const includedPayload = JSON.parse(included.output); assert.equal(includedPayload.executions[0].status, "ok"); assert.equal(includedPayload.report.status, "pass"); - assert.ok(existsSync(join(tmp, "evidence-included", "not-ready-case", "result.json"))); + assert.ok( + existsSync( + join(tmp, "evidence-included", "not-ready-case", "result.json"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -996,13 +1437,18 @@ test("suite new writes a reusable suite skeleton", () => { "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", ); - const result = capture(() => commandSuiteNew({ - root: tmp, - args: ["suite", "new", "new-suite", "--title", "New Suite"], - })); + const result = capture(() => + commandSuiteNew({ + root: tmp, + args: ["suite", "new", "new-suite", "--title", "New Suite"], + }), + ); assert.equal(result.code, 0); - const text = readFileSync(join(skillDir, "suites", "new-suite.yaml"), "utf8"); + const text = readFileSync( + join(skillDir, "suites", "new-suite.yaml"), + "utf8", + ); assert.match(text, /^description:/m); assert.match(text, /^priority: p2/m); assert.match(text, /^cases:/m); @@ -1012,18 +1458,29 @@ test("suite new writes a reusable suite skeleton", () => { }); test("fixture list and check expose reusable fixture readiness", () => { - const list = capture(() => commandFixtureList(ctx(["fixture", "list", "langbot-testing", "--json"]))); + const list = capture(() => + commandFixtureList(ctx(["fixture", "list", "langbot-testing", "--json"])), + ); assert.equal(list.code, 0); const fixtures = JSON.parse(list.output); - assert.ok(fixtures.some((item: { id: string; exists: boolean }) => ( - item.id === "mcp-stdio-echo-server" && item.exists === true - ))); + assert.ok( + fixtures.some( + (item: { id: string; exists: boolean }) => + item.id === "mcp-stdio-echo-server" && item.exists === true, + ), + ); - const check = capture(() => commandFixtureCheck(ctx(["fixture", "check", "langbot-testing", "--json"]))); + const check = capture(() => + commandFixtureCheck(ctx(["fixture", "check", "langbot-testing", "--json"])), + ); assert.equal(check.code, 0); const report = JSON.parse(check.output); assert.equal(report.status, "pass"); - assert.ok(report.fixtures.some((item: { id: string }) => item.id === "qa-plugin-smoke-package")); + assert.ok( + report.fixtures.some( + (item: { id: string }) => item.id === "qa-plugin-smoke-package", + ), + ); }); test("fixture check reports missing manifest paths", () => { @@ -1037,15 +1494,30 @@ test("fixture check reports missing manifest paths", () => { ); writeFileSync( join(skillDir, "fixtures", "fixtures.json"), - JSON.stringify([{ id: "missing-fixture", title: "Missing Fixture", path: "fixtures/missing.txt" }]), + JSON.stringify([ + { + id: "missing-fixture", + title: "Missing Fixture", + path: "fixtures/missing.txt", + }, + ]), ); - const result = capture(() => commandFixtureCheck({ root: tmp, args: ["fixture", "check", "langbot-testing", "--json"] })); + const result = capture(() => + commandFixtureCheck({ + root: tmp, + args: ["fixture", "check", "langbot-testing", "--json"], + }), + ); assert.equal(result.code, 1); const report = JSON.parse(result.output); assert.equal(report.status, "fail"); - assert.ok(report.findings.some((finding: { id?: string }) => finding.id === "missing-fixture")); + assert.ok( + report.findings.some( + (finding: { id?: string }) => finding.id === "missing-fixture", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1056,42 +1528,63 @@ test("fixture check verifies QA AgentRunner source shape", () => { try { const skillDir = join(tmp, "skills", "langbot-testing"); const fixtureDir = join(skillDir, "fixtures", "plugins", "qa-agent-runner"); - mkdirSync(join(fixtureDir, "components", "agent_runner"), { recursive: true }); + mkdirSync(join(fixtureDir, "components", "agent_runner"), { + recursive: true, + }); writeFileSync( join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", ); writeFileSync( join(skillDir, "fixtures", "fixtures.json"), - JSON.stringify([{ - id: "qa-agent-runner-source", - title: "QA AgentRunner", - path: "fixtures/plugins/qa-agent-runner/manifest.yaml", - checks: ["exists", "qa_agent_runner_source"], - }]), + JSON.stringify([ + { + id: "qa-agent-runner-source", + title: "QA AgentRunner", + path: "fixtures/plugins/qa-agent-runner/manifest.yaml", + checks: ["exists", "qa_agent_runner_source"], + }, + ]), + ); + writeFileSync( + join(fixtureDir, "manifest.yaml"), + "spec:\n components:\n AgentRunner: {}\nexecution:\n python:\n attr: QAAgentRunnerPlugin\n", ); - writeFileSync(join(fixtureDir, "manifest.yaml"), "spec:\n components:\n AgentRunner: {}\nexecution:\n python:\n attr: QAAgentRunnerPlugin\n"); - const result = capture(() => commandFixtureCheck({ root: tmp, args: ["fixture", "check", "langbot-testing", "--json"] })); + const result = capture(() => + commandFixtureCheck({ + root: tmp, + args: ["fixture", "check", "langbot-testing", "--json"], + }), + ); assert.equal(result.code, 1); const report = JSON.parse(result.output); - assert.ok(report.findings.some((finding: { kind?: string; path?: string }) => ( - finding.kind === "fixture_check_missing_file" - && finding.path?.endsWith("components/agent_runner/default.py") - ))); + assert.ok( + report.findings.some( + (finding: { kind?: string; path?: string }) => + finding.kind === "fixture_check_missing_file" && + finding.path?.endsWith("components/agent_runner/default.py"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } }); test("fixture check accepts complete QA AgentRunner source shape", () => { - const result = capture(() => commandFixtureCheck(ctx(["fixture", "check", "langbot-testing", "--json"]))); + const result = capture(() => + commandFixtureCheck(ctx(["fixture", "check", "langbot-testing", "--json"])), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); - assert.ok(report.fixtures.some((item: { id: string; checks: string[] }) => ( - item.id === "qa-agent-runner-source" && item.checks.includes("qa_agent_runner_source") - ))); + assert.ok( + report.fixtures.some( + (item: { id: string; checks: string[] }) => + item.id === "qa-agent-runner-source" && + item.checks.includes("qa_agent_runner_source"), + ), + ); }); test("fixture check rejects invalid plugin package files", () => { @@ -1106,21 +1599,32 @@ test("fixture check rejects invalid plugin package files", () => { writeFileSync(join(skillDir, "fixtures", "bad.lbpkg"), "not a zip"); writeFileSync( join(skillDir, "fixtures", "fixtures.json"), - JSON.stringify([{ - id: "bad-package", - title: "Bad Package", - path: "fixtures/bad.lbpkg", - checks: ["exists", "zip_package"], - }]), + JSON.stringify([ + { + id: "bad-package", + title: "Bad Package", + path: "fixtures/bad.lbpkg", + checks: ["exists", "zip_package"], + }, + ]), ); - const result = capture(() => commandFixtureCheck({ root: tmp, args: ["fixture", "check", "langbot-testing", "--json"] })); + const result = capture(() => + commandFixtureCheck({ + root: tmp, + args: ["fixture", "check", "langbot-testing", "--json"], + }), + ); assert.equal(result.code, 1); const report = JSON.parse(result.output); - assert.ok(report.findings.some((finding: { kind?: string; id?: string }) => ( - finding.kind === "fixture_check_invalid_zip" && finding.id === "bad-package" - ))); + assert.ok( + report.findings.some( + (finding: { kind?: string; id?: string }) => + finding.kind === "fixture_check_invalid_zip" && + finding.id === "bad-package", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1142,7 +1646,10 @@ test("debug chat classifier prefers latest response leaf over body counts", () = test("debug chat classifier distinguishes new failure signals from old history", () => { assert.equal( - findNewFailureSignal("Agent runner temporarily unavailable", "Agent runner temporarily unavailable"), + findNewFailureSignal( + "Agent runner temporarily unavailable", + "Agent runner temporarily unavailable", + ), "", ); assert.equal( @@ -1324,12 +1831,27 @@ test("env doctor explains a missing backend listener with a startup hint", async ].join("\n"), ); - const result = await captureAsync(() => commandEnvDoctor({ root: tmp, args: ["env", "doctor"] })); + const result = await captureAsync(() => + commandEnvDoctor({ root: tmp, args: ["env", "doctor"] }), + ); assert.equal(result.code, 1); - assert.match(result.output, /FAIL: LANGBOT_BACKEND_URL: no HTTP service reachable because 127\.0\.0\.1:59998 is not listening/); - assert.match(result.output, new RegExp(`WARN: LANGBOT_BACKEND_URL: start backend: cd ${repoDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} && uv run main.py`)); - assert.match(result.output, new RegExp(`WARN: LANGBOT_FRONTEND_URL: start frontend: cd ${webDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} && pnpm dev`)); + assert.match( + result.output, + /FAIL: LANGBOT_BACKEND_URL: no HTTP service reachable because 127\.0\.0\.1:59998 is not listening/, + ); + assert.match( + result.output, + new RegExp( + `WARN: LANGBOT_BACKEND_URL: start backend: cd ${repoDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} && uv run main.py`, + ), + ); + assert.match( + result.output, + new RegExp( + `WARN: LANGBOT_FRONTEND_URL: start frontend: cd ${webDir.replace(/[.*+?^${}()|[\]\\]/g, "\\$&")} && pnpm dev`, + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1360,15 +1882,72 @@ test("env doctor does not require proxy variables", async () => { ].join("\n"), ); - const result = await captureAsync(() => commandEnvDoctor({ root: tmp, args: ["env", "doctor"] })); + const result = await captureAsync(() => + commandEnvDoctor({ root: tmp, args: ["env", "doctor"] }), + ); assert.equal(result.code, 1); - assert.doesNotMatch(result.output, /missing LANGBOT_PROXY|missing LANGBOT_NO_PROXY/); + assert.doesNotMatch( + result.output, + /missing LANGBOT_PROXY|missing LANGBOT_NO_PROXY/, + ); } finally { rmSync(tmp, { recursive: true, force: true }); } }); +test("env doctor reports missing socksio for active SOCKS proxy", async () => { + const tmp = mkdtempSync(join(tmpdir(), "lbs-env-doctor-socksio-")); + const originalAllProxy = process.env.ALL_PROXY; + const originalAllProxyLower = process.env.all_proxy; + try { + delete process.env.ALL_PROXY; + delete process.env.all_proxy; + const skillsDir = join(tmp, "skills"); + const repoDir = join(tmp, "LangBot"); + const webDir = join(repoDir, "web"); + const venvBin = join(repoDir, ".venv", "bin"); + const browserProfile = join(tmp, "browser-profile"); + const chromium = join(tmp, "chromium"); + mkdirSync(skillsDir, { recursive: true }); + mkdirSync(webDir, { recursive: true }); + mkdirSync(venvBin, { recursive: true }); + mkdirSync(browserProfile, { recursive: true }); + writeFileSync(chromium, ""); + const python = join(venvBin, "python"); + writeFileSync(python, "#!/bin/sh\nexit 1\n"); + chmodSync(python, 0o755); + writeFileSync( + join(skillsDir, ".env"), + [ + "LANGBOT_BACKEND_URL=http://127.0.0.1:59996", + "LANGBOT_FRONTEND_URL=http://127.0.0.1:59996", + "LANGBOT_DEV_FRONTEND_URL=http://127.0.0.1:59996", + `LANGBOT_REPO=${repoDir}`, + `LANGBOT_WEB_REPO=${webDir}`, + `LANGBOT_BROWSER_PROFILE=${browserProfile}`, + `LANGBOT_CHROMIUM_EXECUTABLE=${chromium}`, + "ALL_PROXY=socks5://127.0.0.1:7890", + ].join("\n"), + ); + + const result = await captureAsync(() => + commandEnvDoctor({ root: tmp, args: ["env", "doctor"] }), + ); + + assert.equal(result.code, 1); + assert.match(result.output, /FAIL: SOCKS proxy ALL_PROXY is configured/); + assert.match(result.output, /cannot import socksio/); + assert.match(result.output, /-m pip install socksio/); + } finally { + if (originalAllProxy === undefined) delete process.env.ALL_PROXY; + else process.env.ALL_PROXY = originalAllProxy; + if (originalAllProxyLower === undefined) delete process.env.all_proxy; + else process.env.all_proxy = originalAllProxyLower; + rmSync(tmp, { recursive: true, force: true }); + } +}); + test("env show redacts secret-like values by default", () => { const tmp = mkdtempSync(join(tmpdir(), "lbs-env-show-redact-")); try { @@ -1382,13 +1961,20 @@ test("env show redacts secret-like values by default", () => { ].join("\n"), ); - const text = capture(() => commandEnvShow({ root: tmp, args: ["env", "show"] })); + const text = capture(() => + commandEnvShow({ root: tmp, args: ["env", "show"] }), + ); assert.equal(text.code, 0); assert.match(text.output, /LANGBOT_API_KEY=\[redacted\]/); - assert.match(text.output, /LANGBOT_PROXY_HTTP=http:\/\/\[redacted\]@127\.0\.0\.1:7890/); + assert.match( + text.output, + /LANGBOT_PROXY_HTTP=http:\/\/\[redacted\]@127\.0\.0\.1:7890/, + ); assert.doesNotMatch(text.output, /sk-test-secret|user:pass/); - const json = capture(() => commandEnvShow({ root: tmp, args: ["env", "show", "--json"] })); + const json = capture(() => + commandEnvShow({ root: tmp, args: ["env", "show", "--json"] }), + ); assert.equal(json.code, 0); const parsed = JSON.parse(json.output); assert.equal(parsed.LANGBOT_API_KEY, "[redacted]"); @@ -1399,10 +1985,15 @@ test("env show redacts secret-like values by default", () => { }); test("test plan renders agent-browser QA guidance", () => { - const result = capture(() => commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat"]))); + const result = capture(() => + commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat"])), + ); assert.equal(result.code, 0); assert.match(result.output, /Mode: agent-browser/); - assert.match(result.output, /Use browser\/UI interaction as the primary QA path/); + assert.match( + result.output, + /Use browser\/UI interaction as the primary QA path/, + ); assert.match(result.output, /API\/curl\/log checks are diagnostic only/); assert.match(result.output, /## Browser Steps/); assert.match(result.output, /## Success Signals/); @@ -1414,29 +2005,45 @@ test("test plan renders agent-browser QA guidance", () => { }); test("test plan JSON is parseable and includes troubleshooting patterns", () => { - const result = capture(() => commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"]))); + const result = capture(() => + commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"])), + ); assert.equal(result.code, 0); const plan = JSON.parse(result.output); assert.equal(plan.id, "pipeline-debug-chat"); assert.equal(plan.mode, "agent-browser"); assert.ok(["ready", "missing"].includes(plan.automation_readiness.status)); assert.ok(plan.automation_readiness.defaulted.includes("LANGBOT_E2E_PROMPT")); - assert.ok(plan.automation_readiness.defaulted.includes("LANGBOT_E2E_EXPECTED_TEXT")); + assert.ok( + plan.automation_readiness.defaulted.includes("LANGBOT_E2E_EXPECTED_TEXT"), + ); assert.equal(plan.manual_readiness.status, "manual_check"); assert.ok(plan.success_patterns.includes("Streaming completed")); - assert.ok(plan.troubleshooting.some((entry: { id: string }) => entry.id === "plugin-runtime-timeout")); + assert.ok( + plan.troubleshooting.some( + (entry: { id: string }) => entry.id === "plugin-runtime-timeout", + ), + ); }); test("test plan JSON exposes missing case-specific pipeline readiness", () => { - const result = capture(() => commandTestPlan(ctx(["test", "plan", "local-agent-basic-debug-chat", "--json"]))); + const result = capture(() => + commandTestPlan( + ctx(["test", "plan", "local-agent-basic-debug-chat", "--json"]), + ), + ); assert.equal(result.code, 0); const plan = JSON.parse(result.output); assert.equal(plan.env_readiness.status, "ready"); assert.ok(["ready", "missing"].includes(plan.automation_readiness.status)); assert.ok(plan.automation_readiness.pipeline_env_required); assert.ok( - plan.automation_readiness.missing.includes("LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME") - || plan.automation_readiness.configured.some((key: string) => key.startsWith("LANGBOT_LOCAL_AGENT_PIPELINE_")), + plan.automation_readiness.missing.includes( + "LANGBOT_LOCAL_AGENT_PIPELINE_URL|LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + ) || + plan.automation_readiness.configured.some((key: string) => + key.startsWith("LANGBOT_LOCAL_AGENT_PIPELINE_"), + ), ); }); @@ -1444,31 +2051,53 @@ test("generic pipeline readiness accepts either URL or name target", () => { const originalUrl = process.env.LANGBOT_PIPELINE_URL; const originalName = process.env.LANGBOT_PIPELINE_NAME; try { - withEnv({ - LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", - LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", - }, () => { - process.env.LANGBOT_PIPELINE_URL = "http://127.0.0.1:3000/home/agents?id=only-url"; - process.env.LANGBOT_PIPELINE_NAME = ""; + withEnv( + { + LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", + LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", + }, + () => { + process.env.LANGBOT_PIPELINE_URL = + "http://127.0.0.1:3000/home/agents?id=only-url"; + process.env.LANGBOT_PIPELINE_NAME = ""; - const ready = capture(() => commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"]))); - assert.equal(ready.code, 0); - const plan = JSON.parse(ready.output); - assert.equal(plan.env_readiness.status, "ready"); - assert.equal(plan.automation_readiness.status, "ready"); - assert.ok(plan.automation_readiness.required.includes("LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME")); - }); + const ready = capture(() => + commandTestPlan( + ctx(["test", "plan", "pipeline-debug-chat", "--json"]), + ), + ); + assert.equal(ready.code, 0); + const plan = JSON.parse(ready.output); + assert.equal(plan.env_readiness.status, "ready"); + assert.equal(plan.automation_readiness.status, "ready"); + assert.ok( + plan.automation_readiness.required.includes( + "LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME", + ), + ); + }, + ); process.env.LANGBOT_PIPELINE_URL = ""; process.env.LANGBOT_PIPELINE_NAME = ""; - const missing = capture(() => commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"]))); + const missing = capture(() => + commandTestPlan(ctx(["test", "plan", "pipeline-debug-chat", "--json"])), + ); assert.equal(missing.code, 0); const missingPlan = JSON.parse(missing.output); assert.equal(missingPlan.env_readiness.status, "missing"); - assert.ok(missingPlan.env_readiness.missing.includes("LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME")); + assert.ok( + missingPlan.env_readiness.missing.includes( + "LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME", + ), + ); assert.equal(missingPlan.automation_readiness.status, "missing"); - assert.ok(missingPlan.automation_readiness.missing.includes("LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME")); + assert.ok( + missingPlan.automation_readiness.missing.includes( + "LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME", + ), + ); } finally { if (originalUrl === undefined) delete process.env.LANGBOT_PIPELINE_URL; else process.env.LANGBOT_PIPELINE_URL = originalUrl; @@ -1478,15 +2107,19 @@ test("generic pipeline readiness accepts either URL or name target", () => { }); test("test recommend maps AgentRunner ledger changes to focused probes", () => { - const result = capture(() => commandTestRecommend(ctx([ - "test", - "recommend", - "--file", - "LangBot/src/langbot/pkg/agent/runner/run_ledger_store.py", - "--file", - "LangBot/tests/unit_tests/agent/test_run_ledger_store.py", - "--json", - ]))); + const result = capture(() => + commandTestRecommend( + ctx([ + "test", + "recommend", + "--file", + "LangBot/src/langbot/pkg/agent/runner/run_ledger_store.py", + "--file", + "LangBot/tests/unit_tests/agent/test_run_ledger_store.py", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); const ids = report.recommendations.map((item: { id: string }) => item.id); @@ -1495,18 +2128,30 @@ test("test recommend maps AgentRunner ledger changes to focused probes", () => { assert.ok(ids.includes("agent-runner-ledger-contention")); assert.ok(ids.includes("agent-runner-async-db-readiness")); assert.ok(ids.includes("agent-runner-ledger-concurrency")); - assert.ok(report.commands.every((command: string) => !command.startsWith("bin/lbs test run ") || command.endsWith(" --dry-run"))); - assert.ok(report.notes.some((note: string) => note.includes("Remove --dry-run"))); + assert.ok( + report.commands.every( + (command: string) => + !command.startsWith("bin/lbs test run ") || + command.endsWith(" --dry-run"), + ), + ); + assert.ok( + report.notes.some((note: string) => note.includes("Remove --dry-run")), + ); }); test("test recommend maps AgentRunner result changes to fixture contract", () => { - const result = capture(() => commandTestRecommend(ctx([ - "test", - "recommend", - "--file", - "langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/agent_runner/result.py", - "--json", - ]))); + const result = capture(() => + commandTestRecommend( + ctx([ + "test", + "recommend", + "--file", + "langbot-plugin-sdk/src/langbot_plugin/api/entities/builtin/agent_runner/result.py", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); const ids = report.recommendations.map((item: { id: string }) => item.id); @@ -1516,13 +2161,17 @@ test("test recommend maps AgentRunner result changes to fixture contract", () => }); test("test recommend maps QA AgentRunner fixture changes to live install", () => { - const result = capture(() => commandTestRecommend(ctx([ - "test", - "recommend", - "--file", - "langbot-skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.py", - "--json", - ]))); + const result = capture(() => + commandTestRecommend( + ctx([ + "test", + "recommend", + "--file", + "langbot-skills/skills/langbot-testing/fixtures/plugins/qa-agent-runner/components/agent_runner/default.py", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); const ids = report.recommendations.map((item: { id: string }) => item.id); @@ -1532,13 +2181,17 @@ test("test recommend maps QA AgentRunner fixture changes to live install", () => }); test("test recommend maps QA plugin smoke fixture changes to live install", () => { - const result = capture(() => commandTestRecommend(ctx([ - "test", - "recommend", - "--file", - "langbot-skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/main.py", - "--json", - ]))); + const result = capture(() => + commandTestRecommend( + ctx([ + "test", + "recommend", + "--file", + "langbot-skills/skills/langbot-testing/fixtures/plugins/qa-plugin-smoke/main.py", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); const ids = report.recommendations.map((item: { id: string }) => item.id); @@ -1555,26 +2208,66 @@ test("test recommend keeps git status paths intact", () => { }; try { const repo = join(tmp, "LangBot"); - mkdirSync(join(repo, "src", "langbot", "pkg", "agent", "runner"), { recursive: true }); + mkdirSync(join(repo, "src", "langbot", "pkg", "agent", "runner"), { + recursive: true, + }); spawnSync("git", ["init"], { cwd: repo }); - spawnSync("git", ["config", "user.email", "qa@example.test"], { cwd: repo }); + spawnSync("git", ["config", "user.email", "qa@example.test"], { + cwd: repo, + }); spawnSync("git", ["config", "user.name", "QA"], { cwd: repo }); writeFileSync(join(repo, "README.md"), "test\n"); - writeFileSync(join(repo, "src", "langbot", "pkg", "agent", "runner", "run_ledger_store.py"), "# test\n"); - spawnSync("git", ["add", "README.md", "src/langbot/pkg/agent/runner/run_ledger_store.py"], { cwd: repo }); + writeFileSync( + join( + repo, + "src", + "langbot", + "pkg", + "agent", + "runner", + "run_ledger_store.py", + ), + "# test\n", + ); + spawnSync( + "git", + ["add", "README.md", "src/langbot/pkg/agent/runner/run_ledger_store.py"], + { cwd: repo }, + ); spawnSync("git", ["commit", "-m", "init"], { cwd: repo }); - writeFileSync(join(repo, "src", "langbot", "pkg", "agent", "runner", "run_ledger_store.py"), "# changed\n"); + writeFileSync( + join( + repo, + "src", + "langbot", + "pkg", + "agent", + "runner", + "run_ledger_store.py", + ), + "# changed\n", + ); process.env.LANGBOT_REPO = repo; process.env.LANGBOT_PLUGIN_SDK_REPO = join(tmp, "missing-sdk"); process.env.LANGBOT_AGENT_RUNNER_REPO = join(tmp, "missing-runner"); process.env.LANGBOT_LOCAL_AGENT_REPO = join(tmp, "missing-local"); - const result = capture(() => commandTestRecommend({ root, args: ["test", "recommend", "--json"] })); + const result = capture(() => + commandTestRecommend({ root, args: ["test", "recommend", "--json"] }), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); - assert.ok(report.changed_files.includes("LangBot/src/langbot/pkg/agent/runner/run_ledger_store.py")); - assert.ok(!report.changed_files.some((file: string) => file.includes("LangBot/rc/"))); + assert.ok( + report.changed_files.includes( + "LangBot/src/langbot/pkg/agent/runner/run_ledger_store.py", + ), + ); + assert.ok( + !report.changed_files.some((file: string) => + file.includes("LangBot/rc/"), + ), + ); } finally { for (const [key, value] of Object.entries(originalRepos)) { if (value === undefined) delete process.env[key]; @@ -1585,25 +2278,41 @@ test("test recommend keeps git status paths intact", () => { }); test("test start creates a run handoff with a bounded report command", () => { - const result = capture(() => commandTestStart(ctx(["test", "start", "pipeline-debug-chat"]))); + const result = capture(() => + commandTestStart(ctx(["test", "start", "pipeline-debug-chat"])), + ); assert.equal(result.code, 0); assert.match(result.output, /^# Test Start: pipeline-debug-chat/m); assert.match(result.output, /bin\/lbs test plan pipeline-debug-chat/); - assert.match(result.output, /bin\/lbs test run pipeline-debug-chat --run-id .+ --output reports\/evidence\/.+pipeline-debug-chat/); - assert.match(result.output, /bin\/lbs test report pipeline-debug-chat --since ".+" --console-log reports\/evidence\/.+\/console\.log --evidence-dir reports\/evidence\/.+ --output reports\/.+pipeline-debug-chat\.md/); + assert.match( + result.output, + /bin\/lbs test run pipeline-debug-chat --run-id .+ --output reports\/evidence\/.+pipeline-debug-chat/, + ); + assert.match( + result.output, + /bin\/lbs test report pipeline-debug-chat --since ".+" --console-log reports\/evidence\/.+\/console\.log --evidence-dir reports\/evidence\/.+ --output reports\/.+pipeline-debug-chat\.md/, + ); assert.match(result.output, /Streaming completed/); }); test("test start JSON is parseable for agent orchestration", () => { - const result = capture(() => commandTestStart(ctx(["test", "start", "pipeline-debug-chat", "--json"]))); + const result = capture(() => + commandTestStart(ctx(["test", "start", "pipeline-debug-chat", "--json"])), + ); assert.equal(result.code, 0); const start = JSON.parse(result.output); assert.equal(start.case.id, "pipeline-debug-chat"); assert.match(start.run_id, /pipeline-debug-chat$/); assert.match(start.started_at_local, /\d{4}-\d{2}-\d{2}T/); assert.match(start.report_command, /--since/); - assert.match(start.result_command_template, /bin\/lbs test result pipeline-debug-chat/); - assert.match(start.automation.command, /bin\/lbs test run pipeline-debug-chat/); + assert.match( + start.result_command_template, + /bin\/lbs test result pipeline-debug-chat/, + ); + assert.match( + start.automation.command, + /bin\/lbs test run pipeline-debug-chat/, + ); assert.ok(start.success_patterns.includes("Streaming completed")); assert.ok(start.evidence_required.includes("backend_log")); }); @@ -1612,22 +2321,26 @@ test("test result writes a suite-readable result.json and enforces pass evidence const tmp = mkdtempSync(join(tmpdir(), "lbs-test-result-")); try { const evidenceDir = join(tmp, "pipeline-run"); - const ok = capture(() => commandTestResult(ctx([ - "test", - "result", - "pipeline-debug-chat", - "--result", - "pass", - "--reason", - "Debug Chat returned OK and logs were clean.", - "--evidence-dir", - evidenceDir, - "--started-at", - "2026-05-21T10:30:00.000+08:00", - "--evidence", - "ui,screenshot,console,backend_log", - "--json", - ]))); + const ok = capture(() => + commandTestResult( + ctx([ + "test", + "result", + "pipeline-debug-chat", + "--result", + "pass", + "--reason", + "Debug Chat returned OK and logs were clean.", + "--evidence-dir", + evidenceDir, + "--started-at", + "2026-05-21T10:30:00.000+08:00", + "--evidence", + "ui,screenshot,console,backend_log", + "--json", + ]), + ), + ); assert.equal(ok.code, 0); const record = JSON.parse(ok.output); @@ -1635,21 +2348,29 @@ test("test result writes a suite-readable result.json and enforces pass evidence assert.equal(record.status, "pass"); assert.equal(record.evidence_status, "complete"); assert.deepEqual(record.evidence_missing, []); - assert.equal(JSON.parse(readFileSync(join(evidenceDir, "result.json"), "utf8")).case_id, "pipeline-debug-chat"); - - const missing = captureAll(() => commandTestResult(ctx([ - "test", - "result", + assert.equal( + JSON.parse(readFileSync(join(evidenceDir, "result.json"), "utf8")) + .case_id, "pipeline-debug-chat", - "--result", - "pass", - "--reason", - "Missing backend evidence should not be accepted as pass.", - "--evidence-dir", - join(tmp, "missing-evidence"), - "--evidence", - "ui", - ]))); + ); + + const missing = captureAll(() => + commandTestResult( + ctx([ + "test", + "result", + "pipeline-debug-chat", + "--result", + "pass", + "--reason", + "Missing backend evidence should not be accepted as pass.", + "--evidence-dir", + join(tmp, "missing-evidence"), + "--evidence", + "ui", + ]), + ), + ); assert.equal(missing.code, 1); assert.match(missing.error, /missing required evidence/); } finally { @@ -1658,42 +2379,62 @@ test("test result writes a suite-readable result.json and enforces pass evidence }); test("test run dry-run exposes case automation script and evidence paths", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "pipeline-debug-chat", - "--run-id", - "run-123", - "--output", - "reports/evidence/run-123", - "--dry-run", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "pipeline-debug-chat", + "--run-id", + "run-123", + "--output", + "reports/evidence/run-123", + "--dry-run", + ]), + ), + ); assert.equal(result.code, 0); assert.match(result.output, /^# Test Automation: pipeline-debug-chat/m); assert.match(result.output, /scripts\/e2e\/pipeline-debug-chat\.mjs/); - assert.match(result.output, /console_log: reports\/evidence\/run-123\/console\.log/); - assert.match(result.output, /automation_result_json: reports\/evidence\/run-123\/automation-result\.json/); - assert.match(result.output, /result_json: reports\/evidence\/run-123\/result\.json/); + assert.match( + result.output, + /console_log: reports\/evidence\/run-123\/console\.log/, + ); + assert.match( + result.output, + /automation_result_json: reports\/evidence\/run-123\/automation-result\.json/, + ); + assert.match( + result.output, + /result_json: reports\/evidence\/run-123\/result\.json/, + ); assert.match(result.output, /LANGBOT_PIPELINE_URL/); }); test("test run dry-run JSON is parseable for automation orchestration", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "webui-login-state", - "--run-id", - "login-run", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "webui-login-state", + "--run-id", + "login-run", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.case.id, "webui-login-state"); assert.equal(run.run_id, "login-run"); assert.equal(run.automation.script, "scripts/e2e/webui-login-state.mjs"); assert.equal(run.automation.exists, true); - assert.match(run.automation.automation_result_json, /automation-result\.json$/); + assert.match( + run.automation.automation_result_json, + /automation-result\.json$/, + ); assert.match(run.automation.result_json, /result\.json$/); assert.match(run.automation.report_command, /--console-log/); }); @@ -1706,7 +2447,10 @@ test("test run JSON executes automation unless dry-run is explicit", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "json-exec.yaml"), @@ -1731,10 +2475,12 @@ test("test run JSON executes automation unless dry-run is explicit", () => { ].join("\n"), ); - const result = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "json-exec", "--run-id", "json-run", "--json"], - })); + const result = capture(() => + commandTestRun({ + root: tmp, + args: ["test", "run", "json-exec", "--run-id", "json-run", "--json"], + }), + ); assert.equal(result.code, 0); assert.equal(readFileSync(join(tmp, "json-ran.txt"), "utf8"), "yes"); @@ -1755,7 +2501,10 @@ test("test run lets explicit environment override automation defaults", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "env-override.yaml"), @@ -1769,7 +2518,7 @@ test("test run lets explicit environment override automation defaults", () => { "risk: low", "ci_eligible: false", "automation: scripts/write-env.mjs", - "automation_runner_config_patch_json: '{\"source\":\"default\"}'", + 'automation_runner_config_patch_json: \'{"source":"default"}\'', ].join("\n"), ); writeFileSync( @@ -1784,16 +2533,21 @@ test("test run lets explicit environment override automation defaults", () => { ); process.env.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON = '{"source":"explicit"}'; - const result = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "env-override", "--run-id", "env-run"], - })); + const result = capture(() => + commandTestRun({ + root: tmp, + args: ["test", "run", "env-override", "--run-id", "env-run"], + }), + ); assert.equal(result.code, 0); - const observed = JSON.parse(readFileSync(join(tmp, "env-out.json"), "utf8")); + const observed = JSON.parse( + readFileSync(join(tmp, "env-out.json"), "utf8"), + ); assert.equal(observed.patch, '{"source":"explicit"}'); } finally { - if (originalPatch === undefined) delete process.env.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON; + if (originalPatch === undefined) + delete process.env.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON; else process.env.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON = originalPatch; rmSync(tmp, { recursive: true, force: true }); } @@ -1807,8 +2561,14 @@ test("test run expands env references in automation defaults", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); - writeFileSync(join(tmp, "skills", ".env"), "QA_KB_UUID=kb-from-env\nQA_MODEL_UUID=model-from-env\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); + writeFileSync( + join(tmp, "skills", ".env"), + "QA_KB_UUID=kb-from-env\nQA_MODEL_UUID=model-from-env\n", + ); writeFileSync( join(casesDir, "env-expand.yaml"), [ @@ -1821,7 +2581,7 @@ test("test run expands env references in automation defaults", () => { "risk: low", "ci_eligible: false", "automation: scripts/write-expanded-env.mjs", - "automation_runner_config_patch_json: '{\"knowledge-bases\":[\"${QA_KB_UUID}\"],\"model\":{\"primary\":\"${QA_MODEL_UUID}\"}}'", + 'automation_runner_config_patch_json: \'{"knowledge-bases":["${QA_KB_UUID}"],"model":{"primary":"${QA_MODEL_UUID}"}}\'', ].join("\n"), ); writeFileSync( @@ -1835,10 +2595,20 @@ test("test run expands env references in automation defaults", () => { ].join("\n"), ); - const dryRun = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "env-expand", "--run-id", "env-expand-dry", "--dry-run", "--json"], - })); + const dryRun = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "env-expand", + "--run-id", + "env-expand-dry", + "--dry-run", + "--json", + ], + }), + ); assert.equal(dryRun.code, 0); const plan = JSON.parse(dryRun.output); assert.equal( @@ -1846,13 +2616,20 @@ test("test run expands env references in automation defaults", () => { '{"knowledge-bases":["kb-from-env"],"model":{"primary":"model-from-env"}}', ); - const run = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "env-expand", "--run-id", "env-expand-run"], - })); + const run = capture(() => + commandTestRun({ + root: tmp, + args: ["test", "run", "env-expand", "--run-id", "env-expand-run"], + }), + ); assert.equal(run.code, 0); - const observed = JSON.parse(readFileSync(join(tmp, "expanded-env-out.json"), "utf8")); - assert.equal(observed.patch, '{"knowledge-bases":["kb-from-env"],"model":{"primary":"model-from-env"}}'); + const observed = JSON.parse( + readFileSync(join(tmp, "expanded-env-out.json"), "utf8"), + ); + assert.equal( + observed.patch, + '{"knowledge-bases":["kb-from-env"],"model":{"primary":"model-from-env"}}', + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -1866,7 +2643,10 @@ test("test run setup automation isolates evidence and reloads env", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), "SETUP_VALUE=\n"); writeFileSync( join(casesDir, "setup-main.yaml"), @@ -1882,7 +2662,7 @@ test("test run setup automation isolates evidence and reloads env", () => { "env:", " - SETUP_VALUE", "setup_automation:", - " - \"node:scripts/write-setup-env.mjs --write-env\"", + ' - "node:scripts/write-setup-env.mjs --write-env"', "setup_provides_env:", " - SETUP_VALUE", "automation: scripts/read-setup-env.mjs", @@ -1900,7 +2680,7 @@ test("test run setup automation isolates evidence and reloads env", () => { "risk: low", "ci_eligible: false", "setup_automation:", - " - \"node:scripts/write-setup-env-issue.mjs\"", + ' - "node:scripts/write-setup-env-issue.mjs"', "automation: scripts/read-setup-env.mjs", ].join("\n"), ); @@ -1916,7 +2696,7 @@ test("test run setup automation isolates evidence and reloads env", () => { "risk: low", "ci_eligible: false", "setup_automation:", - " - \"node:scripts/write-setup-pass-then-fail.mjs\"", + ' - "node:scripts/write-setup-pass-then-fail.mjs"', "automation: scripts/read-setup-env.mjs", ].join("\n"), ); @@ -1966,45 +2746,118 @@ test("test run setup automation isolates evidence and reloads env", () => { ].join("\n"), ); - const dryRun = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-main", "--run-id", "setup-run", "--output", join(tmp, "evidence"), "--dry-run", "--json"], - })); + const dryRun = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-main", + "--run-id", + "setup-run", + "--output", + join(tmp, "evidence"), + "--dry-run", + "--json", + ], + }), + ); assert.equal(dryRun.code, 0); const plan = JSON.parse(dryRun.output); assert.equal(plan.setup_automation.length, 1); - assert.match(plan.setup_automation[0].evidence_dir, /setup\/01-write-setup-env$/); - assert.match(plan.setup_automation[0].command, /^node scripts\/write-setup-env\.mjs --write-env$/); + assert.match( + plan.setup_automation[0].evidence_dir, + /setup\/01-write-setup-env$/, + ); + assert.match( + plan.setup_automation[0].command, + /^node scripts\/write-setup-env\.mjs --write-env$/, + ); assert.equal(plan.setup_automation[0].dry_run_command, ""); assert.equal(existsSync(join(tmp, "skills", ".env.local")), false); - const run = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-main", "--run-id", "setup-run", "--output", join(tmp, "evidence")], - })); + const run = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-main", + "--run-id", + "setup-run", + "--output", + join(tmp, "evidence"), + ], + }), + ); assert.equal(run.code, 0); - const observed = JSON.parse(readFileSync(join(tmp, "main-observed.json"), "utf8")); + const observed = JSON.parse( + readFileSync(join(tmp, "main-observed.json"), "utf8"), + ); assert.equal(observed.value, "from-setup"); - const setupResult = JSON.parse(readFileSync(join(tmp, "evidence", "setup", "01-write-setup-env", "automation-result.json"), "utf8")); - const mainResult = JSON.parse(readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8")); + const setupResult = JSON.parse( + readFileSync( + join( + tmp, + "evidence", + "setup", + "01-write-setup-env", + "automation-result.json", + ), + "utf8", + ), + ); + const mainResult = JSON.parse( + readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8"), + ); assert.equal(setupResult.stage, "setup"); assert.equal(mainResult.stage, "main"); - const envIssue = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-env-issue", "--run-id", "setup-env-issue-run", "--output", join(tmp, "evidence-env-issue")], - })); + const envIssue = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-env-issue", + "--run-id", + "setup-env-issue-run", + "--output", + join(tmp, "evidence-env-issue"), + ], + }), + ); assert.equal(envIssue.code, 2); - const parentResult = JSON.parse(readFileSync(join(tmp, "evidence-env-issue", "automation-result.json"), "utf8")); + const parentResult = JSON.parse( + readFileSync( + join(tmp, "evidence-env-issue", "automation-result.json"), + "utf8", + ), + ); assert.equal(parentResult.status, "env_issue"); assert.equal(parentResult.reason, "setup env missing"); - const failAfterPass = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-fail-after-pass", "--run-id", "setup-fail-after-pass-run", "--output", join(tmp, "evidence-fail-after-pass")], - })); + const failAfterPass = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-fail-after-pass", + "--run-id", + "setup-fail-after-pass-run", + "--output", + join(tmp, "evidence-fail-after-pass"), + ], + }), + ); assert.equal(failAfterPass.code, 1); - const failAfterPassResult = JSON.parse(readFileSync(join(tmp, "evidence-fail-after-pass", "automation-result.json"), "utf8")); + const failAfterPassResult = JSON.parse( + readFileSync( + join(tmp, "evidence-fail-after-pass", "automation-result.json"), + "utf8", + ), + ); assert.equal(failAfterPassResult.status, "fail"); assert.equal(failAfterPassResult.reason, "stale pass before crash"); } finally { @@ -2020,7 +2873,10 @@ test("test run setup automation can execute another case outside this source rep const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), "SETUP_VALUE=\n"); writeFileSync( join(casesDir, "setup-child.yaml"), @@ -2048,7 +2904,7 @@ test("test run setup automation can execute another case outside this source rep "risk: low", "ci_eligible: true", "setup_automation:", - " - \"case:setup-child\"", + ' - "case:setup-child"', "setup_provides_env:", " - SETUP_VALUE", "automation: scripts/read-child-env.mjs", @@ -2077,14 +2933,30 @@ test("test run setup automation can execute another case outside this source rep ].join("\n"), ); - const run = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-parent", "--run-id", "setup-parent-run", "--output", join(tmp, "evidence")], - })); + const run = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "setup-parent", + "--run-id", + "setup-parent-run", + "--output", + join(tmp, "evidence"), + ], + }), + ); assert.equal(run.code, 0); - assert.ok(existsSync(join(tmp, "evidence", "setup", "01-setup-child", "result.json"))); - const result = JSON.parse(readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8")); + assert.ok( + existsSync( + join(tmp, "evidence", "setup", "01-setup-child", "result.json"), + ), + ); + const result = JSON.parse( + readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8"), + ); assert.equal(result.value, "from-child"); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -2099,7 +2971,10 @@ test("test run automation inherits parent process environment", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "env-inherit.yaml"), @@ -2126,13 +3001,25 @@ test("test run automation inherits parent process environment", () => { ].join("\n"), ); - const run = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "env-inherit", "--run-id", "env-inherit-run", "--output", join(tmp, "evidence")], - })); + const run = capture(() => + commandTestRun({ + root: tmp, + args: [ + "test", + "run", + "env-inherit", + "--run-id", + "env-inherit-run", + "--output", + join(tmp, "evidence"), + ], + }), + ); assert.equal(run.code, 0); - const result = JSON.parse(readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8")); + const result = JSON.parse( + readFileSync(join(tmp, "evidence", "automation-result.json"), "utf8"), + ); assert.equal(result.status, "pass"); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -2147,7 +3034,10 @@ test("test run dry-run marks missing setup case targets", () => { const scriptsDir = join(tmp, "scripts"); mkdirSync(casesDir, { recursive: true }); mkdirSync(scriptsDir, { recursive: true }); - writeFileSync(join(skillDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(skillDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync(join(tmp, "skills", ".env"), ""); writeFileSync( join(casesDir, "setup-parent.yaml"), @@ -2161,16 +3051,18 @@ test("test run dry-run marks missing setup case targets", () => { "risk: low", "ci_eligible: true", "setup_automation:", - " - \"case:missing-child\"", + ' - "case:missing-child"', "automation: scripts/pass.mjs", ].join("\n"), ); writeFileSync(join(scriptsDir, "pass.mjs"), "process.exit(0);\n"); - const result = capture(() => commandTestRun({ - root: tmp, - args: ["test", "run", "setup-parent", "--dry-run", "--json"], - })); + const result = capture(() => + commandTestRun({ + root: tmp, + args: ["test", "run", "setup-parent", "--dry-run", "--json"], + }), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); @@ -2184,159 +3076,225 @@ test("test run dry-run marks missing setup case targets", () => { }); test("local-agent effective prompt case has runnable automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-effective-prompt-debug-chat", - "--run-id", - "effective-run", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-effective-prompt-debug-chat", + "--run-id", + "effective-run", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.script, "scripts/e2e/pipeline-debug-chat.mjs"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_PROMPT, "qa-effective-prompt"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, "PROMPT_PREPROCESS_OK"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_RESPONSE_TIMEOUT_MS, "180000"); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_PROMPT, + "qa-effective-prompt", + ); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, + "PROMPT_PREPROCESS_OK", + ); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_RESPONSE_TIMEOUT_MS, + "180000", + ); assert.equal(run.automation.pipeline_env_required, true); - assert.ok(run.automation.env_aliases.some((alias: { target: string; source: string }) => ( - alias.target === "LANGBOT_E2E_PIPELINE_URL" && alias.source === "LANGBOT_LOCAL_AGENT_PIPELINE_URL" - ))); + assert.ok( + run.automation.env_aliases.some( + (alias: { target: string; source: string }) => + alias.target === "LANGBOT_E2E_PIPELINE_URL" && + alias.source === "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + ), + ); }); test("local-agent basic case can setup the local-agent pipeline env", () => { - withEnv({ - LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", - LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", - }, () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-basic-debug-chat", - "--dry-run", - "--json", - ]))); - assert.equal(result.code, 0); - const run = JSON.parse(result.output); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", - ]); + withEnv( + { + LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", + LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", + }, + () => { + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-basic-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); + assert.equal(result.code, 0); + const run = JSON.parse(result.output); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + ["node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env"], + ); - const planResult = capture(() => commandTestPlan(ctx(["test", "plan", "local-agent-basic-debug-chat", "--json"]))); - assert.equal(planResult.code, 0); - const plan = JSON.parse(planResult.output); - assert.deepEqual(plan.setup_provides_env, [ - "LANGBOT_LOCAL_AGENT_PIPELINE_URL", - "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", - ]); - assert.equal(plan.automation_readiness.status, "ready"); - }); + const planResult = capture(() => + commandTestPlan( + ctx(["test", "plan", "local-agent-basic-debug-chat", "--json"]), + ), + ); + assert.equal(planResult.code, 0); + const plan = JSON.parse(planResult.output); + assert.deepEqual(plan.setup_provides_env, [ + "LANGBOT_LOCAL_AGENT_PIPELINE_URL", + "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", + ]); + assert.equal(plan.automation_readiness.status, "ready"); + }, + ); }); test("local-agent nonstreaming case disables stream output through automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-nonstreaming-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-nonstreaming-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.script, "scripts/e2e/pipeline-debug-chat.mjs"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_PROMPT, "Reply only NONSTREAM_OK."); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, "NONSTREAM_OK"); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_PROMPT, + "Reply only NONSTREAM_OK.", + ); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, + "NONSTREAM_OK", + ); assert.equal(run.automation.env_defaults.LANGBOT_E2E_STREAM_OUTPUT, "0"); assert.equal(run.automation.pipeline_env_required, true); }); test("local-agent multimodal case exposes image fixture automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-multimodal-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-multimodal-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.script, "scripts/e2e/pipeline-debug-chat.mjs"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, "IMAGE_OK"); - assert.match(run.automation.env_defaults.LANGBOT_E2E_IMAGE_BASE64_PATH, /red-square\.png\.base64$/); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_TEXT, + "IMAGE_OK", + ); + assert.match( + run.automation.env_defaults.LANGBOT_E2E_IMAGE_BASE64_PATH, + /red-square\.png\.base64$/, + ); assert.equal(run.automation.pipeline_env_required, true); }); test("MCP stdio case passes case-specific failure signals to automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "mcp-stdio-tool-call", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "mcp-stdio-tool-call", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.match(run.automation.env_defaults.LANGBOT_E2E_FAILURE_SIGNALS, /qa-plugin-smoke:mcp-ok-local-agent/); - assert.match(run.automation.env_defaults.LANGBOT_E2E_FAILURE_SIGNALS, /model_not_found/); + assert.match( + run.automation.env_defaults.LANGBOT_E2E_FAILURE_SIGNALS, + /qa-plugin-smoke:mcp-ok-local-agent/, + ); + assert.match( + run.automation.env_defaults.LANGBOT_E2E_FAILURE_SIGNALS, + /model_not_found/, + ); }); test("MCP stdio tool-call case setups pipeline and registered MCP server", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "mcp-stdio-tool-call", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "mcp-stdio-tool-call", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", - "case:mcp-stdio-register", - ]); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "case:mcp-stdio-register", + ], + ); - const planResult = capture(() => commandTestPlan(ctx(["test", "plan", "mcp-stdio-tool-call", "--json"]))); + const planResult = capture(() => + commandTestPlan(ctx(["test", "plan", "mcp-stdio-tool-call", "--json"])), + ); assert.equal(planResult.code, 0); const plan = JSON.parse(planResult.output); assert.deepEqual(plan.setup_provides_env, [ "LANGBOT_LOCAL_AGENT_PIPELINE_URL", "LANGBOT_LOCAL_AGENT_PIPELINE_NAME", ]); - assert.ok(!plan.preconditions.some((item: string) => item.includes("points to the local-agent pipeline"))); + assert.ok( + !plan.preconditions.some((item: string) => + item.includes("points to the local-agent pipeline"), + ), + ); }); test("generic pipeline automation can still use the shared pipeline env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "pipeline-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "pipeline-debug-chat", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.pipeline_env_required, false); assert.deepEqual(run.automation.env_aliases, []); - assert.ok(run.automation.required_env.includes("LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME")); + assert.ok( + run.automation.required_env.includes( + "LANGBOT_PIPELINE_URL|LANGBOT_PIPELINE_NAME", + ), + ); }); test("AgentRunner live install case exposes package automation defaults", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "agent-runner-live-install", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "agent-runner-live-install", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal( run.automation.env_defaults.LANGBOT_E2E_PLUGIN_PACKAGE, "skills/langbot-testing/fixtures/plugins/qa-agent-runner/dist/qa-agent-runner-0.1.0.lbpkg", ); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_PLUGIN_ID, "qa/agent-runner"); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID, "plugin:qa/agent-runner/default"); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_PLUGIN_ID, + "qa/agent-runner", + ); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID, + "plugin:qa/agent-runner/default", + ); }); test("QA plugin live install checks the fixture package before installed state", () => { @@ -2367,18 +3325,19 @@ test("QA plugin live install checks the fixture package before installed state", }); test("AgentRunner QA Debug Chat case uses dedicated pipeline env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "agent-runner-qa-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "agent-runner-qa-debug-chat", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); assert.equal(run.automation.script, "scripts/e2e/pipeline-debug-chat.mjs"); assert.equal(run.automation.pipeline_env_required, true); - assert.equal(run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID, "plugin:qa/agent-runner/default"); + assert.equal( + run.automation.env_defaults.LANGBOT_E2E_EXPECTED_RUNNER_ID, + "plugin:qa/agent-runner/default", + ); assert.deepEqual( run.setup_automation.map((item: { entry: string }) => item.entry), [ @@ -2386,141 +3345,245 @@ test("AgentRunner QA Debug Chat case uses dedicated pipeline env", () => { "node:scripts/e2e/ensure-qa-agent-runner-pipeline.mjs --write-env", ], ); - assert.ok(run.automation.env_aliases.some((alias: { target: string; source: string }) => ( - alias.target === "LANGBOT_E2E_PIPELINE_URL" && alias.source === "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL" - ))); + assert.ok( + run.automation.env_aliases.some( + (alias: { target: string; source: string }) => + alias.target === "LANGBOT_E2E_PIPELINE_URL" && + alias.source === "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL", + ), + ); }); test("AgentRunner QA Debug Chat setup automation removes manual readiness", () => { - withEnv({ - LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", - LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", - }, () => { - const planResult = capture(() => commandTestPlan(ctx(["test", "plan", "agent-runner-qa-debug-chat", "--json"]))); - assert.equal(planResult.code, 0); - const plan = JSON.parse(planResult.output); - assert.equal(plan.manual_readiness.status, "not_required"); - assert.deepEqual(plan.setup_provides_env, [ - "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL", - "LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME", - ]); - assert.equal(plan.automation_readiness.status, "ready"); + withEnv( + { + LANGBOT_BROWSER_PROFILE: "/tmp/langbot-test-profile", + LANGBOT_CHROMIUM_EXECUTABLE: "/tmp/langbot-test-chromium", + }, + () => { + const planResult = capture(() => + commandTestPlan( + ctx(["test", "plan", "agent-runner-qa-debug-chat", "--json"]), + ), + ); + assert.equal(planResult.code, 0); + const plan = JSON.parse(planResult.output); + assert.equal(plan.manual_readiness.status, "not_required"); + assert.deepEqual(plan.setup_provides_env, [ + "LANGBOT_QA_AGENT_RUNNER_PIPELINE_URL", + "LANGBOT_QA_AGENT_RUNNER_PIPELINE_NAME", + ]); + assert.equal(plan.automation_readiness.status, "ready"); - const suiteResult = capture(() => commandSuitePlan(ctx(["suite", "plan", "agent-runner-release-gate", "--json"]))); - assert.equal(suiteResult.code, 0); - const suite = JSON.parse(suiteResult.output); - assert.ok(!suite.readiness.manual_check_cases.includes("agent-runner-qa-debug-chat")); - }); + const suiteResult = capture(() => + commandSuitePlan( + ctx(["suite", "plan", "agent-runner-release-gate", "--json"]), + ), + ); + assert.equal(suiteResult.code, 0); + const suite = JSON.parse(suiteResult.output); + assert.ok( + !suite.readiness.manual_check_cases.includes( + "agent-runner-qa-debug-chat", + ), + ); + }, + ); }); test("ACP AgentRunner Debug Chat case setups the ACP pipeline env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "acp-agent-runner-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "acp-agent-runner-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env", - ]); - assert.ok(run.automation.env_aliases.some((alias: { target: string; source: string }) => ( - alias.target === "LANGBOT_E2E_PIPELINE_URL" && alias.source === "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL" - ))); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + ["node:scripts/e2e/ensure-acp-agent-runner-pipeline.mjs --write-env"], + ); + assert.ok( + run.automation.env_aliases.some( + (alias: { target: string; source: string }) => + alias.target === "LANGBOT_E2E_PIPELINE_URL" && + alias.source === "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", + ), + ); - const planResult = capture(() => commandTestPlan(ctx(["test", "plan", "acp-agent-runner-debug-chat", "--json"]))); + const planResult = capture(() => + commandTestPlan( + ctx(["test", "plan", "acp-agent-runner-debug-chat", "--json"]), + ), + ); assert.equal(planResult.code, 0); const plan = JSON.parse(planResult.output); assert.deepEqual(plan.setup_provides_env, [ "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_URL", "LANGBOT_ACP_AGENT_RUNNER_PIPELINE_NAME", ]); - assert.ok(!plan.preconditions.some((item: string) => item.includes("pipeline AI runner"))); + assert.ok( + !plan.preconditions.some((item: string) => + item.includes("pipeline AI runner"), + ), + ); }); test("local-agent plugin cases setup the QA plugin smoke fixture", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-plugin-tool-call-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-plugin-tool-call-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", - "case:qa-plugin-smoke-live-install", - ]); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "case:qa-plugin-smoke-live-install", + ], + ); }); test("local-agent RAG case only requires the KB fixture env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-rag-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx(["test", "run", "local-agent-rag-debug-chat", "--dry-run", "--json"]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.ok(run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID")); - assert.ok(!run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_TEXT_MODEL_UUID")); + assert.ok( + run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID"), + ); + assert.ok( + !run.automation.required_env.includes( + "LANGBOT_LOCAL_AGENT_RAG_TEXT_MODEL_UUID", + ), + ); assert.equal( run.automation.env_defaults.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON, JSON.stringify({ - "knowledge-bases": [ - loadEnv(root).LANGBOT_LOCAL_AGENT_RAG_KB_UUID || "", - ], + "knowledge-bases": [loadEnv(root).LANGBOT_LOCAL_AGENT_RAG_KB_UUID || ""], }), ); }); test("LangRAG retrieve readiness requires a KB UUID alternative", () => { - const result = capture(() => commandTestPlan(ctx(["test", "plan", "langrag-kb-retrieve", "--json"]))); + const result = capture(() => + commandTestPlan(ctx(["test", "plan", "langrag-kb-retrieve", "--json"])), + ); assert.equal(result.code, 0); const plan = JSON.parse(result.output); - assert.ok(plan.automation_readiness.required.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID|LANGBOT_RAG_KB_UUID")); + assert.ok( + plan.automation_readiness.required.includes( + "LANGBOT_LOCAL_AGENT_RAG_KB_UUID|LANGBOT_RAG_KB_UUID", + ), + ); }); test("local-agent RAG multimodal case setups the KB fixture env", () => { - const result = capture(() => commandTestRun(ctx([ - "test", - "run", - "local-agent-rag-multimodal-debug-chat", - "--dry-run", - "--json", - ]))); + const result = capture(() => + commandTestRun( + ctx([ + "test", + "run", + "local-agent-rag-multimodal-debug-chat", + "--dry-run", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const run = JSON.parse(result.output); - assert.ok(run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID")); + assert.ok( + run.automation.required_env.includes("LANGBOT_LOCAL_AGENT_RAG_KB_UUID"), + ); assert.equal( run.automation.env_defaults.LANGBOT_E2E_RUNNER_CONFIG_PATCH_JSON, JSON.stringify({ - "knowledge-bases": [ - loadEnv(root).LANGBOT_LOCAL_AGENT_RAG_KB_UUID || "", - ], + "knowledge-bases": [loadEnv(root).LANGBOT_LOCAL_AGENT_RAG_KB_UUID || ""], }), ); - assert.deepEqual(run.setup_automation.map((item: { entry: string }) => item.entry), [ - "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", - "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", - ]); + assert.deepEqual( + run.setup_automation.map((item: { entry: string }) => item.entry), + [ + "node:scripts/e2e/ensure-local-agent-pipeline.mjs --write-env", + "node:scripts/e2e/ensure-langrag-sentinel-kb.mjs --write-env", + ], + ); }); test("test report renders a reusable evidence template", () => { - const result = capture(() => commandTestReport(ctx(["test", "report", "pipeline-debug-chat", "--no-auto-log"]))); + const result = capture(() => + commandTestReport( + ctx(["test", "report", "pipeline-debug-chat", "--no-auto-log"]), + ), + ); assert.equal(result.code, 0); assert.match(result.output, /^# Test Report: pipeline-debug-chat/m); - assert.match(result.output, /result: pass \| fail \| blocked \| env_issue \| flaky/); + assert.match( + result.output, + /result: pass \| fail \| blocked \| env_issue \| flaky/, + ); assert.match(result.output, /## Log Guard/); assert.match(result.output, /## Automation Result/); assert.match(result.output, /## Required Evidence/); assert.match(result.output, /no log files provided/); }); +test("test report promotes loaded automation evidence into result section", () => { + const tmp = mkdtempSync(join(tmpdir(), "lbs-report-automation-")); + try { + writeFileSync( + join(tmp, "automation-result.json"), + JSON.stringify({ + status: "pass", + reason: "latency thresholds passed", + url: "http://127.0.0.1:5300", + artifacts: { metrics_json: join(tmp, "metrics.json") }, + }), + ); + + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "langbot-live-backend-latency", + "--evidence-dir", + tmp, + "--no-auto-log", + ]), + ), + ); + + assert.equal(result.code, 0); + assert.match( + result.output, + /## Result\n- result: pass\n- reason: latency thresholds passed/, + ); + assert.match(result.output, /- target_tested: http:\/\/127\.0\.0\.1:5300/); + assert.doesNotMatch(result.output, /target_tested: TODO/); + assert.match(result.output, /## Automation Result/); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } +}); + test("validate rejects dangling case references and missing automation scripts", () => { const tmp = mkdtempSync(join(tmpdir(), "lbs-validate-strict-")); try { @@ -2533,11 +3596,22 @@ test("validate rejects dangling case references and missing automation scripts", mkdirSync(join(testingDir, "fixtures"), { recursive: true }); mkdirSync(join(testingDir, "suites"), { recursive: true }); mkdirSync(envSetupDir, { recursive: true }); - for (const schemaName of ["case.schema.json", "suite.schema.json", "troubleshooting.schema.json", "skill-index.schema.json"]) { + for (const schemaName of [ + "case.schema.json", + "suite.schema.json", + "troubleshooting.schema.json", + "skill-index.schema.json", + ]) { writeFileSync(join(schemasDir, schemaName), "{}"); } - writeFileSync(join(envSetupDir, "SKILL.md"), "---\nname: langbot-env-setup\ndescription: Env setup.\n---\n\n# Env\n"); - writeFileSync(join(testingDir, "SKILL.md"), "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n"); + writeFileSync( + join(envSetupDir, "SKILL.md"), + "---\nname: langbot-env-setup\ndescription: Env setup.\n---\n\n# Env\n", + ); + writeFileSync( + join(testingDir, "SKILL.md"), + "---\nname: langbot-testing\ndescription: Testing.\n---\n\n# Testing\n", + ); writeFileSync( join(skillsDir, ".env"), [ @@ -2584,7 +3658,10 @@ test("validate rejects dangling case references and missing automation scripts", " - missing-trouble", ].join("\n"), ); - for (const [id, target] of [["cycle-a", "cycle-b"], ["cycle-b", "cycle-a"]]) { + for (const [id, target] of [ + ["cycle-a", "cycle-b"], + ["cycle-b", "cycle-a"], + ]) { writeFileSync( join(testingDir, "cases", `${id}.yaml`), [ @@ -2627,7 +3704,14 @@ test("validate rejects dangling case references and missing automation scripts", ); writeFileSync( join(testingDir, "fixtures", "fixtures.json"), - JSON.stringify([{ id: "bad-fixture", title: "Bad Fixture", path: "fixtures/missing.txt", related_cases: ["missing-case"] }]), + JSON.stringify([ + { + id: "bad-fixture", + title: "Bad Fixture", + path: "fixtures/missing.txt", + related_cases: ["missing-case"], + }, + ]), ); const result = captureAll(() => commandValidate(tmp)); @@ -2659,21 +3743,43 @@ test("test report JSON scans logs and redacts secrets", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx(["test", "report", "pipeline-debug-chat", "--backend-log", logPath, "--json"]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); assert.doesNotMatch(result.output, /sk-test-secret/); const report = JSON.parse(result.output); assert.equal(report.log_guard.status, "fail"); - assert.ok(report.log_guard.findings.some((finding: { kind: string }) => ( - finding.kind === "case_failure_pattern" - ))); - assert.ok(report.log_guard.findings.some((finding: { troubleshooting_id?: string }) => ( - finding.troubleshooting_id === "plugin-runtime-timeout" - ))); - assert.ok(report.log_guard.findings.some((finding: { kind: string }) => finding.kind === "python_traceback")); + assert.ok( + report.log_guard.findings.some( + (finding: { kind: string }) => finding.kind === "case_failure_pattern", + ), + ); + assert.ok( + report.log_guard.findings.some( + (finding: { troubleshooting_id?: string }) => + finding.troubleshooting_id === "plugin-runtime-timeout", + ), + ); + assert.ok( + report.log_guard.findings.some( + (finding: { kind: string }) => finding.kind === "python_traceback", + ), + ); - const secretFinding = report.log_guard.findings.find((finding: { kind: string }) => finding.kind === "secret_leak"); + const secretFinding = report.log_guard.findings.find( + (finding: { kind: string }) => finding.kind === "secret_leak", + ); assert.ok(secretFinding); assert.match(secretFinding.excerpt, /\[redacted\]/); } finally { @@ -2690,15 +3796,33 @@ test("test report does not treat invalid api key wording as a secret leak", () = "RequesterError: 模型请求失败: 无效的 api-key: Error code: 401 - invalid api key\n", ); - const result = capture(() => commandTestReport(ctx(["test", "report", "mcp-stdio-tool-call", "--backend-log", logPath, "--json"]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "mcp-stdio-tool-call", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); assert.match(result.output, /api-key: Error code/); const report = JSON.parse(result.output); - assert.ok(!report.log_guard.findings.some((finding: { kind: string }) => finding.kind === "secret_leak")); - assert.ok(report.log_guard.findings.some((finding: { troubleshooting_id?: string }) => ( - finding.troubleshooting_id === "local-agent-model-route-unavailable" - ))); + assert.ok( + !report.log_guard.findings.some( + (finding: { kind: string }) => finding.kind === "secret_leak", + ), + ); + assert.ok( + report.log_guard.findings.some( + (finding: { troubleshooting_id?: string }) => + finding.troubleshooting_id === "local-agent-model-route-unavailable", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2716,21 +3840,28 @@ test("test report records declared success signals from logs", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.status, "pass"); assert.equal(report.log_guard.success_signals.length, 2); - assert.ok(report.log_guard.success_signals.some((signal: { pattern: string }) => ( - signal.pattern === "Streaming completed" - ))); + assert.ok( + report.log_guard.success_signals.some( + (signal: { pattern: string }) => + signal.pattern === "Streaming completed", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2742,20 +3873,27 @@ test("test report warns when declared success signals are missing", () => { const logPath = join(tmp, "backend.log"); writeFileSync(logPath, "INFO request started\nINFO request ended\n"); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.status, "warning"); - assert.ok(report.log_guard.findings.some((finding: { kind: string }) => ( - finding.kind === "missing_success_signal" - ))); + assert.ok( + report.log_guard.findings.some( + (finding: { kind: string }) => + finding.kind === "missing_success_signal", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2775,28 +3913,39 @@ test("test report can limit log guard to tail lines", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--tail-lines", - "2", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--tail-lines", + "2", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.scan.mode, "tail-lines"); assert.equal(report.log_guard.scan.tail_lines, 2); assert.equal(report.log_guard.sources[0].line_count, 2); assert.equal(report.log_guard.sources[0].start_line, 3); - assert.ok(report.log_guard.findings.some((finding: { troubleshooting_id?: string }) => ( - finding.troubleshooting_id === "plugin-runtime-timeout" - ))); - assert.ok(!report.log_guard.findings.some((finding: { kind: string; excerpt?: string }) => ( - finding.kind === "error_log" && finding.excerpt?.includes("old failure") - ))); + assert.ok( + report.log_guard.findings.some( + (finding: { troubleshooting_id?: string }) => + finding.troubleshooting_id === "plugin-runtime-timeout", + ), + ); + assert.ok( + !report.log_guard.findings.some( + (finding: { kind: string; excerpt?: string }) => + finding.kind === "error_log" && + finding.excerpt?.includes("old failure"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2816,26 +3965,38 @@ test("test report can limit log guard with since timestamp", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--since", - "2026-05-21T10:30:00+08:00", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--since", + "2026-05-21T10:30:00+08:00", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.scan.mode, "since"); assert.equal(report.log_guard.sources[0].line_count, 3); assert.equal(report.log_guard.sources[0].start_line, 2); assert.equal(report.log_guard.sources[0].timestamped_line_count, 3); - assert.ok(report.log_guard.findings.some((finding: { line?: number; troubleshooting_id?: string }) => ( - finding.line === 2 && finding.troubleshooting_id === "plugin-runtime-timeout" - ))); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("old failure"))); + assert.ok( + report.log_guard.findings.some( + (finding: { line?: number; troubleshooting_id?: string }) => + finding.line === 2 && + finding.troubleshooting_id === "plugin-runtime-timeout", + ), + ); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("old failure"), + ), + ); assert.doesNotMatch(result.output, /sk-since-secret/); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -2856,18 +4017,22 @@ test("test report can limit log guard with since and until timestamps", () => { ].join("\n"), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--backend-log", - logPath, - "--since", - "2026-05-21T10:30:00+08:00", - "--until", - "2026-05-21T10:32:00+08:00", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--backend-log", + logPath, + "--since", + "2026-05-21T10:30:00+08:00", + "--until", + "2026-05-21T10:32:00+08:00", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.scan.mode, "since+until"); @@ -2875,8 +4040,16 @@ test("test report can limit log guard with since and until timestamps", () => { assert.equal(report.log_guard.sources[0].start_line, 2); assert.equal(report.log_guard.sources[0].end_line, 3); assert.equal(report.log_guard.status, "pass"); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("old failure"))); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("later failure"))); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("old failure"), + ), + ); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("later failure"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2891,20 +4064,28 @@ test("test report classifies model route failures as env_issue", () => { "[05-21 10:31:00.000] runner.py (2) - [ERROR] : runner.llm_error model_not_found no available channel for model gpt-test\n", ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "local-agent-plugin-tool-call-debug-chat", - "--backend-log", - logPath, - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "local-agent-plugin-tool-call-debug-chat", + "--backend-log", + logPath, + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.status, "env_issue"); - assert.ok(report.log_guard.findings.some((finding: { severity?: string; troubleshooting_id?: string }) => ( - finding.severity === "env_issue" && finding.troubleshooting_id === "local-agent-model-route-unavailable" - ))); + assert.ok( + report.log_guard.findings.some( + (finding: { severity?: string; troubleshooting_id?: string }) => + finding.severity === "env_issue" && + finding.troubleshooting_id === "local-agent-model-route-unavailable", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2936,15 +4117,19 @@ test("test report infers scan window from automation result evidence", () => { }), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "pipeline-debug-chat", - "--console-log", - consoleLog, - "--no-auto-log", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "pipeline-debug-chat", + "--console-log", + consoleLog, + "--no-auto-log", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.log_guard.scan.mode, "since+until"); @@ -2955,8 +4140,16 @@ test("test report infers scan window from automation result evidence", () => { assert.equal(report.automation_result.status, "loaded"); assert.equal(report.automation_result.result, "pass"); assert.equal(report.automation_result.reason, "UI sentinel appeared."); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("old failure"))); - assert.ok(!report.log_guard.findings.some((finding: { excerpt?: string }) => finding.excerpt?.includes("later failure"))); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("old failure"), + ), + ); + assert.ok( + !report.log_guard.findings.some((finding: { excerpt?: string }) => + finding.excerpt?.includes("later failure"), + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -2968,7 +4161,10 @@ test("test report does not treat final result as automation evidence", () => { const evidenceDir = join(tmp, "evidence", "run-final"); mkdirSync(evidenceDir, { recursive: true }); const consoleLog = join(evidenceDir, "console.log"); - writeFileSync(consoleLog, "[05-21 10:31:00.000] ui.js (1) - [INFO] : opened\n"); + writeFileSync( + consoleLog, + "[05-21 10:31:00.000] ui.js (1) - [INFO] : opened\n", + ); writeFileSync( join(evidenceDir, "result.json"), JSON.stringify({ @@ -2981,20 +4177,27 @@ test("test report does not treat final result as automation evidence", () => { }), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "webui-login-state", - "--console-log", - consoleLog, - "--no-auto-log", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "webui-login-state", + "--console-log", + consoleLog, + "--no-auto-log", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.automation_result.status, "not_provided"); - assert.match(report.automation_result.reason, /only final result\.json is present/); + assert.match( + report.automation_result.reason, + /only final result\.json is present/, + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3006,7 +4209,10 @@ test("test report still scans untimestamped explicit console evidence within an const evidenceDir = join(tmp, "evidence", "run-untimestamped"); mkdirSync(evidenceDir, { recursive: true }); const consoleLog = join(evidenceDir, "console.log"); - writeFileSync(consoleLog, "[error] Uncaught TypeError: Cannot read properties of undefined\n"); + writeFileSync( + consoleLog, + "[error] Uncaught TypeError: Cannot read properties of undefined\n", + ); writeFileSync( join(evidenceDir, "result.json"), JSON.stringify({ @@ -3017,15 +4223,19 @@ test("test report still scans untimestamped explicit console evidence within an }), ); - const result = capture(() => commandTestReport(ctx([ - "test", - "report", - "webui-login-state", - "--console-log", - consoleLog, - "--no-auto-log", - "--json", - ]))); + const result = capture(() => + commandTestReport( + ctx([ + "test", + "report", + "webui-login-state", + "--console-log", + consoleLog, + "--no-auto-log", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); @@ -3033,9 +4243,12 @@ test("test report still scans untimestamped explicit console evidence within an assert.equal(report.log_guard.sources[0].timestamped_line_count, 0); assert.ok(report.log_guard.sources[0].line_count >= 1); assert.equal(report.log_guard.status, "fail"); - assert.ok(report.log_guard.findings.some((finding: { kind: string }) => ( - finding.kind === "frontend_uncaught_error" - ))); + assert.ok( + report.log_guard.findings.some( + (finding: { kind: string }) => + finding.kind === "frontend_uncaught_error", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3045,10 +4258,17 @@ test("test report can write markdown to an output path", () => { const tmp = mkdtempSync(join(tmpdir(), "lbs-report-output-")); try { const output = join(tmp, "reports", "pipeline-debug-chat.md"); - const result = capture(() => commandTestReport(ctx(["test", "report", "pipeline-debug-chat", "--output", output]))); + const result = capture(() => + commandTestReport( + ctx(["test", "report", "pipeline-debug-chat", "--output", output]), + ), + ); assert.equal(result.code, 0); assert.match(result.output, /pipeline-debug-chat\.md$/); - assert.match(readFileSync(output, "utf8"), /^# Test Report: pipeline-debug-chat/m); + assert.match( + readFileSync(output, "utf8"), + /^# Test Report: pipeline-debug-chat/m, + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3067,32 +4287,49 @@ test("log scan reuses case-aware log guard patterns", () => { ].join("\n"), ); - const result = capture(() => commandLogScan(ctx([ - "log", - "scan", - "--backend-log", - logPath, - "--case", - "pipeline-debug-chat", - "--json", - ]))); + const result = capture(() => + commandLogScan( + ctx([ + "log", + "scan", + "--backend-log", + logPath, + "--case", + "pipeline-debug-chat", + "--json", + ]), + ), + ); assert.equal(result.code, 0); const report = JSON.parse(result.output); assert.equal(report.status, "fail"); - assert.ok(report.success_signals.some((signal: { pattern: string }) => signal.pattern === "Streaming completed")); - assert.ok(report.findings.some((finding: { kind: string }) => finding.kind === "case_failure_pattern")); + assert.ok( + report.success_signals.some( + (signal: { pattern: string }) => + signal.pattern === "Streaming completed", + ), + ); + assert.ok( + report.findings.some( + (finding: { kind: string }) => finding.kind === "case_failure_pattern", + ), + ); - const strict = capture(() => commandLogScan(ctx([ - "log", - "scan", - "--backend-log", - logPath, - "--case", - "pipeline-debug-chat", - "--strict", - "--json", - ]))); + const strict = capture(() => + commandLogScan( + ctx([ + "log", + "scan", + "--backend-log", + logPath, + "--case", + "pipeline-debug-chat", + "--strict", + "--json", + ]), + ), + ); assert.equal(strict.code, 1); } finally { rmSync(tmp, { recursive: true, force: true }); @@ -3106,42 +4343,54 @@ test("log guard start and stop bound a QA log window", () => { const outputDir = join(tmp, "guards"); writeFileSync(logPath, "INFO before guard\n"); - const start = capture(() => commandLogGuard(ctx([ - "log", - "guard", - "start", - "--run-id", - "qa-run", - "--output-dir", - outputDir, - "--backend-log", - logPath, - "--case", - "pipeline-debug-chat", - "--json", - ]))); + const start = capture(() => + commandLogGuard( + ctx([ + "log", + "guard", + "start", + "--run-id", + "qa-run", + "--output-dir", + outputDir, + "--backend-log", + logPath, + "--case", + "pipeline-debug-chat", + "--json", + ]), + ), + ); assert.equal(start.code, 0); const session = JSON.parse(start.output); assert.equal(session.run_id, "qa-run"); assert.ok(existsSync(join(outputDir, "qa-run.json"))); appendFileSync(logPath, "Traceback (most recent call last):\n"); - const stop = capture(() => commandLogGuard(ctx([ - "log", - "guard", - "stop", - "--run-id", - "qa-run", - "--output-dir", - outputDir, - "--json", - ]))); + const stop = capture(() => + commandLogGuard( + ctx([ + "log", + "guard", + "stop", + "--run-id", + "qa-run", + "--output-dir", + outputDir, + "--json", + ]), + ), + ); assert.equal(stop.code, 1); const report = JSON.parse(stop.output); assert.equal(report.session.run_id, "qa-run"); assert.equal(report.result.status, "fail"); - assert.ok(report.result.findings.some((finding: { kind: string }) => finding.kind === "python_traceback")); + assert.ok( + report.result.findings.some( + (finding: { kind: string }) => finding.kind === "python_traceback", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } @@ -3153,18 +4402,22 @@ test("log watch observes appended LangBot backend lines", async () => { const logPath = join(tmp, "backend.log"); writeFileSync(logPath, "INFO existing line\n"); - const watching = captureAsync(() => commandLogWatch(ctx([ - "log", - "watch", - "--backend-log", - logPath, - "--duration-ms", - "220", - "--interval-ms", - "20", - "--strict", - "--json", - ]))); + const watching = captureAsync(() => + commandLogWatch( + ctx([ + "log", + "watch", + "--backend-log", + logPath, + "--duration-ms", + "220", + "--interval-ms", + "20", + "--strict", + "--json", + ]), + ), + ); setTimeout(() => { appendFileSync(logPath, "Traceback (most recent call last):\n"); }, 50); @@ -3175,14 +4428,20 @@ test("log watch observes appended LangBot backend lines", async () => { assert.equal(summary.mode, "watch"); assert.equal(summary.status, "fail"); assert.ok(summary.bytes_read > 0); - assert.ok(summary.findings.some((finding: { kind: string }) => finding.kind === "python_traceback")); + assert.ok( + summary.findings.some( + (finding: { kind: string }) => finding.kind === "python_traceback", + ), + ); } finally { rmSync(tmp, { recursive: true, force: true }); } }); test("trouble search finds structured troubleshooting entries", () => { - const result = capture(() => commandTroubleSearch(ctx(["trouble", "search", "proxy"]))); + const result = capture(() => + commandTroubleSearch(ctx(["trouble", "search", "proxy"])), + ); assert.equal(result.code, 0); assert.match(result.output, /proxy-env-mismatch/); }); @@ -3191,7 +4450,10 @@ test("env local overrides shared env defaults", () => { const tmp = mkdtempSync(join(tmpdir(), "lbs-env-")); try { mkdirSync(join(tmp, "skills")); - writeFileSync(join(tmp, "skills", ".env"), "LANGBOT_REPO=/shared\nLANGBOT_BACKEND_URL=http://127.0.0.1:5300\n"); + writeFileSync( + join(tmp, "skills", ".env"), + "LANGBOT_REPO=/shared\nLANGBOT_BACKEND_URL=http://127.0.0.1:5300\n", + ); writeFileSync(join(tmp, "skills", ".env.local"), "LANGBOT_REPO=/local\n"); assert.deepEqual(loadEnv(tmp), { diff --git a/src/langbot/pkg/api/http/controller/groups/pipelines/pipelines.py b/src/langbot/pkg/api/http/controller/groups/pipelines/pipelines.py index c6b2a1b43..2e45add77 100644 --- a/src/langbot/pkg/api/http/controller/groups/pipelines/pipelines.py +++ b/src/langbot/pkg/api/http/controller/groups/pipelines/pipelines.py @@ -86,6 +86,10 @@ class PipelinesRouterGroup(group.RouterGroup): 'available_plugins': plugins, 'bound_mcp_servers': extensions_prefs.get('mcp_servers', []), 'available_mcp_servers': mcp_servers, + 'bound_mcp_resources': extensions_prefs.get('mcp_resources', []), + 'mcp_resource_agent_read_enabled': extensions_prefs.get( + 'mcp_resource_agent_read_enabled', True + ), 'bound_skills': extensions_prefs.get('skills', []), 'available_skills': available_skills, } @@ -99,6 +103,8 @@ class PipelinesRouterGroup(group.RouterGroup): bound_plugins = json_data.get('bound_plugins', []) bound_mcp_servers = json_data.get('bound_mcp_servers', []) bound_skills = json_data.get('bound_skills', []) + bound_mcp_resources = json_data.get('bound_mcp_resources') + mcp_resource_agent_read_enabled = json_data.get('mcp_resource_agent_read_enabled') await self.ap.pipeline_service.update_pipeline_extensions( pipeline_uuid, @@ -108,6 +114,8 @@ class PipelinesRouterGroup(group.RouterGroup): enable_all_mcp_servers, bound_skills=bound_skills, enable_all_skills=enable_all_skills, + bound_mcp_resources=bound_mcp_resources, + mcp_resource_agent_read_enabled=mcp_resource_agent_read_enabled, ) return self.success() diff --git a/src/langbot/pkg/api/http/controller/groups/platform/bots.py b/src/langbot/pkg/api/http/controller/groups/platform/bots.py index ac580b1a3..e3a13b789 100644 --- a/src/langbot/pkg/api/http/controller/groups/platform/bots.py +++ b/src/langbot/pkg/api/http/controller/groups/platform/bots.py @@ -18,7 +18,6 @@ class BotsRouterGroup(group.RouterGroup): @self.route('/', methods=['GET', 'PUT', 'DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY) async def _(bot_uuid: str) -> str: if quart.request.method == 'GET': - # 返回运行时信息,包括webhook地址等 bot = await self.ap.bot_service.get_runtime_bot_info(bot_uuid) if bot is None: return self.http_status(404, -1, 'bot not found') @@ -37,30 +36,21 @@ class BotsRouterGroup(group.RouterGroup): from_index = json_data.get('from_index', -1) max_count = json_data.get('max_count', 10) logs, total_count = await self.ap.bot_service.list_event_logs(bot_uuid, from_index, max_count) - return self.success( - data={ - 'logs': logs, - 'total_count': total_count, - } - ) + return self.success(data={'logs': logs, 'total_count': total_count}) @self.route('//send_message', methods=['POST'], auth_type=group.AuthType.API_KEY) async def _(bot_uuid: str) -> str: - """Send message to a specific target via bot""" json_data = await quart.request.json target_type = json_data.get('target_type') target_id = json_data.get('target_id') message_chain_data = json_data.get('message_chain') - # Validate required fields if not target_type: return self.http_status(400, -1, 'target_type is required') if not target_id: return self.http_status(400, -1, 'target_id is required') if not message_chain_data: return self.http_status(400, -1, 'message_chain is required') - - # Validate target_type if target_type not in ['person', 'group']: return self.http_status(400, -1, 'target_type must be either "person" or "group"') @@ -72,3 +62,29 @@ class BotsRouterGroup(group.RouterGroup): traceback.print_exc() return self.http_status(500, -1, f'Failed to send message: {str(e)}') + + # ============ Bot Admins ============ + + @self.route('//admins', methods=['GET', 'POST'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY) + async def _(bot_uuid: str) -> str: + if quart.request.method == 'GET': + admins = await self.ap.bot_service.get_bot_admins(bot_uuid) + return self.success(data={'admins': admins}) + elif quart.request.method == 'POST': + json_data = await quart.request.json + launcher_type = json_data.get('launcher_type', '').strip() + launcher_id = str(json_data.get('launcher_id', '')).strip() + if not launcher_type or not launcher_id: + return self.http_status(400, -1, 'launcher_type and launcher_id are required') + try: + admin_id = await self.ap.bot_service.add_bot_admin(bot_uuid, launcher_type, launcher_id) + return self.success(data={'id': admin_id}) + except Exception as e: + return self.http_status(409, -1, str(e)) + + @self.route( + '//admins/', methods=['DELETE'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY + ) + async def _(bot_uuid: str, admin_id: int) -> str: + await self.ap.bot_service.delete_bot_admin(bot_uuid, admin_id) + return self.success() diff --git a/src/langbot/pkg/api/http/controller/groups/resources/mcp.py b/src/langbot/pkg/api/http/controller/groups/resources/mcp.py index e6bc2e77d..4ee3f3e84 100644 --- a/src/langbot/pkg/api/http/controller/groups/resources/mcp.py +++ b/src/langbot/pkg/api/http/controller/groups/resources/mcp.py @@ -2,6 +2,7 @@ from __future__ import annotations import quart import traceback +from urllib.parse import unquote from ... import group @@ -66,3 +67,50 @@ class MCPRouterGroup(group.RouterGroup): server_data = await quart.request.json task_id = await self.ap.mcp_service.test_mcp_server(server_name=server_name, server_data=server_data) return self.success(data={'task_id': task_id}) + + @self.route('/servers//resources', methods=['GET'], auth_type=group.AuthType.USER_TOKEN) + async def _(server_name: str) -> str: + """Get resources from an MCP server""" + server_name = unquote(server_name) + try: + resources = await self.ap.mcp_service.get_mcp_server_resources(server_name) + templates = await self.ap.mcp_service.get_mcp_server_resource_templates(server_name) + runtime_info = await self.ap.mcp_service.get_runtime_info(server_name) + return self.success( + data={ + 'resources': resources, + 'resource_templates': templates, + 'resource_capabilities': (runtime_info or {}).get('resource_capabilities', {}), + } + ) + except Exception as e: + return self.http_status(500, -1, f'Failed to get resources: {str(e)}') + + @self.route('/servers//resource-templates', methods=['GET'], auth_type=group.AuthType.USER_TOKEN) + async def _(server_name: str) -> str: + """Get resource templates from an MCP server""" + server_name = unquote(server_name) + try: + templates = await self.ap.mcp_service.get_mcp_server_resource_templates(server_name) + return self.success(data={'resource_templates': templates}) + except Exception as e: + return self.http_status(500, -1, f'Failed to get resource templates: {str(e)}') + + @self.route('/servers//resources/read', methods=['POST'], auth_type=group.AuthType.USER_TOKEN) + async def _(server_name: str) -> str: + """Read a resource from an MCP server""" + server_name = unquote(server_name) + data = await quart.request.json + uri = data.get('uri') + if not uri: + return self.http_status(400, -1, 'URI is required') + try: + envelope = await self.ap.mcp_service.read_mcp_server_resource_envelope( + server_name, + uri, + max_bytes=data.get('max_bytes'), + include_blob=bool(data.get('include_blob', False)), + ) + return self.success(data=envelope) + except Exception as e: + return self.http_status(500, -1, f'Failed to read resource: {str(e)}') diff --git a/src/langbot/pkg/api/http/controller/groups/resources/tools.py b/src/langbot/pkg/api/http/controller/groups/resources/tools.py index de827e544..128a0647d 100644 --- a/src/langbot/pkg/api/http/controller/groups/resources/tools.py +++ b/src/langbot/pkg/api/http/controller/groups/resources/tools.py @@ -1,5 +1,7 @@ from __future__ import annotations +import quart + from ... import group @@ -9,25 +11,41 @@ class ToolsRouterGroup(group.RouterGroup): @self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN) async def _() -> str: """获取所有可用工具列表""" - tools = await self.ap.tool_mgr.get_all_tools() + pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id') + bound_plugins: list[str] | None = None + bound_mcp_servers: list[str] | None = None - tool_list = [] - for tool in tools: - tool_list.append( - { - 'name': tool.name, - 'description': tool.description, - 'human_desc': tool.human_desc, - 'parameters': tool.parameters, - } - ) + if pipeline_uuid: + pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid) + if pipeline is None: + return self.http_status(404, -1, 'pipeline not found') - return self.success(data={'tools': tool_list}) + extensions_prefs = pipeline.get('extensions_preferences', {}) or {} + if not extensions_prefs.get('enable_all_plugins', True): + bound_plugins = [ + f'{plugin.get("author", "")}/{plugin.get("name", "")}' + for plugin in extensions_prefs.get('plugins', []) + if isinstance(plugin, dict) and plugin.get('name') + ] + if not extensions_prefs.get('enable_all_mcp_servers', True): + bound_mcp_servers = [ + server for server in (extensions_prefs.get('mcp_servers', []) or []) if isinstance(server, str) + ] + + return self.success( + data={ + 'tools': await self.ap.tool_mgr.get_tool_catalog( + bound_plugins, + bound_mcp_servers, + include_skill_authoring=True, + ) + } + ) @self.route('/', methods=['GET'], auth_type=group.AuthType.USER_TOKEN) async def _(tool_name: str) -> str: """获取特定工具详情""" - tools = await self.ap.tool_mgr.get_all_tools() + tools = await self.ap.tool_mgr.get_all_tools(include_skill_authoring=True) for tool in tools: if tool.name == tool_name: diff --git a/src/langbot/pkg/api/http/controller/groups/survey.py b/src/langbot/pkg/api/http/controller/groups/survey.py index dcfd7f9ee..a65d51a85 100644 --- a/src/langbot/pkg/api/http/controller/groups/survey.py +++ b/src/langbot/pkg/api/http/controller/groups/survey.py @@ -1,3 +1,5 @@ +import base64 + import quart from .. import group @@ -30,6 +32,50 @@ class SurveyRouterGroup(group.RouterGroup): return self.fail(2, 'Failed to submit response') return self.fail(3, 'Survey not available') + @self.route('/feedback', methods=['POST'], auth_type=group.AuthType.USER_TOKEN) + async def _feedback(user_email: str) -> str: + """Submit on-demand user feedback from the sidebar.""" + json_data = await quart.request.get_json(silent=True) or {} + content = str(json_data.get('content', '')).strip() + attachments = json_data.get('attachments', []) + + if not content: + return self.fail(1, 'content required') + if len(content) > 5000: + return self.fail(2, 'content too long') + if not isinstance(attachments, list): + return self.fail(3, 'attachments must be an array') + if len(attachments) > 3: + return self.fail(4, 'too many attachments') + + normalized_attachments = [] + for item in attachments: + if not isinstance(item, dict): + continue + data_url = str(item.get('data_url', '')) + mime_type = str(item.get('mime_type', ''))[:128] + name = str(item.get('name', ''))[:255] + if not data_url.startswith('data:image/'): + continue + try: + payload = data_url.split(',', 1)[1] + if len(base64.b64decode(payload, validate=True)) > 1024 * 1024: + return self.fail(5, 'attachment too large') + except Exception: + return self.fail(5, 'attachment too large') + normalized_attachments.append({'name': name, 'mime_type': mime_type, 'data_url': data_url}) + + if self.ap.survey: + ok = await self.ap.survey.submit_feedback( + content=content, + attachments=normalized_attachments, + user_email=user_email, + ) + if ok: + return self.success() + return self.fail(6, 'Failed to submit feedback') + return self.fail(7, 'Survey not available') + @self.route('/dismiss', methods=['POST'], auth_type=group.AuthType.USER_TOKEN) async def _dismiss() -> str: """Dismiss survey.""" diff --git a/src/langbot/pkg/api/http/controller/groups/user.py b/src/langbot/pkg/api/http/controller/groups/user.py index e86d6d1e2..886dc5d0d 100644 --- a/src/langbot/pkg/api/http/controller/groups/user.py +++ b/src/langbot/pkg/api/http/controller/groups/user.py @@ -195,6 +195,13 @@ class UserRouterGroup(group.RouterGroup): @self.route('/set-password', methods=['POST'], auth_type=group.AuthType.USER_TOKEN) async def _(user_email: str) -> str: """Set password for Space account (first time) or change password""" + # Check if modifying login info is allowed + allow_modify_login_info = self.ap.instance_config.data.get('system', {}).get( + 'allow_modify_login_info', True + ) + if not allow_modify_login_info: + return self.http_status(403, -1, 'Modifying login info is disabled') + json_data = await quart.request.json new_password = json_data.get('new_password') current_password = json_data.get('current_password') diff --git a/src/langbot/pkg/api/http/service/bot.py b/src/langbot/pkg/api/http/service/bot.py index 9f9e62829..6013ad2b9 100644 --- a/src/langbot/pkg/api/http/service/bot.py +++ b/src/langbot/pkg/api/http/service/bot.py @@ -274,3 +274,35 @@ class BotService: # Send message via adapter await runtime_bot.adapter.send_message(target_type, str(target_id), message_chain) + + # ============ Bot Admins ============ + + async def get_bot_admins(self, bot_uuid: str) -> list[dict]: + from ....entity.persistence import bot as persistence_bot + + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(persistence_bot.BotAdmin).where(persistence_bot.BotAdmin.bot_uuid == bot_uuid) + ) + return [{'id': r.id, 'launcher_type': r.launcher_type, 'launcher_id': r.launcher_id} for r in result.all()] + + async def add_bot_admin(self, bot_uuid: str, launcher_type: str, launcher_id: str) -> int: + from ....entity.persistence import bot as persistence_bot + + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.insert(persistence_bot.BotAdmin).values( + bot_uuid=bot_uuid, + launcher_type=launcher_type, + launcher_id=launcher_id, + ) + ) + return result.inserted_primary_key[0] + + async def delete_bot_admin(self, bot_uuid: str, admin_id: int) -> None: + from ....entity.persistence import bot as persistence_bot + + await self.ap.persistence_mgr.execute_async( + sqlalchemy.delete(persistence_bot.BotAdmin).where( + persistence_bot.BotAdmin.bot_uuid == bot_uuid, + persistence_bot.BotAdmin.id == admin_id, + ) + ) diff --git a/src/langbot/pkg/api/http/service/mcp.py b/src/langbot/pkg/api/http/service/mcp.py index 9db699c20..22e281b6f 100644 --- a/src/langbot/pkg/api/http/service/mcp.py +++ b/src/langbot/pkg/api/http/service/mcp.py @@ -136,6 +136,32 @@ class MCPService: if server_name in self.ap.tool_mgr.mcp_tool_loader.sessions: await self.ap.tool_mgr.mcp_tool_loader.remove_mcp_server(server_name) + async def get_mcp_server_resources(self, server_name: str) -> list[dict]: + """Get resources from a specific MCP server.""" + return await self.ap.tool_mgr.mcp_tool_loader.get_resources(server_name) + + async def get_mcp_server_resource_templates(self, server_name: str) -> list[dict]: + """Get resource templates from a specific MCP server.""" + return await self.ap.tool_mgr.mcp_tool_loader.get_resource_templates(server_name) + + async def read_mcp_server_resource_envelope( + self, + server_name: str, + uri: str, + *, + max_bytes: int | None = None, + include_blob: bool = False, + ) -> dict: + """Read a resource from a specific MCP server with metadata.""" + kwargs = {'include_blob': include_blob, 'source': 'ui_preview'} + if max_bytes is not None: + kwargs['max_bytes'] = max_bytes + return await self.ap.tool_mgr.mcp_tool_loader.read_resource_envelope(server_name, uri, **kwargs) + + async def read_mcp_server_resource(self, server_name: str, uri: str) -> list[dict]: + """Read a resource from a specific MCP server.""" + return await self.ap.tool_mgr.mcp_tool_loader.read_resource(server_name, uri) + async def test_mcp_server(self, server_name: str, server_data: dict) -> int: """测试 MCP 服务器连接并返回任务 ID""" diff --git a/src/langbot/pkg/api/http/service/pipeline.py b/src/langbot/pkg/api/http/service/pipeline.py index dbe7c2dda..b5b48b177 100644 --- a/src/langbot/pkg/api/http/service/pipeline.py +++ b/src/langbot/pkg/api/http/service/pipeline.py @@ -185,6 +185,8 @@ class PipelineService: 'enable_all_mcp_servers': True, 'plugins': [], 'mcp_servers': [], + 'mcp_resources': [], + 'mcp_resource_agent_read_enabled': True, } await self.ap.persistence_mgr.execute_async( @@ -284,6 +286,8 @@ class PipelineService: 'enable_all_mcp_servers': True, 'plugins': [], 'mcp_servers': [], + 'mcp_resources': [], + 'mcp_resource_agent_read_enabled': True, } ), } @@ -308,6 +312,8 @@ class PipelineService: enable_all_mcp_servers: bool = True, bound_skills: list[str] = None, enable_all_skills: bool = True, + bound_mcp_resources: list[dict] = None, + mcp_resource_agent_read_enabled: bool | None = None, ) -> None: """Update the bound plugins and MCP servers for a pipeline""" # Get current pipeline @@ -327,10 +333,14 @@ class PipelineService: extensions_preferences['enable_all_mcp_servers'] = enable_all_mcp_servers extensions_preferences['enable_all_skills'] = enable_all_skills extensions_preferences['plugins'] = bound_plugins + if mcp_resource_agent_read_enabled is not None: + extensions_preferences['mcp_resource_agent_read_enabled'] = mcp_resource_agent_read_enabled if bound_mcp_servers is not None: extensions_preferences['mcp_servers'] = bound_mcp_servers if bound_skills is not None: extensions_preferences['skills'] = bound_skills + if bound_mcp_resources is not None: + extensions_preferences['mcp_resources'] = bound_mcp_resources await self.ap.persistence_mgr.execute_async( sqlalchemy.update(persistence_pipeline.LegacyPipeline) diff --git a/src/langbot/pkg/api/http/service/user.py b/src/langbot/pkg/api/http/service/user.py index 93d7ad6f0..a9185f9bc 100644 --- a/src/langbot/pkg/api/http/service/user.py +++ b/src/langbot/pkg/api/http/service/user.py @@ -20,6 +20,15 @@ class UserService: def __init__(self, ap: app.Application) -> None: self.ap = ap self._create_user_lock = asyncio.Lock() + self._password_hash_lock = asyncio.Semaphore(1) + + async def _hash_password(self, password: str) -> str: + async with self._password_hash_lock: + return await asyncio.to_thread(argon2.PasswordHasher().hash, password) + + async def _verify_password(self, hashed_password: str, password: str) -> None: + async with self._password_hash_lock: + await asyncio.to_thread(argon2.PasswordHasher().verify, hashed_password, password) async def is_initialized(self) -> bool: result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1)) @@ -28,9 +37,7 @@ class UserService: return result_list is not None and len(result_list) > 0 async def create_user(self, user_email: str, password: str) -> None: - ph = argon2.PasswordHasher() - - hashed_password = ph.hash(password) + hashed_password = await self._hash_password(password) await self.ap.persistence_mgr.execute_async( sqlalchemy.insert(user.User).values(user=user_email, password=hashed_password, account_type='local') @@ -69,9 +76,7 @@ class UserService: if not user_obj.password: raise ValueError('请使用 Space 账户登录') - ph = argon2.PasswordHasher() - - ph.verify(user_obj.password, password) + await self._verify_password(user_obj.password, password) return await self.generate_jwt_token(user_email) @@ -93,17 +98,13 @@ class UserService: return jwt.decode(token, jwt_secret, algorithms=['HS256'])['user'] async def reset_password(self, user_email: str, new_password: str) -> None: - ph = argon2.PasswordHasher() - - hashed_password = ph.hash(new_password) + hashed_password = await self._hash_password(new_password) await self.ap.persistence_mgr.execute_async( sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password) ) async def change_password(self, user_email: str, current_password: str, new_password: str) -> None: - ph = argon2.PasswordHasher() - user_obj = await self.get_user_by_email(user_email) if user_obj is None: raise ValueError('User not found') @@ -111,9 +112,9 @@ class UserService: if not user_obj.password: raise ValueError('No local password set, please set a password first') - ph.verify(user_obj.password, current_password) + await self._verify_password(user_obj.password, current_password) - hashed_password = ph.hash(new_password) + hashed_password = await self._hash_password(new_password) await self.ap.persistence_mgr.execute_async( sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password) @@ -232,7 +233,6 @@ class UserService: async def set_password(self, user_email: str, new_password: str, current_password: str | None = None) -> None: """Set or change password for a user""" - ph = argon2.PasswordHasher() user_obj = await self.get_user_by_email(user_email) if user_obj is None: @@ -243,9 +243,9 @@ class UserService: if has_password: if not current_password: raise ValueError('Current password is required') - ph.verify(user_obj.password, current_password) + await self._verify_password(user_obj.password, current_password) - hashed_password = ph.hash(new_password) + hashed_password = await self._hash_password(new_password) await self.ap.persistence_mgr.execute_async( sqlalchemy.update(user.User).where(user.User.user == user_email).values(password=hashed_password) ) diff --git a/src/langbot/pkg/box/service.py b/src/langbot/pkg/box/service.py index c24dc4420..e7f22df14 100644 --- a/src/langbot/pkg/box/service.py +++ b/src/langbot/pkg/box/service.py @@ -82,7 +82,6 @@ class BoxService: return self._enabled async def initialize(self): - self._ensure_default_workspace() if not self._enabled: # Disabled by config: do NOT connect to a remote runtime, do NOT # fork a stdio subprocess. Every consumer of box_service should @@ -99,6 +98,7 @@ class BoxService: await self._runtime_connector.initialize() else: await self.client.initialize() + self._ensure_default_workspace() self._available = True self._connector_error = '' self.ap.logger.info( @@ -1158,6 +1158,9 @@ class BoxService: if self.default_workspace is None: return + if not self.shares_filesystem_with_box: + return + if os.path.isdir(self.default_workspace): return @@ -1182,7 +1185,7 @@ class BoxService: return host_path = os.path.realpath(spec.host_path) - if not os.path.isdir(host_path): + if self.shares_filesystem_with_box and not os.path.isdir(host_path): raise BoxValidationError('host_path must point to an existing directory on the host') if not self.allowed_mount_roots: diff --git a/src/langbot/pkg/command/cmdmgr.py b/src/langbot/pkg/command/cmdmgr.py index a1d7e009c..ee064c219 100644 --- a/src/langbot/pkg/command/cmdmgr.py +++ b/src/langbot/pkg/command/cmdmgr.py @@ -84,7 +84,17 @@ class CommandManager: privilege = 1 - if f'{query.launcher_type.value}_{query.launcher_id}' in self.ap.instance_config.data['admins']: + import sqlalchemy as _sa + from ..entity.persistence.bot import BotAdmin as _BotAdmin + + _admins = await self.ap.persistence_mgr.execute_async( + _sa.select(_BotAdmin).where( + _BotAdmin.bot_uuid == (query.bot_uuid or ''), + _BotAdmin.launcher_type == query.launcher_type.value, + _BotAdmin.launcher_id == str(query.launcher_id), + ) + ) + if _admins.first() is not None: privilege = 2 ctx = command_context.ExecuteContext( diff --git a/src/langbot/pkg/entity/persistence/bot.py b/src/langbot/pkg/entity/persistence/bot.py index ee6bc7a8c..9b9f7e4d6 100644 --- a/src/langbot/pkg/entity/persistence/bot.py +++ b/src/langbot/pkg/entity/persistence/bot.py @@ -3,6 +3,20 @@ import sqlalchemy from .base import Base +class BotAdmin(Base): + """Bot admin — a launcher that has admin privilege for a specific bot's commands""" + + __tablename__ = 'bot_admins' + + id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True) + bot_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + launcher_type = sqlalchemy.Column(sqlalchemy.String(64), nullable=False) + launcher_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now()) + + __table_args__ = (sqlalchemy.UniqueConstraint('bot_uuid', 'launcher_type', 'launcher_id', name='uq_bot_admin'),) + + class Bot(Base): """Bot""" diff --git a/src/langbot/pkg/entity/persistence/pipeline.py b/src/langbot/pkg/entity/persistence/pipeline.py index b3a4b7fe0..d74cf78ee 100644 --- a/src/langbot/pkg/entity/persistence/pipeline.py +++ b/src/langbot/pkg/entity/persistence/pipeline.py @@ -26,7 +26,14 @@ class LegacyPipeline(Base): extensions_preferences = sqlalchemy.Column( sqlalchemy.JSON, nullable=False, - default={'enable_all_plugins': True, 'enable_all_mcp_servers': True, 'plugins': [], 'mcp_servers': []}, + default={ + 'enable_all_plugins': True, + 'enable_all_mcp_servers': True, + 'plugins': [], + 'mcp_servers': [], + 'mcp_resources': [], + 'mcp_resource_agent_read_enabled': True, + }, ) diff --git a/src/langbot/pkg/persistence/alembic/versions/0007_add_bot_admins.py b/src/langbot/pkg/persistence/alembic/versions/0007_add_bot_admins.py new file mode 100644 index 000000000..a13f2caa6 --- /dev/null +++ b/src/langbot/pkg/persistence/alembic/versions/0007_add_bot_admins.py @@ -0,0 +1,84 @@ +"""add bot_admins table and migrate config admins + +Revision ID: 0007_add_bot_admins +Revises: 0006_normalize_mcp_remote_mode +Create Date: 2026-06-26 +""" + +import sqlalchemy as sa +from alembic import op + +revision = '0007_add_bot_admins' +down_revision = '0006_normalize_mcp_remote_mode' +branch_labels = None +depends_on = None + + +def upgrade() -> None: + conn = op.get_bind() + if 'bot_admins' in sa.inspect(conn).get_table_names(): + return + op.create_table( + 'bot_admins', + sa.Column('id', sa.Integer, primary_key=True, autoincrement=True), + sa.Column('bot_uuid', sa.String(255), nullable=False), + sa.Column('launcher_type', sa.String(64), nullable=False), + sa.Column('launcher_id', sa.String(255), nullable=False), + sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.UniqueConstraint('bot_uuid', 'launcher_type', 'launcher_id', name='uq_bot_admin'), + ) + + # Migrate old config-based admins into the first bot (best-effort) + inspector = sa.inspect(conn) + tables = inspector.get_table_names() + + if 'bots' not in tables: + return + + # Read the first bot uuid + row = conn.execute(sa.text('SELECT uuid FROM bots ORDER BY created_at LIMIT 1')).first() + if row is None: + return + first_bot_uuid = row[0] + + # Read instance_config metadata key that holds the admins list + if 'metadata' not in tables: + return + meta_row = conn.execute(sa.text("SELECT value FROM metadata WHERE key = 'instance_config'")).first() + if meta_row is None: + return + + import json + + try: + cfg = json.loads(meta_row[0]) + except Exception: + return + + admins = cfg.get('admins', []) + for entry in admins: + parts = entry.split('_', 1) + if len(parts) != 2: + continue + launcher_type, launcher_id = parts + try: + conn.execute( + sa.text( + 'INSERT OR IGNORE INTO bot_admins (bot_uuid, launcher_type, launcher_id) VALUES (:bu, :lt, :li)' + ), + {'bu': first_bot_uuid, 'lt': launcher_type, 'li': launcher_id}, + ) + except Exception: + pass + + # Remove admins key from stored config + if 'admins' in cfg: + del cfg['admins'] + conn.execute( + sa.text("UPDATE metadata SET value = :v WHERE key = 'instance_config'"), + {'v': json.dumps(cfg)}, + ) + + +def downgrade() -> None: + op.drop_table('bot_admins') diff --git a/src/langbot/pkg/persistence/alembic/versions/0008_add_mcp_resource_preferences.py b/src/langbot/pkg/persistence/alembic/versions/0008_add_mcp_resource_preferences.py new file mode 100644 index 000000000..9b92db97d --- /dev/null +++ b/src/langbot/pkg/persistence/alembic/versions/0008_add_mcp_resource_preferences.py @@ -0,0 +1,95 @@ +"""add mcp resource preferences to pipelines + +Revision ID: 0008_mcp_resource_prefs +Revises: 0007_add_bot_admins +Create Date: 2026-06-30 +""" + +from __future__ import annotations + +import json +from typing import Any + +import sqlalchemy as sa +from alembic import op + +revision = '0008_mcp_resource_prefs' +down_revision = '0007_add_bot_admins' +branch_labels = None +depends_on = None + + +_PIPELINE_TABLE = sa.table( + 'legacy_pipelines', + sa.column('uuid', sa.String(255)), + sa.column('extensions_preferences', sa.JSON()), +) + + +def _has_extensions_preferences_table(conn: sa.Connection) -> bool: + inspector = sa.inspect(conn) + if 'legacy_pipelines' not in inspector.get_table_names(): + return False + columns = {column['name'] for column in inspector.get_columns('legacy_pipelines')} + return 'extensions_preferences' in columns + + +def _decode_preferences(value: Any) -> dict[str, Any]: + if value is None: + return {} + if isinstance(value, dict): + return dict(value) + if isinstance(value, str): + try: + decoded = json.loads(value) + except json.JSONDecodeError: + return {} + if isinstance(decoded, dict): + return decoded + return {} + + +def _update_preferences(conn: sa.Connection, uuid: str, preferences: dict[str, Any]) -> None: + conn.execute( + _PIPELINE_TABLE.update().where(_PIPELINE_TABLE.c.uuid == uuid).values(extensions_preferences=preferences) + ) + + +def upgrade() -> None: + conn = op.get_bind() + if not _has_extensions_preferences_table(conn): + return + + rows = conn.execute(sa.select(_PIPELINE_TABLE.c.uuid, _PIPELINE_TABLE.c.extensions_preferences)).all() + for uuid, raw_preferences in rows: + preferences = _decode_preferences(raw_preferences) + changed = False + + if 'mcp_resources' not in preferences: + preferences['mcp_resources'] = [] + changed = True + if 'mcp_resource_agent_read_enabled' not in preferences: + preferences['mcp_resource_agent_read_enabled'] = True + changed = True + + if changed: + _update_preferences(conn, uuid, preferences) + + +def downgrade() -> None: + conn = op.get_bind() + if not _has_extensions_preferences_table(conn): + return + + rows = conn.execute(sa.select(_PIPELINE_TABLE.c.uuid, _PIPELINE_TABLE.c.extensions_preferences)).all() + for uuid, raw_preferences in rows: + preferences = _decode_preferences(raw_preferences) + changed = False + + for key in ('mcp_resources', 'mcp_resource_agent_read_enabled'): + if key in preferences: + preferences.pop(key) + changed = True + + if changed: + _update_preferences(conn, uuid, preferences) diff --git a/src/langbot/pkg/pipeline/monitoring_helper.py b/src/langbot/pkg/pipeline/monitoring_helper.py index 19467cc84..a3a9654bc 100644 --- a/src/langbot/pkg/pipeline/monitoring_helper.py +++ b/src/langbot/pkg/pipeline/monitoring_helper.py @@ -32,7 +32,7 @@ class MonitoringHelper: """Record the start of query processing, returns message_id""" try: # Check if session exists, if not, record session start - session_id = f'{query.launcher_type}_{query.launcher_id}' + session_id = f'{query.launcher_type.value if hasattr(query.launcher_type, "value") else query.launcher_type}_{query.launcher_id}' # Get sender name from message event sender_name = None @@ -137,7 +137,7 @@ class MonitoringHelper: ): """Record bot response message to monitoring""" try: - session_id = f'{query.launcher_type}_{query.launcher_id}' + session_id = f'{query.launcher_type.value if hasattr(query.launcher_type, "value") else query.launcher_type}_{query.launcher_id}' # Get sender name from message event sender_name = None @@ -202,7 +202,7 @@ class MonitoringHelper: ) -> str: """Record query processing error, returns message_id""" try: - session_id = f'{query.launcher_type}_{query.launcher_id}' + session_id = f'{query.launcher_type.value if hasattr(query.launcher_type, "value") else query.launcher_type}_{query.launcher_id}' # Get sender name from message event sender_name = None @@ -268,7 +268,7 @@ class MonitoringHelper: ): """Record LLM call""" try: - session_id = f'{query.launcher_type}_{query.launcher_id}' + session_id = f'{query.launcher_type.value if hasattr(query.launcher_type, "value") else query.launcher_type}_{query.launcher_id}' await ap.monitoring_service.record_llm_call( bot_id=bot_id, diff --git a/src/langbot/pkg/pipeline/pipelinemgr.py b/src/langbot/pkg/pipeline/pipelinemgr.py index b494acc23..e9a98c0ac 100644 --- a/src/langbot/pkg/pipeline/pipelinemgr.py +++ b/src/langbot/pkg/pipeline/pipelinemgr.py @@ -14,6 +14,7 @@ import langbot_plugin.api.entities.builtin.platform.events as platform_events import langbot_plugin.api.entities.events as events from ..utils import importutil from .config_coercion import coerce_pipeline_config +from ..agent.runner.config_migration import ConfigMigration import langbot_plugin.api.entities.builtin.provider.session as provider_session import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query @@ -95,6 +96,34 @@ class RuntimePipeline: self.enable_all_plugins = extensions_prefs.get('enable_all_plugins', True) self.enable_all_mcp_servers = extensions_prefs.get('enable_all_mcp_servers', True) + pipeline_config = pipeline_entity.config or {} + ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {} + legacy_local_agent_config = ai_config.get('local-agent', {}) if isinstance(ai_config, dict) else {} + if not isinstance(legacy_local_agent_config, dict): + legacy_local_agent_config = {} + + runner_config: dict[str, typing.Any] = {} + runner_id = ConfigMigration.resolve_runner_id(pipeline_config) if isinstance(pipeline_config, dict) else None + if runner_id: + resolved_runner_config = ConfigMigration.resolve_runner_config(pipeline_config, runner_id) + if isinstance(resolved_runner_config, dict): + runner_config = resolved_runner_config + + self.mcp_resource_attachments = runner_config.get( + 'mcp-resources', + legacy_local_agent_config.get( + 'mcp-resources', + extensions_prefs.get('mcp_resources', []), + ), + ) + self.mcp_resource_agent_read_enabled = runner_config.get( + 'mcp-resource-agent-read-enabled', + legacy_local_agent_config.get( + 'mcp-resource-agent-read-enabled', + extensions_prefs.get('mcp_resource_agent_read_enabled', True), + ), + ) + if self.enable_all_plugins: # None indicates to use all available plugins self.bound_plugins = None @@ -114,6 +143,8 @@ class RuntimePipeline: # Store bound plugins and MCP servers in query for filtering query.variables['_pipeline_bound_plugins'] = self.bound_plugins query.variables['_pipeline_bound_mcp_servers'] = self.bound_mcp_servers + query.variables['_pipeline_mcp_resource_attachments'] = self.mcp_resource_attachments + query.variables['_pipeline_mcp_resource_agent_read_enabled'] = self.mcp_resource_agent_read_enabled # Record query start for monitoring try: @@ -176,7 +207,7 @@ class RuntimePipeline: bot_name = query.variables.get('_monitoring_bot_name', 'Unknown') pipeline_name = query.variables.get('_monitoring_pipeline_name', 'Unknown') message_id = query.variables.get('_monitoring_message_id', '') - session_id = f'{query.launcher_type}_{query.launcher_id}' + session_id = f'{query.launcher_type.value if hasattr(query.launcher_type, "value") else query.launcher_type}_{query.launcher_id}' # Update message status to error if message_id: diff --git a/src/langbot/pkg/pipeline/plugin_diagnostics.py b/src/langbot/pkg/pipeline/plugin_diagnostics.py new file mode 100644 index 000000000..3e1195cf2 --- /dev/null +++ b/src/langbot/pkg/pipeline/plugin_diagnostics.py @@ -0,0 +1,274 @@ +from __future__ import annotations + +import traceback +import weakref +from dataclasses import dataclass, field +from typing import Any + +import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query +import langbot_plugin.api.entities.builtin.platform.message as platform_message + + +@dataclass(frozen=True) +class PluginResponseSource: + plugin: dict[str, str] + event_name: str | None = None + is_approximate: bool = False + + +@dataclass +class QueryDiagnosticState: + pending_by_chain_id: dict[int, list[PluginResponseSource]] = field(default_factory=dict) + by_response_index: dict[int, list[PluginResponseSource]] = field(default_factory=dict) + finalizer: weakref.finalize | None = None + + +_QUERY_STATES: dict[int, QueryDiagnosticState] = {} + + +def record_plugin_response_source( + query: pipeline_query.Query, + response_index: int, + response_sources: list[dict[str, Any]] | None, + emitted_plugins: list[dict[str, Any]] | None = None, + event_name: str | None = None, +) -> None: + plugin_sources = _build_plugin_sources(response_sources, emitted_plugins, event_name) + if not plugin_sources: + return + state = _get_or_create_query_state(query) + state.by_response_index[response_index] = plugin_sources + + +def record_last_plugin_response_source( + query: pipeline_query.Query, + response_sources: list[dict[str, Any]] | None, + emitted_plugins: list[dict[str, Any]] | None = None, + event_name: str | None = None, +) -> None: + record_plugin_response_source( + query, + len(query.resp_message_chain) - 1, + response_sources, + emitted_plugins, + event_name, + ) + + +def record_pending_plugin_response_source( + query: pipeline_query.Query, + message_chain: platform_message.MessageChain, + response_sources: list[dict[str, Any]] | None, + emitted_plugins: list[dict[str, Any]] | None = None, + event_name: str | None = None, +) -> None: + plugin_sources = _build_plugin_sources(response_sources, emitted_plugins, event_name) + if not plugin_sources: + return + state = _get_or_create_query_state(query) + state.pending_by_chain_id[id(message_chain)] = plugin_sources + + +def consume_pending_plugin_response_source( + query: pipeline_query.Query, + message_chain: platform_message.MessageChain, + response_index: int, +) -> None: + state = _get_query_state(query) + if state is None: + return + source = state.pending_by_chain_id.pop(id(message_chain), None) + if source is None: + return + state.by_response_index[response_index] = source + + +def clear_response_source(query: pipeline_query.Query, response_index: int) -> None: + state = _get_query_state(query) + if state is None: + return + state.by_response_index.pop(response_index, None) + _discard_query_state_if_empty(query) + + +async def notify_response_delivery_failure( + ap: Any, + query: pipeline_query.Query, + response_index: int, + message_chain: platform_message.MessageChain, + error: Exception, +) -> None: + try: + plugin_refs = _get_response_sources(query, response_index) + if not plugin_refs: + return + connector = getattr(ap, 'plugin_connector', None) + if connector is None or not hasattr(connector, 'notify_plugin_diagnostic'): + return + for source in plugin_refs: + payload = _build_delivery_failure_payload( + plugin_ref=source.plugin, + event_name=source.event_name, + is_approximate=source.is_approximate, + query=query, + response_index=response_index, + message_chain=message_chain, + error=error, + ) + try: + await connector.notify_plugin_diagnostic(payload) + except Exception as diag_error: + _debug(ap, f'Plugin diagnostic forwarding failed: {diag_error}') + except Exception as diag_error: + _debug(ap, f'Plugin diagnostic forwarding skipped: {diag_error}') + + +def get_emitted_plugins(event_ctx: Any) -> list[dict[str, Any]]: + emitted_plugins = getattr(event_ctx, '_emitted_plugins', []) + return emitted_plugins if isinstance(emitted_plugins, list) else [] + + +def get_response_sources(event_ctx: Any) -> list[dict[str, Any]] | None: + event_attrs = vars(event_ctx) + if '_response_sources' not in event_attrs: + return None + response_sources = event_attrs['_response_sources'] + return response_sources if isinstance(response_sources, list) else [] + + +def _get_or_create_query_state(query: pipeline_query.Query) -> QueryDiagnosticState: + query_key = id(query) + state = _QUERY_STATES.get(query_key) + if state is not None: + return state + + state = QueryDiagnosticState() + try: + state.finalizer = weakref.finalize(query, _discard_query_state, query_key) + except TypeError: + state.finalizer = None + _QUERY_STATES[query_key] = state + return state + + +def _get_query_state(query: pipeline_query.Query) -> QueryDiagnosticState | None: + return _QUERY_STATES.get(id(query)) + + +def _discard_query_state(query_key: int) -> None: + _QUERY_STATES.pop(query_key, None) + + +def _discard_query_state_if_empty(query: pipeline_query.Query) -> None: + query_key = id(query) + state = _QUERY_STATES.get(query_key) + if state is None: + return + if state.pending_by_chain_id or state.by_response_index: + return + if state.finalizer is not None: + state.finalizer.detach() + _discard_query_state(query_key) + + +def _get_response_sources( + query: pipeline_query.Query, + response_index: int, +) -> list[PluginResponseSource]: + state = _get_query_state(query) + if state is None: + return [] + return state.by_response_index.get(response_index, []) + + +def _extract_plugin_ref(plugin: Any) -> dict[str, str] | None: + manifest = plugin.get('manifest') if isinstance(plugin, dict) else None + metadata = manifest.get('metadata') if isinstance(manifest, dict) else None + if not isinstance(metadata, dict): + return None + author = metadata.get('author') + name = metadata.get('name') + if not author or not name: + return None + return {'author': str(author), 'name': str(name)} + + +def _extract_response_source_plugin_ref(source: Any) -> dict[str, str] | None: + if not isinstance(source, dict): + return None + if source.get('kind') != 'reply_message_chain': + return None + plugin_ref = source.get('plugin') + if not isinstance(plugin_ref, dict): + return None + author = plugin_ref.get('author') + name = plugin_ref.get('name') + if not author or not name: + return None + return {'author': str(author), 'name': str(name)} + + +def _build_plugin_sources( + response_sources: list[dict[str, Any]] | None, + emitted_plugins: list[dict[str, Any]] | None, + event_name: str | None, +) -> list[PluginResponseSource]: + if response_sources is not None: + plugin_refs = [_extract_response_source_plugin_ref(source) for source in response_sources] + return [ + PluginResponseSource(plugin=plugin, event_name=event_name) for plugin in plugin_refs if plugin is not None + ] + + if emitted_plugins: + plugin_refs = [_extract_plugin_ref(plugin) for plugin in emitted_plugins] + return [ + PluginResponseSource(plugin=plugin, event_name=event_name, is_approximate=True) + for plugin in plugin_refs + if plugin is not None + ] + return [] + + +def _debug(ap: Any, message: str) -> None: + logger = getattr(ap, 'logger', None) + if logger is not None: + logger.debug(message) + + +def _build_delivery_failure_payload( + plugin_ref: dict[str, str], + event_name: str | None, + is_approximate: bool, + query: pipeline_query.Query, + response_index: int, + message_chain: platform_message.MessageChain, + error: Exception, +) -> dict[str, Any]: + details: dict[str, Any] = { + 'message_component_types': [component.__class__.__name__ for component in message_chain], + 'message_preview': str(message_chain)[:200], + } + if is_approximate: + details['attribution_warning'] = ( + 'This diagnostic was delivered to all plugins that handled the event because the ' + 'plugin runtime did not report the exact reply_message_chain source.' + ) + + return { + 'level': 'ERROR', + 'code': 'response_delivery_failed', + 'message': 'Failed to deliver a plugin-provided response message.', + 'plugin': plugin_ref, + 'query': { + 'query_id': query.query_id, + 'event_name': event_name or query.message_event.__class__.__name__, + 'stage': query.current_stage_name or 'SendResponseBackStage', + 'response_index': response_index, + }, + 'details': details, + 'delivery': { + 'error_type': error.__class__.__name__, + 'error_message': str(error), + 'traceback': traceback.format_exception_only(type(error), error)[-1].strip(), + }, + } diff --git a/src/langbot/pkg/pipeline/preproc/preproc.py b/src/langbot/pkg/pipeline/preproc/preproc.py index 4667ae28e..791c06fb4 100644 --- a/src/langbot/pkg/pipeline/preproc/preproc.py +++ b/src/langbot/pkg/pipeline/preproc/preproc.py @@ -136,9 +136,7 @@ class PreProcessor(stage.PipelineStage): strict_thread=True, ) except Exception as e: - self.ap.logger.warning( - f'Unable to load Transcript history view for conversation {conversation_uuid}: {e}' - ) + self.ap.logger.warning(f'Unable to load Transcript history view for conversation {conversation_uuid}: {e}') return None return messages or None @@ -161,6 +159,21 @@ class PreProcessor(stage.PipelineStage): return transcript_messages return conversation.messages.copy() + @staticmethod + def _filter_selected_tools( + tools: list, + runner_config: dict, + ) -> list: + if runner_config.get('enable-all-tools', True) is not False: + return tools + + selected_tools = runner_config.get('tools', []) + if not isinstance(selected_tools, list): + return [] + + selected_tool_names = {tool for tool in selected_tools if isinstance(tool, str)} + return [tool for tool in tools if tool.name in selected_tool_names] + async def process( self, query: pipeline_query.Query, @@ -181,6 +194,7 @@ class PreProcessor(stage.PipelineStage): uses_host_models = config_schema.uses_host_models(descriptor) uses_host_tools = config_schema.uses_host_tools(descriptor) + include_mcp_resource_tools = query.variables.get('_pipeline_mcp_resource_agent_read_enabled', True) llm_model = None if uses_host_models: primary_uuid, fallback_uuids = config_schema.extract_model_selection(descriptor, runner_config) @@ -235,10 +249,12 @@ class PreProcessor(stage.PipelineStage): query.use_llm_model_uuid = llm_model.model_entity.uuid if uses_host_tools and 'func_call' in (llm_model.model_entity.abilities or []): - query.use_funcs = await self.ap.tool_mgr.get_all_tools( + all_tools = await self.ap.tool_mgr.get_all_tools( bound_plugins, bound_mcp_servers, + include_mcp_resource_tools=include_mcp_resource_tools, ) + query.use_funcs = self._filter_selected_tools(all_tools, runner_config) self.ap.logger.debug(f'Bound plugins: {bound_plugins}') self.ap.logger.debug(f'Bound MCP servers: {bound_mcp_servers}') @@ -247,15 +263,19 @@ class PreProcessor(stage.PipelineStage): # If primary model doesn't support func_call but fallback models exist, # load tools anyway since fallback models may support them if uses_host_tools and not query.use_funcs and query.variables.get('_fallback_model_uuids'): - query.use_funcs = await self.ap.tool_mgr.get_all_tools( + all_tools = await self.ap.tool_mgr.get_all_tools( bound_plugins, bound_mcp_servers, + include_mcp_resource_tools=include_mcp_resource_tools, ) + query.use_funcs = self._filter_selected_tools(all_tools, runner_config) elif uses_host_tools: - query.use_funcs = await self.ap.tool_mgr.get_all_tools( + all_tools = await self.ap.tool_mgr.get_all_tools( bound_plugins, bound_mcp_servers, + include_mcp_resource_tools=include_mcp_resource_tools, ) + query.use_funcs = self._filter_selected_tools(all_tools, runner_config) self.ap.logger.debug(f'Bound plugins: {bound_plugins}') self.ap.logger.debug(f'Bound MCP servers: {bound_mcp_servers}') diff --git a/src/langbot/pkg/pipeline/process/handlers/chat.py b/src/langbot/pkg/pipeline/process/handlers/chat.py index 7a3f755fb..ad80b6aae 100644 --- a/src/langbot/pkg/pipeline/process/handlers/chat.py +++ b/src/langbot/pkg/pipeline/process/handlers/chat.py @@ -9,6 +9,7 @@ from datetime import datetime from .. import handler from ... import entities +from ... import plugin_diagnostics import langbot_plugin.api.entities.events as events from ....agent.runner.config_migration import ConfigMigration @@ -64,6 +65,13 @@ class ChatMessageHandler(handler.MessageHandler): if event_ctx.is_prevented_default(): if event_ctx.event.reply_message_chain is not None: mc = event_ctx.event.reply_message_chain + plugin_diagnostics.record_pending_plugin_response_source( + query, + mc, + plugin_diagnostics.get_response_sources(event_ctx), + plugin_diagnostics.get_emitted_plugins(event_ctx), + event.event_name, + ) query.resp_messages.append(mc) yield entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) diff --git a/src/langbot/pkg/pipeline/process/handlers/command.py b/src/langbot/pkg/pipeline/process/handlers/command.py index 6d686acd4..09fa5379b 100644 --- a/src/langbot/pkg/pipeline/process/handlers/command.py +++ b/src/langbot/pkg/pipeline/process/handlers/command.py @@ -4,6 +4,7 @@ import typing from .. import handler from ... import entities +from ... import plugin_diagnostics import langbot_plugin.api.entities.builtin.provider.message as provider_message import langbot_plugin.api.entities.builtin.provider.session as provider_session import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query @@ -52,6 +53,13 @@ class CommandHandler(handler.MessageHandler): if event_ctx.is_prevented_default(): if event_ctx.event.reply_message_chain is not None: mc = event_ctx.event.reply_message_chain + plugin_diagnostics.record_pending_plugin_response_source( + query, + mc, + plugin_diagnostics.get_response_sources(event_ctx), + plugin_diagnostics.get_emitted_plugins(event_ctx), + event.event_name, + ) query.resp_messages.append(mc) diff --git a/src/langbot/pkg/pipeline/respback/respback.py b/src/langbot/pkg/pipeline/respback/respback.py index 574404bcf..0c85fbb45 100644 --- a/src/langbot/pkg/pipeline/respback/respback.py +++ b/src/langbot/pkg/pipeline/respback/respback.py @@ -9,6 +9,7 @@ import langbot_plugin.api.entities.builtin.platform.message as platform_message import langbot_plugin.api.entities.builtin.provider.message as provider_message from .. import stage, entities +from .. import plugin_diagnostics import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query @@ -39,20 +40,35 @@ class SendResponseBackStage(stage.PipelineStage): has_chunks = any(isinstance(msg, provider_message.MessageChunk) for msg in query.resp_messages) # TODO 命令与流式的兼容性问题 - if await query.adapter.is_stream_output_supported() and has_chunks: - is_final = [msg.is_final for msg in query.resp_messages][0] - await query.adapter.reply_message_chunk( - message_source=query.message_event, - bot_message=query.resp_messages[-1], - message=query.resp_message_chain[-1], - quote_origin=quote_origin, - is_final=is_final, - ) - else: - await query.adapter.reply_message( - message_source=query.message_event, - message=query.resp_message_chain[-1], - quote_origin=quote_origin, + response_index = len(query.resp_message_chain) - 1 + message_chain = query.resp_message_chain[-1] + + try: + if await query.adapter.is_stream_output_supported() and has_chunks: + is_final = [msg.is_final for msg in query.resp_messages][0] + await query.adapter.reply_message_chunk( + message_source=query.message_event, + bot_message=query.resp_messages[-1], + message=message_chain, + quote_origin=quote_origin, + is_final=is_final, + ) + else: + await query.adapter.reply_message( + message_source=query.message_event, + message=message_chain, + quote_origin=quote_origin, + ) + except Exception as e: + await plugin_diagnostics.notify_response_delivery_failure( + self.ap, + query, + response_index, + message_chain, + e, ) + plugin_diagnostics.clear_response_source(query, response_index) + raise + plugin_diagnostics.clear_response_source(query, response_index) return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) diff --git a/src/langbot/pkg/pipeline/wrapper/wrapper.py b/src/langbot/pkg/pipeline/wrapper/wrapper.py index a158c1840..50db693d4 100644 --- a/src/langbot/pkg/pipeline/wrapper/wrapper.py +++ b/src/langbot/pkg/pipeline/wrapper/wrapper.py @@ -3,6 +3,7 @@ from __future__ import annotations import typing from .. import entities +from .. import plugin_diagnostics from .. import stage import langbot_plugin.api.entities.builtin.platform.message as platform_message @@ -78,6 +79,11 @@ class ResponseWrapper(stage.PipelineStage): # 如果 resp_messages[-1] 已经是 MessageChain 了 if isinstance(query.resp_messages[-1], platform_message.MessageChain): query.resp_message_chain.append(query.resp_messages[-1]) + plugin_diagnostics.consume_pending_plugin_response_source( + query, + query.resp_messages[-1], + len(query.resp_message_chain) - 1, + ) yield entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) @@ -129,8 +135,10 @@ class ResponseWrapper(stage.PipelineStage): else: if event_ctx.event.reply_message_chain is not None: reply_chain = event_ctx.event.reply_message_chain + is_plugin_reply = True else: reply_chain = result.get_content_platform_message_chain() + is_plugin_reply = False # Attach files the agent produced in the sandbox # outbox, but only on the terminal assistant message. @@ -138,6 +146,13 @@ class ResponseWrapper(stage.PipelineStage): await self._append_outbound_attachments(query, reply_chain) query.resp_message_chain.append(reply_chain) + if is_plugin_reply: + plugin_diagnostics.record_last_plugin_response_source( + query, + plugin_diagnostics.get_response_sources(event_ctx), + plugin_diagnostics.get_emitted_plugins(event_ctx), + event.event_name, + ) yield entities.StageProcessResult( result_type=entities.ResultType.CONTINUE, @@ -180,6 +195,12 @@ class ResponseWrapper(stage.PipelineStage): else: if event_ctx.event.reply_message_chain is not None: query.resp_message_chain.append(event_ctx.event.reply_message_chain) + plugin_diagnostics.record_last_plugin_response_source( + query, + plugin_diagnostics.get_response_sources(event_ctx), + plugin_diagnostics.get_emitted_plugins(event_ctx), + event.event_name, + ) else: query.resp_message_chain.append( diff --git a/src/langbot/pkg/platform/sources/aiocqhttp.py b/src/langbot/pkg/platform/sources/aiocqhttp.py index 3cb55d89d..fc57a5661 100644 --- a/src/langbot/pkg/platform/sources/aiocqhttp.py +++ b/src/langbot/pkg/platform/sources/aiocqhttp.py @@ -16,6 +16,14 @@ from ...utils import image import langbot_plugin.api.definition.abstract.platform.event_logger as abstract_platform_logger +def _normalize_base64_payload(value: str) -> str: + if value.startswith('base64://'): + return value.removeprefix('base64://') + if value.startswith('data:') and ';base64,' in value: + return value.split(';base64,', 1)[1] + return value + + class AiocqhttpMessageConverter(abstract_platform_adapter.AbstractMessageConverter): @staticmethod async def yiri2target( @@ -35,7 +43,7 @@ class AiocqhttpMessageConverter(abstract_platform_adapter.AbstractMessageConvert elif type(msg) is platform_message.Image: arg = '' if msg.base64: - arg = msg.base64 + arg = _normalize_base64_payload(msg.base64) msg_list.append(aiocqhttp.MessageSegment.image(f'base64://{arg}')) elif msg.url: arg = msg.url @@ -50,7 +58,7 @@ class AiocqhttpMessageConverter(abstract_platform_adapter.AbstractMessageConvert elif type(msg) is platform_message.Voice: arg = '' if msg.base64: - arg = msg.base64 + arg = _normalize_base64_payload(msg.base64) msg_list.append(aiocqhttp.MessageSegment.record(f'base64://{arg}')) elif msg.url: arg = msg.url @@ -62,7 +70,10 @@ class AiocqhttpMessageConverter(abstract_platform_adapter.AbstractMessageConvert for node in msg.node_list: msg_list.extend((await AiocqhttpMessageConverter.yiri2target(node.message_chain))[0]) elif isinstance(msg, platform_message.File): - msg_list.append({'type': 'file', 'data': {'file': msg.url, 'name': msg.name}}) + file = msg.url or msg.path + if not file and msg.base64: + file = f'base64://{_normalize_base64_payload(msg.base64)}' + msg_list.append({'type': 'file', 'data': {'file': file, 'name': msg.name}}) elif isinstance(msg, platform_message.Face): if msg.face_type == 'face': msg_list.append(aiocqhttp.MessageSegment.face(msg.face_id)) @@ -433,9 +444,7 @@ class AiocqhttpAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter) elif isinstance(component, platform_message.Image): img_data = {} if component.base64: - b64 = component.base64 - if b64.startswith('data:'): - b64 = b64.split(',', 1)[-1] if ',' in b64 else b64 + b64 = _normalize_base64_payload(component.base64) img_data['file'] = f'base64://{b64}' elif component.url: img_data['file'] = component.url diff --git a/src/langbot/pkg/plugin/connector.py b/src/langbot/pkg/plugin/connector.py index e6506bcc6..337d8b352 100644 --- a/src/langbot/pkg/plugin/connector.py +++ b/src/langbot/pkg/plugin/connector.py @@ -751,6 +751,8 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): event_ctx = context.EventContext.from_event(event) if not self.is_enable_plugin: + event_ctx._emitted_plugins = [] + event_ctx._response_sources = [] return event_ctx # Pass include_plugins to runtime for filtering @@ -759,9 +761,21 @@ class PluginRuntimeConnector(ManagedRuntimeConnector): ) event_ctx = context.EventContext.model_validate(event_ctx_result['event_context']) + event_ctx._emitted_plugins = event_ctx_result.get('emitted_plugins', []) + if 'response_sources' in event_ctx_result: + event_ctx._response_sources = event_ctx_result['response_sources'] return event_ctx + async def notify_plugin_diagnostic(self, diagnostic: dict[str, Any]) -> None: + """Best-effort diagnostic forwarding to the plugin runtime.""" + if not self.is_enable_plugin: + return + try: + await self.handler.notify_plugin_diagnostic(diagnostic) + except Exception as e: + self.ap.logger.debug(f'Plugin diagnostic forwarding skipped: {e}') + async def list_tools(self, bound_plugins: list[str] | None = None) -> list[ComponentManifest]: if not self.is_enable_plugin: return [] diff --git a/src/langbot/pkg/plugin/handler.py b/src/langbot/pkg/plugin/handler.py index 43fa27aef..0d74e7e28 100644 --- a/src/langbot/pkg/plugin/handler.py +++ b/src/langbot/pkg/plugin/handler.py @@ -51,6 +51,15 @@ def _serialize_plugin_api_result(value: Any) -> Any: return value +class _RawAction: + def __init__(self, value: str): + self.value = value + + +def _langbot_to_runtime_action(enum_name: str, fallback_value: str) -> Any: + return getattr(LangBotToRuntimeAction, enum_name, _RawAction(fallback_value)) + + def _make_rag_error_response(error: Exception, error_type: str, **extra_context) -> handler.ActionResponse: """Create a clean error response for RAG operations. @@ -1541,6 +1550,18 @@ class RuntimeConnectionHandler(handler.Handler): return result + async def notify_plugin_diagnostic(self, diagnostic: dict[str, Any]) -> dict[str, Any]: + """Notify the plugin runtime about a best-effort plugin diagnostic. + + This intentionally uses the raw protocol string instead of a SDK enum so + LangBot can keep running with older langbot-plugin versions. + """ + return await self.call_action( + _langbot_to_runtime_action('PLUGIN_DIAGNOSTIC', 'plugin_diagnostic'), + diagnostic, + timeout=5, + ) + async def list_tools(self, include_plugins: list[str] | None = None) -> list[dict[str, Any]]: """List tools""" result = await self.call_action( diff --git a/src/langbot/pkg/provider/tools/loaders/mcp.py b/src/langbot/pkg/provider/tools/loaders/mcp.py index 2cc83b1c7..e059a756d 100644 --- a/src/langbot/pkg/provider/tools/loaders/mcp.py +++ b/src/langbot/pkg/provider/tools/loaders/mcp.py @@ -1,6 +1,10 @@ from __future__ import annotations +import base64 import enum +import json +import re +import time import typing from contextlib import AsyncExitStack import traceback @@ -10,10 +14,11 @@ import asyncio import httpx import uuid as uuid_module -from mcp import ClientSession, StdioServerParameters +from mcp import ClientSession, StdioServerParameters, types as mcp_types from mcp.client.stdio import stdio_client from mcp.client.sse import sse_client from mcp.client.streamable_http import streamable_http_client +from pydantic import AnyUrl from .. import loader from ....core import app @@ -22,6 +27,157 @@ import langbot_plugin.api.entities.builtin.provider.message as provider_message from ....entity.persistence import mcp as persistence_mcp from .mcp_stdio import BoxStdioSessionRuntime, MCPServerBoxConfig, MCPSessionErrorPhase # noqa: F401 +# Synthesized LLM tools for MCP resources (not from server tools/list). +# Dispatched in MCPLoader.invoke_tool; placeholder func on LLMTool is never used. +# Prefixed with langbot_ to avoid clashing with MCP server tool names. +MCP_TOOL_LIST_RESOURCES = 'langbot_mcp_list_resources' +MCP_TOOL_READ_RESOURCE = 'langbot_mcp_read_resource' + +MCP_RESOURCE_DISCOVERY_MAX_PAGES = 20 +MCP_RESOURCE_CACHE_TTL_SECONDS = 30 +MCP_RESOURCE_PREVIEW_MAX_BYTES = 64 * 1024 +MCP_RESOURCE_AGENT_READ_MAX_BYTES = 64 * 1024 +MCP_RESOURCE_AGENT_READ_MAX_TOKENS = 12000 +MCP_RESOURCE_CONTEXT_MAX_TOKENS = 8000 +MCP_RESOURCE_CONTEXT_MAX_BYTES = 96 * 1024 +MCP_RESOURCE_TRACE_QUERY_KEY = '_mcp_resource_reads' +MCP_RESOURCE_LINKS_QUERY_KEY = '_mcp_resource_links' +MCP_RESOURCE_CONTEXT_QUERY_KEY = '_mcp_resource_context' + +TEXT_LIKE_MIME_TYPES = { + 'application/json', + 'application/ld+json', + 'application/xml', + 'application/yaml', + 'application/x-yaml', + 'application/toml', + 'application/javascript', + 'application/typescript', + 'application/sql', + 'application/graphql', +} + +MCP_LIST_RESOURCES_SCHEMA: dict[str, typing.Any] = { + 'type': 'object', + 'properties': { + 'server_name': { + 'type': 'string', + 'description': 'MCP server name as configured in LangBot (see admin / pipeline bindings).', + } + }, + 'required': ['server_name'], +} + +MCP_READ_RESOURCE_SCHEMA: dict[str, typing.Any] = { + 'type': 'object', + 'properties': { + 'server_name': { + 'type': 'string', + 'description': 'MCP server name as configured in LangBot.', + }, + 'uri': { + 'type': 'string', + 'description': 'Resource URI from langbot_mcp_list_resources output or a listed resource template.', + }, + }, + 'required': ['server_name', 'uri'], +} + + +def _mcp_model_dump(obj: typing.Any) -> typing.Any: + if obj is None: + return None + if hasattr(obj, 'model_dump'): + return obj.model_dump(mode='json', by_alias=True, exclude_none=True) + if isinstance(obj, (str, int, float, bool)): + return obj + if isinstance(obj, list): + return [_mcp_model_dump(item) for item in obj] + if isinstance(obj, dict): + return {str(k): _mcp_model_dump(v) for k, v in obj.items()} + return str(obj) + + +def _truncate_text(text: str, max_bytes: int, max_tokens: int | None = None) -> tuple[str, bool, int]: + raw = text.encode('utf-8') + original_bytes = len(raw) + truncated = False + + if max_bytes > 0 and len(raw) > max_bytes: + raw = raw[:max_bytes] + text = raw.decode('utf-8', errors='ignore') + truncated = True + + if max_tokens is not None and max_tokens > 0: + max_chars = max_tokens * 4 + if len(text) > max_chars: + text = text[:max_chars] + truncated = True + + return text, truncated, original_bytes + + +def _blob_size(blob: str) -> int: + try: + return len(base64.b64decode(blob, validate=False)) + except Exception: + return len(blob.encode('utf-8', errors='ignore')) + + +def _resource_to_dict(resource: mcp_types.Resource | mcp_types.ResourceLink) -> dict: + return { + 'uri': str(resource.uri), + 'name': resource.name, + 'title': resource.title or '', + 'description': resource.description or '', + 'mime_type': resource.mimeType or '', + 'size': resource.size, + 'icons': _mcp_model_dump(resource.icons) or [], + 'annotations': _mcp_model_dump(resource.annotations) or {}, + '_meta': _mcp_model_dump(getattr(resource, 'meta', None)) or {}, + } + + +def _resource_template_to_dict(resource_template: mcp_types.ResourceTemplate) -> dict: + return { + 'uri_template': resource_template.uriTemplate, + 'name': resource_template.name, + 'title': resource_template.title or '', + 'description': resource_template.description or '', + 'mime_type': resource_template.mimeType or '', + 'icons': _mcp_model_dump(resource_template.icons) or [], + 'annotations': _mcp_model_dump(resource_template.annotations) or {}, + '_meta': _mcp_model_dump(getattr(resource_template, 'meta', None)) or {}, + } + + +def _is_text_like_mime(mime_type: str) -> bool: + if not mime_type: + return False + normalized = mime_type.split(';', 1)[0].strip().lower() + return normalized.startswith('text/') or normalized in TEXT_LIKE_MIME_TYPES or normalized.endswith('+json') + + +def _uri_matches_template(uri: str, uri_template: str) -> bool: + if uri_template == uri: + return True + if not uri_template or '{' not in uri_template: + return False + + pattern_parts: list[str] = [] + pos = 0 + for match in re.finditer(r'\{[^{}]+\}', uri_template): + pattern_parts.append(re.escape(uri_template[pos : match.start()])) + pattern_parts.append(r'[^\s]+') + pos = match.end() + pattern_parts.append(re.escape(uri_template[pos:])) + return re.fullmatch(''.join(pattern_parts), uri) is not None + + +async def _mcp_resource_tool_placeholder(**kwargs: typing.Any) -> list[provider_message.ContentElement]: + """LLMTool requires a func; real execution goes through MCPLoader.invoke_tool.""" + raise RuntimeError('MCP resource tool execution must be routed through MCPLoader.invoke_tool') + class MCPSessionStatus(enum.Enum): CONNECTING = 'connecting' @@ -46,6 +202,12 @@ class RuntimeMCPSession: functions: list[resource_tool.LLMTool] = [] + resources: list[dict] = [] + + resource_templates: list[dict] = [] + + resource_capabilities: dict = {} + enable: bool # connected: bool @@ -82,6 +244,10 @@ class RuntimeMCPSession: self.exit_stack = AsyncExitStack() self.functions = [] + self.resources = [] + self.resource_templates = [] + self.resource_capabilities = {} + self._resource_cache: dict[tuple[str, int, int | None, bool], dict] = {} self.status = MCPSessionStatus.CONNECTING @@ -253,6 +419,7 @@ class RuntimeMCPSession: await self.exit_stack.aclose() self.exit_stack = AsyncExitStack() self.functions.clear() + self.resources.clear() self.session = None except Exception as e: self.ap.logger.error(f'Error cleaning up MCP session {self.server_name}: {e}\n{traceback.format_exc()}') @@ -348,6 +515,15 @@ class RuntimeMCPSession: return self.functions.clear() + self.resources.clear() + self.resource_templates.clear() + self._resource_cache.clear() + + try: + capabilities = self.session.get_server_capabilities() + self.resource_capabilities = _mcp_model_dump(getattr(capabilities, 'resources', None)) or {} + except Exception: + self.resource_capabilities = {} tools = await self.session.list_tools() @@ -356,28 +532,7 @@ class RuntimeMCPSession: for tool in tools.tools: async def func(*, _tool=tool, **kwargs): - if not self.session: - raise Exception('MCP session is not connected') - - result = await self.session.call_tool(_tool.name, kwargs) - if result.isError: - error_texts = [] - for content in result.content: - if content.type == 'text': - error_texts.append(content.text) - raise Exception('\n'.join(error_texts) if error_texts else 'Unknown error from MCP tool') - - result_contents: list[provider_message.ContentElement] = [] - for content in result.content: - if content.type == 'text': - result_contents.append(provider_message.ContentElement.from_text(content.text)) - elif content.type == 'image': - result_contents.append(provider_message.ContentElement.from_image_base64(content.image_base64)) - elif content.type == 'resource': - # TODO: Handle resource content - pass - - return result_contents + return await self.invoke_mcp_tool(_tool.name, kwargs) func.__name__ = tool.name @@ -391,9 +546,335 @@ class RuntimeMCPSession: ) ) + await self._refresh_resources() + + async def _refresh_resources(self): + if not self.session: + return + + try: + cursor: str | None = None + for _ in range(MCP_RESOURCE_DISCOVERY_MAX_PAGES): + resources_result = await self.session.list_resources(cursor) + for resource in resources_result.resources: + self.resources.append(_resource_to_dict(resource)) + cursor = getattr(resources_result, 'nextCursor', None) + if not cursor: + break + self.ap.logger.debug(f'Refresh MCP resources: {len(self.resources)} resources found') + except Exception as e: + self.ap.logger.debug(f'MCP server {self.server_name} does not support resources or failed to list: {e}') + + try: + cursor = None + for _ in range(MCP_RESOURCE_DISCOVERY_MAX_PAGES): + templates_result = await self.session.list_resource_templates(cursor) + for template in templates_result.resourceTemplates: + self.resource_templates.append(_resource_template_to_dict(template)) + cursor = getattr(templates_result, 'nextCursor', None) + if not cursor: + break + self.ap.logger.debug(f'Refresh MCP resource templates: {len(self.resource_templates)} templates found') + except Exception as e: + self.ap.logger.debug( + f'MCP server {self.server_name} does not support resource templates or failed to list: {e}' + ) + + def _record_query_resource_link( + self, + query: pipeline_query.Query | None, + resource_link: dict, + source_tool: str, + ) -> None: + if query is None: + return + try: + link = { + **resource_link, + 'server_name': self.server_name, + 'server_uuid': self.server_uuid, + 'source_tool': source_tool, + } + query.variables.setdefault(MCP_RESOURCE_LINKS_QUERY_KEY, []).append(link) + except Exception: + pass + + def _content_to_provider_elements( + self, + content: typing.Any, + *, + query: pipeline_query.Query | None = None, + source_tool: str = '', + ) -> list[provider_message.ContentElement]: + content_type = getattr(content, 'type', '') + if content_type == 'text': + return [provider_message.ContentElement.from_text(content.text)] + + if content_type == 'image': + image_data = getattr(content, 'data', None) or getattr(content, 'image_base64', None) + if image_data: + return [provider_message.ContentElement.from_image_base64(image_data)] + return [] + + if content_type == 'audio': + return [ + provider_message.ContentElement.from_text( + json.dumps( + { + 'type': 'audio', + 'mime_type': getattr(content, 'mimeType', ''), + 'message': 'Audio content returned by MCP tool is available to the host but not inlined.', + }, + ensure_ascii=False, + ) + ) + ] + + if content_type == 'resource_link': + resource_link = _resource_to_dict(content) + self._record_query_resource_link(query, resource_link, source_tool) + return [ + provider_message.ContentElement.from_text( + json.dumps( + { + 'type': 'resource_link', + 'server_name': self.server_name, + 'server_uuid': self.server_uuid, + 'resource': resource_link, + 'message': 'Resource link captured. Read it only if the task needs this additional context.', + }, + ensure_ascii=False, + indent=2, + ) + ) + ] + + if content_type == 'resource': + resource = getattr(content, 'resource', None) + if isinstance(resource, mcp_types.TextResourceContents): + text, truncated, original_bytes = _truncate_text( + resource.text, + MCP_RESOURCE_AGENT_READ_MAX_BYTES, + MCP_RESOURCE_AGENT_READ_MAX_TOKENS, + ) + header = { + 'type': 'embedded_resource', + 'server_name': self.server_name, + 'server_uuid': self.server_uuid, + 'uri': str(resource.uri), + 'mime_type': resource.mimeType or '', + 'bytes': original_bytes, + 'truncated': truncated, + } + return [provider_message.ContentElement.from_text(f'{json.dumps(header, ensure_ascii=False)}\n{text}')] + if isinstance(resource, mcp_types.BlobResourceContents): + return [ + provider_message.ContentElement.from_text( + json.dumps( + { + 'type': 'embedded_resource', + 'server_name': self.server_name, + 'server_uuid': self.server_uuid, + 'uri': str(resource.uri), + 'mime_type': resource.mimeType or '', + 'bytes': _blob_size(resource.blob), + 'binary_omitted': True, + }, + ensure_ascii=False, + ) + ) + ] + + return [] + + async def invoke_mcp_tool( + self, + tool_name: str, + arguments: dict, + query: pipeline_query.Query | None = None, + ) -> list[provider_message.ContentElement]: + if not self.session: + raise Exception('MCP session is not connected') + + result = await self.session.call_tool(tool_name, arguments) + if result.isError: + error_texts = [] + for content in result.content: + if getattr(content, 'type', '') == 'text': + error_texts.append(content.text) + raise Exception('\n'.join(error_texts) if error_texts else 'Unknown error from MCP tool') + + result_contents: list[provider_message.ContentElement] = [] + for content in result.content: + result_contents.extend(self._content_to_provider_elements(content, query=query, source_tool=tool_name)) + return result_contents + def get_tools(self) -> list[resource_tool.LLMTool]: return self.functions + def get_resources(self) -> list[dict]: + return self.resources + + def get_resource_templates(self) -> list[dict]: + return self.resource_templates + + def has_resource_support(self) -> bool: + return bool(self.resources or self.resource_templates or self.resource_capabilities) + + def invalidate_resource_cache(self, uri: str | None = None) -> None: + if uri is None: + self._resource_cache.clear() + return + for key in list(self._resource_cache.keys()): + if key[0] == uri: + self._resource_cache.pop(key, None) + + def resource_uri_allowed(self, uri: str) -> bool: + if any(item.get('uri') == uri for item in self.resources): + return True + + for template in self.resource_templates: + uri_template = template.get('uri_template', '') + if _uri_matches_template(uri, uri_template): + return True + + return False + + async def read_resource_envelope( + self, + uri: str, + *, + max_bytes: int = MCP_RESOURCE_PREVIEW_MAX_BYTES, + max_tokens: int | None = None, + include_blob: bool = False, + source: str = 'api', + query: pipeline_query.Query | None = None, + ) -> dict: + """Read a resource by URI with safety limits and audit metadata.""" + if not self.session: + raise Exception('MCP session is not connected') + + if not self.resource_uri_allowed(uri): + raise ValueError( + f'Resource URI is not available from MCP server {self.server_name!r}: {uri!r}. ' + 'Use listed resources or resource templates.' + ) + + cache_key = (uri, max_bytes, max_tokens, include_blob) + now = time.time() + cached = self._resource_cache.get(cache_key) + if cached and now - cached.get('cached_at', 0) <= MCP_RESOURCE_CACHE_TTL_SECONDS: + envelope = { + **cached['envelope'], + 'cache_hit': True, + 'source': source, + } + self._record_resource_read_trace(query, envelope) + return envelope + + result = await self.session.read_resource(AnyUrl(uri)) + contents: list[dict] = [] + total_bytes = 0 + truncated_any = False + warnings: list[str] = [] + remaining_bytes = max_bytes if max_bytes > 0 else None + remaining_tokens = max_tokens if max_tokens is not None and max_tokens > 0 else None + + for content in result.contents: + if isinstance(content, mcp_types.TextResourceContents): + if (remaining_bytes is not None and remaining_bytes <= 0) or ( + remaining_tokens is not None and remaining_tokens <= 0 + ): + text = '' + truncated = True + original_bytes = len(content.text.encode('utf-8')) + else: + text, truncated, original_bytes = _truncate_text( + content.text, + remaining_bytes if remaining_bytes is not None else 0, + remaining_tokens, + ) + total_bytes += original_bytes + truncated_any = truncated_any or truncated + if remaining_bytes is not None: + remaining_bytes = max(0, remaining_bytes - len(text.encode('utf-8'))) + if remaining_tokens is not None: + remaining_tokens = max(0, remaining_tokens - (max(1, len(text) // 4) if text else 0)) + contents.append( + { + 'uri': str(content.uri), + 'mime_type': content.mimeType or '', + 'type': 'text', + 'text': text, + 'bytes': original_bytes, + 'truncated': truncated, + '_meta': _mcp_model_dump(getattr(content, 'meta', None)) or {}, + } + ) + elif isinstance(content, mcp_types.BlobResourceContents): + original_bytes = _blob_size(content.blob) + total_bytes += original_bytes + include_this_blob = include_blob and (remaining_bytes is None or original_bytes <= remaining_bytes) + if not include_this_blob: + truncated_any = True + warnings.append('Binary resource content omitted from response.') + elif remaining_bytes is not None: + remaining_bytes = max(0, remaining_bytes - original_bytes) + contents.append( + { + 'uri': str(content.uri), + 'mime_type': content.mimeType or '', + 'type': 'blob', + 'blob': content.blob if include_this_blob else None, + 'bytes': original_bytes, + 'truncated': not include_this_blob, + 'binary_omitted': not include_this_blob, + '_meta': _mcp_model_dump(getattr(content, 'meta', None)) or {}, + } + ) + + envelope = { + 'server_name': self.server_name, + 'server_uuid': self.server_uuid, + 'uri': uri, + 'source': source, + 'contents': contents, + 'bytes': total_bytes, + 'truncated': truncated_any, + 'cache_hit': False, + 'warnings': warnings, + } + self._resource_cache[cache_key] = {'cached_at': now, 'envelope': envelope} + self._record_resource_read_trace(query, envelope) + return envelope + + def _record_resource_read_trace(self, query: pipeline_query.Query | None, envelope: dict) -> None: + if query is None: + return + try: + from langbot.pkg.telemetry import features as telemetry_features + + telemetry_features.increment(query, 'mcp_resource_reads', envelope.get('source') or 'unknown') + query.variables.setdefault(MCP_RESOURCE_TRACE_QUERY_KEY, []).append( + { + 'server_name': envelope.get('server_name'), + 'server_uuid': envelope.get('server_uuid'), + 'uri': envelope.get('uri'), + 'source': envelope.get('source'), + 'bytes': envelope.get('bytes', 0), + 'truncated': envelope.get('truncated', False), + 'cache_hit': envelope.get('cache_hit', False), + 'content_types': [item.get('type') for item in envelope.get('contents', [])], + } + ) + except Exception: + pass + + async def read_resource(self, uri: str) -> list[dict]: + """Read a resource by URI and return its capped contents.""" + envelope = await self.read_resource_envelope(uri) + return envelope['contents'] + def get_runtime_info_dict(self) -> dict: info = { 'status': self.status.value, @@ -409,6 +890,11 @@ class RuntimeMCPSession: } for tool in self.get_tools() ], + 'resource_count': len(self.get_resources()), + 'resources': self.get_resources(), + 'resource_template_count': len(self.get_resource_templates()), + 'resource_templates': self.get_resource_templates(), + 'resource_capabilities': self.resource_capabilities, } if self._uses_box_stdio(): info['box_session_id'] = self._build_box_session_id() @@ -575,8 +1061,164 @@ class MCPLoader(loader.ToolLoader): return session - async def get_tools(self, bound_mcp_servers: list[str] | None = None) -> list[resource_tool.LLMTool]: - all_functions = [] + @staticmethod + def _get_bound_mcp_from_query(query: pipeline_query.Query) -> list[str] | None: + v = getattr(query, 'variables', None) or {} + return v.get('_pipeline_bound_mcp_servers', None) + + def _eligible_sessions_for_bound(self, bound_mcp_servers: list[str] | None) -> list[RuntimeMCPSession]: + out: list[RuntimeMCPSession] = [] + for session in self.sessions.values(): + if not session.enable: + continue + if session.status != MCPSessionStatus.CONNECTED: + continue + if session.session is None: + continue + if bound_mcp_servers is not None and session.server_uuid not in bound_mcp_servers: + continue + out.append(session) + return out + + def _eligible_resource_sessions_for_bound(self, bound_mcp_servers: list[str] | None) -> list[RuntimeMCPSession]: + return [ + session + for session in self._eligible_sessions_for_bound(bound_mcp_servers) + if session.has_resource_support() + ] + + @staticmethod + def _mcp_synthetic_resource_tools() -> list[resource_tool.LLMTool]: + return [ + resource_tool.LLMTool( + name=MCP_TOOL_LIST_RESOURCES, + human_desc='List MCP resource URIs for a server (MCP resources/list).', + description=( + 'Lists resources and resource templates exposed by an MCP server. ' + 'Call langbot_mcp_read_resource with a listed resource URI or a URI constructed from a listed template. ' + 'Use the server name from LangBot pipeline MCP bindings or admin configuration.' + ), + parameters=MCP_LIST_RESOURCES_SCHEMA, + func=_mcp_resource_tool_placeholder, + ), + resource_tool.LLMTool( + name=MCP_TOOL_READ_RESOURCE, + human_desc='Read a single MCP resource by URI (MCP resources/read).', + description=( + 'Fetches capped text content for a resource. Binary resources return metadata only. ' + 'Only read URIs exposed by langbot_mcp_list_resources for the bound server.' + ), + parameters=MCP_READ_RESOURCE_SCHEMA, + func=_mcp_resource_tool_placeholder, + ), + ] + + async def _invoke_mcp_list_resources(self, parameters: dict, query: pipeline_query.Query) -> typing.Any: + server_name = parameters.get('server_name') if parameters else None + if not server_name or not isinstance(server_name, str): + return [provider_message.ContentElement.from_text('Error: "server_name" (string) is required.')] + + bound = self._get_bound_mcp_from_query(query) + allowed = {s.server_name for s in self._eligible_resource_sessions_for_bound(bound)} + if server_name not in allowed: + return [ + provider_message.ContentElement.from_text( + f'Error: MCP server {server_name!r} is not available for this query. ' + f'Allowed server names: {sorted(allowed)}. ' + 'Check pipeline MCP server bindings and that the server is connected.' + ) + ] + + session = self.get_session(server_name) + if session is None or session.status != MCPSessionStatus.CONNECTED: + return [provider_message.ContentElement.from_text(f'Error: MCP server not connected: {server_name!r}')] + + data = session.get_resources() + templates = session.get_resource_templates() + body = { + 'server_name': server_name, + 'resource_count': len(data), + 'resources': data, + 'resource_template_count': len(templates), + 'resource_templates': templates, + 'resource_capabilities': session.resource_capabilities, + } + return [provider_message.ContentElement.from_text(json.dumps(body, ensure_ascii=False, indent=2))] + + async def _invoke_mcp_read_resource(self, parameters: dict, query: pipeline_query.Query) -> typing.Any: + server_name = parameters.get('server_name') if parameters else None + uri = parameters.get('uri') if parameters else None + if not server_name or not isinstance(server_name, str): + return [provider_message.ContentElement.from_text('Error: "server_name" (string) is required.')] + if not uri or not isinstance(uri, str): + return [provider_message.ContentElement.from_text('Error: "uri" (string) is required.')] + + bound = self._get_bound_mcp_from_query(query) + allowed = {s.server_name for s in self._eligible_resource_sessions_for_bound(bound)} + if server_name not in allowed: + return [ + provider_message.ContentElement.from_text( + f'Error: MCP server {server_name!r} is not available for this query. ' + f'Allowed server names: {sorted(allowed)}.' + ) + ] + + session = self.get_session(server_name) + if session is None or session.status != MCPSessionStatus.CONNECTED: + return [provider_message.ContentElement.from_text(f'Error: MCP server not connected: {server_name!r}')] + + try: + envelope = await session.read_resource_envelope( + uri, + max_bytes=MCP_RESOURCE_AGENT_READ_MAX_BYTES, + max_tokens=MCP_RESOURCE_AGENT_READ_MAX_TOKENS, + include_blob=False, + source='agent_tool', + query=query, + ) + except Exception as e: + self.ap.logger.error(f'read_resource {uri!r} on {server_name}: {e}\n{traceback.format_exc()}') + return [provider_message.ContentElement.from_text(f'Error reading resource: {e!s}')] + + out_chunks: list[str] = [] + for item in envelope.get('contents', []): + if not isinstance(item, dict): + continue + t = item.get('type', '') + if t == 'text' and 'text' in item: + header = { + 'uri': item.get('uri'), + 'mime_type': item.get('mime_type', ''), + 'bytes': item.get('bytes', 0), + 'truncated': item.get('truncated', False), + } + out_chunks.append(f'{json.dumps(header, ensure_ascii=False)}\n{typing.cast(str, item["text"])}') + elif t == 'blob': + out_chunks.append( + json.dumps( + { + 'uri': item.get('uri'), + 'mime_type': item.get('mime_type', ''), + 'bytes': item.get('bytes', 0), + 'binary_omitted': True, + }, + ensure_ascii=False, + ) + ) + if not out_chunks: + return [provider_message.ContentElement.from_text(json.dumps(envelope, ensure_ascii=False, indent=2))] + suffix = '' + if envelope.get('truncated'): + suffix = '\n\n[LangBot: resource content was truncated by configured byte/token limits.]' + return [provider_message.ContentElement.from_text('\n\n'.join(out_chunks) + suffix)] + + async def get_tools( + self, + bound_mcp_servers: list[str] | None = None, + *, + include_resource_tools: bool = True, + ) -> list[resource_tool.LLMTool]: + all_functions: list[resource_tool.LLMTool] = [] for session in self.sessions.values(): # If bound_mcp_servers is specified, only include tools from those servers @@ -587,12 +1229,57 @@ class MCPLoader(loader.ToolLoader): # If no bound servers specified, include all tools all_functions.extend(session.get_tools()) + if include_resource_tools and self._eligible_resource_sessions_for_bound(bound_mcp_servers): + all_functions.extend(self._mcp_synthetic_resource_tools()) + self._last_listed_functions = all_functions return all_functions + async def get_tool_catalog( + self, + bound_mcp_servers: list[str] | None = None, + *, + include_resource_tools: bool = False, + ) -> list[dict[str, typing.Any]]: + items: list[dict[str, typing.Any]] = [] + + for session in self.sessions.values(): + if bound_mcp_servers is not None and session.server_uuid not in bound_mcp_servers: + continue + for tool in session.get_tools(): + items.append( + { + 'name': tool.name, + 'description': tool.description, + 'human_desc': tool.human_desc, + 'parameters': tool.parameters, + 'source': 'mcp', + 'source_name': session.server_name, + 'source_id': session.server_uuid, + } + ) + + if include_resource_tools and self._eligible_resource_sessions_for_bound(bound_mcp_servers): + for tool in self._mcp_synthetic_resource_tools(): + items.append( + { + 'name': tool.name, + 'description': tool.description, + 'human_desc': tool.human_desc, + 'parameters': tool.parameters, + 'source': 'mcp', + 'source_name': 'MCP resources', + 'source_id': '', + } + ) + + return items + async def has_tool(self, name: str) -> bool: """检查工具是否存在""" + if name in (MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE): + return bool(self._eligible_resource_sessions_for_bound(None)) for session in self.sessions.values(): for function in session.get_tools(): if function.name == name: @@ -608,12 +1295,21 @@ class MCPLoader(loader.ToolLoader): async def invoke_tool(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any: """执行工具调用""" + if name == MCP_TOOL_LIST_RESOURCES: + if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is False: + return [provider_message.ContentElement.from_text('Error: MCP resource agent reads are disabled.')] + return await self._invoke_mcp_list_resources(parameters, query) + if name == MCP_TOOL_READ_RESOURCE: + if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is False: + return [provider_message.ContentElement.from_text('Error: MCP resource agent reads are disabled.')] + return await self._invoke_mcp_read_resource(parameters, query) + for session in self.sessions.values(): for function in session.get_tools(): if function.name == name: self.ap.logger.debug(f'Invoking MCP tool: {name} with parameters: {parameters}') try: - result = await function.func(**parameters) + result = await session.invoke_mcp_tool(name, parameters, query=query) self.ap.logger.debug(f'MCP tool {name} executed successfully') return result except Exception as e: @@ -622,6 +1318,159 @@ class MCPLoader(loader.ToolLoader): raise ValueError(f'Tool not found: {name}') + async def get_resources(self, server_name: str) -> list[dict]: + """Get resources from a specific MCP server.""" + session = self.get_session(server_name) + if session is None: + raise ValueError(f'MCP server not found: {server_name}') + return session.get_resources() + + async def get_resource_templates(self, server_name: str) -> list[dict]: + """Get resource templates from a specific MCP server.""" + session = self.get_session(server_name) + if session is None: + raise ValueError(f'MCP server not found: {server_name}') + return session.get_resource_templates() + + async def read_resource_envelope( + self, + server_name: str, + uri: str, + *, + max_bytes: int = MCP_RESOURCE_PREVIEW_MAX_BYTES, + max_tokens: int | None = None, + include_blob: bool = False, + source: str = 'api', + query: pipeline_query.Query | None = None, + ) -> dict: + """Read a resource from a specific MCP server and return metadata plus contents.""" + session = self.get_session(server_name) + if session is None: + raise ValueError(f'MCP server not found: {server_name}') + return await session.read_resource_envelope( + uri, + max_bytes=max_bytes, + max_tokens=max_tokens, + include_blob=include_blob, + source=source, + query=query, + ) + + async def read_resource(self, server_name: str, uri: str) -> list[dict]: + """Read a resource from a specific MCP server.""" + envelope = await self.read_resource_envelope(server_name, uri) + return envelope['contents'] + + def get_session_by_uuid(self, server_uuid: str) -> RuntimeMCPSession | None: + for session in self.sessions.values(): + if session.server_uuid == server_uuid: + return session + return None + + def _resolve_attachment_session(self, attachment: dict) -> RuntimeMCPSession | None: + server_uuid = attachment.get('server_uuid') or attachment.get('server_id') + server_name = attachment.get('server_name') + if server_uuid: + return self.get_session_by_uuid(server_uuid) + if server_name: + return self.get_session(server_name) + return None + + async def build_resource_context_for_query( + self, + query: pipeline_query.Query, + *, + default_max_tokens: int = MCP_RESOURCE_CONTEXT_MAX_TOKENS, + default_max_bytes: int = MCP_RESOURCE_CONTEXT_MAX_BYTES, + ) -> str: + """Build host-controlled MCP resource context for the current query.""" + if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is False: + return '' + + attachments = (query.variables or {}).get('_pipeline_mcp_resource_attachments', []) + if not isinstance(attachments, list) or not attachments: + return '' + + bound = self._get_bound_mcp_from_query(query) + eligible = self._eligible_resource_sessions_for_bound(bound) + eligible_by_uuid = {session.server_uuid: session for session in eligible} + eligible_by_name = {session.server_name: session for session in eligible} + + blocks: list[str] = [] + remaining_tokens = default_max_tokens + + for raw_attachment in attachments: + if remaining_tokens <= 0: + break + if not isinstance(raw_attachment, dict) or raw_attachment.get('enabled') is False: + continue + + attachment = raw_attachment.copy() + mode = attachment.get('mode', 'pinned') + if mode not in ('pinned', 'manual', 'auto'): + continue + + uri = attachment.get('uri') + if not uri or not isinstance(uri, str): + continue + + session = self._resolve_attachment_session(attachment) + if session is None: + continue + if session.server_uuid not in eligible_by_uuid and session.server_name not in eligible_by_name: + continue + + max_tokens = min(int(attachment.get('max_tokens') or remaining_tokens), remaining_tokens) + max_bytes = int(attachment.get('max_bytes') or default_max_bytes) + + try: + envelope = await session.read_resource_envelope( + uri, + max_bytes=max_bytes, + max_tokens=max_tokens, + include_blob=False, + source='preloaded', + query=query, + ) + except Exception as e: + self.ap.logger.warning(f'Failed to preload MCP resource {uri!r} from {session.server_name!r}: {e}') + continue + + for item in envelope.get('contents', []): + if item.get('type') != 'text': + continue + mime_type = item.get('mime_type', '') + text = item.get('text') or '' + if not text: + continue + approx_tokens = max(1, len(text) // 4) + remaining_tokens -= approx_tokens + header_attrs = { + 'server': session.server_name, + 'server_uuid': session.server_uuid, + 'uri': item.get('uri') or uri, + 'mime_type': mime_type, + 'bytes': item.get('bytes', 0), + 'truncated': item.get('truncated', False), + 'mode': mode, + } + attr_text = ' '.join(f'{k}={json.dumps(v, ensure_ascii=False)}' for k, v in header_attrs.items()) + blocks.append(f'\n{text}\n') + if remaining_tokens <= 0: + break + + context = '\n\n'.join(blocks) + if context: + try: + query.variables[MCP_RESOURCE_CONTEXT_QUERY_KEY] = { + 'resource_count': len(blocks), + 'max_tokens': default_max_tokens, + 'traces': query.variables.get(MCP_RESOURCE_TRACE_QUERY_KEY, []), + } + except Exception: + pass + return context + async def remove_mcp_server(self, server_name: str): """移除 MCP 服务器""" if server_name not in self.sessions: diff --git a/src/langbot/pkg/provider/tools/loaders/plugin.py b/src/langbot/pkg/provider/tools/loaders/plugin.py index 44f40e421..2697da44f 100644 --- a/src/langbot/pkg/provider/tools/loaders/plugin.py +++ b/src/langbot/pkg/provider/tools/loaders/plugin.py @@ -32,6 +32,24 @@ class PluginToolLoader(loader.ToolLoader): return all_functions + async def get_tool_catalog(self, bound_plugins: list[str] | None = None) -> list[dict[str, typing.Any]]: + catalog: list[dict[str, typing.Any]] = [] + + for tool in await self.ap.plugin_connector.list_tools(bound_plugins): + catalog.append( + { + 'name': tool.metadata.name, + 'description': tool.spec['llm_prompt'], + 'human_desc': tool.metadata.description.en_US, + 'parameters': tool.spec['parameters'], + 'source': 'plugin', + 'source_name': tool.owner, + 'source_id': tool.owner, + } + ) + + return catalog + async def has_tool(self, name: str) -> bool: """检查工具是否存在""" for tool in await self.ap.plugin_connector.list_tools(): diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index 41bf31f6a..b5cfbdc07 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -58,6 +58,8 @@ class ToolManager: self, bound_plugins: list[str] | None = None, bound_mcp_servers: list[str] | None = None, + include_skill_authoring: bool = False, + include_mcp_resource_tools: bool = True, ) -> list[resource_tool.LLMTool]: all_functions: list[resource_tool.LLMTool] = [] @@ -68,10 +70,51 @@ class ToolManager: # capability-gated surface. all_functions.extend(await self.skill_tool_loader.get_tools()) all_functions.extend(await self.plugin_tool_loader.get_tools(bound_plugins)) - all_functions.extend(await self.mcp_tool_loader.get_tools(bound_mcp_servers)) + all_functions.extend( + await self.mcp_tool_loader.get_tools( + bound_mcp_servers, + include_resource_tools=include_mcp_resource_tools, + ) + ) return all_functions + async def get_tool_catalog( + self, + bound_plugins: list[str] | None = None, + bound_mcp_servers: list[str] | None = None, + include_skill_authoring: bool = False, + include_mcp_resource_tools: bool = False, + ) -> list[dict[str, typing.Any]]: + catalog: list[dict[str, typing.Any]] = [] + + def append_tools(source: str, source_name: str, tools: list[resource_tool.LLMTool]) -> None: + for tool in tools: + catalog.append( + { + 'name': tool.name, + 'description': tool.description, + 'human_desc': tool.human_desc, + 'parameters': tool.parameters, + 'source': source, + 'source_name': source_name, + } + ) + + append_tools('builtin', 'LangBot', await self.native_tool_loader.get_tools()) + if include_skill_authoring: + append_tools('skill', 'LangBot', await self.skill_tool_loader.get_tools()) + catalog.extend(await self.plugin_tool_loader.get_tool_catalog(bound_plugins)) + + if self.mcp_tool_loader: + for item in await self.mcp_tool_loader.get_tool_catalog( + bound_mcp_servers, + include_resource_tools=include_mcp_resource_tools, + ): + catalog.append(item) + + return catalog + async def get_tool_by_name(self, name: str) -> tool_loader.ToolLookupResult | None: """Get tool by name from any active loader.""" for active_loader in ( diff --git a/src/langbot/pkg/survey/manager.py b/src/langbot/pkg/survey/manager.py index bf28404bd..34689625e 100644 --- a/src/langbot/pkg/survey/manager.py +++ b/src/langbot/pkg/survey/manager.py @@ -159,6 +159,21 @@ class SurveyManager: """Clear the pending survey (after user responds or dismisses).""" self._pending_survey = None + async def _build_base_metadata(self, user_email: str | None = None) -> dict: + metadata = { + 'version': constants.semantic_version, + 'instance_id': constants.instance_id, + } + if user_email: + metadata['login_account'] = user_email + try: + user_obj = await self.ap.user_service.get_user_by_email(user_email) + metadata['account_type'] = getattr(user_obj, 'account_type', '') or 'local' + metadata['space_account_uuid'] = getattr(user_obj, 'space_account_uuid', '') or '' + except Exception: + pass + return metadata + async def submit_response(self, survey_id: str, answers: dict, completed: bool = True) -> bool: """Submit a survey response to Space.""" if not self._is_space_configured(): @@ -169,9 +184,7 @@ class SurveyManager: 'survey_id': survey_id, 'instance_id': constants.instance_id, 'answers': answers, - 'metadata': { - 'version': constants.semantic_version, - }, + 'metadata': await self._build_base_metadata(), 'completed': completed, } async with httpx.AsyncClient(timeout=httpx.Timeout(10)) as client: @@ -183,6 +196,33 @@ class SurveyManager: self.ap.logger.warning(f'Failed to submit survey response: {e}') return False + async def submit_feedback( + self, + content: str, + attachments: list[dict], + user_email: str | None = None, + ) -> bool: + """Submit an on-demand user feedback item to Space.""" + if not self._is_space_configured(): + return False + try: + url = f'{self._space_url}/api/v1/survey/feedback' + metadata = await self._build_base_metadata(user_email) + payload = { + 'instance_id': constants.instance_id, + 'content': content, + 'attachments': attachments, + 'metadata': metadata, + } + async with httpx.AsyncClient(timeout=httpx.Timeout(30)) as client: + resp = await client.post(url, json=payload) + if resp.status_code == 200: + return True + self.ap.logger.warning(f'Failed to submit feedback: {resp.status_code} {resp.text[:200]}') + except Exception as e: + self.ap.logger.warning(f'Failed to submit feedback: {e}') + return False + async def dismiss_survey(self, survey_id: str) -> bool: """Dismiss a survey.""" if not self._is_space_configured(): diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index e03a53991..6b14eba40 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -1,4 +1,3 @@ -admins: [] api: port: 5300 webhook_prefix: 'http://127.0.0.1:5300' @@ -157,6 +156,8 @@ box: - './data/box' - '/tmp' workspace_quota_mb: null # Optional disk quota override (>= 0). null = profile default. + docker: + cpu_limit_enabled: true # When false, Docker sandbox containers are started without --cpus. Memory and PID limits still apply. e2b: api_key: '' # Can also be set via E2B_API_KEY env var. api_url: '' # Custom API URL for self-hosted deployments. diff --git a/src/langbot/templates/metadata/pipeline/ai.yaml b/src/langbot/templates/metadata/pipeline/ai.yaml index f169ccb0e..6db180092 100644 --- a/src/langbot/templates/metadata/pipeline/ai.yaml +++ b/src/langbot/templates/metadata/pipeline/ai.yaml @@ -40,4 +40,4 @@ stages: default: 0 # Runner config stages are dynamically added from AgentRunnerRegistry # Each plugin runner's config schema is added as a separate stage - # The stage name matches the runner id for frontend matching \ No newline at end of file + # The stage name matches the runner id for frontend matching diff --git a/tests/integration/pipeline/test_full_flow.py b/tests/integration/pipeline/test_full_flow.py index 6aa704436..767594c33 100644 --- a/tests/integration/pipeline/test_full_flow.py +++ b/tests/integration/pipeline/test_full_flow.py @@ -662,6 +662,100 @@ class TestSendResponseBackStage: assert len(outbound) == 1 assert outbound[0]['type'] == 'reply' + @pytest.mark.asyncio + async def test_send_response_failure_notifies_plugin_diagnostic(self, pipeline_app): + """Plugin-provided deferred replies should report delivery failures.""" + from langbot.pkg.pipeline import plugin_diagnostics + from langbot.pkg.pipeline.respback import respback + from tests.factories.message import text_chain + from langbot_plugin.api.entities.builtin.provider.message import Message + + query = text_query('hello') + query.adapter.reply_message.side_effect = RuntimeError('send failed') + query.pipeline_config = create_minimal_pipeline_config() + query.current_stage_name = 'SendResponseBackStage' + query.resp_messages = [Message(role='assistant', content='test response')] + query.resp_message_chain = [text_chain('test response')] + plugin_diagnostics.record_plugin_response_source( + query, + 0, + [ + { + 'kind': 'reply_message_chain', + 'plugin': {'author': 'tester', 'name': 'demo'}, + } + ], + [{'manifest': {'metadata': {'author': 'observer', 'name': 'not-reply-source'}}}], + 'NormalMessageResponded', + ) + pipeline_app.plugin_connector.notify_plugin_diagnostic = AsyncMock() + + respback_stage = respback.SendResponseBackStage(pipeline_app) + + with pytest.raises(RuntimeError, match='send failed'): + await respback_stage.process(query, 'SendResponseBackStage') + + pipeline_app.plugin_connector.notify_plugin_diagnostic.assert_awaited_once() + payload = pipeline_app.plugin_connector.notify_plugin_diagnostic.await_args.args[0] + assert payload['code'] == 'response_delivery_failed' + assert payload['plugin'] == {'author': 'tester', 'name': 'demo'} + assert payload['query']['event_name'] == 'NormalMessageResponded' + assert payload['delivery']['error_type'] == 'RuntimeError' + assert 'attribution_warning' not in payload['details'] + + @pytest.mark.asyncio + async def test_send_response_failure_warns_for_old_runtime_attribution(self, pipeline_app): + """Older plugin runtimes without response_sources should get approximate diagnostics.""" + from langbot.pkg.pipeline import plugin_diagnostics + from langbot.pkg.pipeline.respback import respback + from tests.factories.message import text_chain + from langbot_plugin.api.entities.builtin.provider.message import Message + + query = text_query('hello') + query.adapter.reply_message.side_effect = RuntimeError('send failed') + query.pipeline_config = create_minimal_pipeline_config() + query.resp_messages = [Message(role='assistant', content='test response')] + query.resp_message_chain = [text_chain('test response')] + plugin_diagnostics.record_plugin_response_source( + query, + 0, + None, + [{'manifest': {'metadata': {'author': 'tester', 'name': 'demo'}}}], + 'NormalMessageResponded', + ) + pipeline_app.plugin_connector.notify_plugin_diagnostic = AsyncMock() + + respback_stage = respback.SendResponseBackStage(pipeline_app) + + with pytest.raises(RuntimeError, match='send failed'): + await respback_stage.process(query, 'SendResponseBackStage') + + payload = pipeline_app.plugin_connector.notify_plugin_diagnostic.await_args.args[0] + assert payload['plugin'] == {'author': 'tester', 'name': 'demo'} + assert 'attribution_warning' in payload['details'] + + @pytest.mark.asyncio + async def test_send_response_failure_ignores_query_variable_spoofing(self, pipeline_app): + """Plugin-controlled query variables must not mask delivery failures.""" + from langbot.pkg.pipeline.respback import respback + from tests.factories.message import text_chain + from langbot_plugin.api.entities.builtin.provider.message import Message + + query = text_query('hello') + query.adapter.reply_message.side_effect = RuntimeError('send failed') + query.pipeline_config = create_minimal_pipeline_config() + query.resp_messages = [Message(role='assistant', content='test response')] + query.resp_message_chain = [text_chain('test response')] + query.variables['_plugin_response_sources'] = {0: ['malformed']} + pipeline_app.plugin_connector.notify_plugin_diagnostic = AsyncMock() + + respback_stage = respback.SendResponseBackStage(pipeline_app) + + with pytest.raises(RuntimeError, match='send failed'): + await respback_stage.process(query, 'SendResponseBackStage') + + pipeline_app.plugin_connector.notify_plugin_diagnostic.assert_not_called() + @pytest.mark.usefixtures('mock_circular_import_chain') class TestStageChainIntegration: diff --git a/tests/unit_tests/api/service/test_mcp_service.py b/tests/unit_tests/api/service/test_mcp_service.py index 17c746e73..8e3e6cd98 100644 --- a/tests/unit_tests/api/service/test_mcp_service.py +++ b/tests/unit_tests/api/service/test_mcp_service.py @@ -90,6 +90,56 @@ class TestMCPServiceGetRuntimeInfo: assert result is None +class TestMCPServiceResources: + """Tests for MCP resource helpers.""" + + async def test_get_resource_templates_delegates_to_loader(self): + ap = SimpleNamespace() + ap.tool_mgr = SimpleNamespace() + ap.tool_mgr.mcp_tool_loader = SimpleNamespace() + ap.tool_mgr.mcp_tool_loader.get_resource_templates = AsyncMock( + return_value=[{'uri_template': 'file:///{path}', 'name': 'files'}] + ) + + service = MCPService(ap) + + result = await service.get_mcp_server_resource_templates('docs') + + assert result == [{'uri_template': 'file:///{path}', 'name': 'files'}] + ap.tool_mgr.mcp_tool_loader.get_resource_templates.assert_awaited_once_with('docs') + + async def test_read_resource_envelope_uses_ui_preview_source(self): + ap = SimpleNamespace() + ap.tool_mgr = SimpleNamespace() + ap.tool_mgr.mcp_tool_loader = SimpleNamespace() + ap.tool_mgr.mcp_tool_loader.read_resource_envelope = AsyncMock( + return_value={ + 'server_name': 'docs', + 'uri': 'file:///README.md', + 'contents': [], + 'source': 'ui_preview', + } + ) + + service = MCPService(ap) + + result = await service.read_mcp_server_resource_envelope( + 'docs', + 'file:///README.md', + max_bytes=4096, + include_blob=True, + ) + + assert result['source'] == 'ui_preview' + ap.tool_mgr.mcp_tool_loader.read_resource_envelope.assert_awaited_once_with( + 'docs', + 'file:///README.md', + include_blob=True, + source='ui_preview', + max_bytes=4096, + ) + + class TestMCPServiceGetMCPServers: """Tests for get_mcp_servers method.""" diff --git a/tests/unit_tests/api/service/test_pipeline_service.py b/tests/unit_tests/api/service/test_pipeline_service.py index 28d2fc117..fade30372 100644 --- a/tests/unit_tests/api/service/test_pipeline_service.py +++ b/tests/unit_tests/api/service/test_pipeline_service.py @@ -348,6 +348,8 @@ class TestPipelineServiceCreatePipeline: 'enable_all_mcp_servers': True, 'plugins': [], 'mcp_servers': [], + 'mcp_resources': [], + 'mcp_resource_agent_read_enabled': True, } @@ -814,6 +816,47 @@ class TestPipelineServiceUpdatePipelineExtensions: # Verify - persistence was called ap.persistence_mgr.execute_async.assert_called() + async def test_update_extensions_preserves_mcp_resource_agent_read_when_omitted(self): + """Does not reset mcp_resource_agent_read_enabled when omitted by older clients.""" + ap = SimpleNamespace() + ap.persistence_mgr = SimpleNamespace() + ap.pipeline_mgr = SimpleNamespace() + ap.pipeline_mgr.remove_pipeline = AsyncMock() + ap.pipeline_mgr.load_pipeline = AsyncMock() + + original_pipeline = _create_mock_pipeline( + extensions_preferences={ + 'enable_all_plugins': True, + 'enable_all_mcp_servers': True, + 'plugins': [], + 'mcp_servers': [], + 'mcp_resources': [{'server_uuid': 'srv-1', 'uri': 'file:///README.md'}], + 'mcp_resource_agent_read_enabled': False, + } + ) + + call_count = 0 + + async def mock_execute(query): + nonlocal call_count + call_count += 1 + if call_count == 1: + return _create_mock_result(first_item=original_pipeline) + return Mock() + + ap.persistence_mgr.execute_async = AsyncMock(side_effect=mock_execute) + ap.persistence_mgr.serialize_model = Mock(return_value={'uuid': 'test-uuid'}) + + service = PipelineService(ap) + service.get_pipeline = AsyncMock(return_value={'uuid': 'test-uuid'}) + + await service.update_pipeline_extensions('test-uuid', bound_plugins=[]) + + assert original_pipeline.extensions_preferences['mcp_resource_agent_read_enabled'] is False + assert original_pipeline.extensions_preferences['mcp_resources'] == [ + {'server_uuid': 'srv-1', 'uri': 'file:///README.md'} + ] + class TestDefaultStageOrder: """Tests for default_stage_order constant.""" diff --git a/tests/unit_tests/box/test_box_service.py b/tests/unit_tests/box/test_box_service.py index 99c74cc85..22bce7f13 100644 --- a/tests/unit_tests/box/test_box_service.py +++ b/tests/unit_tests/box/test_box_service.py @@ -273,6 +273,31 @@ class TestSharesFilesystemWithBox: assert service.shares_filesystem_with_box is False +def test_separated_box_runtime_does_not_create_default_workspace_in_langbot(tmp_path): + logger = Mock() + runtime = BoxRuntime(logger=logger, backends=[FakeBackend(logger)], session_ttl_sec=300) + host_root = tmp_path / 'box' + service = BoxService(make_app(logger, host_root=str(host_root)), client=_InProcessBoxRuntimeClient(logger, runtime)) + service._shares_filesystem_with_box_override = False + + service._ensure_default_workspace() + + assert not (host_root / 'default').exists() + + +def test_separated_box_runtime_allows_box_owned_missing_host_path(tmp_path): + logger = Mock() + runtime = BoxRuntime(logger=logger, backends=[FakeBackend(logger)], session_ttl_sec=300) + host_root = tmp_path / 'box' + service = BoxService(make_app(logger, host_root=str(host_root)), client=_InProcessBoxRuntimeClient(logger, runtime)) + service._shares_filesystem_with_box_override = False + + spec = service.build_spec({'cmd': 'echo hi', 'session_id': 'missing-host-path'}) + + assert spec.host_path == str(host_root / 'default') + assert not (host_root / 'default').exists() + + @pytest.mark.asyncio async def test_box_service_get_sessions_delegates_to_client(): client = Mock() @@ -517,6 +542,7 @@ async def test_box_service_creates_default_workspace_on_initialize(tmp_path): app = make_app(logger, [str(allowed_root)]) app.instance_config.data['box']['local']['default_workspace'] = str(default_workspace) service = BoxService(app, client=_InProcessBoxRuntimeClient(logger, runtime)) + service._shares_filesystem_with_box_override = True await service.initialize() @@ -531,6 +557,7 @@ async def test_box_service_derives_workspace_and_allowed_root_from_host_root(tmp shared_root = tmp_path / 'shared-box-root' app = make_app(logger, host_root=str(shared_root)) service = BoxService(app, client=_InProcessBoxRuntimeClient(logger, runtime)) + service._shares_filesystem_with_box_override = True await service.initialize() diff --git a/tests/unit_tests/pipeline/test_pipelinemgr.py b/tests/unit_tests/pipeline/test_pipelinemgr.py index f2e6780d6..49984542c 100644 --- a/tests/unit_tests/pipeline/test_pipelinemgr.py +++ b/tests/unit_tests/pipeline/test_pipelinemgr.py @@ -162,3 +162,46 @@ async def test_runtime_pipeline_execute(mock_app, sample_query): # Verify stage was called mock_stage.process.assert_called_once() + + +def test_runtime_pipeline_prefers_local_agent_mcp_resources(mock_app): + """Local Agent resource selection should override legacy extension prefs.""" + pipelinemgr = get_pipelinemgr_module() + persistence_pipeline = get_persistence_pipeline_module() + + pipeline_entity = Mock(spec=persistence_pipeline.LegacyPipeline) + pipeline_entity.config = { + 'ai': { + 'local-agent': { + 'mcp-resources': [{'server_uuid': 'srv-new', 'uri': 'file:///new.md'}], + 'mcp-resource-agent-read-enabled': False, + } + } + } + pipeline_entity.extensions_preferences = { + 'mcp_resources': [{'server_uuid': 'srv-old', 'uri': 'file:///old.md'}], + 'mcp_resource_agent_read_enabled': True, + } + + runtime_pipeline = pipelinemgr.RuntimePipeline(mock_app, pipeline_entity, []) + + assert runtime_pipeline.mcp_resource_attachments == [{'server_uuid': 'srv-new', 'uri': 'file:///new.md'}] + assert runtime_pipeline.mcp_resource_agent_read_enabled is False + + +def test_runtime_pipeline_falls_back_to_extension_mcp_resources(mock_app): + """Existing extension prefs remain compatible until a Local Agent value exists.""" + pipelinemgr = get_pipelinemgr_module() + persistence_pipeline = get_persistence_pipeline_module() + + pipeline_entity = Mock(spec=persistence_pipeline.LegacyPipeline) + pipeline_entity.config = {'ai': {'local-agent': {}}} + pipeline_entity.extensions_preferences = { + 'mcp_resources': [{'server_uuid': 'srv-old', 'uri': 'file:///old.md'}], + 'mcp_resource_agent_read_enabled': False, + } + + runtime_pipeline = pipelinemgr.RuntimePipeline(mock_app, pipeline_entity, []) + + assert runtime_pipeline.mcp_resource_attachments == [{'server_uuid': 'srv-old', 'uri': 'file:///old.md'}] + assert runtime_pipeline.mcp_resource_agent_read_enabled is False diff --git a/tests/unit_tests/pipeline/test_preproc.py b/tests/unit_tests/pipeline/test_preproc.py index 2d68a7f40..c0414e3ed 100644 --- a/tests/unit_tests/pipeline/test_preproc.py +++ b/tests/unit_tests/pipeline/test_preproc.py @@ -14,6 +14,7 @@ from __future__ import annotations import pytest from unittest.mock import AsyncMock, Mock from importlib import import_module +from types import SimpleNamespace from tests.factories import ( FakeApp, @@ -461,3 +462,60 @@ class TestPreProcessorVariables: variables = result.new_query.variables assert 'group_name' in variables assert 'sender_name' in variables + + +class TestPreProcessorToolSelection: + """Tests for Local Agent tool selection.""" + + @pytest.mark.asyncio + async def test_local_agent_filters_selected_tools(self): + """Only selected tools should be exposed when all-tools mode is off.""" + preproc = get_preproc_module() + + app = FakeApp() + mock_session = Mock() + mock_session.launcher_type = Mock(value='person') + mock_session.launcher_id = 12345 + app.sess_mgr.get_session = AsyncMock(return_value=mock_session) + + mock_conversation = Mock() + mock_conversation.prompt = Mock(messages=[]) + mock_conversation.prompt.copy = Mock(return_value=Mock(messages=[])) + mock_conversation.messages = [] + mock_conversation.uuid = None + app.sess_mgr.get_conversation = AsyncMock(return_value=mock_conversation) + + mock_model = Mock() + mock_model.model_entity = Mock(uuid='primary-model-uuid', abilities=['func_call']) + app.model_mgr.get_model_by_uuid = AsyncMock(return_value=mock_model) + app.tool_mgr.get_all_tools = AsyncMock( + return_value=[ + SimpleNamespace(name='exec'), + SimpleNamespace(name='plugin_tool'), + SimpleNamespace(name='mcp_tool'), + ] + ) + + mock_event_ctx = Mock() + mock_event_ctx.event = Mock(default_prompt=[], prompt=[]) + app.plugin_connector.emit_event = AsyncMock(return_value=mock_event_ctx) + + stage = preproc.PreProcessor(app) + query = text_query('hello') + query.pipeline_config = { + 'ai': { + 'runner': {'runner': 'local-agent'}, + 'local-agent': { + 'model': {'primary': 'primary-model-uuid', 'fallbacks': []}, + 'prompt': 'default', + 'enable-all-tools': False, + 'tools': ['plugin_tool'], + }, + }, + 'output': {'misc': {'at-sender': False}}, + 'trigger': {'misc': {}}, + } + + result = await stage.process(query, 'PreProcessor') + + assert [tool.name for tool in result.new_query.use_funcs] == ['plugin_tool'] diff --git a/tests/unit_tests/pipeline/test_wrapper.py b/tests/unit_tests/pipeline/test_wrapper.py index 8dea6c8bb..034c14115 100644 --- a/tests/unit_tests/pipeline/test_wrapper.py +++ b/tests/unit_tests/pipeline/test_wrapper.py @@ -36,6 +36,11 @@ def get_entities_module(): return import_module('langbot.pkg.pipeline.entities') +def get_plugin_diagnostics_module(): + """Lazy import for plugin diagnostic attribution helpers.""" + return import_module('langbot.pkg.pipeline.plugin_diagnostics') + + def make_wrapper_config(): """Create a pipeline config for wrapper tests.""" return { @@ -106,6 +111,45 @@ class TestResponseWrapperMessageChain: assert results[0].result_type == entities.ResultType.CONTINUE assert len(results[0].new_query.resp_message_chain) == 1 + @pytest.mark.asyncio + async def test_message_chain_direct_append_consumes_pending_plugin_source(self): + """MessageChain replies from earlier plugin events keep attribution.""" + wrapper = get_wrapper_module() + + app = FakeApp() + stage = wrapper.ResponseWrapper(app) + await stage.initialize(make_wrapper_config()) + + reply_chain = platform_message.MessageChain([platform_message.Plain(text='response')]) + query = text_query('hello') + query.pipeline_config = make_wrapper_config() + query.resp_messages = [reply_chain] + query.resp_message_chain = [] + plugin_diagnostics = get_plugin_diagnostics_module() + plugin_diagnostics.record_pending_plugin_response_source( + query, + reply_chain, + [ + { + 'kind': 'reply_message_chain', + 'plugin': {'author': 'tester', 'name': 'demo'}, + } + ], + [{'manifest': {'metadata': {'author': 'observer', 'name': 'not-reply-source'}}}], + 'PersonNormalMessageReceived', + ) + + results = [] + async for result in stage.process(query, 'ResponseWrapper'): + results.append(result) + + sources = plugin_diagnostics._get_response_sources(results[0].new_query, 0) + assert sources[0].plugin == {'author': 'tester', 'name': 'demo'} + assert sources[0].event_name == 'PersonNormalMessageReceived' + assert sources[0].is_approximate is False + assert '_plugin_response_sources' not in query.variables + assert '_plugin_pending_response_sources' not in query.variables + class TestResponseWrapperCommand: """Tests for command response wrapping.""" @@ -421,6 +465,104 @@ class TestResponseWrapperCustomReply: chain = results[0].new_query.resp_message_chain[0] assert 'Custom reply' in str(chain) + @pytest.mark.asyncio + async def test_custom_reply_records_plugin_source(self): + """Plugin reply_message_chain should keep emitted plugin attribution.""" + wrapper = get_wrapper_module() + + app = FakeApp() + app.sess_mgr.get_session = AsyncMock(return_value=make_session()) + + custom_chain = platform_message.MessageChain([platform_message.Plain(text='Custom reply')]) + mock_event_ctx = Mock() + mock_event_ctx.is_prevented_default = Mock(return_value=False) + mock_event_ctx.event = Mock() + mock_event_ctx.event.reply_message_chain = custom_chain + mock_event_ctx._emitted_plugins = [ + { + 'manifest': {'metadata': {'author': 'observer', 'name': 'not-reply-source'}}, + 'plugin_config': {'token': 'secret-token'}, + }, + ] + mock_event_ctx._response_sources = [ + { + 'kind': 'reply_message_chain', + 'plugin': {'author': 'tester', 'name': 'demo'}, + } + ] + app.plugin_connector.emit_event = AsyncMock(return_value=mock_event_ctx) + + stage = wrapper.ResponseWrapper(app) + pipeline_config = make_wrapper_config() + await stage.initialize(pipeline_config) + + query = text_query('hello') + query.pipeline_config = pipeline_config + query.resp_message_chain = [] + assistant_resp = Mock() + assistant_resp.role = 'assistant' + assistant_resp.content = 'Default reply' + assistant_resp.tool_calls = None + assistant_resp.get_content_platform_message_chain = Mock( + return_value=platform_message.MessageChain([platform_message.Plain(text='Default reply')]) + ) + query.resp_messages = [assistant_resp] + + results = [] + async for result in stage.process(query, 'ResponseWrapper'): + results.append(result) + + plugin_diagnostics = get_plugin_diagnostics_module() + sources = plugin_diagnostics._get_response_sources(results[0].new_query, 0) + assert sources[0].plugin == {'author': 'tester', 'name': 'demo'} + assert sources[0].event_name == 'NormalMessageResponded' + assert sources[0].is_approximate is False + assert 'secret-token' not in str(sources) + assert '_plugin_response_sources' not in query.variables + + @pytest.mark.asyncio + async def test_custom_reply_falls_back_to_emitted_plugins_for_old_runtime(self): + """Older plugin runtimes without response_sources keep approximate attribution.""" + wrapper = get_wrapper_module() + + app = FakeApp() + app.sess_mgr.get_session = AsyncMock(return_value=make_session()) + + custom_chain = platform_message.MessageChain([platform_message.Plain(text='Custom reply')]) + mock_event_ctx = Mock() + mock_event_ctx.is_prevented_default = Mock(return_value=False) + mock_event_ctx.event = Mock() + mock_event_ctx.event.reply_message_chain = custom_chain + mock_event_ctx._emitted_plugins = [ + {'manifest': {'metadata': {'author': 'tester', 'name': 'demo'}}}, + ] + app.plugin_connector.emit_event = AsyncMock(return_value=mock_event_ctx) + + stage = wrapper.ResponseWrapper(app) + pipeline_config = make_wrapper_config() + await stage.initialize(pipeline_config) + + query = text_query('hello') + query.pipeline_config = pipeline_config + query.resp_message_chain = [] + assistant_resp = Mock() + assistant_resp.role = 'assistant' + assistant_resp.content = 'Default reply' + assistant_resp.tool_calls = None + assistant_resp.get_content_platform_message_chain = Mock( + return_value=platform_message.MessageChain([platform_message.Plain(text='Default reply')]) + ) + query.resp_messages = [assistant_resp] + + results = [] + async for result in stage.process(query, 'ResponseWrapper'): + results.append(result) + + plugin_diagnostics = get_plugin_diagnostics_module() + sources = plugin_diagnostics._get_response_sources(results[0].new_query, 0) + assert sources[0].plugin == {'author': 'tester', 'name': 'demo'} + assert sources[0].is_approximate is True + class TestResponseWrapperVariables: """Tests for bound plugins variable.""" diff --git a/tests/unit_tests/platform/test_aiocqhttp_message_converter.py b/tests/unit_tests/platform/test_aiocqhttp_message_converter.py new file mode 100644 index 000000000..55f835c97 --- /dev/null +++ b/tests/unit_tests/platform/test_aiocqhttp_message_converter.py @@ -0,0 +1,105 @@ +import pytest + +import langbot_plugin.api.entities.builtin.platform.message as platform_message +from langbot.pkg.platform.sources.aiocqhttp import AiocqhttpAdapter, AiocqhttpMessageConverter + + +async def _convert_single(component: platform_message.MessageComponent): + chain = platform_message.MessageChain([component]) + message, _, _ = await AiocqhttpMessageConverter.yiri2target(chain) + return message[0] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('payload', 'expected'), + [ + ('data:image/jpeg;base64,raw-image', 'base64://raw-image'), + ('raw-image', 'base64://raw-image'), + ('base64://raw-image', 'base64://raw-image'), + ], +) +async def test_image_base64_payload_is_normalized(payload, expected): + segment = await _convert_single(platform_message.Image(base64=payload)) + + assert segment.type == 'image' + assert segment.data['file'] == expected + + +@pytest.mark.asyncio +async def test_voice_data_uri_base64_payload_is_normalized(): + segment = await _convert_single(platform_message.Voice(base64='data:audio/wav;base64,raw-voice')) + + assert segment.type == 'record' + assert segment.data['file'] == 'base64://raw-voice' + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ('component', 'expected'), + [ + ( + platform_message.File(name='report.txt', base64='data:text/plain;base64,raw-file'), + {'file': 'base64://raw-file', 'name': 'report.txt'}, + ), + ( + platform_message.File(name='report.txt', base64='raw-file'), + {'file': 'base64://raw-file', 'name': 'report.txt'}, + ), + ( + platform_message.File(name='a.txt', url='http://example.com/a.txt'), + {'file': 'http://example.com/a.txt', 'name': 'a.txt'}, + ), + ( + platform_message.File(name='a.txt', path='/tmp/a.txt'), + {'file': '/tmp/a.txt', 'name': 'a.txt'}, + ), + ], +) +async def test_file_message_uses_available_file_source(component, expected): + segment = await _convert_single(component) + + assert segment.type == 'file' + assert segment.data == expected + + +@pytest.mark.asyncio +async def test_forward_image_base64_payload_is_normalized(): + forward = platform_message.Forward( + node_list=[ + platform_message.ForwardMessageNode( + sender_id='10001', + sender_name='Tester', + message_chain=platform_message.MessageChain( + [platform_message.Image(base64='data:image/png;base64,raw-forward-image')] + ), + ) + ] + ) + messages = [] + + class Logger: + async def info(self, _message): + return None + + async def error(self, _message): + return None + + class Bot: + async def call_action(self, action, **kwargs): + assert action == 'send_forward_msg' + messages.append(kwargs) + + platform = AiocqhttpAdapter.model_construct( + bot_account_id='10000', + config={}, + logger=Logger(), + bot=Bot(), + ) + + await platform._send_forward_message(1000, forward) + + assert messages[0]['messages'][0]['data']['content'][0] == { + 'type': 'image', + 'data': {'file': 'base64://raw-forward-image'}, + } diff --git a/tests/unit_tests/plugin/test_connector_methods.py b/tests/unit_tests/plugin/test_connector_methods.py index 5f09ce5a4..34cab5271 100644 --- a/tests/unit_tests/plugin/test_connector_methods.py +++ b/tests/unit_tests/plugin/test_connector_methods.py @@ -13,6 +13,8 @@ import pytest from unittest.mock import Mock, AsyncMock from importlib import import_module +from tests.factories import text_query + def get_connector_module(): """Lazy import to avoid circular import issues.""" @@ -132,6 +134,130 @@ class TestListPlugins: assert result[0]['debug'] is True +class TestPluginDiagnostics: + @pytest.mark.asyncio + async def test_emit_event_preserves_response_sources(self): + connector = create_mock_connector() + query = text_query('hello') + event = query.message_event + object.__setattr__(event, 'query', query) + connector_module = get_connector_module() + original_from_event = connector_module.context.EventContext.from_event + original_model_validate = connector_module.context.EventContext.model_validate + response_sources = [ + { + 'kind': 'reply_message_chain', + 'plugin': {'author': 'tester', 'name': 'demo'}, + } + ] + + async def emit_event_response(event_context, include_plugins=None): + return { + 'event_context': event_context, + 'emitted_plugins': [], + 'response_sources': response_sources, + } + + connector.handler = AsyncMock() + connector.handler.emit_event = AsyncMock(side_effect=emit_event_response) + + fake_event_ctx = Mock() + event_dump = event.model_dump() + event_dump['event_name'] = 'FriendMessage' + fake_event_ctx.model_dump.return_value = { + 'query_id': query.query_id, + 'eid': 0, + 'event_name': 'FriendMessage', + 'event': event_dump, + 'is_prevent_default': False, + 'is_prevent_postorder': False, + } + connector_module.context.EventContext.from_event = Mock(return_value=fake_event_ctx) + parsed_event_ctx = Mock() + connector_module.context.EventContext.model_validate = Mock(return_value=parsed_event_ctx) + try: + event_ctx = await connector.emit_event(event) + finally: + connector_module.context.EventContext.from_event = original_from_event + connector_module.context.EventContext.model_validate = original_model_validate + + assert event_ctx is parsed_event_ctx + assert event_ctx._response_sources == response_sources + + @pytest.mark.asyncio + async def test_emit_event_leaves_response_sources_absent_for_old_runtime(self): + connector = create_mock_connector() + query = text_query('hello') + event = query.message_event + object.__setattr__(event, 'query', query) + connector_module = get_connector_module() + original_from_event = connector_module.context.EventContext.from_event + original_model_validate = connector_module.context.EventContext.model_validate + + async def emit_event_response(event_context, include_plugins=None): + return { + 'event_context': event_context, + 'emitted_plugins': [ + {'manifest': {'metadata': {'author': 'tester', 'name': 'demo'}}}, + ], + } + + connector.handler = AsyncMock() + connector.handler.emit_event = AsyncMock(side_effect=emit_event_response) + + fake_event_ctx = Mock() + event_dump = event.model_dump() + event_dump['event_name'] = 'FriendMessage' + fake_event_ctx.model_dump.return_value = { + 'query_id': query.query_id, + 'eid': 0, + 'event_name': 'FriendMessage', + 'event': event_dump, + 'is_prevent_default': False, + 'is_prevent_postorder': False, + } + connector_module.context.EventContext.from_event = Mock(return_value=fake_event_ctx) + parsed_event_ctx = Mock() + connector_module.context.EventContext.model_validate = Mock(return_value=parsed_event_ctx) + try: + event_ctx = await connector.emit_event(event) + finally: + connector_module.context.EventContext.from_event = original_from_event + connector_module.context.EventContext.model_validate = original_model_validate + + assert '_response_sources' not in vars(event_ctx) + assert event_ctx._emitted_plugins == [ + {'manifest': {'metadata': {'author': 'tester', 'name': 'demo'}}}, + ] + + @pytest.mark.asyncio + async def test_notify_plugin_diagnostic_skips_when_disabled(self): + connector_module = get_connector_module() + + async def mock_disconnect(conn): + pass + + mock_app = create_mock_app() + mock_app.instance_config.data = {'plugin': {'enable': False}} + connector = connector_module.PluginRuntimeConnector(mock_app, mock_disconnect) + connector.handler = AsyncMock() + + await connector.notify_plugin_diagnostic({'code': 'response_delivery_failed'}) + + connector.handler.notify_plugin_diagnostic.assert_not_called() + + @pytest.mark.asyncio + async def test_notify_plugin_diagnostic_is_best_effort(self): + connector = create_mock_connector() + connector.handler = AsyncMock() + connector.handler.notify_plugin_diagnostic = AsyncMock(side_effect=RuntimeError('action not found')) + + await connector.notify_plugin_diagnostic({'code': 'response_delivery_failed'}) + + connector.handler.notify_plugin_diagnostic.assert_awaited_once() + connector.ap.logger.debug.assert_called_once() + + class TestListKnowledgeEngines: """Tests for list_knowledge_engines method.""" diff --git a/tests/unit_tests/plugin/test_handler.py b/tests/unit_tests/plugin/test_handler.py index 989a333a4..a2fdddd33 100644 --- a/tests/unit_tests/plugin/test_handler.py +++ b/tests/unit_tests/plugin/test_handler.py @@ -159,6 +159,36 @@ class TestHandlerRagErrorResponse: assert 'KeyError' in response.message +class TestHandlerPluginDiagnostic: + @pytest.mark.asyncio + async def test_notify_plugin_diagnostic_falls_back_to_raw_protocol_action(self): + """Diagnostic forwarding works before the SDK enum exists.""" + app = SimpleNamespace() + app.logger = SimpleNamespace(debug=MagicMock()) + runtime_handler = make_handler(app) + runtime_handler.call_action = AsyncMock(return_value={}) + + payload = {'code': 'response_delivery_failed'} + await runtime_handler.notify_plugin_diagnostic(payload) + + action = runtime_handler.call_action.await_args.args[0] + assert action.value == 'plugin_diagnostic' + assert runtime_handler.call_action.await_args.args[1] is payload + assert runtime_handler.call_action.await_args.kwargs['timeout'] == 5 + + def test_langbot_to_runtime_action_uses_enum_when_available(self): + """The compatibility helper should prefer SDK enums once available.""" + from langbot.pkg.plugin import handler as plugin_handler + + sentinel = object() + original = plugin_handler.LangBotToRuntimeAction + plugin_handler.LangBotToRuntimeAction = SimpleNamespace(PLUGIN_DIAGNOSTIC=sentinel) + try: + assert plugin_handler._langbot_to_runtime_action('PLUGIN_DIAGNOSTIC', 'plugin_diagnostic') is sentinel + finally: + plugin_handler.LangBotToRuntimeAction = original + + class TestConstantsSemanticVersion: """Tests for version constant access.""" diff --git a/tests/unit_tests/plugin/test_handler_actions.py b/tests/unit_tests/plugin/test_handler_actions.py index dae2fe009..ad52b5c58 100644 --- a/tests/unit_tests/plugin/test_handler_actions.py +++ b/tests/unit_tests/plugin/test_handler_actions.py @@ -68,13 +68,15 @@ class TestRagRerankAction: app.model_mgr.get_rerank_model_by_uuid = AsyncMock(return_value=rerank_model) runtime_handler = make_handler(app) - response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_RERANK.value]({ - 'rerank_model_uuid': 'rerank-1', - 'query': 'hello', - 'documents': ['a', 'b'], - 'top_k': 1, - 'extra_args': {'return_documents': False}, - }) + response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_RERANK.value]( + { + 'rerank_model_uuid': 'rerank-1', + 'query': 'hello', + 'documents': ['a', 'b'], + 'top_k': 1, + 'extra_args': {'return_documents': False}, + } + ) assert response.code == 0 assert response.data['results'] == [{'index': 1, 'relevance_score': 0.9}] @@ -89,16 +91,16 @@ class TestRagRerankAction: @pytest.mark.asyncio async def test_returns_error_when_rerank_model_missing(self, app): """Missing rerank model returns an action error.""" - app.model_mgr.get_rerank_model_by_uuid = AsyncMock( - side_effect=ValueError('not found') - ) + app.model_mgr.get_rerank_model_by_uuid = AsyncMock(side_effect=ValueError('not found')) runtime_handler = make_handler(app) - response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_RERANK.value]({ - 'rerank_model_uuid': 'missing', - 'query': 'hello', - 'documents': ['a'], - }) + response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_RERANK.value]( + { + 'rerank_model_uuid': 'missing', + 'query': 'hello', + 'documents': ['a'], + } + ) assert response.code != 0 assert 'Rerank model with rerank_model_uuid missing not found' in response.message @@ -461,9 +463,7 @@ class TestAgentRunProxyActions: return SimpleNamespace( pipeline_config={'output': {'misc': {'remove-think': remove_think}}}, variables={}, - prompt=SimpleNamespace( - messages=[provider_message.Message(role='system', content='effective prompt')] - ), + prompt=SimpleNamespace(messages=[provider_message.Message(role='system', content='effective prompt')]), ) @pytest.mark.asyncio @@ -489,10 +489,12 @@ class TestAgentRunProxyActions: runtime_handler = make_handler(app) try: - response = await runtime_handler.actions[PluginToRuntimeAction.GET_PROMPT.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - }) + response = await runtime_handler.actions[PluginToRuntimeAction.GET_PROMPT.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + } + ) finally: await registry.unregister(run_id) @@ -533,19 +535,23 @@ class TestAgentRunProxyActions: runtime_handler = make_handler(app) try: - response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - 'llm_model_uuid': 'llm_001', - 'messages': [{'role': 'user', 'content': 'hello'}], - 'funcs': [{ - 'name': 'search', - 'human_desc': 'Search', - 'description': 'Search', - 'parameters': {'type': 'object'}, - }], - 'extra_args': {'temperature': 0.7, 'presence_penalty': 0.1}, - }) + response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + 'llm_model_uuid': 'llm_001', + 'messages': [{'role': 'user', 'content': 'hello'}], + 'funcs': [ + { + 'name': 'search', + 'human_desc': 'Search', + 'description': 'Search', + 'parameters': {'type': 'object'}, + } + ], + 'extra_args': {'temperature': 0.7, 'presence_penalty': 0.1}, + } + ) finally: await registry.unregister(run_id) @@ -601,12 +607,14 @@ class TestAgentRunProxyActions: runtime_handler = make_handler(app) try: - response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - 'llm_model_uuid': 'llm_usage_001', - 'messages': [{'role': 'user', 'content': 'hello'}], - }) + response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + 'llm_model_uuid': 'llm_usage_001', + 'messages': [{'role': 'user', 'content': 'hello'}], + } + ) finally: await registry.unregister(run_id) @@ -645,19 +653,23 @@ class TestAgentRunProxyActions: runtime_handler = make_handler(app) try: - response = await runtime_handler.actions[PluginToRuntimeAction.COUNT_TOKENS.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - 'llm_model_uuid': 'llm_count_001', - 'messages': [{'role': 'user', 'content': 'hello'}], - 'funcs': [{ - 'name': 'search', - 'human_desc': 'Search', - 'description': 'Search', - 'parameters': {'type': 'object'}, - }], - 'extra_args': {'temperature': 0.7}, - }) + response = await runtime_handler.actions[PluginToRuntimeAction.COUNT_TOKENS.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + 'llm_model_uuid': 'llm_count_001', + 'messages': [{'role': 'user', 'content': 'hello'}], + 'funcs': [ + { + 'name': 'search', + 'human_desc': 'Search', + 'description': 'Search', + 'parameters': {'type': 'object'}, + } + ], + 'extra_args': {'temperature': 0.7}, + } + ) finally: await registry.unregister(run_id) @@ -690,12 +702,14 @@ class TestAgentRunProxyActions: runtime_handler = make_handler(app) try: - response = await runtime_handler.actions[PluginToRuntimeAction.COUNT_TOKENS.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - 'llm_model_uuid': 'llm_count_002', - 'messages': [{'role': 'user', 'content': 'hello'}], - }) + response = await runtime_handler.actions[PluginToRuntimeAction.COUNT_TOKENS.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + 'llm_model_uuid': 'llm_count_002', + 'messages': [{'role': 'user', 'content': 'hello'}], + } + ) finally: await registry.unregister(run_id) @@ -740,20 +754,24 @@ class TestAgentRunProxyActions: responses = [] try: - stream = runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM_STREAM.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - 'llm_model_uuid': 'llm_stream_001', - 'messages': [{'role': 'user', 'content': 'hello'}], - 'funcs': [{ - 'name': 'search', - 'human_desc': 'Search', - 'description': 'Search', - 'parameters': {'type': 'object'}, - }], - 'extra_args': {'max_tokens': 256}, - 'remove_think': True, - }) + stream = runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM_STREAM.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + 'llm_model_uuid': 'llm_stream_001', + 'messages': [{'role': 'user', 'content': 'hello'}], + 'funcs': [ + { + 'name': 'search', + 'human_desc': 'Search', + 'description': 'Search', + 'parameters': {'type': 'object'}, + } + ], + 'extra_args': {'max_tokens': 256}, + 'remove_think': True, + } + ) async for response in stream: responses.append(response) finally: @@ -799,12 +817,14 @@ class TestAgentRunProxyActions: responses = [] try: - stream = runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM_STREAM.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - 'llm_model_uuid': 'llm_stream_002', - 'messages': [{'role': 'user', 'content': 'hello'}], - }) + stream = runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM_STREAM.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + 'llm_model_uuid': 'llm_stream_002', + 'messages': [{'role': 'user', 'content': 'hello'}], + } + ) async for response in stream: responses.append(response) finally: @@ -854,12 +874,14 @@ class TestAgentRunProxyActions: responses = [] try: - stream = runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM_STREAM.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - 'llm_model_uuid': 'llm_stream_usage_001', - 'messages': [{'role': 'user', 'content': 'hello'}], - }) + stream = runtime_handler.actions[PluginToRuntimeAction.INVOKE_LLM_STREAM.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + 'llm_model_uuid': 'llm_stream_usage_001', + 'messages': [{'role': 'user', 'content': 'hello'}], + } + ) async for response in stream: responses.append(response) finally: @@ -892,12 +914,14 @@ class TestAgentRunProxyActions: runtime_handler = make_handler(app) try: - response = await runtime_handler.actions[PluginToRuntimeAction.CALL_TOOL.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - 'tool_name': 'test/search', - 'parameters': {'q': 'langbot'}, - }) + response = await runtime_handler.actions[PluginToRuntimeAction.CALL_TOOL.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + 'tool_name': 'test/search', + 'parameters': {'q': 'langbot'}, + } + ) finally: await registry.unregister(run_id) @@ -926,10 +950,12 @@ class TestAgentRunProxyActions: ) provider = SimpleNamespace( - invoke_rerank=AsyncMock(return_value=[ - {'index': 0, 'relevance_score': 0.2}, - {'index': 1, 'relevance_score': 0.9}, - ]), + invoke_rerank=AsyncMock( + return_value=[ + {'index': 0, 'relevance_score': 0.2}, + {'index': 1, 'relevance_score': 0.9}, + ] + ), ) rerank_model = SimpleNamespace( model_entity=SimpleNamespace(extra_args={'top_n': 5, 'return_documents': False}), @@ -939,15 +965,17 @@ class TestAgentRunProxyActions: runtime_handler = make_handler(app) try: - response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_RERANK.value]({ - 'run_id': run_id, - 'caller_plugin_identity': 'test/runner', - 'rerank_model_uuid': 'rerank_001', - 'query': 'hello', - 'documents': ['a', 'b'], - 'top_k': 1, - 'extra_args': {'top_n': 2}, - }) + response = await runtime_handler.actions[PluginToRuntimeAction.INVOKE_RERANK.value]( + { + 'run_id': run_id, + 'caller_plugin_identity': 'test/runner', + 'rerank_model_uuid': 'rerank_001', + 'query': 'hello', + 'documents': ['a', 'b'], + 'top_k': 1, + 'extra_args': {'top_n': 2}, + } + ) finally: await registry.unregister(run_id) diff --git a/tests/unit_tests/plugin/test_plugin_id_parsing.py b/tests/unit_tests/plugin/test_plugin_id_parsing.py index c6d479fbc..f76463785 100644 --- a/tests/unit_tests/plugin/test_plugin_id_parsing.py +++ b/tests/unit_tests/plugin/test_plugin_id_parsing.py @@ -2,7 +2,7 @@ import pytest -from src.langbot.pkg.plugin.connector import PluginRuntimeConnector +from langbot.pkg.plugin.connector import PluginRuntimeConnector def test_parse_plugin_id_accepts_author_name(): diff --git a/tests/unit_tests/provider/test_mcp_resources.py b/tests/unit_tests/provider/test_mcp_resources.py new file mode 100644 index 000000000..565422c46 --- /dev/null +++ b/tests/unit_tests/provider/test_mcp_resources.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +import base64 +from types import SimpleNamespace +from unittest.mock import AsyncMock, Mock + +import pytest +from mcp import types as mcp_types + +from langbot.pkg.provider.tools.loaders.mcp import ( + MCP_RESOURCE_CONTEXT_QUERY_KEY, + MCP_RESOURCE_TRACE_QUERY_KEY, + MCP_TOOL_LIST_RESOURCES, + MCP_TOOL_READ_RESOURCE, + MCPLoader, + MCPSessionStatus, + RuntimeMCPSession, +) +from langbot.pkg.telemetry import features as telemetry_features + + +def _app() -> SimpleNamespace: + return SimpleNamespace(logger=Mock()) + + +def _connected_session( + *, + name: str = 'docs', + uuid: str = 'srv-1', + resources: list[dict] | None = None, + templates: list[dict] | None = None, +) -> RuntimeMCPSession: + session = RuntimeMCPSession(name, {'uuid': uuid, 'mode': 'remote'}, True, _app()) + session.status = MCPSessionStatus.CONNECTED + session.session = SimpleNamespace(read_resource=AsyncMock()) + session.resources = resources or [ + { + 'uri': 'file:///README.md', + 'name': 'README.md', + 'title': '', + 'description': '', + 'mime_type': 'text/markdown', + 'size': None, + 'icons': [], + 'annotations': {}, + '_meta': {}, + } + ] + session.resource_templates = templates or [] + return session + + +def _query() -> SimpleNamespace: + return SimpleNamespace(variables={}) + + +@pytest.mark.asyncio +async def test_read_resource_envelope_truncates_caches_and_records_trace(): + session = _connected_session() + session.session.read_resource.return_value = mcp_types.ReadResourceResult( + contents=[ + mcp_types.TextResourceContents( + uri='file:///README.md', + mimeType='text/markdown', + text='abcdef', + ) + ] + ) + query = _query() + + first = await session.read_resource_envelope( + 'file:///README.md', + max_bytes=4, + source='ui_preview', + query=query, + ) + second = await session.read_resource_envelope( + 'file:///README.md', + max_bytes=4, + source='agent_tool', + query=query, + ) + + assert first['contents'][0]['text'] == 'abcd' + assert first['contents'][0]['bytes'] == 6 + assert first['truncated'] is True + assert first['cache_hit'] is False + assert second['cache_hit'] is True + assert second['source'] == 'agent_tool' + assert session.session.read_resource.await_count == 1 + + traces = query.variables[MCP_RESOURCE_TRACE_QUERY_KEY] + assert [trace['source'] for trace in traces] == ['ui_preview', 'agent_tool'] + assert traces[1]['cache_hit'] is True + assert query.variables[telemetry_features.FEATURES_KEY]['mcp_resource_reads'] == { + 'ui_preview': 1, + 'agent_tool': 1, + } + + +@pytest.mark.asyncio +async def test_read_resource_envelope_shares_byte_budget_across_text_contents(): + session = _connected_session() + session.session.read_resource.return_value = mcp_types.ReadResourceResult( + contents=[ + mcp_types.TextResourceContents( + uri='file:///README.md#first', + mimeType='text/plain', + text='abc', + ), + mcp_types.TextResourceContents( + uri='file:///README.md#second', + mimeType='text/plain', + text='def', + ), + ] + ) + + envelope = await session.read_resource_envelope('file:///README.md', max_bytes=4) + + assert [item['text'] for item in envelope['contents']] == ['abc', 'd'] + assert envelope['contents'][0]['truncated'] is False + assert envelope['contents'][1]['truncated'] is True + assert envelope['bytes'] == 6 + assert envelope['truncated'] is True + + +@pytest.mark.asyncio +async def test_read_resource_envelope_omits_binary_by_default(): + session = _connected_session( + resources=[ + { + 'uri': 'file:///image.png', + 'name': 'image.png', + 'title': '', + 'description': '', + 'mime_type': 'image/png', + 'size': 4, + 'icons': [], + 'annotations': {}, + '_meta': {}, + } + ] + ) + session.session.read_resource.return_value = mcp_types.ReadResourceResult( + contents=[ + mcp_types.BlobResourceContents( + uri='file:///image.png', + mimeType='image/png', + blob=base64.b64encode(b'\x00\x01\x02\x03').decode(), + ) + ] + ) + + envelope = await session.read_resource_envelope('file:///image.png') + + content = envelope['contents'][0] + assert content['type'] == 'blob' + assert content['blob'] is None + assert content['bytes'] == 4 + assert content['binary_omitted'] is True + assert envelope['truncated'] is True + assert envelope['warnings'] == ['Binary resource content omitted from response.'] + + +@pytest.mark.asyncio +async def test_read_resource_envelope_rejects_unlisted_uri(): + session = _connected_session() + + with pytest.raises(ValueError, match='Resource URI is not available'): + await session.read_resource_envelope('file:///secret.txt') + + session.session.read_resource.assert_not_called() + + +def test_resource_uri_allowed_supports_listed_templates_conservatively(): + session = _connected_session( + resources=[], + templates=[ + { + 'uri_template': 'repo://{owner}/{repo}/file/{path}', + 'name': 'repository file', + 'title': '', + 'description': '', + 'mime_type': 'text/plain', + 'icons': [], + 'annotations': {}, + '_meta': {}, + } + ], + ) + + assert session.resource_uri_allowed('repo://langbot-app/LangBot/file/src/main.py') is True + assert session.resource_uri_allowed('repo://langbot-app/LangBot/issues/1') is False + assert session.resource_uri_allowed('https://example.com/secret') is False + + +@pytest.mark.asyncio +async def test_mcp_loader_can_hide_synthetic_resource_tools(): + loader = MCPLoader(_app()) + session = _connected_session() + loader.sessions = {'docs': session} + + with_resource_tools = await loader.get_tools(['srv-1'], include_resource_tools=True) + without_resource_tools = await loader.get_tools(['srv-1'], include_resource_tools=False) + + assert {tool.name for tool in with_resource_tools} == { + MCP_TOOL_LIST_RESOURCES, + MCP_TOOL_READ_RESOURCE, + } + assert without_resource_tools == [] + + +@pytest.mark.asyncio +async def test_mcp_loader_refuses_resource_tool_calls_when_agent_read_disabled(): + loader = MCPLoader(_app()) + session = _connected_session() + loader.sessions = {'docs': session} + query = SimpleNamespace( + variables={ + '_pipeline_bound_mcp_servers': ['srv-1'], + '_pipeline_mcp_resource_agent_read_enabled': False, + } + ) + + result = await loader.invoke_tool( + MCP_TOOL_READ_RESOURCE, + {'server_name': 'docs', 'uri': 'file:///README.md'}, + query, + ) + + assert result[0].text == 'Error: MCP resource agent reads are disabled.' + session.session.read_resource.assert_not_called() + + +@pytest.mark.asyncio +async def test_build_resource_context_for_query_uses_only_bound_attached_text_resources(): + loader = MCPLoader(_app()) + docs = _connected_session(name='docs', uuid='srv-1') + docs.session.read_resource.return_value = mcp_types.ReadResourceResult( + contents=[ + mcp_types.TextResourceContents( + uri='file:///README.md', + mimeType='text/markdown', + text='LangBot MCP resource context', + ) + ] + ) + other = _connected_session(name='other', uuid='srv-2') + other.session.read_resource.return_value = mcp_types.ReadResourceResult( + contents=[ + mcp_types.TextResourceContents( + uri='file:///README.md', + mimeType='text/markdown', + text='must not be injected', + ) + ] + ) + loader.sessions = {'docs': docs, 'other': other} + query = SimpleNamespace( + variables={ + '_pipeline_bound_mcp_servers': ['srv-1'], + '_pipeline_mcp_resource_attachments': [ + {'server_uuid': 'srv-1', 'server_name': 'docs', 'uri': 'file:///README.md', 'mode': 'pinned'}, + {'server_uuid': 'srv-2', 'server_name': 'other', 'uri': 'file:///README.md', 'mode': 'pinned'}, + ], + } + ) + + context = await loader.build_resource_context_for_query(query) + + assert ' bool: return any(tool.name == name for tool in self._tools) @@ -70,6 +92,28 @@ async def test_tool_manager_omits_skill_tools_when_loader_unavailable(): assert [tool.name for tool in tools] == ['exec', 'plugin_tool', 'mcp_tool'] +@pytest.mark.asyncio +async def test_tool_manager_catalog_labels_tool_sources(): + manager = ToolManager(SimpleNamespace()) + manager.native_tool_loader = StubLoader([make_tool('exec')]) + manager.skill_tool_loader = StubLoader([make_tool('activate')]) + manager.plugin_tool_loader = StubLoader( + [make_tool('plugin_tool')], + catalog_source='plugin', + catalog_source_name='fixture-plugin', + ) + manager.mcp_tool_loader = StubLoader([make_tool('mcp_tool')]) + + catalog = await manager.get_tool_catalog(include_skill_authoring=True) + + assert [(item['name'], item['source'], item['source_name']) for item in catalog] == [ + ('exec', 'builtin', 'LangBot'), + ('activate', 'skill', 'LangBot'), + ('plugin_tool', 'plugin', 'fixture-plugin'), + ('mcp_tool', 'mcp', 'fixture-server'), + ] + + @pytest.mark.asyncio async def test_tool_manager_routes_native_tool_calls(): app = SimpleNamespace() diff --git a/tests/unit_tests/test_paths.py b/tests/unit_tests/test_paths.py index c1e84f443..a631ff7f7 100644 --- a/tests/unit_tests/test_paths.py +++ b/tests/unit_tests/test_paths.py @@ -1,6 +1,6 @@ from pathlib import Path -from src.langbot.pkg.utils import paths +from langbot.pkg.utils import paths def test_get_data_root_uses_source_root_in_repo_checkout(): diff --git a/tests/unit_tests/test_preproc.py b/tests/unit_tests/test_preproc.py index b34516c5f..42a3567a2 100644 --- a/tests/unit_tests/test_preproc.py +++ b/tests/unit_tests/test_preproc.py @@ -159,7 +159,11 @@ async def test_preproc_loads_host_tools_for_runner(): result = await stage.process(_make_query(), 'PreProcessor') assert result.result_type == entities_module.ResultType.CONTINUE - app.tool_mgr.get_all_tools.assert_awaited_once_with(None, None) + app.tool_mgr.get_all_tools.assert_awaited_once_with( + None, + None, + include_mcp_resource_tools=True, + ) @pytest.mark.asyncio @@ -180,7 +184,11 @@ async def test_preproc_puts_host_skill_tools_into_query_scope(): result = await stage.process(query, 'PreProcessor') assert result.result_type == entities_module.ResultType.CONTINUE - app.tool_mgr.get_all_tools.assert_awaited_once_with(None, None) + app.tool_mgr.get_all_tools.assert_awaited_once_with( + None, + None, + include_mcp_resource_tools=True, + ) assert [tool.name for tool in query.use_funcs] == ['activate', 'register_skill'] @@ -195,7 +203,30 @@ async def test_preproc_loads_host_tools_regardless_of_skill_service(): result = await stage.process(_make_query(), 'PreProcessor') assert result.result_type == entities_module.ResultType.CONTINUE - app.tool_mgr.get_all_tools.assert_awaited_once_with(None, None) + app.tool_mgr.get_all_tools.assert_awaited_once_with( + None, + None, + include_mcp_resource_tools=True, + ) + + +@pytest.mark.asyncio +async def test_preproc_disables_mcp_resource_tools_when_agent_reading_is_disabled(): + preproc_module, entities_module = _import_preproc_modules() + + app = _make_app(skill_service=SimpleNamespace()) + stage = preproc_module.PreProcessor(app) + query = _make_query() + query.variables['_pipeline_mcp_resource_agent_read_enabled'] = False + + result = await stage.process(query, 'PreProcessor') + + assert result.result_type == entities_module.ResultType.CONTINUE + app.tool_mgr.get_all_tools.assert_awaited_once_with( + None, + None, + include_mcp_resource_tools=False, + ) @pytest.mark.asyncio diff --git a/tests/unit_tests/utils/test_importutil.py b/tests/unit_tests/utils/test_importutil.py index bf0e4e050..cfd7fc23e 100644 --- a/tests/unit_tests/utils/test_importutil.py +++ b/tests/unit_tests/utils/test_importutil.py @@ -138,7 +138,7 @@ class TestReadResourceFile: from langbot.pkg.utils import importutil content = importutil.read_resource_file('templates/config.yaml') - assert 'admins:' in content + assert 'api:' in content assert 'edition: community' in content def test_raises_for_nonexistent_file(self): @@ -157,7 +157,7 @@ class TestReadResourceFileBytes: from langbot.pkg.utils import importutil content = importutil.read_resource_file_bytes('templates/config.yaml') - assert b'admins:' in content + assert b'api:' in content assert b'edition: community' in content def test_raises_for_nonexistent_file_bytes(self): diff --git a/web/.prettierrc.mjs b/web/.prettierrc.mjs old mode 100644 new mode 100755 diff --git a/web/eslint.config.mjs b/web/eslint.config.mjs old mode 100644 new mode 100755 diff --git a/web/fix_router.sh b/web/fix_router.sh old mode 100644 new mode 100755 diff --git a/web/postcss.config.mjs b/web/postcss.config.mjs old mode 100644 new mode 100755 diff --git a/web/scripts/check-i18n.mjs b/web/scripts/check-i18n.mjs old mode 100644 new mode 100755 diff --git a/web/src/app/home/bots/components/bot-admins/BotAdminsDialog.tsx b/web/src/app/home/bots/components/bot-admins/BotAdminsDialog.tsx new file mode 100644 index 000000000..8c187db32 --- /dev/null +++ b/web/src/app/home/bots/components/bot-admins/BotAdminsDialog.tsx @@ -0,0 +1,199 @@ +import { useState, useEffect, useCallback } from 'react'; +import { useTranslation } from 'react-i18next'; +import { httpClient } from '@/app/infra/http/HttpClient'; +import { Button } from '@/components/ui/button'; +import { Input } from '@/components/ui/input'; +import { + Dialog, + DialogContent, + DialogHeader, + DialogTitle, + DialogDescription, +} from '@/components/ui/dialog'; +import { + Select, + SelectContent, + SelectItem, + SelectTrigger, + SelectValue, +} from '@/components/ui/select'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { Trash2, Plus, ShieldCheck } from 'lucide-react'; +import { toast } from 'sonner'; + +export interface BotAdmin { + id: number; + launcher_type: string; + launcher_id: string; +} + +interface BotAdminsDialogProps { + botId: string; + open: boolean; + onOpenChange: (open: boolean) => void; + admins: BotAdmin[]; + onAdminsChange: () => void; +} + +export default function BotAdminsDialog({ + botId, + open, + onOpenChange, + admins, + onAdminsChange, +}: BotAdminsDialogProps) { + const { t } = useTranslation(); + const [newType, setNewType] = useState('person'); + const [newId, setNewId] = useState(''); + const [adding, setAdding] = useState(false); + + async function handleAdd() { + if (!newId.trim()) return; + setAdding(true); + try { + await httpClient.addBotAdmin(botId, newType, newId.trim()); + toast.success(t('bots.admins.addSuccess')); + setNewId(''); + onAdminsChange(); + } catch (e: unknown) { + const err = e as { msg?: string; message?: string }; + toast.error(t('bots.admins.addError') + (err?.msg ?? err?.message ?? '')); + } finally { + setAdding(false); + } + } + + async function handleDelete(id: number) { + try { + await httpClient.deleteBotAdmin(botId, id); + toast.success(t('bots.admins.deleteSuccess')); + onAdminsChange(); + } catch (e: unknown) { + const err = e as { msg?: string; message?: string }; + toast.error( + t('bots.admins.deleteError') + (err?.msg ?? err?.message ?? ''), + ); + } + } + + return ( + + + + + + {t('bots.admins.title')} + + {t('bots.admins.description')} + + +
+ {/* Add row */} +
+ + setNewId(e.target.value)} + onKeyDown={(e) => e.key === 'Enter' && handleAdd()} + /> + +
+ + {/* List */} + {admins.length === 0 ? ( +
+ {t('bots.admins.noAdmins')} +
+ ) : ( + +
+ + + + + + + + + {admins.map((admin) => ( + + + + + + ))} + +
+ {t('bots.admins.launcherType')} + + {t('bots.admins.launcherId')} + +
+ + {admin.launcher_type === 'person' + ? t('bots.admins.typePerson') + : t('bots.admins.typeGroup')} + + + {admin.launcher_id} + + +
+
+
+ )} +
+
+
+ ); +} + +// Shared hook so the session monitor and the dialog stay in sync. +export function useBotAdmins(botId: string) { + const [admins, setAdmins] = useState([]); + + const reload = useCallback(async () => { + try { + const res = await httpClient.getBotAdmins(botId); + setAdmins(res.admins ?? []); + } catch (error) { + console.error('Failed to load bot admins:', error); + } + }, [botId]); + + useEffect(() => { + reload(); + }, [reload]); + + return { admins, reload }; +} diff --git a/web/src/app/home/bots/components/bot-form/EventBindingsEditor.tsx b/web/src/app/home/bots/components/bot-form/EventBindingsEditor.tsx index 19004790e..5d5214757 100644 --- a/web/src/app/home/bots/components/bot-form/EventBindingsEditor.tsx +++ b/web/src/app/home/bots/components/bot-form/EventBindingsEditor.tsx @@ -728,7 +728,11 @@ export default function EventBindingsEditor({ function toggleExpand(id: string) { setExpandedIds((prev) => { const s = new Set(prev); - s.has(id) ? s.delete(id) : s.add(id); + if (s.has(id)) { + s.delete(id); + } else { + s.add(id); + } return s; }); } diff --git a/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx b/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx index 09ae23d50..e8a8703ba 100644 --- a/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx +++ b/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx @@ -18,7 +18,14 @@ import { Workflow, ThumbsUp, ThumbsDown, + ShieldCheck, + ShieldOff, } from 'lucide-react'; +import { toast } from 'sonner'; +import BotAdminsDialog, { + useBotAdmins, +} from '@/app/home/bots/components/bot-admins/BotAdminsDialog'; +import type { BotAdmin } from '@/app/home/bots/components/bot-admins/BotAdminsDialog'; import { copyToClipboard } from '@/app/utils/clipboard'; import { MessageChainComponent, @@ -94,15 +101,60 @@ const BotSessionMonitor = forwardRef< Record >({}); const messagesContainerRef = useRef(null); + const { admins, reload: reloadAdmins } = useBotAdmins(botId); + const [adminsDialogOpen, setAdminsDialogOpen] = useState(false); + const [togglingAdmin, setTogglingAdmin] = useState(null); const parseSessionType = (sessionId: string): string | null => { - const idx = sessionId.indexOf('_'); - if (idx === -1) return null; - const type = sessionId.slice(0, idx); - if (type === 'person' || type === 'group') return type; + const lower = sessionId.toLowerCase(); + if (lower.includes('person')) return 'person'; + if (lower.includes('group')) return 'group'; return null; }; + const isSessionAdmin = (session: SessionInfo): boolean => { + const type = parseSessionType(session.session_id); + const lid = + session.user_id ?? + session.session_id.replace( + /^.*?[._](?:PERSON|GROUP|person|group)[._]/i, + '', + ); + return admins.some( + (a: BotAdmin) => a.launcher_type === type && a.launcher_id === lid, + ); + }; + + const toggleAdmin = async (session: SessionInfo) => { + const type = parseSessionType(session.session_id); + if (!type) return; + const lid = + session.user_id ?? + session.session_id.replace( + /^.*?[._](?:PERSON|GROUP|person|group)[._]/i, + '', + ); + const key = session.session_id; + setTogglingAdmin(key); + try { + const existing = admins.find( + (a: BotAdmin) => a.launcher_type === type && a.launcher_id === lid, + ); + if (existing) { + await httpClient.deleteBotAdmin(botId, existing.id); + toast.success(t('bots.admins.deleteSuccess')); + } else { + await httpClient.addBotAdmin(botId, type, lid); + toast.success(t('bots.admins.addSuccess')); + } + await reloadAdmins(); + } catch { + toast.error(t('bots.admins.addError')); + } finally { + setTogglingAdmin(null); + } + }; + const abbreviateId = (id: string): string => { if (id.length <= 10) return id; return `${id.slice(0, 4)}..${id.slice(-4)}`; @@ -384,257 +436,307 @@ const BotSessionMonitor = forwardRef< ); return ( -
- {/* Left Panel: Session List */} -
- {/* Session List */} - - {loadingSessions && sessions.length === 0 ? ( -
- {t('bots.sessionMonitor.loading')} -
- ) : sessions.length === 0 ? ( -
- {t('bots.sessionMonitor.noSessions')} + <> +
+ {/* Left Panel: Session List */} +
+ {/* Admin header */} +
+ +
+ {/* Session List */} + + {loadingSessions && sessions.length === 0 ? ( +
+ {t('bots.sessionMonitor.loading')} +
+ ) : sessions.length === 0 ? ( +
+ {t('bots.sessionMonitor.noSessions')} +
+ ) : ( +
+ {sessions.map((session) => { + const isSelected = selectedSessionId === session.session_id; + const sessionType = parseSessionType(session.session_id); + const sessionIsAdmin = isSessionAdmin(session); + return ( +
setSelectedSessionId(session.session_id)} + > +
+ + {session.user_name || + session.user_id || + session.session_id.slice(0, 12)} + + + {formatRelativeTime(session.last_activity)} + +
+
+ {sessionType && ( + + {sessionType} + + )} + + {session.user_id && ( + + {abbreviateId(session.user_id)} + + )} + {session.is_active && ( + + + + )} +
+
+ ); + })} +
+ )} +
+
+ + {/* Right Panel: Messages */} +
+ {!selectedSessionId ? ( +
+ {t('bots.sessionMonitor.selectSession')}
) : ( -
- {sessions.map((session) => { - const isSelected = selectedSessionId === session.session_id; - return ( - + + )} + {selectedSession?.is_active && ( + <> + · + + Active - )} -
- - ); - })} -
- )} - -
- - {/* Right Panel: Messages */} -
- {!selectedSessionId ? ( -
- {t('bots.sessionMonitor.selectSession')} -
- ) : ( - <> - {/* Chat Header */} -
-
-
- {selectedSession?.user_name || - selectedSession?.user_id || - selectedSessionId.slice(0, 20)} -
-
- {parseSessionType(selectedSessionId) && ( - {parseSessionType(selectedSessionId)} - )} - {selectedSession?.platform && ( - <> - {parseSessionType(selectedSessionId) && ·} - {selectedSession.platform} - - )} - {selectedSession?.user_id && ( - <> - · - - {selectedSession.user_id} - - - - )} - {selectedSession?.is_active && ( - <> - · - - - Active - - - )} + + )} + {selectedSession && parseSessionType(selectedSessionId) && ( + <> + · + + + )} +
-
- {/* Messages Area */} - -
- {loadingMessages ? ( -
- {t('bots.sessionMonitor.loading')} -
- ) : messages.length === 0 ? ( -
- {t('bots.sessionMonitor.noMessages')} -
- ) : ( - messages.map((msg, msgIndex) => { - const isUser = isUserMessage(msg); - const isDiscarded = - msg.status === 'discarded' || - msg.pipeline_id === PIPELINE_DISCARD; - // For bot replies, find feedback linked to the preceding user message - let msgFeedback: SessionFeedback | undefined; - if (!isUser) { - for (let i = msgIndex - 1; i >= 0; i--) { - if (isUserMessage(messages[i])) { - msgFeedback = feedbackMap[messages[i].id]; - break; + {/* Messages Area */} + +
+ {loadingMessages ? ( +
+ {t('bots.sessionMonitor.loading')} +
+ ) : messages.length === 0 ? ( +
+ {t('bots.sessionMonitor.noMessages')} +
+ ) : ( + messages.map((msg, msgIndex) => { + const isUser = isUserMessage(msg); + const isDiscarded = + msg.status === 'discarded' || + msg.pipeline_id === PIPELINE_DISCARD; + // For bot replies, find feedback linked to the preceding user message + let msgFeedback: SessionFeedback | undefined; + if (!isUser) { + for (let i = msgIndex - 1; i >= 0; i--) { + if (isUserMessage(messages[i])) { + msgFeedback = feedbackMap[messages[i].id]; + break; + } } } - } - return ( -
+ return (
- {renderMessageContent(msg)} - {/* Role label + pipeline + timestamp */}
- - {isUser - ? t('bots.sessionMonitor.userMessage', { - defaultValue: 'User', - }) - : t('bots.sessionMonitor.botMessage', { - defaultValue: 'Assistant', + {renderMessageContent(msg)} + {/* Role label + pipeline + timestamp */} +
+ + {isUser + ? t('bots.sessionMonitor.userMessage', { + defaultValue: 'User', + }) + : t('bots.sessionMonitor.botMessage', { + defaultValue: 'Assistant', + })} + + + {formatTime(msg.timestamp)} + + {isDiscarded ? ( + + + {t('bots.sessionMonitor.discarded', { + defaultValue: 'Discarded', })} - - - {formatTime(msg.timestamp)} - - {isDiscarded ? ( - - - {t('bots.sessionMonitor.discarded', { - defaultValue: 'Discarded', - })} - - ) : msg.pipeline_name ? ( - - - {msg.pipeline_name} - - ) : null} - {msg.status === 'error' && ( - error - )} - {msg.runner_name && ( - - - {msg.runner_name} - - )} - {/* Feedback indicator — same line, pushed right */} - {!isUser && - msgFeedback && - (msgFeedback.feedback_type === 1 ? ( - - - {t('monitoring.feedback.like')} - {msgFeedback.feedback_content && ( - - {msgFeedback.feedback_content} - - )} - ) : ( - - - {t('monitoring.feedback.dislike')} - {msgFeedback.feedback_content && ( - - {msgFeedback.feedback_content} - - )} + ) : msg.pipeline_name ? ( + + + {msg.pipeline_name} - ))} + ) : null} + {msg.status === 'error' && ( + error + )} + {msg.runner_name && ( + + + {msg.runner_name} + + )} + {/* Feedback indicator — same line, pushed right */} + {!isUser && + msgFeedback && + (msgFeedback.feedback_type === 1 ? ( + + + {t('monitoring.feedback.like')} + {msgFeedback.feedback_content && ( + + {msgFeedback.feedback_content} + + )} + + ) : ( + + + {t('monitoring.feedback.dislike')} + {msgFeedback.feedback_content && ( + + {msgFeedback.feedback_content} + + )} + + ))} +
-
- ); - }) - )} -
-
- - )} + ); + }) + )} +
+
+ + )} +
-
+ + + ); }); diff --git a/web/src/app/home/components/dynamic-form/DynamicFormComponent.tsx b/web/src/app/home/components/dynamic-form/DynamicFormComponent.tsx index 671aa0d63..88442d19f 100644 --- a/web/src/app/home/components/dynamic-form/DynamicFormComponent.tsx +++ b/web/src/app/home/components/dynamic-form/DynamicFormComponent.tsx @@ -1,7 +1,7 @@ import { + DynamicFormItemType, IDynamicFormItemSchema, SYSTEM_FIELD_PREFIX, - DynamicFormItemType, } from '@/app/infra/entities/form/dynamic'; import { useForm } from 'react-hook-form'; import { zodResolver } from '@hookform/resolvers/zod'; @@ -59,6 +59,97 @@ function resolveShowIfValue( return externalDependentValues?.[field]; } +type DynamicFormValueSpec = Pick< + IDynamicFormItemSchema, + 'default' | 'name' | 'required' | 'type' +>; + +function getValueSpecs(item: IDynamicFormItemSchema): DynamicFormValueSpec[] { + if (item.type === DynamicFormItemType.RICH_TOOLS_SELECTOR) { + return [ + item, + { + name: 'enable-all-tools', + type: DynamicFormItemType.BOOLEAN, + required: false, + default: true, + }, + ]; + } + + if (item.type === DynamicFormItemType.RESOURCES_SELECTOR) { + return [ + item, + { + name: 'mcp-resources', + type: DynamicFormItemType.UNKNOWN, + required: false, + default: [], + }, + { + name: 'mcp-resource-agent-read-enabled', + type: DynamicFormItemType.BOOLEAN, + required: false, + default: true, + }, + ]; + } + + return [item]; +} + +function getValueSchema(spec: DynamicFormValueSpec) { + if (spec.name === 'mcp-resources') { + return z.array(z.any()); + } + + const normalizedType = normalizeItemType(spec.type); + + switch (normalizedType) { + case DynamicFormItemType.INT: + return z.number(); + case DynamicFormItemType.FLOAT: + return z.number(); + case DynamicFormItemType.BOOLEAN: + return z.boolean(); + case DynamicFormItemType.STRING: + return z.string(); + case DynamicFormItemType.STRING_ARRAY: + return z.array(z.string()); + case DynamicFormItemType.SELECT: + return z.string(); + case DynamicFormItemType.LLM_MODEL_SELECTOR: + return z.string(); + case DynamicFormItemType.EMBEDDING_MODEL_SELECTOR: + return z.string(); + case DynamicFormItemType.RERANK_MODEL_SELECTOR: + return z.string(); + case DynamicFormItemType.KNOWLEDGE_BASE_SELECTOR: + return z.string(); + case DynamicFormItemType.KNOWLEDGE_BASE_MULTI_SELECTOR: + case DynamicFormItemType.RESOURCES_SELECTOR: + case DynamicFormItemType.RICH_TOOLS_SELECTOR: + case DynamicFormItemType.TOOLS_SELECTOR: + return z.array(z.string()); + case DynamicFormItemType.BOT_SELECTOR: + return z.string(); + case DynamicFormItemType.MODEL_FALLBACK_SELECTOR: + return z.object({ + primary: z.string(), + fallbacks: z.array(z.string()), + }); + case DynamicFormItemType.PROMPT_EDITOR: + return z.array( + z.object({ + content: z.string(), + role: z.string(), + }), + ); + default: + return z.string(); + } +} + /** * Display-only component for embed code fields with copy animation. */ @@ -331,9 +422,16 @@ export default function DynamicFormComponent({ // model-fallback-selector) is coerced to the expected shape // so that downstream components never crash. const normalizeFieldValue = ( - item: IDynamicFormItemSchema, + item: DynamicFormValueSpec, value: unknown, ): unknown => { + if ( + item.name === 'mcp-resources' || + item.type === DynamicFormItemType.RESOURCES_SELECTOR || + item.type === DynamicFormItemType.RICH_TOOLS_SELECTOR + ) { + return Array.isArray(value) ? value : []; + } if (item.type === 'model-fallback-selector') { if (value != null && typeof value === 'object' && !Array.isArray(value)) { const obj = value as Record; @@ -377,74 +475,16 @@ export default function DynamicFormComponent({ [itemConfigList], ); + const editableValueSpecs = useMemo( + () => editableItems.flatMap(getValueSpecs), + [editableItems], + ); + // 根据 itemConfigList 动态生成 zod schema const formSchema = z.object( - editableItems.reduce( + editableValueSpecs.reduce( (acc, item) => { - // Normalize type to handle plugin manifest type names - const normalizedType = normalizeItemType(item.type); - - let fieldSchema; - switch (normalizedType) { - case 'integer': - fieldSchema = z.number(); - break; - case 'float': - fieldSchema = z.number(); - break; - case 'boolean': - fieldSchema = z.boolean(); - break; - case 'string': - fieldSchema = z.string(); - break; - case 'array[string]': - fieldSchema = z.array(z.string()); - break; - case 'select': - fieldSchema = z.string(); - break; - case 'llm-model-selector': - fieldSchema = z.string(); - break; - case 'embedding-model-selector': - fieldSchema = z.string(); - break; - case 'rerank-model-selector': - fieldSchema = z.string(); - break; - case 'knowledge-base-selector': - fieldSchema = z.string(); - break; - case 'knowledge-base-multi-selector': - fieldSchema = z.array(z.string()); - break; - case 'bot-selector': - fieldSchema = z.string(); - break; - case 'tools-selector': - fieldSchema = z.array(z.string()); - break; - case 'model-fallback-selector': - fieldSchema = z.object({ - primary: z.string(), - fallbacks: z.array(z.string()), - }); - break; - case 'prompt-editor': - fieldSchema = z.array( - z.object({ - content: z.string(), - role: z.string(), - }), - ); - break; - case 'text': - fieldSchema = z.string(); - break; - default: - fieldSchema = z.string(); - } + let fieldSchema = getValueSchema(item); if ( item.required && @@ -469,7 +509,7 @@ export default function DynamicFormComponent({ const form = useForm({ resolver: zodResolver(formSchema), - defaultValues: editableItems.reduce((acc, item) => { + defaultValues: editableValueSpecs.reduce((acc, item) => { // 优先使用 initialValues,如果没有则使用默认值 const rawValue = initialValues?.[item.name] ?? item.default; return { @@ -508,7 +548,7 @@ export default function DynamicFormComponent({ if (initialValues && hasRealChange) { // 合并默认值和初始值 - const mergedValues = editableItems.reduce( + const mergedValues = editableValueSpecs.reduce( (acc, item) => { const rawValue = initialValues[item.name] ?? item.default; acc[item.name] = normalizeFieldValue(item, rawValue) as object; @@ -523,10 +563,16 @@ export default function DynamicFormComponent({ previousInitialValues.current = initialValues; } - }, [initialValues, form, editableItems]); + }, [initialValues, form, editableValueSpecs]); // Get reactive form values for conditional rendering const watchedValues = form.watch(); + const setFormValue = (name: string, value: unknown) => { + form.setValue(name as keyof FormValues, value as never, { + shouldDirty: true, + shouldValidate: true, + }); + }; // Stable ref for onSubmit to avoid re-triggering the effect when the // parent passes a new closure on every render. @@ -539,7 +585,7 @@ export default function DynamicFormComponent({ // even if the user saves without modifying any field. // form.watch(callback) only fires on subsequent changes, not on mount. const formValues = form.getValues(); - const initialFinalValues = editableItems.reduce( + const initialFinalValues = editableValueSpecs.reduce( (acc, item) => { acc[item.name] = formValues[item.name] ?? item.default; return acc; @@ -559,7 +605,7 @@ export default function DynamicFormComponent({ const subscription = form.watch(() => { const formValues = form.getValues(); - const finalValues = editableItems.reduce( + const finalValues = editableValueSpecs.reduce( (acc, item) => { acc[item.name] = formValues[item.name] ?? item.default; return acc; @@ -570,7 +616,7 @@ export default function DynamicFormComponent({ previousInitialValues.current = finalValues as Record; }); return () => subscription.unsubscribe(); - }, [form, editableItems]); + }, [form, editableValueSpecs]); // State for QR code login dialog const [qrDialogOpen, setQrDialogOpen] = useState(false); @@ -808,6 +854,41 @@ export default function DynamicFormComponent({ ); } + if ( + normalizedConfig.type === DynamicFormItemType.RICH_TOOLS_SELECTOR || + normalizedConfig.type === DynamicFormItemType.RESOURCES_SELECTOR + ) { + return ( + ( + + +
+ } + onFileUploaded={onFileUploaded} + setFormValue={setFormValue} + systemContext={systemContext} + /> +
+
+ +
+ )} + /> + ); + } + // Boolean fields use a special inline layout if (normalizedConfig.type === 'boolean') { return ( @@ -838,7 +919,10 @@ export default function DynamicFormComponent({ } onFileUploaded={onFileUploaded} + setFormValue={setFormValue} + systemContext={systemContext} />
@@ -875,7 +959,10 @@ export default function DynamicFormComponent({ } onFileUploaded={onFileUploaded} + setFormValue={setFormValue} + systemContext={systemContext} />
diff --git a/web/src/app/home/components/dynamic-form/DynamicFormItemComponent.tsx b/web/src/app/home/components/dynamic-form/DynamicFormItemComponent.tsx index 5771838ae..17f605e46 100644 --- a/web/src/app/home/components/dynamic-form/DynamicFormItemComponent.tsx +++ b/web/src/app/home/components/dynamic-form/DynamicFormItemComponent.tsx @@ -64,6 +64,8 @@ import { import SettingsDialog, { SettingsSection, } from '@/app/home/components/settings-dialog/SettingsDialog'; +import ToolResourceSelectors from '@/app/home/components/dynamic-form/ToolResourceSelectors'; +import { LANGBOT_MODELS_PROVIDER_REQUESTER } from '@/app/home/components/models-dialog/types'; function getPluginComponentIconURL(value?: string): string | null { if (!value?.startsWith('plugin:')) { @@ -113,11 +115,17 @@ function SelectOptionContent({ export default function DynamicFormItemComponent({ config, field, + formValues, onFileUploaded, + setFormValue, + systemContext, }: { config: IDynamicFormItemSchema; field: ControllerRenderProps; + formValues?: Record; onFileUploaded?: (fileKey: string) => void; + setFormValue?: (name: string, value: unknown) => void; + systemContext?: Record; }) { const [llmModels, setLlmModels] = useState([]); const [embeddingModels, setEmbeddingModels] = useState([]); @@ -148,10 +156,34 @@ export default function DynamicFormItemComponent({ }); }; + const fetchEmbeddingModels = () => { + httpClient + .getProviderEmbeddingModels() + .then((resp) => { + setEmbeddingModels(resp.models); + }) + .catch((err) => { + toast.error(t('embedding.getModelListError') + err.msg); + }); + }; + + const fetchRerankModels = () => { + httpClient + .getProviderRerankModels() + .then((resp) => { + setRerankModels(resp.models); + }) + .catch((err) => { + toast.error('Failed to load rerank models: ' + err.msg); + }); + }; + const handleModelsDialogChange = (open: boolean) => { setModelsDialogOpen(open); if (!open) { fetchLlmModels(); + fetchEmbeddingModels(); + fetchRerankModels(); } }; @@ -219,27 +251,13 @@ export default function DynamicFormItemComponent({ useEffect(() => { if (config.type === DynamicFormItemType.EMBEDDING_MODEL_SELECTOR) { - httpClient - .getProviderEmbeddingModels() - .then((resp) => { - setEmbeddingModels(resp.models); - }) - .catch((err) => { - toast.error(t('embedding.getModelListError') + err.msg); - }); + fetchEmbeddingModels(); } }, [config.type]); useEffect(() => { if (config.type === DynamicFormItemType.RERANK_MODEL_SELECTOR) { - httpClient - .getProviderRerankModels() - .then((resp) => { - setRerankModels(resp.models); - }) - .catch((err) => { - toast.error('Failed to load rerank models: ' + err.msg); - }); + fetchRerankModels(); } }, [config.type]); @@ -293,6 +311,16 @@ export default function DynamicFormItemComponent({ } }, [config.type]); + const handleCompositePatch = (patch: Record) => { + for (const [name, value] of Object.entries(patch)) { + if (setFormValue) { + setFormValue(name, value); + } else if (name === field.name) { + field.onChange(value); + } + } + }; + switch (config.type) { case DynamicFormItemType.INT: case DynamicFormItemType.FLOAT: @@ -461,10 +489,10 @@ export default function DynamicFormItemComponent({ case DynamicFormItemType.LLM_MODEL_SELECTOR: // Separate space models from regular models const spaceModels = llmModels.filter( - (m) => m.provider?.requester === 'space-chat-completions', + (m) => m.provider?.requester === LANGBOT_MODELS_PROVIDER_REQUESTER, ); const regularModels = llmModels.filter( - (m) => m.provider?.requester !== 'space-chat-completions', + (m) => m.provider?.requester !== LANGBOT_MODELS_PROVIDER_REQUESTER, ); // Group regular models by provider @@ -565,7 +593,7 @@ export default function DynamicFormItemComponent({
))} {/* Blurred remaining models with login overlay */} -
+
setModelsDialogOpen(true)} + onClick={() => { + setSettingsSection('models'); + setModelsDialogOpen(true); + }} > @@ -658,9 +689,15 @@ export default function DynamicFormItemComponent({
); - case DynamicFormItemType.EMBEDDING_MODEL_SELECTOR: - // Group embedding models by provider - const groupedEmbeddingModels = embeddingModels.reduce( + case DynamicFormItemType.EMBEDDING_MODEL_SELECTOR: { + const spaceEmbeddingModels = embeddingModels.filter( + (m) => m.provider?.requester === LANGBOT_MODELS_PROVIDER_REQUESTER, + ); + const regularEmbeddingModels = embeddingModels.filter( + (m) => m.provider?.requester !== LANGBOT_MODELS_PROVIDER_REQUESTER, + ); + + const groupedEmbeddingModels = regularEmbeddingModels.reduce( (acc, model) => { const providerName = model.provider?.name || 'Unknown'; if (!acc[providerName]) acc[providerName] = []; @@ -670,29 +707,169 @@ export default function DynamicFormItemComponent({ {} as Record, ); + const groupedSpaceEmbeddingModels = spaceEmbeddingModels.reduce( + (acc, model) => { + const providerName = + model.provider?.name || model.provider?.requester || 'Unknown'; + if (!acc[providerName]) acc[providerName] = []; + acc[providerName].push(model); + return acc; + }, + {} as Record, + ); + + const previewEmbeddingModelNames = [ + 'text-embedding-3-large', + 'text-embedding-3-small', + 'bge-m3', + 'jina-embeddings-v3', + 'qwen3-embedding-8b', + ]; + return ( -
- + + + + + {Object.entries(groupedEmbeddingModels).map( + ([providerName, models]) => ( + + {providerName} + {models.map((model) => ( + + {model.name} + + ))} + + ), + )} + {showSpaceLoginCTA ? ( + + + + + {t('models.langbotModels')} + + e.preventDefault()} + > + + + + {t('models.spaceTrialTooltip')} + + + + +
e.preventDefault()} + > + {(spaceEmbeddingModels.length > 0 + ? spaceEmbeddingModels.map((m) => m.name) + : previewEmbeddingModelNames + ) + .slice(0, 3) + .map((name) => ( +
+ {name} +
+ ))} +
+
+ {(spaceEmbeddingModels.length > 0 + ? spaceEmbeddingModels.map((m) => m.name) + : previewEmbeddingModelNames + ) + .slice(3) + .map((name) => ( +
+ {name} +
+ ))} +
+
+ +
+
+
- ), - )} -
- + ) : !systemInfo.disable_models_service ? ( + Object.entries(groupedSpaceEmbeddingModels).map( + ([providerName, models]) => ( + + + + + {providerName} + + + {models.map((model) => ( + + {model.name} + + ))} + + ), + ) + ) : null} + + +
+ + + + + {t('models.title')} + +
); + } case DynamicFormItemType.RERANK_MODEL_SELECTOR: const groupedRerankModels = rerankModels.reduce( @@ -736,10 +913,10 @@ export default function DynamicFormItemComponent({ case DynamicFormItemType.MODEL_FALLBACK_SELECTOR: { // Separate space models from regular models const fbSpaceModels = llmModels.filter( - (m) => m.provider?.requester === 'space-chat-completions', + (m) => m.provider?.requester === LANGBOT_MODELS_PROVIDER_REQUESTER, ); const fbRegularModels = llmModels.filter( - (m) => m.provider?.requester !== 'space-chat-completions', + (m) => m.provider?.requester !== LANGBOT_MODELS_PROVIDER_REQUESTER, ); // Group regular models by provider @@ -870,7 +1047,7 @@ export default function DynamicFormItemComponent({
))} {/* Blurred remaining models with login overlay */} -
+
); + case DynamicFormItemType.RICH_TOOLS_SELECTOR: + return ( + + ); + + case DynamicFormItemType.RESOURCES_SELECTOR: + return ( + + ); + case DynamicFormItemType.PROMPT_EDITOR: { // Guard: field.value may be undefined when the form resets or // initialValues haven't propagated yet. Fall back to a default diff --git a/web/src/app/home/components/dynamic-form/DynamicFormItemConfig.ts b/web/src/app/home/components/dynamic-form/DynamicFormItemConfig.ts index a5cc5a1f7..a7d368363 100644 --- a/web/src/app/home/components/dynamic-form/DynamicFormItemConfig.ts +++ b/web/src/app/home/components/dynamic-form/DynamicFormItemConfig.ts @@ -68,6 +68,13 @@ export function getDefaultValues( return acc; } acc[item.name] = item.default; + if (item.type === DynamicFormItemType.RICH_TOOLS_SELECTOR) { + acc['enable-all-tools'] = true; + } + if (item.type === DynamicFormItemType.RESOURCES_SELECTOR) { + acc['mcp-resources'] = []; + acc['mcp-resource-agent-read-enabled'] = true; + } return acc; }, diff --git a/web/src/app/home/components/dynamic-form/ToolResourceSelectors.tsx b/web/src/app/home/components/dynamic-form/ToolResourceSelectors.tsx new file mode 100644 index 000000000..9b138c140 --- /dev/null +++ b/web/src/app/home/components/dynamic-form/ToolResourceSelectors.tsx @@ -0,0 +1,1074 @@ +import { useEffect, useMemo, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { backendClient } from '@/app/infra/http'; +import { + KnowledgeBase, + MCPResource, + MCPServer, + PluginTool, +} from '@/app/infra/entities/api'; +import { extractI18nObject } from '@/i18n/I18nProvider'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Checkbox } from '@/components/ui/checkbox'; +import { + Dialog, + DialogContent, + DialogFooter, + DialogHeader, + DialogTitle, +} from '@/components/ui/dialog'; +import { Label } from '@/components/ui/label'; +import { Switch } from '@/components/ui/switch'; +import { + Tooltip, + TooltipContent, + TooltipTrigger, +} from '@/components/ui/tooltip'; +import { + CircleHelp, + Database, + FileText, + Plus, + Server, + Wrench, + X, +} from 'lucide-react'; + +type ToolSource = NonNullable; + +type BoundMCPResource = { + server_uuid?: string; + server_name?: string; + uri: string; + mode?: string; + enabled?: boolean; + max_bytes?: number; + max_tokens?: number; +}; + +type PipelineExtensions = Awaited< + ReturnType +>; +type AvailablePlugin = PipelineExtensions['available_plugins'][number]; + +type ToolProviderGroup = { + key: string; + name: string; + tools: PluginTool[]; +}; + +type ToolSourceGroup = { + key: ToolSource | string; + label: string; + groups: ToolProviderGroup[]; + total: number; +}; + +type MCPToolOwner = { + serverId?: string; + serverName: string; +}; + +type PluginToolOwner = { + pluginId: string; +}; + +type BoundPlugin = PipelineExtensions['bound_plugins'][number]; + +const BUILTIN_TOOL_NAMES = new Set([ + 'exec', + 'read', + 'write', + 'edit', + 'glob', + 'grep', +]); +const TOOL_SOURCE_ORDER = ['plugin', 'mcp', 'skill', 'builtin']; + +function getMCPResourceKey(server: MCPServer, uri: string) { + return `${server.uuid || server.name}:${uri}`; +} + +function isSameMCPResource( + resource: BoundMCPResource, + server: MCPServer, + uri: string, +) { + return ( + (resource.server_uuid === server.uuid || + (!resource.server_uuid && resource.server_name === server.name)) && + resource.uri === uri + ); +} + +function getBoundPluginId(plugin: BoundPlugin) { + return `${plugin.author || ''}/${plugin.name}`; +} + +function InfoTooltip({ label }: { label: string }) { + return ( + + + + + + {label} + + + ); +} + +function normalizeToolSource(tool: PluginTool): ToolSource | string { + if (tool.source) { + return tool.source; + } + if (BUILTIN_TOOL_NAMES.has(tool.name)) { + return 'builtin'; + } + if (tool.name.startsWith('langbot_mcp_')) { + return 'mcp'; + } + return 'plugin'; +} + +function hasReliableSourceMetadata(tool: PluginTool) { + if (!tool.source) { + return false; + } + if (tool.source === 'plugin' || tool.source === 'mcp') { + return Boolean(tool.source_id || tool.source_name); + } + return true; +} + +function buildToolSourceGroups( + tools: PluginTool[], + sourceLabels: Record, +) { + const sourceGroups = new Map>(); + + for (const tool of tools) { + const source = normalizeToolSource(tool); + const providerName = + tool.source_name || + (source === 'builtin' ? 'LangBot' : sourceLabels[source] || source); + const providerKey = `${source}:${tool.source_id || providerName}`; + + if (!sourceGroups.has(source)) { + sourceGroups.set(source, new Map()); + } + + const providerGroups = sourceGroups.get(source)!; + if (!providerGroups.has(providerKey)) { + providerGroups.set(providerKey, { + key: providerKey, + name: providerName, + tools: [], + }); + } + providerGroups.get(providerKey)!.tools.push(tool); + } + + return Array.from(sourceGroups.entries()) + .sort(([left], [right]) => { + const leftIndex = TOOL_SOURCE_ORDER.indexOf(left); + const rightIndex = TOOL_SOURCE_ORDER.indexOf(right); + return ( + (leftIndex === -1 ? TOOL_SOURCE_ORDER.length : leftIndex) - + (rightIndex === -1 ? TOOL_SOURCE_ORDER.length : rightIndex) + ); + }) + .map( + ([source, providerGroups]): ToolSourceGroup => ({ + key: source, + label: sourceLabels[source] || source, + groups: Array.from(providerGroups.values()).sort((left, right) => + left.name.localeCompare(right.name), + ), + total: Array.from(providerGroups.values()).reduce( + (count, group) => count + group.tools.length, + 0, + ), + }), + ); +} + +function buildMCPToolOwners(servers: MCPServer[]) { + const owners = new Map(); + const ambiguousNames = new Set(); + + for (const server of servers) { + for (const tool of server.runtime_info?.tools || []) { + if (!tool.name) { + continue; + } + const owner = { + serverId: server.uuid, + serverName: server.name, + }; + + if (owners.has(tool.name)) { + ambiguousNames.add(tool.name); + continue; + } + owners.set(tool.name, owner); + } + } + + for (const name of ambiguousNames) { + owners.delete(name); + } + + return owners; +} + +function buildPluginToolOwners(plugins: AvailablePlugin[]) { + const owners = new Map(); + const ambiguousNames = new Set(); + + for (const plugin of plugins) { + const metadata = plugin.manifest?.manifest?.metadata; + const pluginName = metadata?.name; + if (!pluginName) { + continue; + } + + const pluginId = metadata.author + ? `${metadata.author}/${pluginName}` + : pluginName; + + for (const component of plugin.components || []) { + const manifest = component.manifest?.manifest; + if (manifest?.kind !== 'Tool') { + continue; + } + + const toolName = manifest.metadata?.name; + if (!toolName) { + continue; + } + + if (owners.has(toolName)) { + ambiguousNames.add(toolName); + continue; + } + owners.set(toolName, { pluginId }); + } + } + + for (const name of ambiguousNames) { + owners.delete(name); + } + + return owners; +} + +function annotateMCPToolSource( + tool: PluginTool, + owner?: MCPToolOwner, +): PluginTool { + if (!owner || hasReliableSourceMetadata(tool)) { + return tool; + } + return { + ...tool, + source: 'mcp', + source_name: owner.serverName, + source_id: owner.serverId, + }; +} + +function annotatePluginToolSource( + tool: PluginTool, + owner?: PluginToolOwner, +): PluginTool { + if (!owner || hasReliableSourceMetadata(tool)) { + return tool; + } + return { + ...tool, + source: 'plugin', + source_name: owner.pluginId, + source_id: owner.pluginId, + }; +} + +function annotateToolSource( + tool: PluginTool, + pluginOwner?: PluginToolOwner, + mcpOwner?: MCPToolOwner, +): PluginTool { + if (hasReliableSourceMetadata(tool)) { + return tool; + } + + if (tool.source === 'plugin') { + return annotatePluginToolSource(tool, pluginOwner); + } + if (tool.source === 'mcp') { + return annotateMCPToolSource(tool, mcpOwner); + } + if (mcpOwner && !pluginOwner) { + return annotateMCPToolSource(tool, mcpOwner); + } + if (pluginOwner && !mcpOwner) { + return annotatePluginToolSource(tool, pluginOwner); + } + + return tool; +} + +export default function ToolResourceSelectors({ + pipelineId, + value, + onChange, + mode = 'all', +}: { + pipelineId?: string; + value: Record; + onChange: (patch: Record) => void; + mode?: 'tools' | 'resources' | 'all'; +}) { + const { t } = useTranslation(); + const [tools, setTools] = useState([]); + const [knowledgeBases, setKnowledgeBases] = useState([]); + const [extensions, setExtensions] = useState(null); + const [toolsDialogOpen, setToolsDialogOpen] = useState(false); + const [kbDialogOpen, setKbDialogOpen] = useState(false); + const [tempSelectedToolNames, setTempSelectedToolNames] = useState( + [], + ); + const [tempSelectedKBIds, setTempSelectedKBIds] = useState([]); + + useEffect(() => { + if (mode !== 'resources') { + backendClient.getTools(pipelineId).then((resp) => setTools(resp.tools)); + } + if (mode !== 'tools') { + backendClient + .getKnowledgeBases() + .then((resp) => setKnowledgeBases(resp.bases)); + } + if (pipelineId) { + backendClient + .getPipelineExtensions(pipelineId) + .then((resp) => setExtensions(resp)); + } + }, [mode, pipelineId]); + + const enableAllTools = value['enable-all-tools'] !== false; + const selectedToolNames = Array.isArray(value.tools) ? value.tools : []; + const selectedKBIds = Array.isArray(value['knowledge-bases']) + ? value['knowledge-bases'] + : []; + const selectedMCPResources: BoundMCPResource[] = Array.isArray( + value['mcp-resources'], + ) + ? value['mcp-resources'] + : extensions?.bound_mcp_resources || []; + const mcpResourceReadEnabled = + typeof value['mcp-resource-agent-read-enabled'] === 'boolean' + ? value['mcp-resource-agent-read-enabled'] + : (extensions?.mcp_resource_agent_read_enabled ?? true); + + const scopedMCPServers = useMemo(() => { + if (!extensions) return []; + const boundServerIds = new Set(extensions.bound_mcp_servers || []); + return extensions.enable_all_mcp_servers + ? extensions.available_mcp_servers + : extensions.available_mcp_servers.filter((server) => + boundServerIds.has(server.uuid || ''), + ); + }, [extensions]); + + const scopedMCPServerIds = useMemo( + () => + new Set( + scopedMCPServers + .map((server) => server.uuid) + .filter((uuid): uuid is string => !!uuid), + ), + [scopedMCPServers], + ); + + const scopedMCPServerNames = useMemo( + () => new Set(scopedMCPServers.map((server) => server.name)), + [scopedMCPServers], + ); + + const mcpToolOwners = useMemo( + () => buildMCPToolOwners(scopedMCPServers), + [scopedMCPServers], + ); + + const pluginToolOwners = useMemo( + () => buildPluginToolOwners(extensions?.available_plugins || []), + [extensions], + ); + + const scopedPluginIds = useMemo( + () => + new Set( + extensions?.enable_all_plugins + ? (extensions.available_plugins || []).map((plugin) => { + const metadata = plugin.manifest?.manifest?.metadata; + return metadata?.name + ? `${metadata.author || ''}/${metadata.name}` + : ''; + }) + : (extensions?.bound_plugins || []).map(getBoundPluginId), + ), + [extensions], + ); + + const availableTools = useMemo( + () => + tools + .map((tool) => + annotateToolSource( + tool, + pluginToolOwners.get(tool.name), + mcpToolOwners.get(tool.name), + ), + ) + .filter((tool) => { + const source = normalizeToolSource(tool); + + if (source === 'plugin') { + if (!extensions) { + return false; + } + if (extensions.enable_all_plugins) { + return true; + } + return ( + (tool.source_id && scopedPluginIds.has(tool.source_id)) || + (tool.source_name && scopedPluginIds.has(tool.source_name)) + ); + } + + if (source === 'mcp') { + if (!extensions) { + return false; + } + if (extensions.enable_all_mcp_servers) { + return true; + } + return ( + (tool.source_id && scopedMCPServerIds.has(tool.source_id)) || + (tool.source_name && scopedMCPServerNames.has(tool.source_name)) + ); + } + + return true; + }), + [ + extensions, + mcpToolOwners, + pluginToolOwners, + scopedMCPServerIds, + scopedMCPServerNames, + scopedPluginIds, + tools, + ], + ); + + const availableToolNames = useMemo( + () => new Set(availableTools.map((tool) => tool.name)), + [availableTools], + ); + + const resourceServers = useMemo(() => { + return scopedMCPServers.filter( + (server) => + server.runtime_info?.status === 'connected' && + (server.runtime_info.resources || []).length > 0, + ); + }, [scopedMCPServers]); + + const availableMCPResourceKeys = useMemo(() => { + const keys = new Set(); + for (const server of resourceServers) { + for (const resource of server.runtime_info?.resources || []) { + keys.add(getMCPResourceKey(server, resource.uri)); + } + } + return keys; + }, [resourceServers]); + + const scopedSelectedMCPResources = selectedMCPResources.filter((resource) => { + const server = scopedMCPServers.find( + (item) => + item.uuid === resource.server_uuid || + (!resource.server_uuid && item.name === resource.server_name), + ); + return server + ? availableMCPResourceKeys.has(getMCPResourceKey(server, resource.uri)) + : false; + }); + + const selectedTools = selectedToolNames + .map((name: string) => availableTools.find((tool) => tool.name === name)) + .filter((tool): tool is PluginTool => !!tool); + + const selectedKnowledgeBases = selectedKBIds + .map((kbId: string) => knowledgeBases.find((base) => base.uuid === kbId)) + .filter((base): base is KnowledgeBase => !!base); + + const sourceLabels = useMemo>( + () => ({ + builtin: t('pipelines.localAgent.builtinTools'), + plugin: t('pipelines.localAgent.pluginTools'), + mcp: t('pipelines.localAgent.mcpTools'), + skill: t('pipelines.localAgent.skillTools'), + }), + [t], + ); + + const availableToolGroups = useMemo( + () => buildToolSourceGroups(availableTools, sourceLabels), + [availableTools, sourceLabels], + ); + const selectedToolGroups = useMemo( + () => buildToolSourceGroups(selectedTools, sourceLabels), + [selectedTools, sourceLabels], + ); + + const handleToggleToolMode = (checked: boolean) => { + onChange({ 'enable-all-tools': checked }); + }; + + const handleConfirmTools = () => { + onChange({ + tools: tempSelectedToolNames.filter((name) => + availableToolNames.has(name), + ), + }); + setToolsDialogOpen(false); + }; + + const handleConfirmKnowledgeBases = () => { + onChange({ 'knowledge-bases': tempSelectedKBIds }); + setKbDialogOpen(false); + }; + + const handleToggleMCPResource = ( + server: MCPServer, + resource: MCPResource, + checked: boolean, + ) => { + const next = checked + ? [ + ...scopedSelectedMCPResources.filter( + (item) => !isSameMCPResource(item, server, resource.uri), + ), + { + server_uuid: server.uuid, + server_name: server.name, + uri: resource.uri, + mode: 'pinned', + enabled: true, + }, + ] + : scopedSelectedMCPResources.filter( + (item) => !isSameMCPResource(item, server, resource.uri), + ); + onChange({ 'mcp-resources': next }); + }; + + const isMCPResourceSelected = (server: MCPServer, uri: string) => + scopedSelectedMCPResources.some( + (resource) => + isSameMCPResource(resource, server, uri) && resource.enabled !== false, + ); + + return ( +
+ {mode !== 'resources' && ( +
+
+
+
+

+ {t('pipelines.localAgent.toolsTitle')} +

+ +
+

+ {t('pipelines.localAgent.toolsDescription')} +

+
+
+ + +
+
+ + {enableAllTools ? ( +
+

+ {t('pipelines.localAgent.allToolsEnabled')} +

+
+ ) : selectedTools.length === 0 ? ( +
+

+ {t('pipelines.localAgent.noToolsSelected')} +

+
+ ) : ( +
+ {selectedToolGroups.map((sourceGroup) => ( +
+
+ {sourceGroup.label} + + {sourceGroup.total} + +
+ {sourceGroup.groups.map((providerGroup) => ( +
+
+ + + {providerGroup.name} + + + {providerGroup.tools.length} + +
+
+ {providerGroup.tools.map((tool) => ( +
+
+
+ {tool.name} +
+ {tool.human_desc && ( +
+ {tool.human_desc} +
+ )} +
+ +
+ ))} +
+
+ ))} +
+ ))} +
+ )} + + +
+ )} + + {mode !== 'tools' && ( +
+
+

+ {t('pipelines.localAgent.resourcesTitle')} +

+

+ {t('pipelines.localAgent.resourcesDescription')} +

+
+ +
+
+
+ + + {t('pipelines.localAgent.knowledgeBases')} + +
+ +
+ {selectedKnowledgeBases.length === 0 ? ( +
+

+ {t('knowledge.noKnowledgeBaseSelected')} +

+
+ ) : ( +
+ {selectedKnowledgeBases.map((base) => ( +
+
+
+ {base.emoji && {base.emoji}} + {base.name} + {base.knowledge_engine?.name && ( + + {extractI18nObject(base.knowledge_engine.name)} + + )} +
+ {base.description && ( +
+ {base.description} +
+ )} +
+ +
+ ))} +
+ )} +
+ +
+
+
+ + + {t('pipelines.localAgent.mcpResources')} + + +
+
+ + + + onChange({ 'mcp-resource-agent-read-enabled': checked }) + } + /> +
+
+ + {resourceServers.length === 0 ? ( +
+

+ {t('pipelines.localAgent.noMCPResourcesAvailable')} +

+
+ ) : ( +
+ {resourceServers.map((server) => ( +
+
+ + {server.name} + + {server.runtime_info?.resources?.length || 0} + +
+
+ {(server.runtime_info?.resources || []).map( + (resource) => ( + + ), + )} +
+
+ ))} +
+ )} +
+
+ )} + + {mode !== 'resources' && ( + + + + {t('pipelines.localAgent.selectTools')} + +
+ {availableToolGroups.map((sourceGroup) => { + return ( +
+
+ + {sourceGroup.label} + + {sourceGroup.key === 'mcp' && ( + + )} + {sourceGroup.key === 'skill' && ( + + )} + + {sourceGroup.total} + +
+
+ {sourceGroup.groups.map((providerGroup) => ( +
+
+ + + {providerGroup.name} + + + {providerGroup.tools.length} + +
+
+ {providerGroup.tools.map((tool) => { + const selected = tempSelectedToolNames.includes( + tool.name, + ); + return ( +
+ setTempSelectedToolNames((prev) => + prev.includes(tool.name) + ? prev.filter( + (name) => name !== tool.name, + ) + : [...prev, tool.name], + ) + } + > + +
+
+ {tool.name} +
+ {tool.human_desc && ( +
+ {tool.human_desc} +
+ )} +
+
+ ); + })} +
+
+ ))} +
+
+ ); + })} + {availableToolGroups.length === 0 && ( +
+

+ {t('pipelines.localAgent.noToolsSelected')} +

+
+ )} +
+ + + + +
+
+ )} + + {mode !== 'tools' && ( + + + + + {t('pipelines.localAgent.selectKnowledgeBases')} + + +
+ {knowledgeBases.map((base) => { + const kbId = base.uuid || ''; + const selected = tempSelectedKBIds.includes(kbId); + return ( +
+ setTempSelectedKBIds((prev) => + prev.includes(kbId) + ? prev.filter((id) => id !== kbId) + : [...prev, kbId], + ) + } + > + + +
+
+ {base.emoji && {base.emoji}} + {base.name} +
+ {base.description && ( +
+ {base.description} +
+ )} +
+
+ ); + })} + {knowledgeBases.length === 0 && ( +
+

+ {t('knowledge.noKnowledgeBaseSelected')} +

+
+ )} +
+ + + + +
+
+ )} +
+ ); +} diff --git a/web/src/app/home/components/home-sidebar/FeedbackPopover.tsx b/web/src/app/home/components/home-sidebar/FeedbackPopover.tsx new file mode 100644 index 000000000..5aba435fb --- /dev/null +++ b/web/src/app/home/components/home-sidebar/FeedbackPopover.tsx @@ -0,0 +1,206 @@ +import { useCallback, useEffect, useRef, useState } from 'react'; +import { ImagePlus, Loader2, Paperclip, Send, X } from 'lucide-react'; +import { useTranslation } from 'react-i18next'; +import { toast } from 'sonner'; +import { Button } from '@/components/ui/button'; +import { Textarea } from '@/components/ui/textarea'; +import { httpClient } from '@/app/infra/http/HttpClient'; + +const MAX_ATTACHMENTS = 3; +const MAX_IMAGE_BYTES = 1024 * 1024; + +type FeedbackAttachment = { + name: string; + mime_type: string; + data_url: string; +}; + +function readImageFile(file: File): Promise { + return new Promise((resolve, reject) => { + if (!file.type.startsWith('image/')) { + reject(new Error('not_image')); + return; + } + if (file.size > MAX_IMAGE_BYTES) { + reject(new Error('too_large')); + return; + } + const reader = new FileReader(); + reader.onload = () => { + const dataUrl = String(reader.result || ''); + if (!dataUrl.startsWith('data:image/')) { + reject(new Error('not_image')); + return; + } + resolve({ + name: file.name || 'pasted-image.png', + mime_type: file.type || 'image/png', + data_url: dataUrl, + }); + }; + reader.onerror = () => reject(reader.error || new Error('read_failed')); + reader.readAsDataURL(file); + }); +} + +const FEEDBACK_I18N_PREFIX = 'monitoring.feedback'; + +export function FeedbackPopoverContent({ + onSubmitted, +}: { + onSubmitted?: () => void; +}) { + const { t } = useTranslation(); + const tf = useCallback( + (key: string) => t(`${FEEDBACK_I18N_PREFIX}.${key}`), + [t], + ); + const [content, setContent] = useState(''); + const [attachments, setAttachments] = useState([]); + const [submitting, setSubmitting] = useState(false); + const fileInputRef = useRef(null); + + const addFiles = useCallback( + async (files: File[]) => { + const slots = MAX_ATTACHMENTS - attachments.length; + if (slots <= 0) { + toast.error(tf('tooManyImages')); + return; + } + const picked = files.slice(0, slots); + const next: FeedbackAttachment[] = []; + for (const file of picked) { + try { + next.push(await readImageFile(file)); + } catch (error) { + const msg = error instanceof Error ? error.message : ''; + toast.error( + msg === 'too_large' ? tf('imageTooLarge') : tf('imageOnly'), + ); + } + } + if (next.length > 0) { + setAttachments((prev) => [...prev, ...next].slice(0, MAX_ATTACHMENTS)); + } + }, + [attachments.length, tf], + ); + + useEffect(() => { + const onPaste = (event: ClipboardEvent) => { + const files = Array.from(event.clipboardData?.files || []).filter( + (file) => file.type.startsWith('image/'), + ); + if (files.length > 0) { + event.preventDefault(); + void addFiles(files); + } + }; + window.addEventListener('paste', onPaste); + return () => window.removeEventListener('paste', onPaste); + }, [addFiles]); + + const handleSubmit = async () => { + const trimmed = content.trim(); + if (!trimmed) { + toast.error(tf('contentRequired')); + return; + } + try { + setSubmitting(true); + await httpClient.submitFeedback({ + content: trimmed, + attachments, + }); + toast.success(tf('submitSuccess')); + setContent(''); + setAttachments([]); + onSubmitted?.(); + } catch { + toast.error(tf('submitFailed')); + } finally { + setSubmitting(false); + } + }; + + return ( +
e.stopPropagation()}> +
+
{tf('title')}
+

+ {tf('description')} +

+
+