From 7bf20490f4b74768d066fdcc7bdeb51c32478638 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Sat, 5 Sep 2026 19:49:35 +0800 Subject: [PATCH 001/174] feat(skill): use SDK store without box execution --- ARCHITECTURE.md | 5 +- docker/docker-compose.yaml | 8 +- docs/review/box-architecture.md | 44 +-- .../pkg/api/http/controller/groups/skills.py | 36 +-- src/langbot/pkg/api/http/service/skill.py | 114 +++----- src/langbot/pkg/box/service.py | 2 +- src/langbot/pkg/core/app.py | 2 + src/langbot/pkg/core/stages/build_app.py | 2 + .../pkg/provider/tools/loaders/native.py | 30 +- .../provider/tools/loaders/skill_authoring.py | 263 ++++++++++++++---- src/langbot/pkg/provider/tools/toolmgr.py | 40 ++- src/langbot/pkg/skill/__init__.py | 4 +- src/langbot/pkg/skill/manager.py | 27 +- src/langbot/pkg/skill/repository.py | 192 +++++++++++++ src/langbot/templates/config.yaml | 10 +- .../api/test_skills_entitlements.py | 16 +- tests/unit_tests/box/test_box_service.py | 6 +- tests/unit_tests/provider/test_skill_tools.py | 210 ++++++++------ .../provider/test_tool_manager_native.py | 23 +- tests/unit_tests/test_skill_repository.py | 175 ++++++++++++ tests/unit_tests/test_skill_service.py | 140 +++------- .../pipeline-extensions/PipelineExtension.tsx | 14 +- .../app/home/skills/SkillDetailContent.tsx | 33 +-- web/src/app/home/skills/page.tsx | 19 +- web/src/i18n/locales/en-US.ts | 6 +- web/src/i18n/locales/es-ES.ts | 6 +- web/src/i18n/locales/ja-JP.ts | 6 +- web/src/i18n/locales/ru-RU.ts | 6 +- web/src/i18n/locales/th-TH.ts | 6 +- web/src/i18n/locales/vi-VN.ts | 6 +- web/src/i18n/locales/zh-Hans.ts | 8 +- web/src/i18n/locales/zh-Hant.ts | 8 +- 32 files changed, 940 insertions(+), 527 deletions(-) create mode 100644 src/langbot/pkg/skill/repository.py create mode 100644 tests/unit_tests/test_skill_repository.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index e6eaa9a72..18c764aab 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -190,14 +190,15 @@ The Pipeline wrapper never scans a Box. Binding ends with the run; the reusable remains subject to Runtime idle expiry. Persistent workspace files survive expiry, but processes and container-local state do not. -Box is the sandbox subsystem used by native agent tools, stdio MCP servers, skill authoring, and managed processes. +Box is the optional sandbox subsystem used by native execution tools, stdio MCP servers, agent-side skill authoring, and managed processes. Skill storage and read-only access are Core responsibilities and remain available without Box. In this repo: - `pkg/box/service.py` is the application-facing facade for exec, sessions, managed processes, skill CRUD, status, reconnects, quotas, mounts, and sandbox profiles. - `pkg/box/connector.py` connects to the Box Runtime over stdio, Windows subprocess+WebSocket, or remote WebSocket. - `pkg/provider/tools/loaders/native.py`, `mcp_stdio.py`, and skill loaders depend on Box availability. -- `pkg/skill/manager.py` loads skills from the Box runtime, falling back to local `data/skills` when needed. +- `pkg/skill/repository.py` is the thin async/Workspace adapter over the Plugin SDK's execution-independent `SkillStore`; it preserves the existing `data/box/skills` layout shared with Box for optional execution mounts. +- `pkg/skill/manager.py` caches the Core repository catalog for progressive disclosure. Activation and read-only resource tools do not require Box; script execution and Workspace mutation still do. Durable Box Workspace storage is shared across placement generations, but sandbox sessions and managed processes are generation-scoped. LangBot validates diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index cb5f1c88e..fa40faa40 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -27,8 +27,8 @@ services: # The Box sandbox runtime is optional. It is only started when you run # ``docker compose --profile box up`` (or ``docker compose --profile all - # up``). With Box off, LangBot keeps the dashboard / skills list visible - # (read-only) but disables sandbox tools, skill add/edit and stdio MCP — + # up``). With Box off, LangBot keeps skill management, activation, and + # read-only resources available but disables execution tools and stdio MCP — # set ``box.enabled: false`` in ``data/config.yaml`` (or # ``BOX__ENABLED=false`` in the langbot service env below) to match. langbot_box: @@ -73,6 +73,10 @@ services: container_name: langbot volumes: - ./data:/app/data + # Core owns the SkillRepository even when the Box profile is disabled. + # Keep this path identical to langbot_box so optional execution can + # consume the same Workspace-scoped package revisions. + - ${LANGBOT_BOX_ROOT:-${PWD}/data/box}:${LANGBOT_BOX_ROOT:-${PWD}/data/box} restart: on-failure environment: - TZ=Asia/Shanghai diff --git a/docs/review/box-architecture.md b/docs/review/box-architecture.md index 4a390ebf1..a76db67ff 100644 --- a/docs/review/box-architecture.md +++ b/docs/review/box-architecture.md @@ -43,7 +43,7 @@ │ │ Action RPC (stdio 或 WebSocket) │ │ │ │ SkillManager (skill_mgr) │ -│ └─ 从 Box runtime 拉取 skills, 不可用时回落 data/skills │ +│ └─ 从 Core SkillRepository 加载 Workspace-scoped skills │ └──────────────────────────────────────────────────────────────────┘ │ ▼ @@ -176,11 +176,10 @@ BoxService ``` SkillManager ├─ initialize() 调用 reload_skills() - ├─ reload_skills() 先从 Box runtime list_skills()， - │ 不可用则回落 data/skills/ 扫描 - ├─ refresh_skill_from_disk() 单 skill 重新加载 + ├─ reload_skills() 从 Core SkillRepository 加载 + ├─ refresh_skill_from_disk() 检查单 skill 的缓存状态 ├─ get_skill_by_name(name) - └─ get_managed_skills_root() 返回 Box 视角的 skills_root 路径 + └─ build_skill_aware_prompt_addition() 生成渐进披露索引 ``` skill 元数据通过 `parse_frontmatter` 解析 `SKILL.md` 头部（`name` / `description` / `instructions`），不再做整体扫描的代价（典型 < 50 个）。 @@ -348,23 +347,24 @@ stdio 模式同样会在 5410 启动 aiohttp，专门承担 managed process atta `BoxSpec` 校验器: `workdir` 默认继承 `mount_path`；`host_path` 支持 POSIX 和 Windows 路径；设置 `host_path` 时 `workdir` 必须在 `mount_path` 下。 -### 3.7 BoxSkillStore (`box/skill_store.py`, 647 行) +### 3.7 SkillStore (`langbot_plugin.skill_store`) -新增模块（commit `4ab3502`），把 skill 持久化收归 Box runtime： +Skill 包存储最初位于 Box Runtime；issue #2410 将通用实现抽到 Plugin SDK 顶层，Core 与 Box 使用同一套存储和 revision 语义： ``` -BoxSkillStore +SkillStore ├─ list_skills() / get_skill(name) ├─ create_skill(data) / update_skill(name, data) / delete_skill(name) ├─ scan_skill_directory(path) 扫描目录返回候选 skill 包列表 - ├─ list_skill_files(name, path) 浏览 skill 内文件树 - ├─ read_skill_file(name, path) / write_skill_file(name, path, content) + ├─ list_skill_resources(name, path, revision) 按 revision 浏览只读资源 + ├─ read_skill_resource(name, path, revision) 按 revision 读取 UTF-8 资源 + ├─ read_skill_file(...) / write_skill_file(...) 管理侧文件接口 ├─ preview_skill_zip(zip_bytes, ...) 不落盘预览 zip 内容 └─ install_skill_zip(zip_bytes, ...) 解压、校验、复制到 skills_root └─ 支持 source_subdir / target_suffix（commit 1aa043f） ``` -GitHub 安装路径：HTTP 层（`api/http/service/skill.py`）先 `git clone` 拉取，再走 `install_skill_zip` 或 directory 路径。Skill 文件存放于 `box.local.skills_root`（默认 `skills`，相对 `host_root`），容器内对应 `/workspace/.skills/`。 +`langbot_plugin.box.skill_store.BoxSkillStore` 仅保留为旧 Box 配置到通用 `SkillStore` 的兼容适配器，不再拥有存储实现。GitHub 安装路径由 Core HTTP 层下载归档，再交给 SkillRepository。Skill 文件继续存放于 `box.local.skills_root`（默认 `skills`，相对 `host_root`），执行时只读挂载到 `/workspace/.skills/`。 ### 3.8 Security (`box/security.py`, 52 行) @@ -472,7 +472,7 @@ BuildAppStage.run(ap) ├─ ap.tool_mgr = tool_mgr │ ├─ ... (platform, pipeline) ... - ├─ SkillManager.initialize() (从 Box runtime 加载 skill 列表) + ├─ SkillManager.initialize() (从 Core SkillRepository 加载 skill 列表) └─ ... (RAG, HTTP, plugins) ... ``` @@ -489,7 +489,7 @@ except Exception as e: logger.warning(f"Box runtime unavailable: {e}") ``` -**静默降级**: Box 初始化失败不会阻止应用启动，仅导致 6 个 native tool、所有 Skill 工具和 MCP-in-Box 工具不暴露给 LLM。与 Plugin 的行为不同（Plugin 失败会抛异常）。 +**静默降级**: Box 初始化失败不会阻止应用启动。6 个 native tool、`register_skill` 和 MCP-in-Box 工具不暴露给 LLM；`activate` 与 Skill 只读资源工具继续由 Core 提供。与 Plugin 的行为不同（Plugin 失败会抛异常）。 ### 5.3 销毁流程 @@ -517,9 +517,9 @@ box: enabled: true # 整个 Box 子系统的总开关。设为 false 时： # - 不连接远程 Box runtime，不 fork 本地 stdio 子进程 # - sandbox 工具 (exec/read/write/edit/glob/grep) 不暴露给 LLM - # - skill 添加/编辑 / GitHub 安装 / 文件写入全部拒绝 + # - Agent 从 sandbox 注册 skill 的能力不可用 # - stdio 模式的 MCP server 启动时报错（http/sse 模式不受影响） - # - skill 列表/读取保持只读可用 + # - skill 管理、激活和只读资源保持可用 # BOX__ENABLED 环境变量可覆盖（统一约定） backend: 'local' # 'local' (探测) / 'docker' / 'nsjail' / 'e2b' # 由 box.backend / BOX__BACKEND 选择后端 @@ -531,7 +531,7 @@ box: image: '' # 覆盖 profile 默认 image host_root: './data/box' # 工作区挂载根，Docker 部署需绝对路径 default_workspace: '' # 默认 '/default' - skills_root: 'skills' # Box 管理的 skill 包目录（相对 host_root） + skills_root: 'skills' # Core 管理且与 Box 共享的 skill 包目录 allowed_mount_roots: # 默认 [''] - './data/box' - '/tmp' @@ -570,15 +570,17 @@ volumes: | 消费方 | Box 可用 | Box 不可用(disabled 或 failed) | |---|---|---| | native exec/read/write/edit/glob/grep 工具 | 暴露给 LLM | **不暴露** | -| `activate` / `register_skill` 工具 | 暴露给 LLM | **不暴露** | +| `activate` / Skill resource 工具 | 暴露给 LLM | 暴露给 LLM | +| `register_skill` 工具 | 暴露给 LLM | **不暴露**；直接调用返回 `sandbox_unavailable` | | stdio MCP server | 在 Box 内启动 | **`_init_stdio_python_server` 抛 RuntimeError** 拒绝;不退化到宿主 stdio | | http/sse MCP server | 正常 | 正常(不依赖 Box) | -| Skill 列表/读取 (`list_skills`/`get_skill`/`read_skill_file`) | 走 Box runtime | 走 LangBot 本地 `data/skills/` 只读 fallback | -| Skill 创建/编辑/安装/写文件 | 走 Box runtime | **HTTP 400** + 明确错误信息(`_require_box_for_write`) | -| Pipeline 扩展页 `enable_all_skills` / 绑定 skill | 可编辑 | **前端禁用** + banner | +| Skill 列表/读取 (`list_skills`/`get_skill`/`read_skill_file`) | 走 Core SkillRepository | 走 Core SkillRepository | +| Skill 创建/编辑/安装/写文件 | 走 Core SkillRepository | 走 Core SkillRepository | +| Pipeline AI 配置中 `box-session-id-template` | 正常生效 | **前端 banner** 提示字段无效 | +| Pipeline 扩展页 `enable_all_skills` / 绑定 skill | 可编辑 | 可编辑 | | 仪表盘 Box 状态卡片 | 绿点 / "已连接" | 灰点 / "已禁用"(disabled) 或 红点 / "已断开"(failed) | -> 后端拒写的边界条件:如果 `ap.box_service` **完全没装**(老式 dev mode,没经过 BuildAppStage),`_require_box_for_write` 视作 no-op,保留 `data/skills/` 本地路径——以兼容历史测试与最小化设置。生产环境总会装 `ap.box_service`,因此该 fallback 不会被触发。 +> Core 的 SkillRepository 是 `langbot_plugin.skill_store.SkillStore` 的异步 Workspace 适配层，保持原 `data/box/skills/tenants/...` 布局，升级时无需移动已安装 Skill。Box 只在执行发生时消费同一份只读 package revision。 ### Session scope diff --git a/src/langbot/pkg/api/http/controller/groups/skills.py b/src/langbot/pkg/api/http/controller/groups/skills.py index 6e6be90c6..214230f0c 100644 --- a/src/langbot/pkg/api/http/controller/groups/skills.py +++ b/src/langbot/pkg/api/http/controller/groups/skills.py @@ -2,9 +2,6 @@ from __future__ import annotations import quart -from langbot.pkg.cloud.entitlements import EntitlementFeatureUnavailableError -from langbot_plugin.box.errors import BoxError - from ...authz import Permission from ...context import RequestContext from .....operation_trace import service as settings_service @@ -25,12 +22,7 @@ class SkillsRouterGroup(group.RouterGroup): async def list_skills(request_context: RequestContext) -> quart.Response: try: skills = await self.ap.skill_service.list_skills(request_context) - except EntitlementFeatureUnavailableError: - # Plans without managed sandbox support have no runnable skills. - # Treat that capability absence as an empty collection so the - # shared UI can render normally instead of surfacing a 500. - return self.success(data={'skills': []}) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) return self.success(data={'skills': skills}) @@ -48,7 +40,7 @@ class SkillsRouterGroup(group.RouterGroup): try: skill = await self.ap.skill_service.create_skill(request_context, data) return self.success(data={'skill': skill}) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) @self.route( @@ -60,7 +52,7 @@ class SkillsRouterGroup(group.RouterGroup): async def get_skill(skill_name: str, request_context: RequestContext) -> quart.Response: try: skill = await self.ap.skill_service.get_skill(request_context, skill_name) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) if not skill: return self.http_status(404, -1, 'Skill not found') @@ -80,11 +72,11 @@ class SkillsRouterGroup(group.RouterGroup): data = await quart.request.json try: previous = await self.ap.skill_service.get_skill(request_context, skill_name) - except (ValueError, BoxError): + except ValueError: previous = None try: skill = await self.ap.skill_service.update_skill(request_context, skill_name, data) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) changes = settings_service.changed_fields( @@ -101,7 +93,7 @@ class SkillsRouterGroup(group.RouterGroup): try: await self.ap.skill_service.delete_skill(request_context, skill_name) return self.success() - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) @self.route( @@ -123,7 +115,7 @@ class SkillsRouterGroup(group.RouterGroup): include_hidden=include_hidden, ) return self.success(data=result) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) @self.route( @@ -136,7 +128,7 @@ class SkillsRouterGroup(group.RouterGroup): try: result = await self.ap.skill_service.read_skill_file(request_context, skill_name, path) return self.success(data=result) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) @self.route( @@ -154,7 +146,7 @@ class SkillsRouterGroup(group.RouterGroup): try: result = await self.ap.skill_service.write_skill_file(request_context, skill_name, path, content) return self.success(data=result) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) @self.route( @@ -191,7 +183,7 @@ class SkillsRouterGroup(group.RouterGroup): try: skill = await self.ap.skill_service.install_from_github(request_context, data) return self.success(data={'skills': skill}) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) except Exception: raise @@ -215,7 +207,7 @@ class SkillsRouterGroup(group.RouterGroup): try: preview = await self.ap.skill_service.preview_install_from_github(request_context, data) return self.success(data={'skills': preview}) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) except Exception: raise @@ -243,7 +235,7 @@ class SkillsRouterGroup(group.RouterGroup): source_paths=form.getlist('source_paths'), ) return self.success(data={'skills': skill}) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) except Exception: raise @@ -266,7 +258,7 @@ class SkillsRouterGroup(group.RouterGroup): filename=file.filename or '', ) return self.success(data={'skills': preview}) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) except Exception: raise @@ -285,5 +277,5 @@ class SkillsRouterGroup(group.RouterGroup): try: result = await self.ap.skill_service.scan_directory_async(request_context, path) return self.success(data=result) - except (ValueError, BoxError) as exc: + except ValueError as exc: return self.http_status(400, -1, str(exc)) diff --git a/src/langbot/pkg/api/http/service/skill.py b/src/langbot/pkg/api/http/service/skill.py index be09c7849..fcd4e5313 100644 --- a/src/langbot/pkg/api/http/service/skill.py +++ b/src/langbot/pkg/api/http/service/skill.py @@ -11,7 +11,6 @@ from typing import Optional from urllib.parse import quote, unquote, urlparse import httpx -from langbot_plugin.entities.io.errors import ActionCallTimeoutError from ....core import app from ....skill.utils import parse_frontmatter @@ -26,6 +25,7 @@ _PUBLIC_SKILL_FIELDS = ( 'description', 'instructions', 'package_root', + 'revision', 'created_at', 'updated_at', ) @@ -54,38 +54,11 @@ class SkillService: def __init__(self, ap: app.Application) -> None: self.ap = ap - def _box_service(self): - box_service = getattr(self.ap, 'box_service', None) - if box_service is not None and getattr(box_service, 'available', False): - return box_service - return None - - def _require_box(self, action: str): - """Return the Box service or raise if it is not available. - - Box is the only source of truth for skills. Every read and write - operation goes through it — there is no local-filesystem fallback. - """ - box_service = self._box_service() - if box_service is not None: - return box_service - ap_box = getattr(self.ap, 'box_service', None) - if ap_box is None: - reason = 'not initialised' - elif not getattr(ap_box, 'enabled', True): - reason = 'disabled in config (box.enabled = false)' - else: - connector_error = getattr(ap_box, '_connector_error', '') or 'currently unavailable' - reason = f'unavailable: {connector_error}' - raise ValueError( - f'{action} requires the Box runtime, which is {reason}. ' - f'Enable Box in config.yaml (box.enabled = true) and ensure the ' - f'runtime is reachable before retrying.' - ) - - def _require_box_for_write(self, action: str) -> None: - """Backwards-compatible alias preserved for clarity at call sites.""" - self._require_box(action) + def _repository(self): + repository = getattr(self.ap, 'skill_repository', None) + if repository is None: + raise ValueError('Skill repository is not initialised') + return repository async def _execution_context(self, context: TenantContext) -> ExecutionContext: workspace_uuid = require_workspace_uuid(context) @@ -114,27 +87,11 @@ class SkillService: async def list_skills(self, context: TenantContext) -> list[dict]: execution_context = await self._execution_context(context) - # When Box is unavailable, surface an empty list rather than raising — - # the skills page should render cleanly, and the UI separately renders - # a "Box disabled / unavailable" banner via useBoxStatus. - box_service = self._box_service() - if box_service is None: - return [] - try: - skills = await box_service.list_skills(execution_context) - except ActionCallTimeoutError as exc: - logger = getattr(self.ap, 'logger', None) - if logger is not None: - logger.warning(f'Box skill listing timed out: {exc}') - return [] - return [self._serialize_skill(skill) for skill in skills] + return [self._serialize_skill(skill) for skill in await self._repository().list_skills(execution_context)] async def get_skill(self, context: TenantContext, skill_name: str) -> Optional[dict]: execution_context = await self._execution_context(context) - box_service = self._box_service() - if box_service is None: - return None - skill = await box_service.get_skill(execution_context, skill_name) + skill = await self._repository().get_skill(execution_context, skill_name, snapshot=True) return self._serialize_skill(skill) if skill else None async def get_skill_by_name(self, context: TenantContext, name: str) -> Optional[dict]: @@ -142,22 +99,25 @@ class SkillService: async def create_skill(self, context: TenantContext, data: dict) -> dict: execution_context = await self._execution_context(context) - box_service = self._require_box('Creating a skill') - created = await box_service.create_skill(execution_context, data) + created = await self._repository().create_skill(execution_context, data) + await self._reload_skills(execution_context) + return self._serialize_skill(created) + + async def import_skill_directory(self, context: TenantContext, path: str, data: dict) -> dict: + execution_context = await self._execution_context(context) + created = await self._repository().import_skill_directory(execution_context, path, data) await self._reload_skills(execution_context) return self._serialize_skill(created) async def update_skill(self, context: TenantContext, skill_name: str, data: dict) -> dict: execution_context = await self._execution_context(context) - box_service = self._require_box('Editing a skill') - updated = await box_service.update_skill(execution_context, skill_name, data) + updated = await self._repository().update_skill(execution_context, skill_name, data) await self._reload_skills(execution_context) return self._serialize_skill(updated) async def delete_skill(self, context: TenantContext, skill_name: str) -> bool: execution_context = await self._execution_context(context) - box_service = self._require_box('Deleting a skill') - await box_service.delete_skill(execution_context, skill_name) + await self._repository().delete_skill(execution_context, skill_name) await self._reload_skills(execution_context) return True @@ -170,24 +130,27 @@ class SkillService: max_entries: int = 200, ) -> dict: execution_context = await self._execution_context(context) - box_service = self._require_box('Browsing skill files') - return await box_service.list_skill_files(execution_context, skill_name, path, include_hidden, max_entries) + return await self._repository().list_skill_files( + execution_context, + skill_name, + path, + include_hidden, + max_entries, + ) async def read_skill_file(self, context: TenantContext, skill_name: str, path: str) -> dict: execution_context = await self._execution_context(context) - box_service = self._require_box('Reading a skill file') - return await box_service.read_skill_file(execution_context, skill_name, path) + return await self._repository().read_skill_file(execution_context, skill_name, path) async def write_skill_file(self, context: TenantContext, skill_name: str, path: str, content: str) -> dict: execution_context = await self._execution_context(context) - box_service = self._require_box('Editing skill files') - result = await box_service.write_skill_file(execution_context, skill_name, path, content) + result = await self._repository().write_skill_file(execution_context, skill_name, path, content) await self._reload_skills(execution_context) return result async def install_from_github(self, context: TenantContext, data: dict) -> list[dict]: execution_context = await self._execution_context(context) - box_service = self._require_box('Installing a skill from GitHub') + repository = self._repository() owner = str(data['owner']).strip() repo = str(data['repo']).strip() release_tag = str(data.get('release_tag', '')).strip() @@ -206,7 +169,7 @@ class SkillService: zip_bytes = await self._download_github_asset(asset_url) filename = f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip' - installed = await box_service.install_skill_zip( + installed = await repository.install_skill_zip( execution_context, zip_bytes, filename, @@ -219,7 +182,7 @@ class SkillService: async def preview_install_from_github(self, context: TenantContext, data: dict) -> list[dict]: execution_context = await self._execution_context(context) - box_service = self._require_box('Previewing a skill from GitHub') + repository = self._repository() owner = str(data['owner']).strip() repo = str(data['repo']).strip() release_tag = str(data.get('release_tag', '')).strip() @@ -236,7 +199,7 @@ class SkillService: source_subdir = str(data.get('source_subdir', '') or '').strip() zip_bytes = await self._download_github_asset(asset_url) - return await box_service.preview_skill_zip( + return await repository.preview_skill_zip( execution_context, zip_bytes, f'{repo}-{release_tag.lstrip("v").replace("/", "-") or "source"}.zip', @@ -253,8 +216,7 @@ class SkillService: source_path: str = '', ) -> list[dict]: execution_context = await self._execution_context(context) - box_service = self._require_box('Installing a skill from upload') - installed = await box_service.install_skill_zip( + installed = await self._repository().install_skill_zip( execution_context, file_bytes, filename, @@ -272,8 +234,7 @@ class SkillService: filename: str, ) -> list[dict]: execution_context = await self._execution_context(context) - box_service = self._require_box('Previewing a skill upload') - return await box_service.preview_skill_zip(execution_context, file_bytes, filename) + return await self._repository().preview_skill_zip(execution_context, file_bytes, filename) async def _install_github_skill_md( self, @@ -284,14 +245,14 @@ class SkillService: repo: str, data: dict, ) -> list[dict]: - box_service = self._require_box('Installing a skill from GitHub') + repository = self._repository() zip_bytes, filename, _package_name = await self._download_github_skill_directory_as_zip( asset_url, owner=owner, repo=repo, ) - installed = await box_service.install_skill_zip( + installed = await repository.install_skill_zip( context, zip_bytes, filename, @@ -310,13 +271,13 @@ class SkillService: owner: str, repo: str, ) -> list[dict]: - box_service = self._require_box('Previewing a skill from GitHub') + repository = self._repository() zip_bytes, _filename, package_name = await self._download_github_skill_directory_as_zip( asset_url, owner=owner, repo=repo, ) - return await box_service.preview_skill_zip(context, zip_bytes, f'{package_name}.zip', target_suffix='') + return await repository.preview_skill_zip(context, zip_bytes, f'{package_name}.zip', target_suffix='') async def reload_skills(self, context: TenantContext) -> list[dict]: execution_context = await self._execution_context(context) @@ -325,8 +286,7 @@ class SkillService: async def scan_directory_async(self, context: TenantContext, path: str) -> dict: execution_context = await self._execution_context(context) - box_service = self._require_box('Scanning a skill directory') - return await box_service.scan_skill_directory(execution_context, path) + return await self._repository().scan_skill_directory(execution_context, path) async def _reload_skills(self, context: TenantContext) -> None: skill_mgr = getattr(self.ap, 'skill_mgr', None) diff --git a/src/langbot/pkg/box/service.py b/src/langbot/pkg/box/service.py index 4f51b15aa..6d0c08839 100644 --- a/src/langbot/pkg/box/service.py +++ b/src/langbot/pkg/box/service.py @@ -698,7 +698,7 @@ class BoxService: { 'host_path': package_root, 'mount_path': f'/workspace/.skills/{skill_name}', - 'mode': 'rw', + 'mode': 'ro', } ) return mounts diff --git a/src/langbot/pkg/core/app.py b/src/langbot/pkg/core/app.py index fa7a14f79..0e7f0e61f 100644 --- a/src/langbot/pkg/core/app.py +++ b/src/langbot/pkg/core/app.py @@ -47,6 +47,7 @@ from . import entities as core_entities from ..rag.knowledge import kbmgr as rag_mgr from ..rag.service import RAGRuntimeService from ..vector import mgr as vectordb_mgr +from ..skill import repository as skill_repository from ..telemetry import telemetry as telemetry_module from ..survey import manager as survey_module from ..skill import manager as skill_mgr @@ -91,6 +92,7 @@ class Application: # TODO move to pipeline tool_mgr: llm_tool_mgr.ToolManager = None box_service: box_service_module.BoxService = None + skill_repository: skill_repository.SkillRepository = None # ======= Config manager ======= diff --git a/src/langbot/pkg/core/stages/build_app.py b/src/langbot/pkg/core/stages/build_app.py index 579f1049a..f515500c9 100644 --- a/src/langbot/pkg/core/stages/build_app.py +++ b/src/langbot/pkg/core/stages/build_app.py @@ -28,6 +28,7 @@ from ...api.http.service import knowledge as knowledge_service from ...api.http.service import mcp as mcp_service from ...api.http.service import apikey as apikey_service from ...api.http.service import webhook as webhook_service +from ...skill import repository as skill_repository from ...api.http.service import monitoring as monitoring_service from ...operation_trace import service as operation_trace_service from ...api.http.service import skill as skill_service @@ -137,6 +138,7 @@ class BuildAppStage(stage.BootingStage): webhook_service_inst = webhook_service.WebhookService(ap) ap.webhook_service = webhook_service_inst + ap.skill_repository = skill_repository.SkillRepository(ap) skill_service_inst = skill_service.SkillService(ap) ap.skill_service = skill_service_inst diff --git a/src/langbot/pkg/provider/tools/loaders/native.py b/src/langbot/pkg/provider/tools/loaders/native.py index 602c22f62..254854fae 100644 --- a/src/langbot/pkg/provider/tools/loaders/native.py +++ b/src/langbot/pkg/provider/tools/loaders/native.py @@ -33,7 +33,7 @@ EDIT_TOOL_NAME = 'edit' GLOB_TOOL_NAME = 'glob' GREP_TOOL_NAME = 'grep' -_ALL_TOOL_NAMES = {EXEC_TOOL_NAME, READ_TOOL_NAME, WRITE_TOOL_NAME, EDIT_TOOL_NAME, GLOB_TOOL_NAME, GREP_TOOL_NAME} +SANDBOX_TOOL_NAMES = {EXEC_TOOL_NAME, READ_TOOL_NAME, WRITE_TOOL_NAME, EDIT_TOOL_NAME, GLOB_TOOL_NAME, GREP_TOOL_NAME} # Skip these dirs during grep walk to avoid noise _SKIP_DIRS = {'.git', 'node_modules', '__pycache__', '.venv', 'venv', '.tox', 'dist', 'build'} @@ -261,7 +261,11 @@ class NativeToolLoader(loader.ToolLoader): return list(self._tools) async def has_tool(self, name: str) -> bool: - return name in _ALL_TOOL_NAMES and await self._is_sandbox_available() + return name in SANDBOX_TOOL_NAMES and await self._is_sandbox_available() + + @staticmethod + def recognizes_tool(name: str) -> bool: + return name in SANDBOX_TOOL_NAMES async def invoke_tool(self, name: str, parameters: dict, query: pipeline_query.Query): from ....box.runner import binding_for @@ -1135,7 +1139,8 @@ else: include_visible=True, include_activated=True, ) - if skill_request is not None and hasattr(self.ap.box_service, 'read_skill_file'): + skill_repository = getattr(self.ap, 'skill_repository', None) + if skill_request is not None and skill_repository is not None: selected_skill, relative = skill_request if self._can_interpret_skill_host_paths(): host_location = self._resolve_skill_host_location(selected_skill, relative) @@ -1148,7 +1153,7 @@ else: pass try: - result = await self.ap.box_service.read_skill_file( + result = await skill_repository.read_skill_file( self._execution_context(query), selected_skill['name'], relative, @@ -1156,7 +1161,7 @@ else: return self._build_read_result_from_text(str(result.get('content', '')), parameters) except Exception: try: - result = await self.ap.box_service.list_skill_files( + result = await skill_repository.list_skill_files( self._execution_context(query), selected_skill['name'], relative, @@ -1190,12 +1195,13 @@ else: include_visible=False, include_activated=True, ) - if skill_request is not None and hasattr(self.ap.box_service, 'write_skill_file'): + skill_repository = getattr(self.ap, 'skill_repository', None) + if skill_request is not None and skill_repository is not None: if encoding != 'text': return {'ok': False, 'error': 'base64 writes to skill packages are not supported.'} selected_skill, relative = skill_request execution_context = self._execution_context(query) - await self.ap.box_service.write_skill_file(execution_context, selected_skill['name'], relative, content) + await skill_repository.write_skill_file(execution_context, selected_skill['name'], relative, content) await self.ap.skill_mgr.reload_skills(execution_context) return {'ok': True, 'path': path} @@ -1228,14 +1234,10 @@ else: include_visible=False, include_activated=True, ) - if ( - skill_request is not None - and hasattr(self.ap.box_service, 'read_skill_file') - and hasattr(self.ap.box_service, 'write_skill_file') - ): + if skill_request is not None and getattr(self.ap, 'skill_repository', None) is not None: selected_skill, relative = skill_request try: - result = await self.ap.box_service.read_skill_file( + result = await self.ap.skill_repository.read_skill_file( self._execution_context(query), selected_skill['name'], relative, @@ -1250,7 +1252,7 @@ else: return {'ok': False, 'error': f'old_string matches {count} locations; provide a more unique string.'} new_content = content.replace(old_string, new_string, 1) execution_context = self._execution_context(query) - await self.ap.box_service.write_skill_file( + await self.ap.skill_repository.write_skill_file( execution_context, selected_skill['name'], relative, diff --git a/src/langbot/pkg/provider/tools/loaders/skill_authoring.py b/src/langbot/pkg/provider/tools/loaders/skill_authoring.py index 4a9b91459..dc0074d47 100644 --- a/src/langbot/pkg/provider/tools/loaders/skill_authoring.py +++ b/src/langbot/pkg/provider/tools/loaders/skill_authoring.py @@ -6,7 +6,6 @@ import typing import langbot_plugin.api.entities.builtin.resource.tool as resource_tool from .. import loader -from .availability import is_box_backend_available from ....api.http.context import ExecutionContext # Align with Claude Code's Skill tool design: @@ -15,12 +14,23 @@ from ....api.http.context import ExecutionContext # - This protects KV Cache and follows industry standard ACTIVATE_SKILL_TOOL_NAME = 'activate' +LIST_SKILL_RESOURCES_TOOL_NAME = 'list_skill_resources' +READ_SKILL_RESOURCE_TOOL_NAME = 'read_skill_resource' REGISTER_SKILL_TOOL_NAME = 'register_skill' -SKILL_TOOL_NAMES = { +READ_ONLY_SKILL_TOOL_NAMES = { ACTIVATE_SKILL_TOOL_NAME, + LIST_SKILL_RESOURCES_TOOL_NAME, + READ_SKILL_RESOURCE_TOOL_NAME, +} +SANDBOX_SKILL_TOOL_NAMES = { REGISTER_SKILL_TOOL_NAME, } +SKILL_TOOL_NAMES = READ_ONLY_SKILL_TOOL_NAMES | SANDBOX_SKILL_TOOL_NAMES +_SKILL_EXECUTION_AVAILABLE_KEY = '_skill_execution_available' +_SKILL_RESOURCE_BYTES_READ_KEY = '_skill_resource_bytes_read' +_MAX_SKILL_RESOURCE_FILE_BYTES = 256 * 1024 +_MAX_SKILL_RESOURCE_RUN_BYTES = 1024 * 1024 class SkillToolLoader(loader.ToolLoader): @@ -28,62 +38,73 @@ class SkillToolLoader(loader.ToolLoader): def __init__(self, ap): super().__init__(ap) - self._tools: list[resource_tool.LLMTool] = [] - self._sandbox_available: bool = False + self._read_only_tools: list[resource_tool.LLMTool] = [] + self._sandbox_tools: list[resource_tool.LLMTool] = [] async def initialize(self): - # Check if sandbox backend is available (same check as native tools) - self._sandbox_available = await self._check_sandbox_available() - if self._sandbox_available: - self._tools = [ + if self._is_available(): + self._read_only_tools = [ self._build_activate_skill_tool(), - self._build_register_skill_tool(), + self._build_list_skill_resources_tool(), + self._build_read_skill_resource_tool(), ] + self._sandbox_tools = [self._build_register_skill_tool()] else: - self.ap.logger.info( - 'Skill tools (activate/register_skill) are NOT available. ' - 'No sandbox backend (Docker/nsjail/E2B) is ready. ' - 'Trusted local development may explicitly select box.backend=host.' - ) + self.ap.logger.info('Skill tools are unavailable because the Core SkillRepository is not initialized.') - async def _check_sandbox_available(self) -> bool: - """Check if the box backend is truly available (not just the runtime).""" - return await is_box_backend_available(self.ap) - - async def get_tools(self, bound_plugins: list[str] | None = None) -> list[resource_tool.LLMTool]: - if not await self._is_available(): + async def get_tools( + self, + bound_plugins: list[str] | None = None, + *, + sandbox_available: bool | None = None, + ) -> list[resource_tool.LLMTool]: + if not self._is_available(): return [] - if not self._tools: - self._tools = [ - self._build_activate_skill_tool(), - self._build_register_skill_tool(), - ] - return list(self._tools) + if not self._read_only_tools: + await self.initialize() + tools = list(self._read_only_tools) + if sandbox_available: + tools.extend(self._sandbox_tools) + return tools - async def has_tool(self, name: str) -> bool: - return await self._is_available() and name in SKILL_TOOL_NAMES + async def get_tool(self, name: str, *, sandbox_available: bool | None = None): + for tool in await self.get_tools(sandbox_available=sandbox_available): + if tool.name == name: + return tool + return None - async def _is_available(self) -> bool: - """Check if skill tools should be available. - - Skill tools require both a skill manager and a sandbox backend. - """ - if not self._has_skill_manager(): + async def has_tool(self, name: str, *, sandbox_available: bool | None = None) -> bool: + if not self._is_available() or name not in SKILL_TOOL_NAMES: return False - self._sandbox_available = await self._check_sandbox_available() - return self._sandbox_available + return name in READ_ONLY_SKILL_TOOL_NAMES or bool(sandbox_available) + + @staticmethod + def is_sandbox_tool(name: str) -> bool: + return name in SANDBOX_SKILL_TOOL_NAMES + + @staticmethod + def recognizes_tool(name: str) -> bool: + return name in SKILL_TOOL_NAMES + + def _is_available(self) -> bool: + return self._has_skill_manager() and getattr(self.ap, 'skill_repository', None) is not None async def invoke_tool(self, name: str, parameters: dict, query) -> typing.Any: - require_sandbox = getattr( - getattr(self.ap, 'box_service', None), - 'require_workspace_sandbox', - None, - ) - if callable(require_sandbox): - await require_sandbox(self._execution_context(query)) if name == ACTIVATE_SKILL_TOOL_NAME: return await self._invoke_activate_skill(parameters, query) + if name == LIST_SKILL_RESOURCES_TOOL_NAME: + return await self._invoke_list_skill_resources(parameters, query) + if name == READ_SKILL_RESOURCE_TOOL_NAME: + return await self._invoke_read_skill_resource(parameters, query) if name == REGISTER_SKILL_TOOL_NAME: + require_sandbox = getattr( + getattr(self.ap, 'box_service', None), + 'require_workspace_sandbox', + None, + ) + if not callable(require_sandbox): + return self._sandbox_unavailable_result(name) + await require_sandbox(self._execution_context(query)) return await self._invoke_register_skill(parameters, query) raise ValueError(f'Unknown skill tool: {name}') @@ -108,6 +129,27 @@ class SkillToolLoader(loader.ToolLoader): def _has_skill_manager(self) -> bool: return getattr(self.ap, 'skill_mgr', None) is not None + @staticmethod + def _sandbox_unavailable_result(name: str) -> dict: + return { + 'ok': False, + 'code': 'sandbox_unavailable', + 'tool': name, + 'message': 'This operation requires Box execution, but Box is not configured or available.', + } + + async def _execution_available(self, query) -> bool: + variables = getattr(query, 'variables', None) + if isinstance(variables, dict) and _SKILL_EXECUTION_AVAILABLE_KEY in variables: + return bool(variables[_SKILL_EXECUTION_AVAILABLE_KEY]) + checker = getattr(getattr(self.ap, 'box_service', None), 'is_workspace_sandbox_available', None) + if not callable(checker): + return False + try: + return bool(await checker(self._execution_context(query))) + except Exception: + return False + async def _invoke_activate_skill(self, parameters: dict, query) -> typing.Any: """Activate a skill and return SKILL.md content via Tool Result.""" skill_name = str(parameters.get('skill_name', '') or '').strip() @@ -116,43 +158,112 @@ class SkillToolLoader(loader.ToolLoader): from . import skill as skill_loader - skill_data = skill_loader.get_visible_skill(self.ap, query, skill_name) - if skill_data is None: + visible_skill = skill_loader.get_visible_skill(self.ap, query, skill_name) + if visible_skill is None: visible_skills = skill_loader.get_visible_skills(self.ap, query) available_names = ', '.join(sorted(visible_skills.keys())) or 'none' raise ValueError(f'Skill "{skill_name}" not found. Available skills: {available_names}') - # Register activated skill for sandbox mount path resolution + skill_data = await self.ap.skill_repository.get_skill( + self._execution_context(query), + skill_name, + snapshot=True, + ) + if skill_data is None: + raise ValueError(f'Skill "{skill_name}" is no longer available; reload the skill catalog.') + skill_loader.register_activated_skill(query, skill_data) await skill_loader.persist_activated_skill(self.ap, query, skill_name) - # Return SKILL.md content as Tool Result (injects into context) instructions = skill_data.get('instructions', '') - package_root = skill_data.get('package_root', '') - mount_path = skill_loader.get_virtual_skill_mount_path(skill_name) + revision = str(skill_data.get('revision', '') or '') + execution_available = await self._execution_available(query) + mount_path = skill_loader.get_virtual_skill_mount_path(skill_name) if execution_available else None - # Build Tool Result content result_content = f'The "{skill_name}" skill is activated\n' result_content += '\n' result_content += f'{skill_name}\n' - result_content += f'{mount_path}\n' - result_content += f'{package_root}\n' + result_content += f'{revision}\n' + result_content += 'true\n' + result_content += f'{str(execution_available).lower()}\n' result_content += f'\n## Instructions\n{instructions}\n' result_content += '\n## Runtime Context\n' - result_content += f'The skill package is mounted at {mount_path}. Use the standard tools to interact with it:\n' - result_content += f'- Use `read` to inspect files under {mount_path}\n' - result_content += f'- Use `exec` with workdir set to {mount_path} to run commands in that package\n' - result_content += '- Use `write` and `edit` on that path when the instructions require updating files\n' + result_content += '- Use `list_skill_resources` and `read_skill_resource` for read-only package resources.\n' + if execution_available: + result_content += ( + f'- Box execution is available; executable package files will be mounted at {mount_path}.\n' + ) + else: + result_content += ( + '- Box execution is unavailable. Do not attempt to run scripts or modify Workspace files.\n' + ) result_content += '\n' return { 'activated': True, 'skill_name': skill_name, 'mount_path': mount_path, + 'revision': revision, + 'capabilities': { + 'instructions_readable': True, + 'resources_readable': True, + 'execution_available': execution_available, + }, 'activated_skill_names': skill_loader.get_activated_skill_names(query), 'content': result_content, } + @staticmethod + def _activated_skill(parameters: dict, query) -> dict: + from . import skill as skill_loader + + skill_name = str(parameters.get('skill_name', '') or '').strip() + if not skill_name: + raise ValueError('skill_name is required') + skill_data = skill_loader.get_activated_skill(query, skill_name) + if skill_data is None: + raise ValueError(f'Skill "{skill_name}" must be activated before its resources can be read.') + requested_revision = str(parameters.get('revision', '') or '').strip() + activated_revision = str(skill_data.get('revision', '') or '').strip() + if requested_revision and requested_revision != activated_revision: + raise ValueError('revision must match the activated skill revision') + return skill_data + + async def _invoke_list_skill_resources(self, parameters: dict, query) -> dict: + skill_data = self._activated_skill(parameters, query) + return await self.ap.skill_repository.list_skill_resources( + self._execution_context(query), + skill_data['name'], + str(parameters.get('path', '.') or '.'), + expected_revision=skill_data.get('revision'), + ) + + async def _invoke_read_skill_resource(self, parameters: dict, query) -> dict: + skill_data = self._activated_skill(parameters, query) + path = str(parameters.get('path', '') or '').strip() + if not path: + raise ValueError('path is required') + result = await self.ap.skill_repository.read_skill_resource( + self._execution_context(query), + skill_data['name'], + path, + expected_revision=skill_data.get('revision'), + ) + content = str(result.get('content', '')) + size = len(content.encode('utf-8')) + if size > _MAX_SKILL_RESOURCE_FILE_BYTES: + raise ValueError('Skill resource exceeds the per-file read limit') + variables = getattr(query, 'variables', None) + if not isinstance(variables, dict): + variables = {} + query.variables = variables + total = int(variables.get(_SKILL_RESOURCE_BYTES_READ_KEY, 0) or 0) + size + if total > _MAX_SKILL_RESOURCE_RUN_BYTES: + raise ValueError('Skill resource reads exceed the per-run limit') + variables[_SKILL_RESOURCE_BYTES_READ_KEY] = total + result['size'] = size + return result + async def _invoke_register_skill(self, parameters: dict, query) -> typing.Any: """Register a skill from sandbox directory to data/skills/.""" from . import skill as skill_loader @@ -179,14 +290,14 @@ class SkillToolLoader(loader.ToolLoader): raise ValueError('skill name is required') # Create the skill - created = await skill_service.create_skill( + created = await skill_service.import_skill_directory( execution_context, + host_path, { 'name': skill_name, 'display_name': str(parameters.get('display_name') or scanned.get('display_name', '')).strip(), 'description': str(parameters.get('description') or scanned.get('description', '')).strip(), 'instructions': str(parameters.get('instructions') or scanned.get('instructions', '')), - 'package_root': host_path, }, ) skill_loader.register_created_skill_visibility(query, skill_name) @@ -253,6 +364,42 @@ class SkillToolLoader(loader.ToolLoader): func=lambda parameters: parameters, ) + def _build_list_skill_resources_tool(self) -> resource_tool.LLMTool: + return resource_tool.LLMTool( + name=LIST_SKILL_RESOURCES_TOOL_NAME, + human_desc='List activated skill resources', + description='List read-only files in an activated skill package without starting a sandbox.', + parameters={ + 'type': 'object', + 'properties': { + 'skill_name': {'type': 'string', 'description': 'The activated skill name.'}, + 'path': {'type': 'string', 'description': 'Relative directory path. Defaults to the package root.'}, + 'revision': {'type': 'string', 'description': 'Optional revision returned by activate.'}, + }, + 'required': ['skill_name'], + 'additionalProperties': False, + }, + func=lambda parameters: parameters, + ) + + def _build_read_skill_resource_tool(self) -> resource_tool.LLMTool: + return resource_tool.LLMTool( + name=READ_SKILL_RESOURCE_TOOL_NAME, + human_desc='Read an activated skill resource', + description='Read a UTF-8 text resource from an activated skill package without starting a sandbox.', + parameters={ + 'type': 'object', + 'properties': { + 'skill_name': {'type': 'string', 'description': 'The activated skill name.'}, + 'path': {'type': 'string', 'description': 'File path relative to the skill package root.'}, + 'revision': {'type': 'string', 'description': 'Optional revision returned by activate.'}, + }, + 'required': ['skill_name', 'path'], + 'additionalProperties': False, + }, + func=lambda parameters: parameters, + ) + def _build_register_skill_tool(self) -> resource_tool.LLMTool: return resource_tool.LLMTool( name=REGISTER_SKILL_TOOL_NAME, diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index 0dda17b56..fe6f8dfd4 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -78,6 +78,15 @@ class ToolManager: except Exception: return False + @staticmethod + def _sandbox_unavailable_result(name: str) -> dict[str, typing.Any]: + return { + 'ok': False, + 'code': 'sandbox_unavailable', + 'tool': name, + 'message': 'This operation requires Box execution, but Box is not configured or available.', + } + async def initialize(self): from langbot.pkg.utils import importutil from langbot.pkg.provider.tools import loaders @@ -114,8 +123,8 @@ class ToolManager: sandbox_available = await self._workspace_sandbox_available(context) if sandbox_available: all_functions.extend(await self.native_tool_loader.get_tools()) - if include_skill_authoring and sandbox_available: - all_functions.extend(await self.skill_tool_loader.get_tools()) + if include_skill_authoring: + all_functions.extend(await self.skill_tool_loader.get_tools(sandbox_available=sandbox_available)) all_functions.extend(await self.plugin_tool_loader.get_tools(bound_plugins)) all_functions.extend( await self.mcp_tool_loader.get_tools( @@ -154,8 +163,12 @@ class ToolManager: sandbox_available = await self._workspace_sandbox_available(context) if sandbox_available: append_tools('builtin', 'LangBot', await self.native_tool_loader.get_tools()) - if include_skill_authoring and sandbox_available: - append_tools('skill', 'LangBot', await self.skill_tool_loader.get_tools()) + if include_skill_authoring: + append_tools( + 'skill', + 'LangBot', + await self.skill_tool_loader.get_tools(sandbox_available=sandbox_available), + ) catalog.extend(await self.plugin_tool_loader.get_tool_catalog(bound_plugins)) if self.mcp_tool_loader: @@ -297,10 +310,9 @@ class ToolManager: tool = await active_loader.get_tool(name) if tool: return tool - if sandbox_available: - tool = await self.skill_tool_loader.get_tool(name) - if tool: - return tool + tool = await self.skill_tool_loader.get_tool(name, sandbox_available=sandbox_available) + if tool: + return tool return await self.mcp_tool_loader.get_tool(context, name) @@ -576,7 +588,10 @@ class ToolManager: query=query, invoke=lambda: self.mcp_tool_loader.invoke_tool(name, parameters, query), ) - if sandbox_available and await self.skill_tool_loader.has_tool(name): + if await self.skill_tool_loader.has_tool(name, sandbox_available=sandbox_available): + variables = getattr(query, 'variables', None) + if isinstance(variables, dict): + variables['_skill_execution_available'] = sandbox_available telemetry_features.increment(query, 'tool_calls', 'skill') return await self._invoke_tool_with_monitoring( source='skill', @@ -585,6 +600,13 @@ class ToolManager: query=query, invoke=lambda: self.skill_tool_loader.invoke_tool(name, parameters, query), ) + recognizes_native = getattr(self.native_tool_loader, 'recognizes_tool', None) + is_sandbox_skill_tool = getattr(self.skill_tool_loader, 'is_sandbox_tool', None) + if not sandbox_available and ( + (callable(recognizes_native) and recognizes_native(name) is True) + or (callable(is_sandbox_skill_tool) and is_sandbox_skill_tool(name) is True) + ): + return self._sandbox_unavailable_result(name) raise ToolNotFoundError(name) async def shutdown(self): diff --git a/src/langbot/pkg/skill/__init__.py b/src/langbot/pkg/skill/__init__.py index b96f23ca1..96e673f3a 100644 --- a/src/langbot/pkg/skill/__init__.py +++ b/src/langbot/pkg/skill/__init__.py @@ -1,3 +1,5 @@ from .manager import SkillManager +from .repository import SkillRepository -__all__ = ['SkillManager'] + +__all__ = ['SkillManager', 'SkillRepository'] diff --git a/src/langbot/pkg/skill/manager.py b/src/langbot/pkg/skill/manager.py index aeb688b7f..d0fef97c0 100644 --- a/src/langbot/pkg/skill/manager.py +++ b/src/langbot/pkg/skill/manager.py @@ -1,14 +1,12 @@ from __future__ import annotations -import os - from ..api.http.context import ExecutionContext from ..api.http.service.tenant import TenantContext, require_workspace_uuid from ..core import app class SkillManager: - """Workspace-scoped in-memory view of Box-managed skill packages.""" + """Workspace-scoped in-memory view of Core-managed skill packages.""" ap: app.Application @@ -63,33 +61,20 @@ class SkillManager: self._skills_by_scope.pop(existing_key, None) self._skills_by_scope[key] = {} - box_service = getattr(self.ap, 'box_service', None) - if box_service is None or not getattr(box_service, 'available', False): - self.ap.logger.info( - f'Box runtime unavailable; skill cache is empty for Workspace {execution_context.workspace_uuid}.' - ) + repository = getattr(self.ap, 'skill_repository', None) + if repository is None: + self.ap.logger.info('Skill repository unavailable; skill cache will remain empty.') return - validate_locally = bool(getattr(box_service, 'shares_filesystem_with_box', False)) try: - dropped = 0 skills: dict[str, dict] = {} - for skill_data in await box_service.list_skills(execution_context): + for skill_data in await repository.list_skills(execution_context): skill_name = skill_data.get('name') if not skill_name: continue - package_root = str(skill_data.get('package_root', '') or '').strip() - if validate_locally and package_root and not os.path.isdir(package_root): - self.ap.logger.warning( - f'Skill "{skill_name}" reported by Box runtime but package_root ' - f'missing on LangBot filesystem ({package_root}); dropping from cache.' - ) - dropped += 1 - continue skills[skill_name] = skill_data self._skills_by_scope[key] = skills - suffix = f' ({dropped} dropped due to missing package_root)' if dropped else '' - self.ap.logger.info(f'Loaded {len(skills)} skills for Workspace {execution_context.workspace_uuid}{suffix}') + self.ap.logger.info(f'Loaded {len(skills)} skills for Workspace {execution_context.workspace_uuid}') except Exception as exc: self.ap.logger.warning(f'Failed to load skills for Workspace {execution_context.workspace_uuid}: {exc}') diff --git a/src/langbot/pkg/skill/repository.py b/src/langbot/pkg/skill/repository.py new file mode 100644 index 000000000..2d04f7cf3 --- /dev/null +++ b/src/langbot/pkg/skill/repository.py @@ -0,0 +1,192 @@ +from __future__ import annotations + +import asyncio +import os + +from langbot_plugin.skill_store import ( + SkillRevisionMismatchError, + SkillStore, + skill_namespace, +) + +from ..api.http.context import ExecutionContext +from ..api.http.service.tenant import TenantContext, require_workspace_uuid +from ..utils.bounded_executor import blocking_work_scope, run_blocking_atomic + + +class SkillRepository: + """Async, Workspace-scoped adapter around the SDK SkillStore.""" + + def __init__(self, ap) -> None: + self.ap = ap + config = getattr(getattr(ap, 'instance_config', None), 'data', {}) or {} + self._local_config = (config.get('box') or {}).get('local') or {} + self._store = SkillStore(self._skills_root()) + self._lock = asyncio.Lock() + + def _host_root(self) -> str: + configured = str(self._local_config.get('host_root') or './data/box').strip() + return os.path.realpath(os.path.abspath(os.path.expanduser(configured))) + + def _skills_root(self) -> str: + configured = str(self._local_config.get('skills_root') or 'skills').strip() + if not os.path.isabs(configured): + configured = os.path.join(self._host_root(), configured) + return os.path.realpath(os.path.abspath(os.path.expanduser(configured))) + + def _default_workspace(self) -> str: + configured = str(self._local_config.get('default_workspace') or '').strip() + if not configured: + configured = os.path.join(self._host_root(), 'default') + elif not os.path.isabs(configured): + configured = os.path.join(self._host_root(), configured) + return os.path.realpath(os.path.abspath(os.path.expanduser(configured))) + + @staticmethod + def _execution_context(context: TenantContext) -> ExecutionContext: + workspace_uuid = require_workspace_uuid(context) + instance_uuid = str(getattr(context, 'instance_uuid', '') or '').strip() + generation = getattr(context, 'placement_generation', None) + if not instance_uuid: + raise ValueError('Skill operations require an explicit instance UUID') + if isinstance(generation, bool) or not isinstance(generation, int) or generation <= 0: + raise ValueError('Skill operations require a positive placement generation') + return ExecutionContext( + instance_uuid=instance_uuid, + workspace_uuid=workspace_uuid, + placement_generation=generation, + bot_uuid=getattr(context, 'bot_uuid', None), + pipeline_uuid=getattr(context, 'pipeline_uuid', None), + query_uuid=getattr(context, 'query_uuid', None), + entitlement_revision=getattr(context, 'entitlement_revision', 0), + ) + + @classmethod + def _namespace(cls, context: TenantContext) -> str: + execution_context = cls._execution_context(context) + return skill_namespace( + execution_context.instance_uuid, + execution_context.workspace_uuid, + ) + + async def _validated_execution_context(self, context: TenantContext) -> ExecutionContext: + execution_context = self._execution_context(context) + binding = await self.ap.workspace_service.get_execution_binding( + execution_context.workspace_uuid, + expected_generation=execution_context.placement_generation, + ) + if ( + binding.instance_uuid != execution_context.instance_uuid + or str(getattr(binding, 'workspace_uuid', '') or '') != execution_context.workspace_uuid + or getattr(binding, 'placement_generation', None) != execution_context.placement_generation + ): + raise ValueError('Skill execution context belongs to a stale Workspace placement') + return execution_context + + def _workspace_root(self, namespace: str) -> str: + return os.path.join(self._default_workspace(), 'tenants', namespace) + + async def _call(self, context: TenantContext, method_name: str, *args, **kwargs): + execution_context = await self._validated_execution_context(context) + namespace = self._namespace(execution_context) + + def invoke(): + method = getattr(self._store.scoped(namespace), method_name) + return method(*args, **kwargs) + + async with self._lock: + with blocking_work_scope(f'skill:{namespace}'): + return await run_blocking_atomic(invoke) + + async def list_skills(self, context: TenantContext) -> list[dict]: + return await self._call(context, 'list_skills') + + async def get_skill(self, context: TenantContext, name: str, *, snapshot: bool = False) -> dict | None: + return await self._call(context, 'get_skill_snapshot' if snapshot else 'get_skill', name) + + async def create_skill(self, context: TenantContext, skill: dict) -> dict: + return await self._call(context, 'create_skill', skill) + + async def import_skill_directory(self, context: TenantContext, path: str, skill: dict) -> dict: + namespace = self._namespace(context) + return await self._call( + context, + 'import_skill_directory', + path, + skill, + source_root=self._workspace_root(namespace), + ) + + async def update_skill(self, context: TenantContext, name: str, skill: dict) -> dict: + return await self._call(context, 'update_skill', name, skill) + + async def delete_skill(self, context: TenantContext, name: str) -> None: + await self._call(context, 'delete_skill', name) + + async def scan_skill_directory(self, context: TenantContext, path: str) -> dict: + namespace = self._namespace(context) + return await self._call( + context, + 'scan_import_directory', + path, + source_root=self._workspace_root(namespace), + ) + + async def list_skill_files( + self, + context: TenantContext, + name: str, + path: str = '.', + include_hidden: bool = False, + max_entries: int = 200, + ) -> dict: + return await self._call(context, 'list_skill_files', name, path, include_hidden, max_entries) + + async def read_skill_file(self, context: TenantContext, name: str, path: str) -> dict: + return await self._call(context, 'read_skill_file', name, path) + + async def list_skill_resources( + self, + context: TenantContext, + name: str, + path: str = '.', + *, + expected_revision: str | None = None, + ) -> dict: + return await self._call( + context, + 'list_skill_resources', + name, + path, + False, + 200, + expected_revision=expected_revision, + ) + + async def read_skill_resource( + self, + context: TenantContext, + name: str, + path: str, + *, + expected_revision: str | None = None, + ) -> dict: + return await self._call( + context, + 'read_skill_resource', + name, + path, + expected_revision=expected_revision, + ) + + async def write_skill_file(self, context: TenantContext, name: str, path: str, content: str) -> dict: + return await self._call(context, 'write_skill_file', name, path, content) + + async def preview_skill_zip(self, context: TenantContext, file_bytes: bytes, filename: str, **kwargs) -> list[dict]: + return await self._call(context, 'preview_zip_upload', file_bytes=file_bytes, filename=filename, **kwargs) + + async def install_skill_zip(self, context: TenantContext, file_bytes: bytes, filename: str, **kwargs) -> list[dict]: + return await self._call(context, 'install_zip_upload', file_bytes=file_bytes, filename=filename, **kwargs) + + +__all__ = ['SkillRepository', 'SkillRevisionMismatchError'] diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index 91303c4d1..9d3ce0f42 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -346,10 +346,10 @@ monitoring: box: # Master switch for the Box sandbox runtime. When false, LangBot does NOT # attempt to connect to a remote Box runtime nor start a local stdio Box - # subprocess. Disabling Box also disables every feature that depends on it: - # the native sandbox tools (exec/read/write/edit/glob/grep), the activate - # skill tool, skill add/edit, and stdio-mode MCP servers. Skills can still - # be listed read-only and http/sse MCP servers continue to work. + # subprocess. Disabling Box disables execution-backed features: native + # sandbox tools (exec/read/write/edit/glob/grep), agent-side skill + # registration, and stdio-mode MCP servers. Skill management, activation, + # and read-only package resources remain available without Box. enabled: true # 'host' runs commands directly as the Box Runtime user without sandbox # isolation. It is never auto-selected and is only for trusted local @@ -398,7 +398,7 @@ box: image: '' # Custom local sandbox image. Leave empty to use the profile default. host_root: './data/box' # Base host directory for local workspace mounts. Docker deployments should override this with an absolute host path. default_workspace: '' # Defaults to '/default'. Relative paths are resolved under host_root. - skills_root: 'skills' # Box-owned skill package directory. Relative paths are resolved under host_root. + skills_root: 'skills' # Core-owned skill repository shared with Box for optional execution mounts. allowed_mount_roots: # Defaults to [''] when left empty. - './data/box' - '/tmp' diff --git a/tests/integration/api/test_skills_entitlements.py b/tests/integration/api/test_skills_entitlements.py index baafe1f18..a936597ab 100644 --- a/tests/integration/api/test_skills_entitlements.py +++ b/tests/integration/api/test_skills_entitlements.py @@ -1,4 +1,4 @@ -"""Skills API behavior when a workspace plan has no managed sandbox.""" +"""Skills API behavior is independent from managed sandbox entitlement.""" from __future__ import annotations @@ -9,10 +9,7 @@ import pytest import quart from langbot.pkg.api.http.controller.groups.skills import SkillsRouterGroup -from langbot.pkg.cloud.entitlements import ( - EntitlementFeatureUnavailableError, - EntitlementUnavailableError, -) +from langbot.pkg.cloud.entitlements import EntitlementUnavailableError pytestmark = pytest.mark.integration WORKSPACE_UUID = '11111111-1111-4111-8111-111111111111' @@ -32,10 +29,7 @@ async def skills_api(): application.user_service.get_authenticated_account = AsyncMock(return_value=account) application.workspace_collaboration_service.resolve_account_workspace = AsyncMock(return_value=access) application.skill_service.list_skills = AsyncMock( - side_effect=EntitlementFeatureUnavailableError( - 'managed_sandbox', - entitlement_revision=1, - ) + return_value=[{'name': 'docs-only', 'description': 'No execution required'}] ) quart_app = quart.Quart(__name__) @@ -45,7 +39,7 @@ async def skills_api(): @pytest.mark.asyncio -async def test_list_skills_is_empty_when_plan_has_no_managed_sandbox(skills_api): +async def test_list_skills_remains_available_without_managed_sandbox(skills_api): application, client = skills_api response = await client.get( '/api/v1/skills', @@ -57,7 +51,7 @@ async def test_list_skills_is_empty_when_plan_has_no_managed_sandbox(skills_api) assert response.status_code == 200 payload = await response.get_json() - assert payload['data'] == {'skills': []} + assert payload['data'] == {'skills': [{'name': 'docs-only', 'description': 'No execution required'}]} application.skill_service.list_skills.assert_awaited_once() diff --git a/tests/unit_tests/box/test_box_service.py b/tests/unit_tests/box/test_box_service.py index 655eba7e1..25701979d 100644 --- a/tests/unit_tests/box/test_box_service.py +++ b/tests/unit_tests/box/test_box_service.py @@ -1840,7 +1840,7 @@ class TestBuildSkillExtraMounts: { 'host_path': live_dir, 'mount_path': '/workspace/.skills/alive', - 'mode': 'rw', + 'mode': 'ro', } ] # Warning logged so operators can see what was dropped @@ -1865,8 +1865,8 @@ class TestBuildSkillExtraMounts: mounts = service.build_skill_extra_mounts(make_query()) assert mounts == [ - {'host_path': '/box/skills/a', 'mount_path': '/workspace/.skills/a', 'mode': 'rw'}, - {'host_path': '/box/skills/b', 'mount_path': '/workspace/.skills/b', 'mode': 'rw'}, + {'host_path': '/box/skills/a', 'mount_path': '/workspace/.skills/a', 'mode': 'ro'}, + {'host_path': '/box/skills/b', 'mount_path': '/workspace/.skills/b', 'mode': 'ro'}, ] # No skill is dropped, so no "missing" warning should be logged. assert not any('package_root missing' in str(call.args[0]) for call in logger.warning.call_args_list) diff --git a/tests/unit_tests/provider/test_skill_tools.py b/tests/unit_tests/provider/test_skill_tools.py index b0e7288ca..17895e2a0 100644 --- a/tests/unit_tests/provider/test_skill_tools.py +++ b/tests/unit_tests/provider/test_skill_tools.py @@ -74,15 +74,10 @@ def _make_skill_data( class TestSkillManagerCache: - """The Box runtime is the only source of truth — SkillManager just holds - an in-memory cache populated by ``reload_skills``. There is no local - filesystem reader anymore.""" + """SkillManager caches the Core-owned SkillRepository catalog.""" def test_refresh_skill_from_disk_reports_cache_presence(self): - """Box is the only source of truth for skill content. refresh_skill_from_disk - now just reports whether the skill is still in the in-memory cache — - the actual content refresh is driven by SkillService awaiting - ``reload_skills`` after every Box mutation.""" + """Disk mutations are reflected by an explicit repository reload.""" from langbot.pkg.skill.manager import SkillManager ap = _make_ap() @@ -99,67 +94,26 @@ class TestSkillManagerCache: assert mgr.refresh_skill_from_disk(_CONTEXT, '') is False @pytest.mark.asyncio - async def test_reload_skills_drops_box_skills_with_missing_package_root(self): - """When LangBot shares a filesystem with Box (local stdio mode) and Box - reports a skill whose package_root is gone from that shared filesystem, - the cache must drop it instead of keeping a stale entry that would later - produce a bad mount.""" + async def test_reload_skills_uses_repository_when_box_is_disabled(self): from langbot.pkg.skill.manager import SkillManager - with tempfile.TemporaryDirectory() as live_dir: - ghost_dir = os.path.join(live_dir, '_does_not_exist') - box_service = SimpleNamespace( - available=True, - shares_filesystem_with_box=True, - list_skills=AsyncMock( - return_value=[ - _make_skill_data(name='alive', package_root=live_dir), - _make_skill_data(name='ghost', package_root=ghost_dir), - ] - ), - ) - - ap = _make_ap() - ap.box_service = box_service - mgr = SkillManager(ap) - - await mgr.reload_skills(_CONTEXT) - - assert list(mgr.get_skills(_CONTEXT)) == ['alive'] - # Warning fired with the dropped skill name so operators can see it. - warning_messages = [str(call.args[0]) for call in ap.logger.warning.call_args_list] - assert any('ghost' in msg and 'package_root missing' in msg for msg in warning_messages) - - @pytest.mark.asyncio - async def test_reload_skills_trusts_box_paths_when_filesystem_not_shared(self): - """In separated deployments (Docker Compose, k8s sidecar, - --standalone-box, remote endpoint) the package_root reported by Box - lives on the Box runtime's filesystem and is not resolvable on the - LangBot side. The cache must keep every Box-reported skill rather than - dropping them all via a local isdir() check.""" - from langbot.pkg.skill.manager import SkillManager - - box_service = SimpleNamespace( - available=True, - shares_filesystem_with_box=False, + repository = SimpleNamespace( list_skills=AsyncMock( return_value=[ - _make_skill_data(name='alpha', package_root='/box/skills/alpha'), - _make_skill_data(name='beta', package_root='/box/skills/beta'), + _make_skill_data(name='alpha', package_root='/skills/alpha'), + _make_skill_data(name='beta', package_root='/skills/beta'), ] ), ) - ap = _make_ap() - ap.box_service = box_service + ap.box_service = SimpleNamespace(available=False, enabled=False) + ap.skill_repository = repository mgr = SkillManager(ap) await mgr.reload_skills(_CONTEXT) assert sorted(mgr.get_skills(_CONTEXT)) == ['alpha', 'beta'] - # No skill dropped → no "package_root missing" warning. - warning_messages = [str(call.args[0]) for call in ap.logger.warning.call_args_list] - assert not any('package_root missing' in msg for msg in warning_messages) + repository.list_skills.assert_awaited_once_with(_CONTEXT) class TestSkillActivationHelper: @@ -329,7 +283,7 @@ class TestSkillPathHelpers: class TestSkillToolLoader: - """The skill tool surface is now just ``activate`` + ``register_skill``. + """Skill activation and resources are independent from sandbox execution. The legacy CRUD authoring tools (create/list/get/update/delete/ import_skill_from_directory/reload_skills) were removed; skill CRUD is @@ -345,8 +299,11 @@ class TestSkillToolLoader: from langbot.pkg.provider.tools.loaders.skill import ACTIVATED_SKILLS_KEY skill = _make_skill_data(name='demo', package_root='/data/skills/demo', instructions='Step 1') + skill['revision'] = 'sha256:demo' ap = _make_ap() ap.skill_mgr = _make_skill_manager({'demo': skill}) + ap.skill_repository = SimpleNamespace(get_skill=AsyncMock(return_value=skill)) + ap.box_service = SimpleNamespace(is_workspace_sandbox_available=AsyncMock(return_value=False)) loader = SkillToolLoader(ap) query = _make_query() @@ -355,9 +312,13 @@ class TestSkillToolLoader: assert result['activated'] is True assert result['skill_name'] == 'demo' - assert result['mount_path'] == '/workspace/.skills/demo' + assert result['mount_path'] is None + assert result['revision'] == 'sha256:demo' + assert result['capabilities']['resources_readable'] is True + assert result['capabilities']['execution_available'] is False assert result['activated_skill_names'] == ['demo'] assert 'Step 1' in result['content'] + assert '' not in result['content'] assert set(query.variables[ACTIVATED_SKILLS_KEY].keys()) == {'demo'} @pytest.mark.asyncio @@ -391,7 +352,11 @@ class TestSkillToolLoader: os.makedirs(repo_dir) ap = _make_ap() - ap.box_service = SimpleNamespace(default_workspace=tmpdir, available=True) + ap.box_service = SimpleNamespace( + default_workspace=tmpdir, + available=True, + require_workspace_sandbox=AsyncMock(return_value=_CONTEXT), + ) ap.skill_service = SimpleNamespace( scan_directory_async=AsyncMock( return_value={ @@ -401,7 +366,7 @@ class TestSkillToolLoader: 'instructions': 'Do work', } ), - create_skill=AsyncMock( + import_skill_directory=AsyncMock( return_value=_make_skill_data(name='cloned-skill', package_root=os.path.realpath(repo_dir)) ), ) @@ -414,14 +379,14 @@ class TestSkillToolLoader: ) ap.skill_service.scan_directory_async.assert_awaited_once_with(_CONTEXT, os.path.realpath(repo_dir)) - ap.skill_service.create_skill.assert_awaited_once_with( + ap.skill_service.import_skill_directory.assert_awaited_once_with( _CONTEXT, + os.path.realpath(repo_dir), { 'name': 'cloned-skill', 'display_name': 'Cloned Skill', 'description': 'Imported from clone', 'instructions': 'Do work', - 'package_root': os.path.realpath(repo_dir), }, ) assert result['registered'] is True @@ -437,8 +402,15 @@ class TestSkillToolLoader: with tempfile.TemporaryDirectory() as tmpdir: ap = _make_ap() - ap.box_service = SimpleNamespace(default_workspace=tmpdir, available=True) - ap.skill_service = SimpleNamespace(scan_directory_async=AsyncMock(), create_skill=AsyncMock()) + ap.box_service = SimpleNamespace( + default_workspace=tmpdir, + available=True, + require_workspace_sandbox=AsyncMock(return_value=_CONTEXT), + ) + ap.skill_service = SimpleNamespace( + scan_directory_async=AsyncMock(), + import_skill_directory=AsyncMock(), + ) loader = SkillToolLoader(ap) @@ -458,7 +430,11 @@ class TestSkillToolLoader: with tempfile.TemporaryDirectory() as tmpdir: ap = _make_ap() # no skill_service attribute - ap.box_service = SimpleNamespace(default_workspace=tmpdir, available=True) + ap.box_service = SimpleNamespace( + default_workspace=tmpdir, + available=True, + require_workspace_sandbox=AsyncMock(return_value=_CONTEXT), + ) loader = SkillToolLoader(ap) @@ -470,21 +446,22 @@ class TestSkillToolLoader: ) @pytest.mark.asyncio - async def test_tools_hidden_when_sandbox_backend_unavailable(self): + async def test_read_only_tools_remain_when_sandbox_unavailable(self): from langbot.pkg.provider.tools.loaders.skill_authoring import SkillToolLoader ap = _make_ap() ap.skill_mgr = SimpleNamespace(skills={}) - ap.box_service = SimpleNamespace( - available=True, - get_backend_status=AsyncMock(return_value={'backend': {'available': False}}), - ) + ap.skill_repository = SimpleNamespace() loader = SkillToolLoader(ap) await loader.initialize() - assert await loader.get_tools() == [] - assert await loader.has_tool('activate') is False + assert sorted(tool.name for tool in await loader.get_tools(sandbox_available=False)) == [ + 'activate', + 'list_skill_resources', + 'read_skill_resource', + ] + assert await loader.has_tool('activate') is True assert await loader.has_tool('register_skill') is False @pytest.mark.asyncio @@ -493,38 +470,89 @@ class TestSkillToolLoader: ap = _make_ap() ap.skill_mgr = _make_skill_manager({'demo': _make_skill_data(name='demo')}) - ap.box_service = SimpleNamespace( - available=True, - get_backend_status=AsyncMock(return_value={'backend': {'available': True}}), - ) + ap.skill_repository = SimpleNamespace() loader = SkillToolLoader(ap) await loader.initialize() - tools = await loader.get_tools() + tools = await loader.get_tools(sandbox_available=True) - assert sorted(tool.name for tool in tools) == ['activate', 'register_skill'] + assert sorted(tool.name for tool in tools) == [ + 'activate', + 'list_skill_resources', + 'read_skill_resource', + 'register_skill', + ] assert await loader.has_tool('activate') is True - assert await loader.has_tool('register_skill') is True + assert await loader.has_tool('register_skill', sandbox_available=True) is True @pytest.mark.asyncio - async def test_tools_reappear_after_box_backend_recovers(self): + async def test_register_skill_appears_after_sandbox_recovers(self): from langbot.pkg.provider.tools.loaders.skill_authoring import SkillToolLoader ap = _make_ap() ap.skill_mgr = SimpleNamespace(skills={'demo': _make_skill_data(name='demo')}) - ap.box_service = SimpleNamespace( - available=False, - get_backend_status=AsyncMock(return_value={'backend': {'available': True}}), - ) + ap.skill_repository = SimpleNamespace() loader = SkillToolLoader(ap) await loader.initialize() - assert await loader.get_tools() == [] + assert 'register_skill' not in {tool.name for tool in await loader.get_tools(sandbox_available=False)} + assert 'register_skill' in {tool.name for tool in await loader.get_tools(sandbox_available=True)} - ap.box_service.available = True + @pytest.mark.asyncio + async def test_resources_require_activation_and_use_pinned_revision(self): + from langbot.pkg.provider.tools.loaders.skill import register_activated_skill + from langbot.pkg.provider.tools.loaders.skill_authoring import SkillToolLoader - assert sorted(tool.name for tool in await loader.get_tools()) == ['activate', 'register_skill'] + skill = _make_skill_data(name='demo', instructions='Read references') + skill['revision'] = 'sha256:demo' + ap = _make_ap() + ap.skill_mgr = _make_skill_manager({'demo': skill}) + ap.skill_repository = SimpleNamespace( + list_skill_resources=AsyncMock( + return_value={'entries': [{'path': 'references/a.md'}], 'revision': 'sha256:demo'} + ), + read_skill_resource=AsyncMock( + return_value={ + 'path': 'references/a.md', + 'content': 'reference text', + 'revision': 'sha256:demo', + 'mime_type': 'text/markdown', + } + ), + ) + loader = SkillToolLoader(ap) + query = _make_query() + + with pytest.raises(ValueError, match='must be activated'): + await loader.invoke_tool( + 'read_skill_resource', + {'skill_name': 'demo', 'path': 'references/a.md'}, + query, + ) + + register_activated_skill(query, skill) + listed = await loader.invoke_tool('list_skill_resources', {'skill_name': 'demo'}, query) + read = await loader.invoke_tool( + 'read_skill_resource', + {'skill_name': 'demo', 'path': 'references/a.md', 'revision': 'sha256:demo'}, + query, + ) + + assert listed['entries'][0]['path'] == 'references/a.md' + assert read['content'] == 'reference text' + ap.skill_repository.list_skill_resources.assert_awaited_once_with( + _CONTEXT, + 'demo', + '.', + expected_revision='sha256:demo', + ) + ap.skill_repository.read_skill_resource.assert_awaited_once_with( + _CONTEXT, + 'demo', + 'references/a.md', + expected_revision='sha256:demo', + ) class TestNativeToolLoaderSkillPaths: @@ -568,7 +596,7 @@ class TestNativeToolLoaderSkillPaths: assert result['truncated'] is False @pytest.mark.asyncio - async def test_external_runtime_read_never_interprets_package_root_on_core_host(self): + async def test_external_runtime_read_uses_core_skill_repository(self): from langbot.pkg.provider.tools.loaders.native import NativeToolLoader from langbot.pkg.provider.tools.loaders.skill import PIPELINE_BOUND_SKILLS_KEY @@ -580,7 +608,9 @@ class TestNativeToolLoaderSkillPaths: ap.box_service = SimpleNamespace( available=True, shares_filesystem_with_box=False, - read_skill_file=AsyncMock(return_value={'content': 'runtime-owned-content'}), + ) + ap.skill_repository = SimpleNamespace( + read_skill_file=AsyncMock(return_value={'content': 'repository-content'}) ) ap.skill_mgr = _make_skill_manager({'demo': _make_skill_data(name='demo', package_root=tmpdir)}) loader = NativeToolLoader(ap) @@ -596,9 +626,9 @@ class TestNativeToolLoaderSkillPaths: ) assert result['ok'] is True - assert result['content'] == 'runtime-owned-content' + assert result['content'] == 'repository-content' assert 'core-host-secret' not in repr(result) - ap.box_service.read_skill_file.assert_awaited_once_with(_CONTEXT, 'demo', 'SKILL.md') + ap.skill_repository.read_skill_file.assert_awaited_once_with(_CONTEXT, 'demo', 'SKILL.md') @pytest.mark.asyncio async def test_external_runtime_rejects_skill_host_fallback_without_protocol_capability(self): diff --git a/tests/unit_tests/provider/test_tool_manager_native.py b/tests/unit_tests/provider/test_tool_manager_native.py index 7dd09f32f..b3b06dd08 100644 --- a/tests/unit_tests/provider/test_tool_manager_native.py +++ b/tests/unit_tests/provider/test_tool_manager_native.py @@ -56,10 +56,20 @@ class StubLoader: for tool in self._tools ] - async def has_tool(self, *args) -> bool: + async def get_tool(self, name: str, **_kwargs): + return next((tool for tool in self._tools if tool.name == name), None) + + async def has_tool(self, *args, **_kwargs) -> bool: name = args[-1] return any(tool.name == name for tool in self._tools) + def recognizes_tool(self, name: str) -> bool: + return any(tool.name == name for tool in self._tools) + + @staticmethod + def is_sandbox_tool(name: str) -> bool: + return name == 'register_skill' + async def invoke_tool(self, name: str, parameters: dict, query): return self._invoke_result(name, parameters, query) if callable(self._invoke_result) else self._invoke_result @@ -146,7 +156,7 @@ async def test_tool_manager_routes_native_tool_calls(): @pytest.mark.asyncio -async def test_tool_manager_hides_sandbox_and_skill_tools_without_workspace_entitlement(): +async def test_tool_manager_keeps_read_only_skill_tools_without_workspace_entitlement(): box_service = SimpleNamespace(is_workspace_sandbox_available=AsyncMock(return_value=False)) manager = ToolManager(SimpleNamespace(box_service=box_service)) manager.native_tool_loader = StubLoader([make_tool('exec')]) @@ -157,8 +167,8 @@ async def test_tool_manager_hides_sandbox_and_skill_tools_without_workspace_enti tools = await manager.get_all_tools(_CONTEXT, include_skill_authoring=True) catalog = await manager.get_tool_catalog(_CONTEXT, include_skill_authoring=True) - assert [tool.name for tool in tools] == ['plugin_tool', 'mcp_tool'] - assert [item['name'] for item in catalog] == ['plugin_tool', 'mcp_tool'] + assert [tool.name for tool in tools] == ['activate', 'plugin_tool', 'mcp_tool'] + assert [item['name'] for item in catalog] == ['activate', 'plugin_tool', 'mcp_tool'] assert box_service.is_workspace_sandbox_available.await_count == 2 @@ -177,9 +187,10 @@ async def test_tool_manager_rechecks_workspace_entitlement_before_native_invocat query_uuid=None, ) - with pytest.raises(Exception, match='exec'): - await manager.execute_func_call('exec', {'command': 'pwd'}, query=query) + result = await manager.execute_func_call('exec', {'command': 'pwd'}, query=query) + assert result['code'] == 'sandbox_unavailable' + assert result['tool'] == 'exec' box_service.is_workspace_sandbox_available.assert_awaited_once_with(_CONTEXT) diff --git a/tests/unit_tests/test_skill_repository.py b/tests/unit_tests/test_skill_repository.py new file mode 100644 index 000000000..9fa3a7ded --- /dev/null +++ b/tests/unit_tests/test_skill_repository.py @@ -0,0 +1,175 @@ +from types import SimpleNamespace + +import pytest + +from langbot.pkg.api.http.context import ExecutionContext +from langbot.pkg.skill.repository import SkillRepository, SkillRevisionMismatchError + + +_CONTEXT = ExecutionContext( + instance_uuid='instance-a', + workspace_uuid='workspace-a', + placement_generation=1, +) + + +async def _binding(workspace_uuid, *, expected_generation): + return SimpleNamespace( + instance_uuid=_CONTEXT.instance_uuid, + workspace_uuid=workspace_uuid, + placement_generation=expected_generation, + ) + + +def _repository(tmp_path) -> SkillRepository: + app = SimpleNamespace( + workspace_service=SimpleNamespace( + get_execution_binding=_binding, + ), + instance_config=SimpleNamespace( + data={ + 'box': { + 'enabled': False, + 'local': { + 'host_root': str(tmp_path / 'box'), + 'skills_root': 'skills', + }, + } + } + ), + ) + return SkillRepository(app) + + +@pytest.mark.asyncio +async def test_repository_crud_and_reads_do_not_require_box(tmp_path): + repository = _repository(tmp_path) + + await repository.create_skill( + _CONTEXT, + { + 'name': 'docs-only', + 'display_name': 'Docs only', + 'description': 'Read-only guidance', + 'instructions': 'Read references/guide.md.', + }, + ) + await repository.write_skill_file( + _CONTEXT, + 'docs-only', + 'references/guide.md', + '# Guide\n\nNo execution needed.', + ) + + skill = await repository.get_skill(_CONTEXT, 'docs-only', snapshot=True) + assert skill is not None + assert skill['revision'].startswith('sha256:') + assert [item['name'] for item in await repository.list_skills(_CONTEXT)] == ['docs-only'] + + listed = await repository.list_skill_resources( + _CONTEXT, + 'docs-only', + 'references', + expected_revision=skill['revision'], + ) + assert listed['entries'][0]['path'] == 'references/guide.md' + assert listed['entries'][0]['mime_type'] == 'text/markdown' + + resource = await repository.read_skill_resource( + _CONTEXT, + 'docs-only', + 'references/guide.md', + expected_revision=skill['revision'], + ) + assert resource['content'].startswith('# Guide') + assert resource['revision'] == skill['revision'] + + +@pytest.mark.asyncio +async def test_repository_rejects_traversal_and_stale_revision(tmp_path): + repository = _repository(tmp_path) + await repository.create_skill( + _CONTEXT, + {'name': 'safe', 'description': 'Safe', 'instructions': 'Use the reference.'}, + ) + await repository.write_skill_file(_CONTEXT, 'safe', 'reference.md', 'first') + skill = await repository.get_skill(_CONTEXT, 'safe', snapshot=True) + assert skill is not None + + with pytest.raises(ValueError, match='stay within'): + await repository.read_skill_resource(_CONTEXT, 'safe', '../secret.txt') + + await repository.write_skill_file(_CONTEXT, 'safe', 'reference.md', 'second') + with pytest.raises(SkillRevisionMismatchError, match='reactivate'): + await repository.read_skill_resource( + _CONTEXT, + 'safe', + 'reference.md', + expected_revision=skill['revision'], + ) + + +@pytest.mark.asyncio +async def test_repository_scopes_skills_by_workspace(tmp_path): + repository = _repository(tmp_path) + other_context = ExecutionContext( + instance_uuid='instance-a', + workspace_uuid='workspace-b', + placement_generation=1, + ) + + await repository.create_skill(_CONTEXT, {'name': 'private', 'instructions': 'A'}) + + assert [skill['name'] for skill in await repository.list_skills(_CONTEXT)] == ['private'] + assert await repository.list_skills(other_context) == [] + + +@pytest.mark.asyncio +async def test_repository_rejects_stale_workspace_placement(tmp_path): + repository = _repository(tmp_path) + + async def stale_binding(workspace_uuid, *, expected_generation): + return SimpleNamespace( + instance_uuid=_CONTEXT.instance_uuid, + workspace_uuid=workspace_uuid, + placement_generation=expected_generation + 1, + ) + + repository.ap.workspace_service.get_execution_binding = stale_binding + with pytest.raises(ValueError, match='stale Workspace placement'): + await repository.list_skills(_CONTEXT) + + +@pytest.mark.asyncio +async def test_repository_imports_only_from_the_fenced_workspace(tmp_path): + repository = _repository(tmp_path) + namespace = repository._namespace(_CONTEXT) + source = tmp_path / 'box' / 'default' / 'tenants' / namespace / 'draft' + source.mkdir(parents=True) + (source / 'SKILL.md').write_text( + '---\nname: draft\ndescription: Draft skill\n---\n\nFollow the guide.', + encoding='utf-8', + ) + (source / 'guide.md').write_text('Imported resource', encoding='utf-8') + + scanned = await repository.scan_skill_directory(_CONTEXT, str(source)) + imported = await repository.import_skill_directory( + _CONTEXT, + str(source), + { + 'name': scanned['name'], + 'display_name': scanned['display_name'], + 'description': scanned['description'], + 'instructions': scanned['instructions'], + }, + ) + + assert imported['name'] == 'draft' + resource = await repository.read_skill_file(_CONTEXT, 'draft', 'guide.md') + assert resource['content'] == 'Imported resource' + + outside = tmp_path / 'outside' + outside.mkdir() + (outside / 'SKILL.md').write_text('Outside', encoding='utf-8') + with pytest.raises(ValueError, match='trusted source root'): + await repository.scan_skill_directory(_CONTEXT, str(outside)) diff --git a/tests/unit_tests/test_skill_service.py b/tests/unit_tests/test_skill_service.py index 1855ca337..0b0ead5ab 100644 --- a/tests/unit_tests/test_skill_service.py +++ b/tests/unit_tests/test_skill_service.py @@ -5,7 +5,6 @@ import zipfile import httpx import pytest -from langbot_plugin.entities.io.errors import ActionCallTimeoutError from langbot.pkg.api.http.context import ExecutionContext from langbot.pkg.api.http.service.skill import SkillService @@ -24,120 +23,73 @@ def _workspace_service(): ) -class TestRequireBoxForWrite: - """Box is the only source of truth for skills — there is no local - filesystem fallback. Every write and (most) read methods refuse cleanly - when the Box runtime is disabled, unreachable, or simply not installed.""" +class TestSkillRepositoryBoundary: + """Skill management and reads remain available without Box execution.""" - def _ap_with_disabled_box(self): + @staticmethod + def _ap_with_repository(): + repository = SimpleNamespace( + list_skills=AsyncMock(return_value=[{'name': 'x', 'instructions': 'Do work'}]), + get_skill=AsyncMock(return_value={'name': 'x', 'instructions': 'Do work', 'revision': 'sha256:x'}), + create_skill=AsyncMock(return_value={'name': 'x', 'instructions': 'Do work'}), + update_skill=AsyncMock(return_value={'name': 'x', 'instructions': 'Updated'}), + delete_skill=AsyncMock(), + read_skill_file=AsyncMock(return_value={'path': 'a.txt', 'content': 'hello'}), + write_skill_file=AsyncMock(return_value={'path': 'a.txt'}), + ) return SimpleNamespace( skill_mgr=SimpleNamespace(reload_skills=AsyncMock()), workspace_service=_workspace_service(), - box_service=SimpleNamespace( - available=False, - enabled=False, - _connector_error='Box runtime is disabled in config (box.enabled = false)', - ), - ) - - def _ap_with_failed_box(self): - return SimpleNamespace( - skill_mgr=SimpleNamespace(reload_skills=AsyncMock()), - workspace_service=_workspace_service(), - box_service=SimpleNamespace( - available=False, - enabled=True, - _connector_error='docker daemon not running', - ), + box_service=SimpleNamespace(available=False, enabled=False), + skill_repository=repository, ) @pytest.mark.asyncio - async def test_create_skill_refused_when_box_disabled(self): - service = SkillService(self._ap_with_disabled_box()) - with pytest.raises(ValueError, match='disabled in config'): - await service.create_skill(_CONTEXT, {'name': 'x'}) + async def test_list_and_read_work_when_box_disabled(self): + ap = self._ap_with_repository() + service = SkillService(ap) + + assert await service.list_skills(_CONTEXT) == [{'name': 'x', 'instructions': 'Do work'}] + assert await service.read_skill_file(_CONTEXT, 'x', 'a.txt') == { + 'path': 'a.txt', + 'content': 'hello', + } + ap.skill_repository.read_skill_file.assert_awaited_once_with(_CONTEXT, 'x', 'a.txt') @pytest.mark.asyncio - async def test_create_skill_refused_when_box_failed(self): - service = SkillService(self._ap_with_failed_box()) - with pytest.raises(ValueError, match='docker daemon not running'): - await service.create_skill(_CONTEXT, {'name': 'x'}) + async def test_create_update_and_write_work_when_box_disabled(self): + ap = self._ap_with_repository() + service = SkillService(ap) + + await service.create_skill(_CONTEXT, {'name': 'x'}) + await service.update_skill(_CONTEXT, 'x', {'instructions': 'Updated'}) + await service.write_skill_file(_CONTEXT, 'x', 'a.txt', 'hello') + + ap.skill_repository.create_skill.assert_awaited_once_with(_CONTEXT, {'name': 'x'}) + ap.skill_repository.update_skill.assert_awaited_once_with(_CONTEXT, 'x', {'instructions': 'Updated'}) + ap.skill_repository.write_skill_file.assert_awaited_once_with(_CONTEXT, 'x', 'a.txt', 'hello') @pytest.mark.asyncio - async def test_update_skill_refused_when_box_disabled(self): - service = SkillService(self._ap_with_disabled_box()) - with pytest.raises(ValueError, match='Editing a skill requires the Box runtime'): - await service.update_skill(_CONTEXT, 'x', {}) + async def test_get_skill_returns_repository_revision(self): + ap = self._ap_with_repository() + service = SkillService(ap) + + skill = await service.get_skill(_CONTEXT, 'x') + + assert skill['revision'] == 'sha256:x' + ap.skill_repository.get_skill.assert_awaited_once_with(_CONTEXT, 'x', snapshot=True) @pytest.mark.asyncio - async def test_write_skill_file_refused_when_box_disabled(self): - service = SkillService(self._ap_with_disabled_box()) - with pytest.raises(ValueError, match='Editing skill files requires the Box runtime'): - await service.write_skill_file(_CONTEXT, 'x', 'a.txt', 'hi') - - @pytest.mark.asyncio - async def test_install_from_github_refused_when_box_disabled(self): - service = SkillService(self._ap_with_disabled_box()) - with pytest.raises(ValueError, match='Installing a skill from GitHub'): - await service.install_from_github( - _CONTEXT, - {'owner': 'o', 'repo': 'r', 'asset_url': 'https://example/x.zip'}, - ) - - @pytest.mark.asyncio - async def test_install_from_zip_upload_refused_when_box_disabled(self): - service = SkillService(self._ap_with_disabled_box()) - with pytest.raises(ValueError, match='Installing a skill from upload'): - await service.install_from_zip_upload( - _CONTEXT, - file_bytes=b'', - filename='x.zip', - ) - - @pytest.mark.asyncio - async def test_create_skill_refused_when_box_service_missing_entirely(self): - """No ap.box_service attribute at all (truly minimal setup): - Box is the only source of truth, so creation must still refuse.""" + async def test_missing_repository_is_explicit(self): service = SkillService( SimpleNamespace( skill_mgr=SimpleNamespace(reload_skills=AsyncMock()), workspace_service=_workspace_service(), ) ) - with pytest.raises(ValueError, match='not initialised'): + with pytest.raises(ValueError, match='repository is not initialised'): await service.create_skill(_CONTEXT, {'name': 'x'}) - @pytest.mark.asyncio - async def test_list_skills_returns_empty_when_box_unavailable(self): - """list_skills should render an empty surface (not crash) so the - skills page can show a banner instead of a broken state.""" - service = SkillService(self._ap_with_disabled_box()) - assert await service.list_skills(_CONTEXT) == [] - - @pytest.mark.asyncio - async def test_list_skills_returns_empty_when_box_action_times_out(self): - """A transient Box list failure should not break unrelated extension surfaces.""" - box_service = SimpleNamespace( - available=True, - list_skills=AsyncMock(side_effect=ActionCallTimeoutError('Action box_list_skills call timed out')), - ) - service = SkillService( - SimpleNamespace( - skill_mgr=SimpleNamespace(reload_skills=AsyncMock()), - workspace_service=_workspace_service(), - box_service=box_service, - ) - ) - - assert await service.list_skills(_CONTEXT) == [] - - @pytest.mark.asyncio - async def test_read_skill_file_refused_when_box_unavailable(self): - service = SkillService(self._ap_with_disabled_box()) - with pytest.raises(ValueError, match='Reading a skill file'): - await service.read_skill_file(_CONTEXT, 'x', 'a.txt') - - class TestGithubSkillArchiveLimits: @staticmethod def _service() -> SkillService: diff --git a/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx b/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx index a26779109..52dd26082 100644 --- a/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx +++ b/web/src/app/home/pipelines/components/pipeline-extensions/PipelineExtension.tsx @@ -24,8 +24,6 @@ import { import { Plugin } from '@/app/infra/entities/plugin'; import { MCPServer, Skill } from '@/app/infra/entities/api'; import PluginComponentList from '@/app/home/plugins/components/plugin-installed/PluginComponentList'; -import { BoxUnavailableNotice } from '@/app/home/components/BoxUnavailableNotice'; -import { useBoxStatus } from '@/app/infra/hooks/useBoxStatus'; function InfoTooltip({ label }: { label: string }) { return ( @@ -52,11 +50,6 @@ export default function PipelineExtension({ pipelineId: string; }) { const { t } = useTranslation(); - const { - available: boxAvailable, - hint: boxHint, - reason: boxReason, - } = useBoxStatus(); const [loading, setLoading] = useState(true); const [enableAllPlugins, setEnableAllPlugins] = useState(true); const [enableAllMCPServers, setEnableAllMCPServers] = useState(true); @@ -558,13 +551,9 @@ export default function PipelineExtension({ id="enable-all-skills" checked={enableAllSkills} onCheckedChange={handleToggleEnableAllSkills} - disabled={!boxAvailable} /> - {!boxAvailable && ( - - )}
{enableAllSkills ? (
@@ -602,7 +591,6 @@ export default function PipelineExtension({ variant="ghost" size="icon" onClick={() => handleRemoveSkill(skill.name)} - disabled={!boxAvailable} > @@ -616,7 +604,7 @@ export default function PipelineExtension({ onClick={handleOpenSkillDialog} variant="outline" className="w-full" - disabled={enableAllSkills || !boxAvailable} + disabled={enableAllSkills} > {t('pipelines.extensions.addSkill')} diff --git a/web/src/app/home/skills/SkillDetailContent.tsx b/web/src/app/home/skills/SkillDetailContent.tsx index e98881753..5b45e41f6 100644 --- a/web/src/app/home/skills/SkillDetailContent.tsx +++ b/web/src/app/home/skills/SkillDetailContent.tsx @@ -1,4 +1,3 @@ -import EntityLoadState from '@/components/EntityLoadState'; import { useEffect, useState } from 'react'; import { useNavigate } from 'react-router-dom'; import { useTranslation } from 'react-i18next'; @@ -22,8 +21,6 @@ import { import { useSidebarData } from '@/app/home/components/home-sidebar/SidebarDataContext'; import { httpClient } from '@/app/infra/http/HttpClient'; import SkillForm from '@/app/home/skills/components/skill-form/SkillForm'; -import { BoxUnavailableNotice } from '@/app/home/components/BoxUnavailableNotice'; -import { useBoxStatus } from '@/app/infra/hooks/useBoxStatus'; import { Sparkles, Trash2 } from 'lucide-react'; import { useCurrentWorkspace } from '@/app/infra/http'; @@ -37,13 +34,6 @@ export default function SkillDetailContent({ id }: { id: string }) { const { refreshSkills, skills, setDetailEntityName } = useSidebarData(); const [showDeleteConfirm, setShowDeleteConfirm] = useState(false); const skill = skills.find((item) => item.id === id); - const { - loading: boxLoading, - available: boxAvailable, - hint: boxHint, - reason: boxReason, - } = useBoxStatus(); - useEffect(() => { if (isCreateMode) { setDetailEntityName(t('skills.createSkill')); @@ -79,8 +69,6 @@ export default function SkillDetailContent({ id }: { id: string }) { } } - if (boxLoading) return ; - if (isCreateMode) { return (
@@ -97,23 +85,12 @@ export default function SkillDetailContent({ id }: { id: string }) {
{canManage && ( - )}
- {!boxAvailable && ( -
- -
- )} -
{t('common.save')}
- {!boxAvailable && ( -
- -
- )} -
{ if (!detailId && !isCreateView) { navigate('/home/add-extension', { replace: true }); @@ -65,20 +57,11 @@ export default function SkillsPage() { -
- {!boxAvailable && ( -
- -
- )}
Date: Sat, 5 Sep 2026 23:09:46 +0800 Subject: [PATCH 002/174] refactor(skill): compose generic box mounts in core --- ARCHITECTURE.md | 20 +- docker/docker-compose.yaml | 4 +- docker/kubernetes.yaml | 14 +- docs/review/box-architecture.md | 45 +-- docs/review/box-issues.md | 4 +- docs/review/box-test-coverage.md | 1 + src/langbot/pkg/box/connector.py | 7 +- src/langbot/pkg/box/service.py | 271 ++++-------------- src/langbot/pkg/box/workspace.py | 188 +----------- .../pkg/provider/tools/loaders/mcp_stdio.py | 3 +- .../pkg/provider/tools/loaders/native.py | 35 +-- .../pkg/provider/tools/loaders/skill.py | 33 ++- src/langbot/pkg/skill/repository.py | 13 +- src/langbot/pkg/telemetry/heartbeat.py | 2 +- src/langbot/pkg/utils/python_workspace.py | 195 +++++++++++++ src/langbot/templates/config.yaml | 8 +- .../test_cloud_box_admission_integration.py | 55 ++-- tests/unit_tests/box/test_box_service.py | 60 ++-- tests/unit_tests/box/test_workspace.py | 2 +- tests/unit_tests/provider/test_skill_tools.py | 57 ++-- tests/unit_tests/test_skill_repository.py | 28 +- 21 files changed, 475 insertions(+), 570 deletions(-) create mode 100644 src/langbot/pkg/utils/python_workspace.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 18c764aab..3a4aaf42e 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -194,10 +194,12 @@ Box is the optional sandbox subsystem used by native execution tools, stdio MCP In this repo: -- `pkg/box/service.py` is the application-facing facade for exec, sessions, managed processes, skill CRUD, status, reconnects, quotas, mounts, and sandbox profiles. +- `pkg/box/service.py` is the application-facing facade for exec, sessions, managed processes, status, reconnects, quotas, generic mounts, and sandbox profiles. - `pkg/box/connector.py` connects to the Box Runtime over stdio, Windows subprocess+WebSocket, or remote WebSocket. -- `pkg/provider/tools/loaders/native.py`, `mcp_stdio.py`, and skill loaders depend on Box availability. -- `pkg/skill/repository.py` is the thin async/Workspace adapter over the Plugin SDK's execution-independent `SkillStore`; it preserves the existing `data/box/skills` layout shared with Box for optional execution mounts. +- `pkg/provider/tools/loaders/native.py` is the Core orchestration seam: the + Skill loader supplies generic read-only mounts to Box execution. `mcp_stdio.py` + and execution-backed tools depend on Box availability. +- `pkg/skill/repository.py` is the thin async/Workspace adapter over the Plugin SDK's execution-independent `SkillStore`; `skills.root` owns its location independently of Box. - `pkg/skill/manager.py` caches the Core repository catalog for progressive disclosure. Activation and read-only resource tools do not require Box; script execution and Workspace mutation still do. Durable Box Workspace storage is shared across placement generations, but @@ -209,11 +211,17 @@ retires stale processes and closes already-attached relays. In `langbot-plugin-sdk`: - `src/langbot_plugin/box/server.py` implements `lbp box` and the WebSocket endpoints on `:5410`. -- `src/langbot_plugin/box/runtime.py` owns sandbox sessions and managed processes. +- `src/langbot_plugin/box/runtime.py` owns sandbox sessions, generic read-only mounts, and managed processes. - `backend.py`, `nsjail_backend.py`, and `e2b_backend.py` implement sandbox backends. -- `skill_store.py` manages skill packages from the Box side. +- `src/langbot_plugin/skill_store.py` is consumed by Core, not Box. Core turns + selected package roots into generic read-only mounts; Box does not understand + Skill names, metadata, revisions, files, or CRUD. -Important config keys live under `box:` in `src/langbot/templates/config.yaml`: `box.enabled`, `box.backend`, `box.runtime.endpoint`, and `box.local.*`. Start LangBot with `--standalone-box` when connecting to an externally launched Box runtime. +Skill storage uses `skills.root`. Box execution config lives under `box:`: +`box.enabled`, `box.backend`, `box.runtime.endpoint`, and `box.local.*`. The old +`box.local.skills_root` key is read only as an online-upgrade fallback and is +marked for removal in the next major version. Start LangBot with +`--standalone-box` when connecting to an externally launched Box runtime. ## HTTP API, Web UI, and MCP Server diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml index fa40faa40..757b7114d 100644 --- a/docker/docker-compose.yaml +++ b/docker/docker-compose.yaml @@ -96,7 +96,9 @@ services: # box.* and are forwarded to the Box runtime via INIT RPC. - BOX__LOCAL__HOST_ROOT=${LANGBOT_BOX_ROOT:-${PWD}/data/box} - BOX__LOCAL__DEFAULT_WORKSPACE=default - - BOX__LOCAL__SKILLS_ROOT=skills + # TODO(next-major): default LANGBOT_SKILLS_ROOT to ${PWD}/data/skills + # after the historical Box storage path no longer needs zero-copy upgrades. + - SKILLS__ROOT=${LANGBOT_SKILLS_ROOT:-${LANGBOT_BOX_ROOT:-${PWD}/data/box}/skills} - BOX__LOCAL__ALLOWED_MOUNT_ROOTS=${LANGBOT_BOX_ROOT:-${PWD}/data/box} - BOX__DOCKER__CPU_LIMIT_ENABLED=${LANGBOT_BOX_DOCKER_CPU_LIMIT_ENABLED:-true} ports: diff --git a/docker/kubernetes.yaml b/docker/kubernetes.yaml index 7ce145610..53a629e20 100644 --- a/docker/kubernetes.yaml +++ b/docker/kubernetes.yaml @@ -214,8 +214,9 @@ spec: # Deployment for LangBot Box (sandbox) runtime # # The Box runtime backs LangBot's sandbox tools (exec / read / write / edit / -# glob / grep), the `activate` skill tool, skill add/edit, and stdio-mode MCP -# servers. It is OPTIONAL: if you do not deploy it, set `BOX__ENABLED=false` on +# glob / grep), Skill script execution, and stdio-mode MCP servers. Skill +# activation, resources, and management remain Core-owned without Box. Box is +# OPTIONAL: if you do not deploy it, set `BOX__ENABLED=false` on # the langbot Deployment (or `box.enabled: false` in config.yaml) so the # dashboard renders cleanly with sandbox features disabled. # @@ -448,14 +449,15 @@ spec: key: token # box.local.* config — forwarded to the Box runtime via INIT RPC. The # host_root MUST match the box-root hostPath mountPath below AND the box - # Deployment's box-root mountPath, so that skill package paths resolve - # identically on both sides and on the node's Docker daemon. + # Deployment's box-root mountPath, so generic package mount paths + # resolve identically on both sides and on the node's Docker daemon. - name: BOX__LOCAL__HOST_ROOT value: "/app/data/box" - name: BOX__LOCAL__DEFAULT_WORKSPACE value: "default" - - name: BOX__LOCAL__SKILLS_ROOT - value: "skills" + - name: SKILLS__ROOT + # TODO(next-major): use /app/data/skills after the legacy path window. + value: "/app/data/box/skills" - name: BOX__LOCAL__ALLOWED_MOUNT_ROOTS value: "/app/data/box" volumeMounts: diff --git a/docs/review/box-architecture.md b/docs/review/box-architecture.md index a76db67ff..f31f0e27e 100644 --- a/docs/review/box-architecture.md +++ b/docs/review/box-architecture.md @@ -24,6 +24,7 @@ │ │ │ (shared 容器, 多 process) │ │ │ │ │ │ │ ├──> SkillToolLoader (activate 工具) │ +│ │ │ └─ build_execution_mounts() │ │ │ │ │ │ │ ├──> SkillAuthoringToolLoader │ │ │ │ │ @@ -35,7 +36,7 @@ │ ├─ Workspace quota 检查 │ │ ├─ 输出截断 (head+tail) │ │ ├─ Host scope 哈希 (resolve_box_session_id) │ -│ ├─ 技能挂载组装 (build_skill_extra_mounts) │ +│ ├─ 通用只读挂载接收 (read_only_mounts) │ │ ├─ 重连循环 (_reconnect_loop, 指数退避) │ │ └─ BoxRuntimeConnector │ │ ├─ 心跳 loop (20s ping) │ @@ -61,10 +62,8 @@ │ NsjailBackend ──┘ (本地 CLI 或 fallback 到容器内 CLI) │ │ E2BBackend (云沙箱, 需要 E2B_API_KEY) │ │ │ -│ BoxSkillStore │ -│ ├─ list / get / create / update / delete │ -│ ├─ scan_skill_directory / read_skill_file / write_skill_file │ -│ └─ preview_skill_zip / install_skill_zip (zip 或 GitHub) │ +│ Generic mount admission │ +│ └─ allow-list + read-only + normalized target validation │ │ │ │ aiohttp 单端口服务 (默认 :5410): │ │ /rpc/ws — Action RPC │ @@ -89,6 +88,7 @@ - 一个 session_id 对应一个容器/沙箱实例。同一 session 内可并存多条 mount 与多个 managed process - Runner 无权指定 session scope。SDK/Python `call_tool` 与 scoped MCP bridge 都发出同一个 `PluginToRuntimeAction.CALL_TOOL`，最终由 Host 的 ToolManager 执行，并使用当前 run 保存的同一个 execution Query - Box 内托管的 stdio MCP server 使用独立的长期 `mcp-shared` session；它不是 Runner 本次事件的 sandbox session（详见 [box-session-scope.md](./box-session-scope.md)） +- Skill 仅是 Core 组装 mount 的业务来源；Box 与默认 exec / MCP Server 只共享通用 session 和 mount 机制（详见 [box-session-scope.md](./box-session-scope.md)） --- @@ -96,7 +96,7 @@ ### 2.1 BoxService (`pkg/box/service.py`, 722 行) -应用层门面，协调 Profile、安全校验、配额、连接、Skill 挂载与 Host scope 哈希： +应用层门面，协调 Profile、安全校验、配额、连接、Host scope 哈希与 Core 生成的只读挂载： 主要公开方法（按定义顺序）： @@ -108,7 +108,7 @@ BoxService ├─ available (property) 连接状态 │ ├─ resolve_box_session_id(query) 哈希 Host 私有 scope，生成固定长度 session_id - ├─ build_skill_extra_mounts(query) 组装 pipeline-bound skill 的挂载列表 + ├─ execute_tool(..., read_only_mounts=...) 接收 Core 组装的通用只读挂载 │ ├─ execute_tool(parameters, query) Agent 调用 exec 时的入口 │ ├─ _apply_profile / build_spec @@ -125,12 +125,6 @@ BoxService ├─ stop_managed_process(session_id, pid) 单独停止某个 managed process ├─ get_managed_process_websocket_url(...) 返回 WS attach URL │ - ├─ list_skills() / get_skill(name) Skill 元数据 - ├─ create_skill / update_skill / delete_skill Skill CRUD - ├─ scan_skill_directory(path) 扫描目录 - ├─ list_skill_files / read_skill_file / write_skill_file - ├─ preview_skill_zip / install_skill_zip zip / GitHub 安装 - │ ├─ shutdown() / dispose() 清理：RPC SHUTDOWN + 进程终止 ├─ get_status() / get_sessions() / get_recent_errors() └─ get_system_guidance() LLM 系统提示 @@ -142,7 +136,7 @@ BoxService **Session 所有权**: `resolve_box_session_id(query)` 只接受 Host 已确定的私有 scope 或 Query launcher/session identity，并输出 `lb-box-` + 64 位小写 SHA-256 十六进制摘要（固定 71 个 ASCII 字符）。哈希输入是 canonical JSON，包含 instance、workspace、bot、platform adapter、target type/id 与 thread；原始用户、群组、conversation 或 event id 不会出现在 Box session id 中。相同 Host scope 稳定复用，不同 target/thread/workspace/bot/adapter/instance 相互隔离；缺少可用 identity 时 fail closed。Pipeline、Agent 或 Runner 配置都不能覆盖该规则。 -**Skill 挂载合并**: `execute_tool()` 调用时，`build_skill_extra_mounts(query)` 会把当前 pipeline-bound 的所有 skill 的 `package_root` 作为 `extra_mounts` 加入 BoxSpec，挂在 `/workspace/.skills/`。LLM 通过 `activate` 工具显式激活某个 skill 后，工具调用才允许引用这个 skill 的虚拟路径。 +**Skill 挂载合并**: native loader 调用 `skill.build_execution_mounts()`，把当前 pipeline-bound 的所有 skill 的 `package_root` 转成普通只读 mount，再通过 `BoxService.execute_tool(..., read_only_mounts=...)` 交给 Box，挂在 `/workspace/.skills/`。LLM 通过 `activate` 工具显式激活某个 skill 后，工具调用才允许引用这个 skill 的虚拟路径；BoxService 和 Box Runtime 都不知道这些 mount 来自 Skill。 ### 2.2 BoxRuntimeConnector (`pkg/box/connector.py`, 357 行) @@ -299,7 +293,7 @@ start_managed_process(session, spec): 单端口 aiohttp 服务（默认 5410），通过路径区分（commit `8c71ec5` 合并端口）： -1. **Action RPC** (`/rpc/ws`): `BoxServerHandler` 处理所有 action，包括 `INIT` 配置注入、skill store 操作等 +1. **Action RPC** (`/rpc/ws`): `BoxServerHandler` 处理 `INIT`、exec、session、managed-process 与状态等通用 action 2. **WS Relay** (`/v1/sessions/{id}/managed-process/ws` 与 `/v1/sessions/{id}/managed-process/{pid}/ws`): 双向桥接 WebSocket ↔ 指定 managed process stdin/stdout stdio 模式同样会在 5410 启动 aiohttp，专门承担 managed process attach；Action RPC 走 stdin/stdout。 @@ -308,7 +302,7 @@ stdio 模式同样会在 5410 启动 aiohttp，专门承担 managed process atta `ActionRPCBoxClient` 封装 `Handler.call_action()` 调用： -- 25+ 方法对应 25+ 个 RPC action（exec / session / managed-process / skill / status / shutdown） +- 方法对应 exec / session / managed-process / status / shutdown 等通用 RPC action，不暴露 Skill CRUD - 错误还原: `_translate_action_error()` 通过字符串前缀匹配还原 SDK 侧异常类型 - `execute()` timeout = 300s，其他默认 15s - `BoxRuntimeClient` 是 ABC，供后续可能的非 RPC 实现复用 @@ -349,7 +343,7 @@ stdio 模式同样会在 5410 启动 aiohttp，专门承担 managed process atta ### 3.7 SkillStore (`langbot_plugin.skill_store`) -Skill 包存储最初位于 Box Runtime；issue #2410 将通用实现抽到 Plugin SDK 顶层，Core 与 Box 使用同一套存储和 revision 语义： +Skill 包存储最初位于 Box Runtime；issue #2410 将通用实现抽到 Plugin SDK 顶层，由 Core 独占存储和 revision 语义： ``` SkillStore @@ -364,7 +358,15 @@ SkillStore └─ 支持 source_subdir / target_suffix（commit 1aa043f） ``` -`langbot_plugin.box.skill_store.BoxSkillStore` 仅保留为旧 Box 配置到通用 `SkillStore` 的兼容适配器，不再拥有存储实现。GitHub 安装路径由 Core HTTP 层下载归档，再交给 SkillRepository。Skill 文件继续存放于 `box.local.skills_root`（默认 `skills`，相对 `host_root`），执行时只读挂载到 `/workspace/.skills/`。 +GitHub 安装路径由 Core HTTP 层下载归档，再交给 SkillRepository。Skill 文件位于独立的 `skills.root`，执行时由 Core 组装成通用只读 `BoxMountSpec` 并挂载到 `/workspace/.skills/`。Box 的正常模型、客户端和 Runtime 不包含 `skill_name`、Skill CRUD、revision 或 `SKILL.md` 语义。 + +滚动升级只保留一个隔离桥：`box/legacy_skill_compat.py` 让旧 Core 暂时调用新 Box，并把旧 `skill_name` 转为普通只读 mount。部署顺序必须先升级 Box、再升级 Core。该模块有 `TODO(next-major)`，下一大版本删除；正常架构不依赖它。 + +下一大版本的删除清单（当前均有 `TODO(next-major)`）： + +1. 删除 SDK `box/legacy_skill_compat.py` 及 Server 中唯一的注册/转换钩子。 +2. 删除 Core 对 `box.local.skills_root` 的配置 fallback。 +3. 新安装默认从历史 `./data/box/skills` 切到 `./data/skills`；届时 Box 部署只需只读访问 Core 明确下发的通用 artifact root。 ### 3.8 Security (`box/security.py`, 52 行) @@ -513,6 +515,9 @@ Box 额外做了 RPC SHUTDOWN 通知 Runtime 主动清理容器，比 Plugin 的 ### config.yaml (重构后) ```yaml +skills: + root: './data/box/skills' # Core-owned；Box 关闭时仍可管理/读取 + box: enabled: true # 整个 Box 子系统的总开关。设为 false 时： # - 不连接远程 Box runtime，不 fork 本地 stdio 子进程 @@ -531,7 +536,6 @@ box: image: '' # 覆盖 profile 默认 image host_root: './data/box' # 工作区挂载根，Docker 部署需绝对路径 default_workspace: '' # 默认 '/default' - skills_root: 'skills' # Core 管理且与 Box 共享的 skill 包目录 allowed_mount_roots: # 默认 [''] - './data/box' - '/tmp' @@ -576,11 +580,10 @@ volumes: | http/sse MCP server | 正常 | 正常(不依赖 Box) | | Skill 列表/读取 (`list_skills`/`get_skill`/`read_skill_file`) | 走 Core SkillRepository | 走 Core SkillRepository | | Skill 创建/编辑/安装/写文件 | 走 Core SkillRepository | 走 Core SkillRepository | -| Pipeline AI 配置中 `box-session-id-template` | 正常生效 | **前端 banner** 提示字段无效 | | Pipeline 扩展页 `enable_all_skills` / 绑定 skill | 可编辑 | 可编辑 | | 仪表盘 Box 状态卡片 | 绿点 / "已连接" | 灰点 / "已禁用"(disabled) 或 红点 / "已断开"(failed) | -> Core 的 SkillRepository 是 `langbot_plugin.skill_store.SkillStore` 的异步 Workspace 适配层，保持原 `data/box/skills/tenants/...` 布局，升级时无需移动已安装 Skill。Box 只在执行发生时消费同一份只读 package revision。 +> Core 的 SkillRepository 是 `langbot_plugin.skill_store.SkillStore` 的异步 Workspace 适配层。默认 `skills.root` 保持原 `data/box/skills/tenants/...` 布局，升级时无需移动已安装 Skill；旧 `box.local.skills_root` 仅作为在线升级 fallback，并将在下一大版本删除。Box 只消费 Core 下发的通用只读 mount。 ### Session scope diff --git a/docs/review/box-issues.md b/docs/review/box-issues.md index 15650c7c5..7c3922499 100644 --- a/docs/review/box-issues.md +++ b/docs/review/box-issues.md @@ -52,8 +52,8 @@ ### S5. 挂载校验缺口 — Med-High - **位置**: SDK `box/security.py` `_BLOCKED_HOST_PATHS_POSIX`；`box/backend.py` 的 `extra_mounts` 处理 -- **现状**: ① SDK 黑名单仍不含 `/`（前缀匹配，`host_path="/"` 可通过，挂载整个宿主 fs）；用户 home、`/usr`、`/opt`、`/tmp` 也未拦截。② `validate_sandbox_security` 只校验 `spec.host_path`，**从不遍历 `spec.extra_mounts`**——LangBot 侧 `allowed_mount_roots` 也只校验 `host_path`。当前 `extra_mounts` 仅由 `build_skill_extra_mounts` 内部填充（agent 不可达），但缺乏纵深防御：一旦 S1 的无认证 RPC 被触达，extra_mounts 可挂任意宿主路径，两层都不拦。 -- **要求**: SDK 黑名单加入 `/`（或改白名单）；`extra_mounts` 在 SDK 与 LangBot 两侧都纳入挂载校验。 +- **现状**: grant-enforced 模式已经由 Core 与 Runtime 双重校验通用只读 mount（绝对路径 allow-list、存在性、只读模式、规范化且位于 `/workspace` 下的目标）；它不再有 Skill 特例。遗留风险仅在 admission-disabled 的低信任直连场景：通用 `extra_mounts` 仍未统一套用 grant-enforced 白名单。 +- **要求**: admission-disabled 的外部控制面也复用同一套通用 mount 校验；SDK 黑名单加入 `/`（或全面改白名单）。 ### S6. 容器加固缺失 — Med diff --git a/docs/review/box-test-coverage.md b/docs/review/box-test-coverage.md index 11c53a8d2..99b370d7e 100644 --- a/docs/review/box-test-coverage.md +++ b/docs/review/box-test-coverage.md @@ -53,6 +53,7 @@ | BoxService 输出截断 | 优秀 | 短/精确边界/长输出、独立 stderr | | BoxService 可观测性 | 优秀 | 状态报告、error ring buffer、buffer 上限 | | BoxService Host-owned session | 优秀 | 覆盖 `lb-box-` 固定格式、原始 identity 不泄露、同 scope 稳定、不同 conversation/scope 隔离、缺 identity fail closed | +| BoxService mount contract | 良好 | 通用 `read_only_mounts`；Skill mount 由 Core loader 组装 | | RPC client/server 协议 | 优秀 | execute/get_sessions/delete/create/conflict error | | BoxRuntimeConnector | 良好 | local/remote 模式、Docker 平台、relay URL、心跳与重连回调 | | BoxWorkspaceSession | 良好 | payload 构建、managed process 路径重写、stage host file | diff --git a/src/langbot/pkg/box/connector.py b/src/langbot/pkg/box/connector.py index 21bd97ae4..0b736e308 100644 --- a/src/langbot/pkg/box/connector.py +++ b/src/langbot/pkg/box/connector.py @@ -149,10 +149,9 @@ class BoxRuntimeConnector(ManagedRuntimeConnector): - An explicit ``runtime.endpoint`` was configured When this is True the Box runtime lives in a separate process with its - own filesystem view (container, pod sidecar, or remote host), so paths - it reports (e.g. skill ``package_root``) are NOT resolvable on the - LangBot side. When False, Box runs as a stdio child process that shares - LangBot's filesystem. + own filesystem view (container, pod sidecar, or remote host), so only + explicitly shared paths are usable on both sides. When False, Box runs + as a stdio child process that shares LangBot's filesystem. """ return bool( self.configured_runtime_endpoint diff --git a/src/langbot/pkg/box/service.py b/src/langbot/pkg/box/service.py index 6d0c08839..a7e588974 100644 --- a/src/langbot/pkg/box/service.py +++ b/src/langbot/pkg/box/service.py @@ -28,8 +28,10 @@ from langbot_plugin.box.errors import BoxAdmissionError, BoxError, BoxValidation from langbot_plugin.box.models import ( BUILTIN_PROFILES, BoxExecutionResult, + BoxHostMountMode, BoxManagedProcessInfo, BoxManagedProcessSpec, + BoxMountSpec, BoxProfile, BoxSpec, ) @@ -169,7 +171,7 @@ class BoxService: self._connector_error = 'Box runtime is disabled in config (box.enabled = false)' self.ap.logger.info( 'Box runtime disabled by config; sandbox features (exec/read/write/edit, ' - 'skill add/edit, stdio MCP) will be unavailable.' + 'stdio MCP, executable package scripts) will be unavailable.' ) return try: @@ -275,17 +277,6 @@ class BoxService: await self._purge_attachment_dirs() self._available = True self._connector_error = '' - skill_mgr = getattr(self.ap, 'skill_mgr', None) - reload_skills = getattr(skill_mgr, 'reload_skills', None) - if callable(reload_skills) and not self._cloud_managed: - binding = await self.ap.workspace_service.get_execution_binding() - await reload_skills( - ExecutionContext( - instance_uuid=binding.instance_uuid, - workspace_uuid=binding.workspace_uuid, - placement_generation=binding.placement_generation, - ) - ) self.ap.logger.info('Box runtime reconnected, sandbox features restored.') return except Exception as exc: @@ -365,19 +356,17 @@ class BoxService: """Whether LangBot and the Box runtime share a filesystem view. This is True only when Box runs as a local stdio child process of - LangBot (same container/host). In that case paths the Box runtime - reports — notably skill ``package_root`` — resolve identically on the - LangBot side, so LangBot may validate them against its own filesystem. + LangBot (same container/host). In that case host paths resolve + identically on both sides and Core may perform local filesystem work. It is False for every separated deployment (Docker Compose, k8s sidecar, ``--standalone-box``, or an explicit ``runtime.endpoint``), - where the Box runtime owns its own filesystem and LangBot must trust - the paths it reports rather than checking them locally. + where only explicitly shared and identically mounted roots can cross + the process boundary. When Box is wired up with an injected client (tests, custom embeds) - there is no connector to introspect; we conservatively report False so - LangBot never wrongly drops Box-reported skills. An explicit override - can be set via ``_shares_filesystem_with_box`` (used by tests and any + there is no connector to introspect; we conservatively report False. + An explicit override can be set via ``_shares_filesystem_with_box`` (used by tests and any embedder that knows the real topology). """ if self._shares_filesystem_with_box_override is not None: @@ -490,11 +479,22 @@ class BoxService: self, context: TenantContext, spec_payload: dict, + *, + trusted_read_only_mounts: list[dict] | None = None, ) -> dict: """Reject tenant-owned policy fields and apply the Cloud hard policy.""" payload = dict(spec_payload) + trusted_mounts = self._normalize_trusted_read_only_mounts( + trusted_read_only_mounts or [] + ) if not self._cloud_managed: + if trusted_mounts: + if payload.get('extra_mounts'): + raise BoxValidationError( + 'extra_mounts and trusted_read_only_mounts cannot both be supplied' + ) + payload['extra_mounts'] = trusted_mounts return payload policy = self._admission_policy if policy is None: @@ -544,7 +544,7 @@ class BoxService: 'network': 'off', 'host_path': canonical_host_path, 'mount_path': '/workspace', - 'extra_mounts': [], + 'extra_mounts': trusted_mounts, 'persistent': True, 'timeout_sec': min(timeout, policy.max_timeout_sec), 'cpus': policy.cpus, @@ -556,6 +556,26 @@ class BoxService: ) return payload + def _normalize_trusted_read_only_mounts(self, mounts: list[dict]) -> list[dict]: + """Validate Core-composed artifacts before crossing into Box.""" + + normalized: list[dict] = [] + for raw_mount in mounts: + mount = BoxMountSpec.model_validate(raw_mount) + if mount.mode != BoxHostMountMode.READ_ONLY: + raise BoxAdmissionError('Core-composed additional mounts must be read-only') + host_path = os.path.realpath(mount.host_path) + if not os.path.isdir(host_path): + raise BoxAdmissionError('Core-composed read-only mount source is unavailable') + if not any(_is_path_under(host_path, root) for root in self.allowed_mount_roots): + raise BoxAdmissionError( + 'Core-composed read-only mount source is outside allowed_mount_roots' + ) + normalized.append( + mount.model_copy(update={'host_path': host_path}).model_dump(mode='json') + ) + return normalized + def _reject_cloud_managed_process(self) -> None: if self._cloud_managed: raise BoxAdmissionError('Managed processes are disabled for Cloud sandboxes') @@ -577,6 +597,7 @@ class BoxService: query: pipeline_query.Query, *, skip_host_mount_validation: bool = False, + trusted_read_only_mounts: list[dict] | None = None, ) -> dict: if not self._available: raise BoxError( @@ -589,7 +610,11 @@ class BoxService: raise BoxValidationError('session_id must match the bound Box') spec_payload = {**binding.spec, **spec_payload, 'session_id': binding.session_id} execution_context = await self._validated_execution_context(self._query_execution_context(query)) - spec_payload = self._managed_policy_payload(execution_context, spec_payload) + spec_payload = self._managed_policy_payload( + execution_context, + spec_payload, + trusted_read_only_mounts=trusted_read_only_mounts, + ) await self._require_validated_workspace_sandbox(execution_context) if spec_payload.get('host_path') in (None, ''): tenant_workspace = self._tenant_workspace(execution_context) @@ -645,70 +670,12 @@ class BoxService: return binding_for(query).session_id - def build_skill_extra_mounts(self, query: pipeline_query.Query) -> list[dict]: - """Build extra_mounts entries for all pipeline-bound skills. - - This ensures that when a container is first created it already has - all skill packages mounted, regardless of which skill is currently - activated. - - Path validation is filesystem-topology dependent. When LangBot and the - Box runtime share a filesystem (local stdio mode), a skill whose - ``package_root`` is missing or no longer a directory is skipped with a - warning instead of being passed through to the backend. Without that - guard the three backends behave inconsistently on a stale mount: nsjail - refuses to start the sandbox (failing every exec in the session), - Docker silently auto-creates a root-owned empty directory on the host, - and E2B silently skips the upload — none of which surfaces an - actionable error. - - When Box runs as a separate process (Docker Compose, k8s sidecar, - ``--standalone-box``, or a remote ``runtime.endpoint``), the - ``package_root`` reported by ``list_skills`` is the Box runtime's own - filesystem path and is NOT resolvable on the LangBot side. Validating - it locally would wrongly drop every skill, so LangBot trusts the path - and lets the Box runtime resolve it. The Box runtime only ever reports - skills it discovered on its own filesystem, so the path is valid there - by construction. - """ - if self._cloud_managed: - return [] - skill_mgr = getattr(self.ap, 'skill_mgr', None) - if skill_mgr is None: - return [] - - from ..provider.tools.loaders import skill as skill_loader - - validate_locally = self.shares_filesystem_with_box - - visible_skills = skill_loader.get_visible_skills(self.ap, query) - mounts: list[dict] = [] - for skill_name, skill_data in visible_skills.items(): - package_root = str(skill_data.get('package_root', '') or '').strip() - if not package_root: - continue - if validate_locally and not os.path.isdir(package_root): - self.ap.logger.warning( - f'Skill "{skill_name}" package_root missing on filesystem ' - f'({package_root}); skipping mount to prevent sandbox failures. ' - f'The skill cache may be stale — consider reloading skills.' - ) - continue - mounts.append( - { - 'host_path': package_root, - 'mount_path': f'/workspace/.skills/{skill_name}', - 'mode': 'ro', - } - ) - return mounts - async def execute_tool( self, parameters: dict, query: pipeline_query.Query, *, - skill_name: str | None = None, + read_only_mounts: list[dict] | None = None, ) -> dict: """Execute an agent-facing ``exec`` tool call. @@ -716,8 +683,6 @@ class BoxService: ``BoxSpec.cmd`` field and injects the session id from the query. """ spec_payload: dict = {'cmd': parameters['command']} - if skill_name is not None: - spec_payload['skill_name'] = skill_name # Pass through allowed agent-facing fields for key in ('workdir', 'timeout_sec', 'env'): @@ -727,11 +692,11 @@ class BoxService: # Inject context the agent must not control spec_payload.setdefault('session_id', self.resolve_box_session_id(query)) - # Mount all pipeline-bound skills so they are available in the container - if 'extra_mounts' not in spec_payload: - spec_payload['extra_mounts'] = self.build_skill_extra_mounts(query) - - return await self.execute_spec_payload(spec_payload, query) + return await self.execute_spec_payload( + spec_payload, + query, + trusted_read_only_mounts=read_only_mounts, + ) async def execute_in_context( self, @@ -1265,8 +1230,6 @@ class BoxService: 'timeout_sec': 120, 'session_id': self.resolve_box_session_id(query), } - if 'extra_mounts' not in spec_payload: - spec_payload['extra_mounts'] = self.build_skill_extra_mounts(query) try: spec = self.build_spec(spec_payload) result = await self.client.execute(spec) @@ -1528,126 +1491,6 @@ class BoxService: self._runtime_connector.get_relay_headers(action_context), ) - async def list_skills(self, context: TenantContext) -> list[dict]: - execution_context = await self._validated_skill_execution_context(context) - return await self.client.list_skills(action_context=self._action_context(execution_context)) - - async def get_skill(self, context: TenantContext, name: str) -> dict | None: - execution_context = await self._validated_skill_execution_context(context) - return await self.client.get_skill(name, action_context=self._action_context(execution_context)) - - async def create_skill(self, context: TenantContext, skill: dict) -> dict: - execution_context = await self._validated_skill_execution_context(context) - payload = dict(skill) - payload.pop('workspace_uuid', None) - if self._cloud_managed and str(payload.get('package_root', '') or '').strip(): - raise BoxAdmissionError('Cloud skill package_root is runtime-owned') - if self._cloud_managed: - payload.pop('package_root', None) - return await self.client.create_skill(payload, action_context=self._action_context(execution_context)) - - async def update_skill(self, context: TenantContext, name: str, skill: dict) -> dict: - execution_context = await self._validated_skill_execution_context(context) - payload = dict(skill) - payload.pop('workspace_uuid', None) - if self._cloud_managed: - # The runtime already owns the package path for an existing skill. - # A serialized read response may contain it, but it is never an - # authority-bearing update field in shared Cloud mode. - payload.pop('package_root', None) - return await self.client.update_skill( - name, - payload, - action_context=self._action_context(execution_context), - ) - - async def delete_skill(self, context: TenantContext, name: str) -> None: - execution_context = await self._validated_skill_execution_context(context) - await self.client.delete_skill(name, action_context=self._action_context(execution_context)) - - async def scan_skill_directory(self, context: TenantContext, path: str) -> dict: - execution_context = await self._validated_skill_execution_context(context) - if self._cloud_managed: - raise BoxAdmissionError('Scanning arbitrary host skill directories is disabled in Cloud') - return await self.client.scan_skill_directory(path, action_context=self._action_context(execution_context)) - - async def _validated_skill_execution_context(self, context: TenantContext) -> ExecutionContext: - execution_context = await self._validated_execution_context(context) - await self._require_validated_workspace_sandbox(execution_context) - return execution_context - - async def list_skill_files( - self, - context: TenantContext, - name: str, - path: str = '.', - include_hidden: bool = False, - max_entries: int = 200, - ) -> dict: - execution_context = await self._validated_skill_execution_context(context) - return await self.client.list_skill_files( - name, - path, - include_hidden, - max_entries, - action_context=self._action_context(execution_context), - ) - - async def read_skill_file(self, context: TenantContext, name: str, path: str) -> dict: - execution_context = await self._validated_skill_execution_context(context) - return await self.client.read_skill_file( - name, - path, - action_context=self._action_context(execution_context), - ) - - async def write_skill_file(self, context: TenantContext, name: str, path: str, content: str) -> dict: - execution_context = await self._validated_skill_execution_context(context) - return await self.client.write_skill_file( - name, - path, - content, - action_context=self._action_context(execution_context), - ) - - async def preview_skill_zip( - self, - context: TenantContext, - file_bytes: bytes, - filename: str, - source_subdir: str = '', - target_suffix: str = 'upload', - ) -> list[dict]: - execution_context = await self._validated_skill_execution_context(context) - return await self.client.preview_skill_zip( - file_bytes, - filename, - source_subdir, - target_suffix, - action_context=self._action_context(execution_context), - ) - - async def install_skill_zip( - self, - context: TenantContext, - file_bytes: bytes, - filename: str, - source_paths: list[str] | None = None, - source_path: str = '', - source_subdir: str = '', - target_suffix: str = 'upload', - ) -> list[dict]: - execution_context = await self._validated_skill_execution_context(context) - return await self.client.install_skill_zip( - file_bytes, - filename, - source_paths, - source_path, - source_subdir, - target_suffix, - action_context=self._action_context(execution_context), - ) - def _serialize_result(self, result: BoxExecutionResult) -> dict: stdout, stdout_truncated = self._truncate(result.stdout) stderr, stderr_truncated = self._truncate(result.stderr) @@ -1784,14 +1627,6 @@ class BoxService: default_workspace = os.path.join(self.host_root, default_workspace) return os.path.realpath(os.path.abspath(default_workspace)) - def get_skills_root(self) -> str | None: - skills_root = str(self._local_config().get('skills_root', '') or 'skills').strip() - if not skills_root: - skills_root = 'skills' - if not os.path.isabs(skills_root) and self.host_root is not None: - skills_root = os.path.join(self.host_root, skills_root) - return os.path.realpath(os.path.abspath(skills_root)) - def _load_enabled(self) -> bool: """Read ``box.enabled`` (top-level, not ``box.local.*``). Default True — disabling is opt-in. Accepts bool, ``'true'``/``'false'`` strings, diff --git a/src/langbot/pkg/box/workspace.py b/src/langbot/pkg/box/workspace.py index 3dd91c67b..2e1357f62 100644 --- a/src/langbot/pkg/box/workspace.py +++ b/src/langbot/pkg/box/workspace.py @@ -1,43 +1,27 @@ """Reusable workspace/session helpers built on top of Box. -This module is the middle layer between the raw Box runtime primitives and -application-specific flows such as skills or MCP stdio. +This module is the middle layer between raw Box runtime primitives and +application-specific consumers. It intentionally stays generic: - path and virtualenv rewriting are workspace concerns - Python project detection/bootstrap are workspace concerns - session exec / managed-process helpers are workspace concerns -Higher layers add their own semantics on top, for example: -- skills choose a stable per-skill session id and use repeated exec -- MCP stdio chooses how to prepare dependencies and attaches to a managed process +Higher layers add their own semantics on top; BoxWorkspaceSession retains only +workspace, execution, and managed-process concepts. """ from __future__ import annotations import os -import textwrap from typing import Any -PYTHON_MANIFEST_FILES = ( - 'requirements.txt', - 'pyproject.toml', - 'setup.py', - 'setup.cfg', -) +from ..utils.python_workspace import list_python_manifest_files _VENV_DIRS = frozenset({'.venv', 'venv', 'env', '.env'}) _VENV_BIN_DIRS = frozenset({'bin', 'Scripts'}) -def normalize_host_path(path: str | None) -> str: - if path is None: - return '' - stripped = str(path).strip() - if not stripped: - return '' - return os.path.realpath(os.path.abspath(stripped)) - - def rewrite_mounted_path(path: str, host_path: str | None, *, mount_path: str = '/workspace') -> str: """Translate a host path into the path visible inside the sandbox mount.""" if not host_path or not path: @@ -98,13 +82,6 @@ def rewrite_venv_command(command: str, host_path: str | None, *, mount_path: str return rewrite_mounted_path(normalized_command, host_path, mount_path=mount_path) -def list_python_manifest_files(host_path: str | None) -> list[str]: - normalized_root = normalize_host_path(host_path) - if not normalized_root: - return [] - return [filename for filename in PYTHON_MANIFEST_FILES if os.path.isfile(os.path.join(normalized_root, filename))] - - def classify_python_workspace(host_path: str | None) -> str | None: """Return the generic Python workspace shape, without app-specific policy.""" manifest_files = set(list_python_manifest_files(host_path)) @@ -117,161 +94,6 @@ def classify_python_workspace(host_path: str | None) -> str | None: return None -def should_prepare_python_env(host_path: str | None) -> bool: - normalized_root = normalize_host_path(host_path) - if not normalized_root: - return False - if os.path.isdir(os.path.join(normalized_root, '.venv')): - return True - return bool(list_python_manifest_files(normalized_root)) - - -def wrap_python_command_with_env( - command: str, - *, - mount_path: str = '/workspace', - state_path: str | None = None, -) -> str: - """Wrap a command with a reusable sandbox-local Python env bootstrap. - - ``mount_path`` is always the source tree used for manifest hashing and - installation. ``state_path`` may point at a separate writable directory - for read-only source mounts; when omitted, legacy mutable-workspace behavior - stores the environment beside the source. - """ - writable_state_path = state_path or mount_path - bootstrap = textwrap.dedent( - f""" - set -e - - _LB_VENV_DIR="{writable_state_path}/.venv" - _LB_META_DIR="{writable_state_path}/.langbot" - _LB_META_FILE="$_LB_META_DIR/python-env.json" - _LB_LOCK_DIR="$_LB_META_DIR/python-env.lock" - _LB_TMP_DIR="{writable_state_path}/.tmp" - _LB_PIP_CACHE_DIR="{writable_state_path}/.cache/pip" - - mkdir -p "$_LB_META_DIR" "$_LB_TMP_DIR" "$_LB_PIP_CACHE_DIR" - _LB_SYSTEM_PYTHON="$(command -v python3 || command -v python || true)" - if [ -z "$_LB_SYSTEM_PYTHON" ]; then - echo "python3 or python is required to prepare the workspace Python environment" >&2 - exit 127 - fi - - export TMPDIR="$_LB_TMP_DIR" - export TEMP="$_LB_TMP_DIR" - export TMP="$_LB_TMP_DIR" - export PIP_CACHE_DIR="$_LB_PIP_CACHE_DIR" - - _lb_python_meta() {{ - "$_LB_SYSTEM_PYTHON" - <<'PY' - import hashlib - import json - import os - import sys - - root = "{mount_path}" - max_manifest_bytes = 10 * 1024 * 1024 - digest = hashlib.sha256() - manifest_files = [] - for rel in ("requirements.txt", "pyproject.toml", "setup.py", "setup.cfg"): - path = os.path.join(root, rel) - if not os.path.isfile(path): - continue - if os.path.getsize(path) > max_manifest_bytes: - raise RuntimeError( - f"Python project manifest exceeds {{max_manifest_bytes}} bytes: {{rel}}" - ) - manifest_files.append(rel) - with open(path, "rb") as handle: - digest.update(rel.encode("utf-8")) - digest.update(b"\\0") - while chunk := handle.read(1024 * 1024): - digest.update(chunk) - digest.update(b"\\0") - - print( - json.dumps( - {{ - "python_executable": sys.executable, - "python_version": list(sys.version_info[:3]), - "manifest_files": manifest_files, - "manifest_sha256": digest.hexdigest(), - }}, - sort_keys=True, - ) - ) - PY - }} - - _LB_CURRENT_META="$(_lb_python_meta)" - _LB_NEEDS_BOOTSTRAP=0 - - if [ ! -x "$_LB_VENV_DIR/bin/python" ]; then - _LB_NEEDS_BOOTSTRAP=1 - elif [ ! -f "$_LB_META_FILE" ]; then - _LB_NEEDS_BOOTSTRAP=1 - elif [ "$(cat "$_LB_META_FILE")" != "$_LB_CURRENT_META" ]; then - _LB_NEEDS_BOOTSTRAP=1 - fi - - if [ "$_LB_NEEDS_BOOTSTRAP" -eq 1 ]; then - _LB_LOCK_WAIT=0 - while ! mkdir "$_LB_LOCK_DIR" 2>/dev/null; do - if [ "$_LB_LOCK_WAIT" -ge 120 ]; then - _LB_LOCK_OWNER="$(cat "$_LB_LOCK_DIR/pid" 2>/dev/null || true)" - if [ -n "$_LB_LOCK_OWNER" ] && kill -0 "$_LB_LOCK_OWNER" 2>/dev/null; then - echo "Timed out waiting for active Python environment lock: $_LB_LOCK_DIR" >&2 - exit 1 - fi - echo "Timed out waiting for Python environment lock, clearing stale lock: $_LB_LOCK_DIR" >&2 - rm -rf "$_LB_LOCK_DIR" 2>/dev/null || true - if mkdir "$_LB_LOCK_DIR" 2>/dev/null; then - break - fi - echo "Timed out waiting for Python environment lock: $_LB_LOCK_DIR" >&2 - exit 1 - fi - sleep 1 - _LB_LOCK_WAIT=$((_LB_LOCK_WAIT + 1)) - done - printf '%s\\n' "$$" > "$_LB_LOCK_DIR/pid" 2>/dev/null || true - - _lb_cleanup_lock() {{ - rm -rf "$_LB_LOCK_DIR" >/dev/null 2>&1 || true - }} - trap _lb_cleanup_lock EXIT INT TERM - - _LB_CURRENT_META="$(_lb_python_meta)" - _LB_NEEDS_BOOTSTRAP=0 - if [ ! -x "$_LB_VENV_DIR/bin/python" ]; then - _LB_NEEDS_BOOTSTRAP=1 - elif [ ! -f "$_LB_META_FILE" ]; then - _LB_NEEDS_BOOTSTRAP=1 - elif [ "$(cat "$_LB_META_FILE")" != "$_LB_CURRENT_META" ]; then - _LB_NEEDS_BOOTSTRAP=1 - fi - - if [ "$_LB_NEEDS_BOOTSTRAP" -eq 1 ]; then - rm -rf "$_LB_VENV_DIR" - "$_LB_SYSTEM_PYTHON" -m venv "$_LB_VENV_DIR" - . "$_LB_VENV_DIR/bin/activate" - python -m pip install --upgrade pip setuptools wheel - if [ -f "{mount_path}/requirements.txt" ]; then - python -m pip install -r "{mount_path}/requirements.txt" - elif [ -f "{mount_path}/pyproject.toml" ] || [ -f "{mount_path}/setup.py" ] || [ -f "{mount_path}/setup.cfg" ]; then - python -m pip install "{mount_path}" - fi - printf '%s' "$_LB_CURRENT_META" > "$_LB_META_FILE" - fi - fi - - export VIRTUAL_ENV="$_LB_VENV_DIR" - export PATH="$_LB_VENV_DIR/bin:$PATH" - {command} - """ - ).strip() - return bootstrap + '\n' class BoxWorkspaceSession: diff --git a/src/langbot/pkg/provider/tools/loaders/mcp_stdio.py b/src/langbot/pkg/provider/tools/loaders/mcp_stdio.py index ae01cc632..258b849b4 100644 --- a/src/langbot/pkg/provider/tools/loaders/mcp_stdio.py +++ b/src/langbot/pkg/provider/tools/loaders/mcp_stdio.py @@ -18,12 +18,11 @@ from ....box.workspace import ( BoxWorkspaceSession, classify_python_workspace, infer_workspace_host_path, - normalize_host_path, rewrite_mounted_path, rewrite_venv_command, unwrap_venv_path, - wrap_python_command_with_env, ) +from ....utils.python_workspace import normalize_host_path, wrap_python_command_with_env if TYPE_CHECKING: from .mcp import RuntimeMCPSession diff --git a/src/langbot/pkg/provider/tools/loaders/native.py b/src/langbot/pkg/provider/tools/loaders/native.py index 254854fae..17ec96017 100644 --- a/src/langbot/pkg/provider/tools/loaders/native.py +++ b/src/langbot/pkg/provider/tools/loaders/native.py @@ -333,20 +333,11 @@ class NativeToolLoader(loader.ToolLoader): if not package_root: raise ValueError(f'Activated skill "{selected_skill_name}" has no package_root.') - # Pass only the logical name across the authenticated Core→Runtime - # boundary. In Cloud mode the shared Box Runtime resolves the - # Workspace-scoped package root and constructs the read-only mount; - # Core host paths are never accepted as mount authority. # Wrap command with Python venv bootstrap if the skill has a Python project. # The venv is created inside the skill's mount path. skill_mount = f'/workspace/.skills/{selected_skill_name}' python_project = selected_skill.get('python_project') is True - if 'python_project' not in selected_skill and bool( - getattr(self.ap.box_service, 'shares_filesystem_with_box', False) - ): - # Backward compatibility for a same-process OSS Runtime that - # predates trusted Box metadata. Never probe a path reported by - # an external Runtime from the Core filesystem. + if 'python_project' not in selected_skill: python_project = skill_loader.should_prepare_skill_python_env(package_root) if python_project: parameters = dict(parameters) @@ -362,12 +353,9 @@ class NativeToolLoader(loader.ToolLoader): result = await self.ap.box_service.execute_tool( parameters, query, - skill_name=selected_skill_name, + read_only_mounts=skill_loader.build_execution_mounts(self.ap, query), ) result = self._normalize_exec_result(result) - - if selected_skill is not None: - self._refresh_skill_from_disk(query, selected_skill) return result def _resolve_host_location( @@ -394,8 +382,7 @@ class NativeToolLoader(loader.ToolLoader): if selected_skill is not None: if not self._can_interpret_skill_host_paths(): raise ValueError( - 'Skill package paths are owned by the Box Runtime; ' - 'this operation requires a Runtime skill-file API.' + 'Secure Core host file operations are unavailable on this platform.' ) host_root = selected_skill.get('package_root') workspace_anchor = None @@ -434,11 +421,9 @@ class NativeToolLoader(loader.ToolLoader): return selected_skill, relative def _can_interpret_skill_host_paths(self) -> bool: - """Require an explicitly proven shared Core/Runtime filesystem view.""" + """Return whether Core can use its no-follow host file primitives.""" - return _SECURE_HOST_FILE_OPS_AVAILABLE and bool( - getattr(self.ap.box_service, 'shares_filesystem_with_box', False) - ) + return _SECURE_HOST_FILE_OPS_AVAILABLE def _should_use_box_workspace_files(self, selected_skill: dict | None) -> bool: if selected_skill is not None: @@ -1142,16 +1127,6 @@ else: skill_repository = getattr(self.ap, 'skill_repository', None) if skill_request is not None and skill_repository is not None: selected_skill, relative = skill_request - if self._can_interpret_skill_host_paths(): - host_location = self._resolve_skill_host_location(selected_skill, relative) - else: - host_location = None - if host_location is not None: - try: - return await asyncio.to_thread(self._read_host_location, host_location, parameters) - except FileNotFoundError: - pass - try: result = await skill_repository.read_skill_file( self._execution_context(query), diff --git a/src/langbot/pkg/provider/tools/loaders/skill.py b/src/langbot/pkg/provider/tools/loaders/skill.py index b6a97ceaf..adc016a73 100644 --- a/src/langbot/pkg/provider/tools/loaders/skill.py +++ b/src/langbot/pkg/provider/tools/loaders/skill.py @@ -1,10 +1,14 @@ from __future__ import annotations +import os import re import typing -from ....box import workspace as box_workspace from ....api.http.context import ExecutionContext +from ....utils.python_workspace import ( + should_prepare_python_env, + wrap_python_command_with_env, +) if typing.TYPE_CHECKING: from ....core import app @@ -69,6 +73,29 @@ def register_created_skill_visibility(query: pipeline_query.Query, skill_name: s if isinstance(bound_skills, list) and skill_name not in bound_skills: bound_skills.append(skill_name) +def build_execution_mounts(ap: app.Application, query: pipeline_query.Query) -> list[dict]: + """Translate visible Core-owned packages into generic read-only mounts.""" + + mounts: list[dict] = [] + for skill_name, skill_data in get_visible_skills(ap, query).items(): + package_root = str(skill_data.get('package_root', '') or '').strip() + if not package_root: + continue + if not os.path.isdir(package_root): + ap.logger.warning( + f'Skill "{skill_name}" package_root missing on the Core filesystem ' + f'({package_root}); skipping its execution mount. Reload the skill catalog.' + ) + continue + mounts.append( + { + 'host_path': package_root, + 'mount_path': get_virtual_skill_mount_path(skill_name), + 'mode': 'ro', + } + ) + return mounts + def get_activated_skills(query: pipeline_query.Query) -> dict[str, dict]: if query.variables is None: @@ -275,7 +302,7 @@ def build_skill_session_id(skill_data: dict, query: pipeline_query.Query) -> str def should_prepare_skill_python_env(package_root: str | None) -> bool: - return box_workspace.should_prepare_python_env(package_root) + return should_prepare_python_env(package_root) def wrap_skill_command_with_python_env( @@ -284,7 +311,7 @@ def wrap_skill_command_with_python_env( mount_path: str = '/workspace', state_path: str | None = None, ) -> str: - return box_workspace.wrap_python_command_with_env( + return wrap_python_command_with_env( command, mount_path=mount_path, state_path=state_path, diff --git a/src/langbot/pkg/skill/repository.py b/src/langbot/pkg/skill/repository.py index 2d04f7cf3..9f73ba6e9 100644 --- a/src/langbot/pkg/skill/repository.py +++ b/src/langbot/pkg/skill/repository.py @@ -21,6 +21,7 @@ class SkillRepository: self.ap = ap config = getattr(getattr(ap, 'instance_config', None), 'data', {}) or {} self._local_config = (config.get('box') or {}).get('local') or {} + self._skills_config = config.get('skills') or {} self._store = SkillStore(self._skills_root()) self._lock = asyncio.Lock() @@ -29,9 +30,15 @@ class SkillRepository: return os.path.realpath(os.path.abspath(os.path.expanduser(configured))) def _skills_root(self) -> str: - configured = str(self._local_config.get('skills_root') or 'skills').strip() - if not os.path.isabs(configured): - configured = os.path.join(self._host_root(), configured) + configured = str(self._skills_config.get('root') or '').strip() + if not configured: + # Online-upgrade bridge for installations whose persisted config + # predates the standalone Skill domain. + # TODO(next-major): remove box.local.skills_root fallback. + legacy = str(self._local_config.get('skills_root') or '').strip() + configured = legacy or 'skills' + if not os.path.isabs(configured): + configured = os.path.join(self._host_root(), configured) return os.path.realpath(os.path.abspath(os.path.expanduser(configured))) def _default_workspace(self) -> str: diff --git a/src/langbot/pkg/telemetry/heartbeat.py b/src/langbot/pkg/telemetry/heartbeat.py index 45dcbf268..69b73a97b 100644 --- a/src/langbot/pkg/telemetry/heartbeat.py +++ b/src/langbot/pkg/telemetry/heartbeat.py @@ -198,7 +198,7 @@ async def build_heartbeat_payload( except Exception: features['plugin_count'] = -1 - # Skill count (from Box runtime via skill manager) + # Skill count (from the Core SkillRepository cache) try: skill_mgr = getattr(ap, 'skill_mgr', None) if skill_mgr is not None: diff --git a/src/langbot/pkg/utils/python_workspace.py b/src/langbot/pkg/utils/python_workspace.py new file mode 100644 index 000000000..b21f7a694 --- /dev/null +++ b/src/langbot/pkg/utils/python_workspace.py @@ -0,0 +1,195 @@ +"""Python project detection and sandbox-local environment bootstrap helpers.""" + +from __future__ import annotations + +import os +import textwrap + + +PYTHON_MANIFEST_FILES = ( + 'requirements.txt', + 'pyproject.toml', + 'setup.py', + 'setup.cfg', +) + + +def normalize_host_path(path: str | None) -> str: + if path is None: + return '' + stripped = str(path).strip() + if not stripped: + return '' + return os.path.realpath(os.path.abspath(stripped)) + + +def list_python_manifest_files(host_path: str | None) -> list[str]: + normalized_root = normalize_host_path(host_path) + if not normalized_root: + return [] + return [ + filename + for filename in PYTHON_MANIFEST_FILES + if os.path.isfile(os.path.join(normalized_root, filename)) + ] + + +def should_prepare_python_env(host_path: str | None) -> bool: + normalized_root = normalize_host_path(host_path) + if not normalized_root: + return False + if os.path.isdir(os.path.join(normalized_root, '.venv')): + return True + return bool(list_python_manifest_files(normalized_root)) + + +def wrap_python_command_with_env( + command: str, + *, + mount_path: str = '/workspace', + state_path: str | None = None, +) -> str: + """Wrap a command with a reusable sandbox-local Python env bootstrap.""" + + writable_state_path = state_path or mount_path + bootstrap = textwrap.dedent( + f""" + set -e + + _LB_VENV_DIR="{writable_state_path}/.venv" + _LB_META_DIR="{writable_state_path}/.langbot" + _LB_META_FILE="$_LB_META_DIR/python-env.json" + _LB_LOCK_DIR="$_LB_META_DIR/python-env.lock" + _LB_TMP_DIR="{writable_state_path}/.tmp" + _LB_PIP_CACHE_DIR="{writable_state_path}/.cache/pip" + + mkdir -p "$_LB_META_DIR" "$_LB_TMP_DIR" "$_LB_PIP_CACHE_DIR" + _LB_SYSTEM_PYTHON="$(command -v python3 || command -v python || true)" + if [ -z "$_LB_SYSTEM_PYTHON" ]; then + echo "python3 or python is required to prepare the workspace Python environment" >&2 + exit 127 + fi + + export TMPDIR="$_LB_TMP_DIR" + export TEMP="$_LB_TMP_DIR" + export TMP="$_LB_TMP_DIR" + export PIP_CACHE_DIR="$_LB_PIP_CACHE_DIR" + + _lb_python_meta() {{ + "$_LB_SYSTEM_PYTHON" - <<'PY' + import hashlib + import json + import os + import sys + + root = "{mount_path}" + max_manifest_bytes = 10 * 1024 * 1024 + digest = hashlib.sha256() + manifest_files = [] + for rel in ("requirements.txt", "pyproject.toml", "setup.py", "setup.cfg"): + path = os.path.join(root, rel) + if not os.path.isfile(path): + continue + if os.path.getsize(path) > max_manifest_bytes: + raise RuntimeError( + f"Python project manifest exceeds {{max_manifest_bytes}} bytes: {{rel}}" + ) + manifest_files.append(rel) + with open(path, "rb") as handle: + digest.update(rel.encode("utf-8")) + digest.update(b"\0") + while chunk := handle.read(1024 * 1024): + digest.update(chunk) + digest.update(b"\0") + + print( + json.dumps( + {{ + "python_executable": sys.executable, + "python_version": list(sys.version_info[:3]), + "manifest_files": manifest_files, + "manifest_sha256": digest.hexdigest(), + }}, + sort_keys=True, + ) + ) + PY + }} + + _LB_CURRENT_META="$(_lb_python_meta)" + _LB_NEEDS_BOOTSTRAP=0 + + if [ ! -x "$_LB_VENV_DIR/bin/python" ]; then + _LB_NEEDS_BOOTSTRAP=1 + elif [ ! -f "$_LB_META_FILE" ]; then + _LB_NEEDS_BOOTSTRAP=1 + elif [ "$(cat "$_LB_META_FILE")" != "$_LB_CURRENT_META" ]; then + _LB_NEEDS_BOOTSTRAP=1 + fi + + if [ "$_LB_NEEDS_BOOTSTRAP" -eq 1 ]; then + _LB_LOCK_WAIT=0 + while ! mkdir "$_LB_LOCK_DIR" 2>/dev/null; do + if [ "$_LB_LOCK_WAIT" -ge 120 ]; then + _LB_LOCK_OWNER="$(cat "$_LB_LOCK_DIR/pid" 2>/dev/null || true)" + if [ -n "$_LB_LOCK_OWNER" ] && kill -0 "$_LB_LOCK_OWNER" 2>/dev/null; then + echo "Timed out waiting for active Python environment lock: $_LB_LOCK_DIR" >&2 + exit 1 + fi + echo "Timed out waiting for Python environment lock, clearing stale lock: $_LB_LOCK_DIR" >&2 + rm -rf "$_LB_LOCK_DIR" 2>/dev/null || true + if mkdir "$_LB_LOCK_DIR" 2>/dev/null; then + break + fi + echo "Timed out waiting for Python environment lock: $_LB_LOCK_DIR" >&2 + exit 1 + fi + sleep 1 + _LB_LOCK_WAIT=$((_LB_LOCK_WAIT + 1)) + done + printf '%s\n' "$$" > "$_LB_LOCK_DIR/pid" 2>/dev/null || true + + _lb_cleanup_lock() {{ + rm -rf "$_LB_LOCK_DIR" >/dev/null 2>&1 || true + }} + trap _lb_cleanup_lock EXIT INT TERM + + _LB_CURRENT_META="$(_lb_python_meta)" + _LB_NEEDS_BOOTSTRAP=0 + if [ ! -x "$_LB_VENV_DIR/bin/python" ]; then + _LB_NEEDS_BOOTSTRAP=1 + elif [ ! -f "$_LB_META_FILE" ]; then + _LB_NEEDS_BOOTSTRAP=1 + elif [ "$(cat "$_LB_META_FILE")" != "$_LB_CURRENT_META" ]; then + _LB_NEEDS_BOOTSTRAP=1 + fi + + if [ "$_LB_NEEDS_BOOTSTRAP" -eq 1 ]; then + rm -rf "$_LB_VENV_DIR" + "$_LB_SYSTEM_PYTHON" -m venv "$_LB_VENV_DIR" + . "$_LB_VENV_DIR/bin/activate" + python -m pip install --upgrade pip setuptools wheel + if [ -f "{mount_path}/requirements.txt" ]; then + python -m pip install -r "{mount_path}/requirements.txt" + elif [ -f "{mount_path}/pyproject.toml" ] || [ -f "{mount_path}/setup.py" ] || [ -f "{mount_path}/setup.cfg" ]; then + python -m pip install "{mount_path}" + fi + printf '%s' "$_LB_CURRENT_META" > "$_LB_META_FILE" + fi + fi + + export VIRTUAL_ENV="$_LB_VENV_DIR" + export PATH="$_LB_VENV_DIR/bin:$PATH" + {command} + """ + ).strip() + return bootstrap + '\n' + + +__all__ = [ + 'PYTHON_MANIFEST_FILES', + 'list_python_manifest_files', + 'normalize_host_path', + 'should_prepare_python_env', + 'wrap_python_command_with_env', +] diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index 9d3ce0f42..81af5c004 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -343,6 +343,13 @@ monitoring: # Prevent one large Workspace backlog from monopolizing PostgreSQL. # Supports MONITORING__AUTO_CLEANUP__MAX_BATCHES_PER_TABLE_PER_RUN. max_batches_per_table_per_run: 4 +skills: + # Core-owned SkillStore. Skill discovery, activation, resources, CRUD and + # revisions remain available when Box is disabled. + # TODO(next-major): change the fresh-install default to './data/skills' + # after the online-upgrade window for the historical Box path closes. + root: './data/box/skills' + box: # Master switch for the Box sandbox runtime. When false, LangBot does NOT # attempt to connect to a remote Box runtime nor start a local stdio Box @@ -398,7 +405,6 @@ box: image: '' # Custom local sandbox image. Leave empty to use the profile default. host_root: './data/box' # Base host directory for local workspace mounts. Docker deployments should override this with an absolute host path. default_workspace: '' # Defaults to '/default'. Relative paths are resolved under host_root. - skills_root: 'skills' # Core-owned skill repository shared with Box for optional execution mounts. allowed_mount_roots: # Defaults to [''] when left empty. - './data/box' - '/tmp' diff --git a/tests/integration_tests/box/test_cloud_box_admission_integration.py b/tests/integration_tests/box/test_cloud_box_admission_integration.py index 1080dc68d..78252c3c9 100644 --- a/tests/integration_tests/box/test_cloud_box_admission_integration.py +++ b/tests/integration_tests/box/test_cloud_box_admission_integration.py @@ -30,6 +30,9 @@ from langbot.pkg.cloud.entitlements import ( EntitlementSnapshot, EntitlementUnavailableError, ) +from langbot.pkg.skill.manager import SkillManager +from langbot.pkg.skill.repository import SkillRepository +from langbot.pkg.provider.tools.loaders import skill as skill_loader pytestmark = pytest.mark.integration @@ -56,7 +59,7 @@ class _AdmissionBackend(BaseSandboxBackend): 'mount_isolation': True, 'network_isolation': True, 'hard_workspace_quota': True, - 'hard_skill_storage_quota': True, + 'hard_read_only_mount_quota': True, 'bounded_ephemeral_storage': True, 'inode_quota': True, } @@ -246,9 +249,18 @@ async def _stack(tmp_path): deployment=SimpleNamespace(multi_workspace_enabled=True), entitlement_resolver=EntitlementResolver('instance-a', entitlements), workspace_service=workspace_service, - instance_config=SimpleNamespace(data={'box': box_config, 'system': {'limitation': {}}}), + instance_config=SimpleNamespace( + data={ + 'skills': {'root': str(shared_root / 'skills')}, + 'box': box_config, + 'system': {'limitation': {}}, + } + ), ) + app.skill_repository = SkillRepository(app) + app.skill_mgr = SkillManager(app) service = BoxService(app, client=client) + app.box_service = service await service.initialize() return service, runtime, backend, entitlements, server_task, client_task @@ -329,7 +341,7 @@ async def test_two_workspaces_get_isolated_physical_sessions_and_paths(tmp_path) @pytest.mark.asyncio -async def test_cloud_skills_reject_host_paths_and_require_managed_entitlement(tmp_path): +async def test_cloud_core_skills_mount_generically_and_do_not_require_box_entitlement(tmp_path): service, runtime, backend, entitlements, server_task, client_task = await _stack(tmp_path) first = _context('workspace-a') second = _context('workspace-b') @@ -341,32 +353,35 @@ async def test_cloud_skills_reject_host_paths_and_require_managed_entitlement(tm managed=False, ) try: - private = await service.create_skill( + repository = service.ap.skill_repository + await repository.create_skill( second, { 'name': 'private', 'instructions': 'workspace-b secret', }, ) - own_skill = await service.create_skill( + own_skill = await repository.create_skill( first, { 'name': 'runner', 'instructions': 'Run scripts/main.py', }, ) - await service.write_skill_file(first, 'runner', 'scripts/main.py', "print('ok')") - await service.write_skill_file(first, 'runner', 'requirements.txt', 'requests==2.32.0\n') - refreshed_skill = await service.get_skill(first, 'runner') + await repository.write_skill_file(first, 'runner', 'scripts/main.py', "print('ok')") + await repository.write_skill_file(first, 'runner', 'requirements.txt', 'requests==2.32.0\n') + refreshed_skill = await repository.get_skill(first, 'runner') assert refreshed_skill is not None assert refreshed_skill['python_project'] is True + await service.ap.skill_mgr.reload_skills(first) + query = await _bound_query(service, first, 91) await service.execute_tool( { 'command': 'python /workspace/.skills/runner/scripts/main.py', 'workdir': '/workspace/.skills/runner', }, - await _bound_query(service, first, 91), - skill_name='runner', + query, + read_only_mounts=skill_loader.build_execution_mounts(service.ap, query), ) mounted_spec = backend.started_specs[-1] @@ -375,20 +390,14 @@ async def test_cloud_skills_reject_host_paths_and_require_managed_entitlement(tm assert mounted_spec.extra_mounts[0].mount_path == '/workspace/.skills/runner' assert mounted_spec.extra_mounts[0].mode.value == 'ro' - with pytest.raises(BoxAdmissionError, match='Scanning arbitrary host'): - await service.scan_skill_directory(first, private['package_root']) - with pytest.raises(BoxAdmissionError, match='package_root is runtime-owned'): - await service.create_skill( - first, - { - 'name': 'stolen', - 'package_root': private['package_root'], - }, - ) - - assert await service.get_skill(first, 'private') is None + assert await repository.get_skill(first, 'private') is None + await repository.create_skill( + ineligible, + {'name': 'docs-only', 'instructions': 'Read this without Box.'}, + ) + assert [skill['name'] for skill in await repository.list_skills(ineligible)] == ['docs-only'] with pytest.raises(EntitlementUnavailableError): - await service.list_skills(ineligible) + await service.execute_tool({'command': 'true'}, _query(ineligible, 92)) finally: server_task.cancel() client_task.cancel() diff --git a/tests/unit_tests/box/test_box_service.py b/tests/unit_tests/box/test_box_service.py index 25701979d..943c035a0 100644 --- a/tests/unit_tests/box/test_box_service.py +++ b/tests/unit_tests/box/test_box_service.py @@ -42,6 +42,7 @@ from langbot_plugin.entities.io.context import ActionContext from langbot.pkg.api.http.context import ExecutionContext from langbot.pkg.box.service import BoxService from langbot.pkg.box.runner import RunBoxBinding +from langbot.pkg.provider.tools.loaders import skill as skill_loader _UTC = dt.timezone.utc _CONTEXT = ExecutionContext( @@ -307,7 +308,7 @@ class TestSharesFilesystemWithBox: - stdio (local child process) → shared filesystem → True - WebSocket (Docker / sidecar / --standalone-box / remote) → separated → False - This drives whether LangBot validates Box-reported skill paths locally. + This drives whether LangBot can safely perform local workspace operations. Getting it wrong silently drops every skill in separated deployments. """ @@ -344,7 +345,7 @@ class TestSharesFilesystemWithBox: def test_false_when_client_injected_without_connector(self): # Injected client (no connector) → unknown topology → conservative False - # so LangBot never wrongly drops Box-reported skills. + # so LangBot does not assume a shared local filesystem. service = BoxService(make_app(Mock()), client=Mock(spec=BoxRuntimeClient)) assert service._runtime_connector is None @@ -572,7 +573,6 @@ async def test_box_service_reconnect_restores_workspace_and_runs_cleanup( monkeypatch: pytest.MonkeyPatch, ): app = make_app(Mock()) - app.skill_mgr = SimpleNamespace(reload_skills=AsyncMock()) service = BoxService(app, client=Mock(spec=BoxRuntimeClient)) connector = Mock() connector.reconnect = AsyncMock() @@ -585,16 +585,15 @@ async def test_box_service_reconnect_restores_workspace_and_runs_cleanup( connector.reconnect.assert_awaited_once() service._ensure_default_workspace.assert_called_once() service._purge_attachment_dirs.assert_awaited_once() - app.skill_mgr.reload_skills.assert_awaited_once_with(_CONTEXT) + app.skill_mgr.reload_skills.assert_not_awaited() assert service.available is True @pytest.mark.asyncio -async def test_cloud_box_service_reconnect_does_not_reload_unscoped_skills( +async def test_cloud_box_service_reconnect_restores_runtime_only( monkeypatch: pytest.MonkeyPatch, ): app = make_app(Mock()) - app.skill_mgr = SimpleNamespace(reload_skills=AsyncMock()) service = BoxService(app, client=Mock(spec=BoxRuntimeClient)) service._cloud_managed = True connector = Mock() @@ -607,7 +606,6 @@ async def test_cloud_box_service_reconnect_does_not_reload_unscoped_skills( connector.reconnect.assert_awaited_once() service._verify_cloud_runtime.assert_awaited_once() - app.skill_mgr.reload_skills.assert_not_awaited() assert service.available is True @@ -1803,7 +1801,7 @@ def test_disconnect_callback_does_not_schedule_without_running_event_loop(): assert service._reconnecting is False -class TestBuildSkillExtraMounts: +class TestBuildSkillExecutionMounts: """Robustness of skill mount construction against a stale skill cache. The three sandbox backends behave inconsistently when a skill's @@ -1813,16 +1811,10 @@ class TestBuildSkillExtraMounts: the backend never sees a bad mount. """ - def _make_service(self, logger, skills, *, shares_filesystem=True): + def _make_app(self, logger, skills): app = make_app(logger) app.skill_mgr = SimpleNamespace(skills=skills, get_skills=Mock(return_value=skills)) - client = Mock(spec=BoxRuntimeClient) - service = BoxService(app, client=client) - # Tests construct BoxService with an injected client (no connector), so - # set the topology explicitly. Most cases exercise the shared-fs (local - # stdio) path where local package_root validation applies. - service._shares_filesystem_with_box_override = shares_filesystem - return service + return app def test_skips_skill_with_missing_package_root(self): logger = Mock() @@ -1831,10 +1823,10 @@ class TestBuildSkillExtraMounts: 'alive': {'name': 'alive', 'package_root': live_dir}, 'ghost': {'name': 'ghost', 'package_root': '/nonexistent/path/should/never/exist'}, } - service = self._make_service(logger, skills) + app = self._make_app(logger, skills) query = make_query() - mounts = service.build_skill_extra_mounts(query) + mounts = skill_loader.build_execution_mounts(app, query) assert mounts == [ { @@ -1849,27 +1841,19 @@ class TestBuildSkillExtraMounts: for call in logger.warning.call_args_list ) - def test_trusts_box_paths_when_filesystem_not_shared(self): - """In separated deployments (Docker Compose, k8s sidecar, - --standalone-box, remote endpoint) the Box runtime owns its own - filesystem. package_root values it reports are NOT resolvable on the - LangBot side, so LangBot must trust them rather than dropping every - skill via a local isdir() check.""" + def test_rejects_missing_core_paths_when_filesystem_not_shared(self): + """Core owns package paths even when Box is a separate process.""" logger = Mock() skills = { 'a': {'name': 'a', 'package_root': '/box/skills/a'}, 'b': {'name': 'b', 'package_root': '/box/skills/b'}, } - service = self._make_service(logger, skills, shares_filesystem=False) + app = self._make_app(logger, skills) - mounts = service.build_skill_extra_mounts(make_query()) + mounts = skill_loader.build_execution_mounts(app, make_query()) - assert mounts == [ - {'host_path': '/box/skills/a', 'mount_path': '/workspace/.skills/a', 'mode': 'ro'}, - {'host_path': '/box/skills/b', 'mount_path': '/workspace/.skills/b', 'mode': 'ro'}, - ] - # No skill is dropped, so no "missing" warning should be logged. - assert not any('package_root missing' in str(call.args[0]) for call in logger.warning.call_args_list) + assert mounts == [] + assert len(logger.warning.call_args_list) == 2 def test_skips_skill_with_empty_package_root(self): logger = Mock() @@ -1877,25 +1861,23 @@ class TestBuildSkillExtraMounts: 'no_root': {'name': 'no_root', 'package_root': ''}, 'whitespace': {'name': 'whitespace', 'package_root': ' '}, } - service = self._make_service(logger, skills) + app = self._make_app(logger, skills) - assert service.build_skill_extra_mounts(make_query()) == [] + assert skill_loader.build_execution_mounts(app, make_query()) == [] def test_empty_package_root_skipped_even_when_not_shared(self): """An empty package_root is always invalid regardless of topology.""" logger = Mock() skills = {'no_root': {'name': 'no_root', 'package_root': ''}} - service = self._make_service(logger, skills, shares_filesystem=False) + app = self._make_app(logger, skills) - assert service.build_skill_extra_mounts(make_query()) == [] + assert skill_loader.build_execution_mounts(app, make_query()) == [] def test_returns_empty_when_no_skill_manager(self): logger = Mock() app = make_app(logger) # no skill_mgr attribute - service = BoxService(app, client=Mock(spec=BoxRuntimeClient)) - - assert service.build_skill_extra_mounts(make_query()) == [] + assert skill_loader.build_execution_mounts(app, make_query()) == [] # ── Attachment passthrough (inbound / outbound) ───────────────────────────── diff --git a/tests/unit_tests/box/test_workspace.py b/tests/unit_tests/box/test_workspace.py index 48720a48a..f2a204413 100644 --- a/tests/unit_tests/box/test_workspace.py +++ b/tests/unit_tests/box/test_workspace.py @@ -13,8 +13,8 @@ from langbot.pkg.box.workspace import ( classify_python_workspace, infer_workspace_host_path, rewrite_mounted_path, - wrap_python_command_with_env, ) +from langbot.pkg.utils.python_workspace import wrap_python_command_with_env _CONTEXT = ExecutionContext( diff --git a/tests/unit_tests/provider/test_skill_tools.py b/tests/unit_tests/provider/test_skill_tools.py index 17895e2a0..663335149 100644 --- a/tests/unit_tests/provider/test_skill_tools.py +++ b/tests/unit_tests/provider/test_skill_tools.py @@ -631,7 +631,7 @@ class TestNativeToolLoaderSkillPaths: ap.skill_repository.read_skill_file.assert_awaited_once_with(_CONTEXT, 'demo', 'SKILL.md') @pytest.mark.asyncio - async def test_external_runtime_rejects_skill_host_fallback_without_protocol_capability(self): + async def test_core_owned_skill_path_does_not_depend_on_runtime_topology(self): from langbot.pkg.provider.tools.loaders.native import NativeToolLoader from langbot.pkg.provider.tools.loaders.skill import PIPELINE_BOUND_SKILLS_KEY @@ -651,18 +651,20 @@ class TestNativeToolLoaderSkillPaths: variables={PIPELINE_BOUND_SKILLS_KEY: ['demo']}, ) - with pytest.raises(ValueError, match='owned by the Box Runtime'): - await loader.invoke_tool( - 'grep', - { - 'path': '/workspace/.skills/demo', - 'pattern': 'core-host-secret', - }, - query, - ) + result = await loader.invoke_tool( + 'grep', + { + 'path': '/workspace/.skills/demo', + 'pattern': 'core-host-secret', + }, + query, + ) + + assert result['ok'] is True + assert result['total'] == 1 @pytest.mark.asyncio - async def test_exec_in_activated_skill_mount_rewrites_command_and_refreshes(self): + async def test_exec_in_activated_skill_mount_rewrites_command_without_mutating_skill(self): from langbot.pkg.provider.tools.loaders.native import NativeToolLoader from langbot.pkg.provider.tools.loaders.skill import register_activated_skill @@ -673,11 +675,15 @@ class TestNativeToolLoaderSkillPaths: default_workspace=tmpdir, execute_tool=AsyncMock(return_value={'ok': True}), ) - ap.skill_mgr = SimpleNamespace(refresh_skill_from_disk=Mock()) + skill_data = _make_skill_data(name='demo', package_root=tmpdir) + ap.skill_mgr = _make_skill_manager( + {'demo': skill_data}, + refresh_skill_from_disk=Mock(), + ) loader = NativeToolLoader(ap) query = _make_bound_query(query_id='q1', launcher_type='person', launcher_id='123') - register_activated_skill(query, _make_skill_data(name='demo', package_root=tmpdir)) + register_activated_skill(query, skill_data) result = await loader.invoke_tool( 'exec', @@ -692,8 +698,8 @@ class TestNativeToolLoaderSkillPaths: tool_parameters = ap.box_service.execute_tool.await_args.args[0] assert tool_parameters['command'] == 'python /workspace/.skills/demo/scripts/run.py' assert tool_parameters['workdir'] == '/workspace/.skills/demo' - assert ap.box_service.execute_tool.await_args.kwargs['skill_name'] == 'demo' - ap.skill_mgr.refresh_skill_from_disk.assert_called_once_with(_CONTEXT, 'demo') + assert 'skill_name' not in ap.box_service.execute_tool.await_args.kwargs + ap.skill_mgr.refresh_skill_from_disk.assert_not_called() @pytest.mark.asyncio async def test_external_runtime_python_skill_uses_trusted_metadata_and_writable_env(self): @@ -706,17 +712,18 @@ class TestNativeToolLoaderSkillPaths: shares_filesystem_with_box=False, execute_tool=AsyncMock(return_value={'ok': True}), ) - ap.skill_mgr = SimpleNamespace(refresh_skill_from_disk=Mock()) + skill_data = _make_skill_data( + name='demo', + package_root='/box-runtime/skills/tenants/workspace/demo', + python_project=True, + ) + ap.skill_mgr = _make_skill_manager( + {'demo': skill_data}, + refresh_skill_from_disk=Mock(), + ) loader = NativeToolLoader(ap) query = _make_bound_query(query_id='q-external', launcher_type='person', launcher_id='123') - register_activated_skill( - query, - _make_skill_data( - name='demo', - package_root='/box-runtime/skills/tenants/workspace/demo', - python_project=True, - ), - ) + register_activated_skill(query, skill_data) result = await loader.invoke_tool( 'exec', @@ -733,7 +740,7 @@ class TestNativeToolLoaderSkillPaths: assert '_LB_VENV_DIR="/workspace/.skill-envs/demo/.venv"' in wrapped assert 'root = "/workspace/.skills/demo"' in wrapped assert '/box-runtime/skills/tenants/workspace/demo' not in wrapped - assert ap.box_service.execute_tool.await_args.kwargs['skill_name'] == 'demo' + assert 'skill_name' not in ap.box_service.execute_tool.await_args.kwargs @pytest.mark.asyncio async def test_write_requires_skill_activation(self): diff --git a/tests/unit_tests/test_skill_repository.py b/tests/unit_tests/test_skill_repository.py index 9fa3a7ded..783dfa09a 100644 --- a/tests/unit_tests/test_skill_repository.py +++ b/tests/unit_tests/test_skill_repository.py @@ -28,11 +28,11 @@ def _repository(tmp_path) -> SkillRepository: ), instance_config=SimpleNamespace( data={ + 'skills': {'root': str(tmp_path / 'skill-store')}, 'box': { 'enabled': False, 'local': { 'host_root': str(tmp_path / 'box'), - 'skills_root': 'skills', }, } } @@ -41,6 +41,32 @@ def _repository(tmp_path) -> SkillRepository: return SkillRepository(app) +def test_repository_prefers_standalone_skill_root(tmp_path): + repository = _repository(tmp_path) + + assert repository._store.root == str((tmp_path / 'skill-store').resolve()) + + +def test_repository_keeps_old_box_root_only_for_online_upgrade(tmp_path): + app = SimpleNamespace( + workspace_service=SimpleNamespace(get_execution_binding=_binding), + instance_config=SimpleNamespace( + data={ + 'box': { + 'local': { + 'host_root': str(tmp_path / 'box'), + 'skills_root': 'legacy-skills', + } + } + } + ), + ) + + repository = SkillRepository(app) + + assert repository._store.root == str((tmp_path / 'box' / 'legacy-skills').resolve()) + + @pytest.mark.asyncio async def test_repository_crud_and_reads_do_not_require_box(tmp_path): repository = _repository(tmp_path) From 8256ad01ecc4741257fe80f8a6051a070a757dba Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Sat, 5 Sep 2026 23:18:17 +0800 Subject: [PATCH 003/174] style: format core skill orchestration --- src/langbot/pkg/box/service.py | 16 ++++------------ src/langbot/pkg/box/workspace.py | 3 +-- src/langbot/pkg/provider/tools/loaders/native.py | 4 +--- src/langbot/pkg/utils/python_workspace.py | 6 +----- 4 files changed, 7 insertions(+), 22 deletions(-) diff --git a/src/langbot/pkg/box/service.py b/src/langbot/pkg/box/service.py index a7e588974..4802cc438 100644 --- a/src/langbot/pkg/box/service.py +++ b/src/langbot/pkg/box/service.py @@ -485,15 +485,11 @@ class BoxService: """Reject tenant-owned policy fields and apply the Cloud hard policy.""" payload = dict(spec_payload) - trusted_mounts = self._normalize_trusted_read_only_mounts( - trusted_read_only_mounts or [] - ) + trusted_mounts = self._normalize_trusted_read_only_mounts(trusted_read_only_mounts or []) if not self._cloud_managed: if trusted_mounts: if payload.get('extra_mounts'): - raise BoxValidationError( - 'extra_mounts and trusted_read_only_mounts cannot both be supplied' - ) + raise BoxValidationError('extra_mounts and trusted_read_only_mounts cannot both be supplied') payload['extra_mounts'] = trusted_mounts return payload policy = self._admission_policy @@ -568,12 +564,8 @@ class BoxService: if not os.path.isdir(host_path): raise BoxAdmissionError('Core-composed read-only mount source is unavailable') if not any(_is_path_under(host_path, root) for root in self.allowed_mount_roots): - raise BoxAdmissionError( - 'Core-composed read-only mount source is outside allowed_mount_roots' - ) - normalized.append( - mount.model_copy(update={'host_path': host_path}).model_dump(mode='json') - ) + raise BoxAdmissionError('Core-composed read-only mount source is outside allowed_mount_roots') + normalized.append(mount.model_copy(update={'host_path': host_path}).model_dump(mode='json')) return normalized def _reject_cloud_managed_process(self) -> None: diff --git a/src/langbot/pkg/box/workspace.py b/src/langbot/pkg/box/workspace.py index 2e1357f62..95f66ef93 100644 --- a/src/langbot/pkg/box/workspace.py +++ b/src/langbot/pkg/box/workspace.py @@ -18,6 +18,7 @@ import os from typing import Any from ..utils.python_workspace import list_python_manifest_files + _VENV_DIRS = frozenset({'.venv', 'venv', 'env', '.env'}) _VENV_BIN_DIRS = frozenset({'bin', 'Scripts'}) @@ -94,8 +95,6 @@ def classify_python_workspace(host_path: str | None) -> str | None: return None - - class BoxWorkspaceSession: """High-level handle for one reusable workspace-backed Box session. diff --git a/src/langbot/pkg/provider/tools/loaders/native.py b/src/langbot/pkg/provider/tools/loaders/native.py index 17ec96017..57d8e1f4c 100644 --- a/src/langbot/pkg/provider/tools/loaders/native.py +++ b/src/langbot/pkg/provider/tools/loaders/native.py @@ -381,9 +381,7 @@ class NativeToolLoader(loader.ToolLoader): box_service = self.ap.box_service if selected_skill is not None: if not self._can_interpret_skill_host_paths(): - raise ValueError( - 'Secure Core host file operations are unavailable on this platform.' - ) + raise ValueError('Secure Core host file operations are unavailable on this platform.') host_root = selected_skill.get('package_root') workspace_anchor = None else: diff --git a/src/langbot/pkg/utils/python_workspace.py b/src/langbot/pkg/utils/python_workspace.py index b21f7a694..20a23ffa2 100644 --- a/src/langbot/pkg/utils/python_workspace.py +++ b/src/langbot/pkg/utils/python_workspace.py @@ -27,11 +27,7 @@ def list_python_manifest_files(host_path: str | None) -> list[str]: normalized_root = normalize_host_path(host_path) if not normalized_root: return [] - return [ - filename - for filename in PYTHON_MANIFEST_FILES - if os.path.isfile(os.path.join(normalized_root, filename)) - ] + return [filename for filename in PYTHON_MANIFEST_FILES if os.path.isfile(os.path.join(normalized_root, filename))] def should_prepare_python_env(host_path: str | None) -> bool: From 1ab1a29b53397d486dc84e006adbc06ae9bede7e Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Sun, 6 Sep 2026 12:37:09 +0800 Subject: [PATCH 004/174] refactor(skill): move orchestration into core --- docs/review/box-architecture.md | 11 ++++------- src/langbot/pkg/skill/repository.py | 12 ++++++++++-- tests/unit_tests/test_skill_repository.py | 12 ++++++++++-- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/docs/review/box-architecture.md b/docs/review/box-architecture.md index f31f0e27e..5cbae1566 100644 --- a/docs/review/box-architecture.md +++ b/docs/review/box-architecture.md @@ -358,15 +358,12 @@ SkillStore └─ 支持 source_subdir / target_suffix（commit 1aa043f） ``` -GitHub 安装路径由 Core HTTP 层下载归档，再交给 SkillRepository。Skill 文件位于独立的 `skills.root`，执行时由 Core 组装成通用只读 `BoxMountSpec` 并挂载到 `/workspace/.skills/`。Box 的正常模型、客户端和 Runtime 不包含 `skill_name`、Skill CRUD、revision 或 `SKILL.md` 语义。 +GitHub 安装路径由 Core HTTP 层下载归档，再交给 SkillRepository。Skill 文件位于独立的 `skills.root`，执行时由 Core 组装成通用只读 `BoxMountSpec` 并挂载到 `/workspace/.skills/`。Box 的模型、客户端和 Runtime 不包含 `skill_name`、Skill CRUD、revision 或 `SKILL.md` 语义。Core 与 Box Runtime SDK 按同一发布单元同步升级；Box 不保留旧 Skill RPC，也会拒绝旧的 Skill-aware payload 字段。 -滚动升级只保留一个隔离桥：`box/legacy_skill_compat.py` 让旧 Core 暂时调用新 Box，并把旧 `skill_name` 转为普通只读 mount。部署顺序必须先升级 Box、再升级 Core。该模块有 `TODO(next-major)`，下一大版本删除；正常架构不依赖它。 +仍保留的兼容仅用于已有数据的在线升级，不用于混版本协议： -下一大版本的删除清单（当前均有 `TODO(next-major)`）： - -1. 删除 SDK `box/legacy_skill_compat.py` 及 Server 中唯一的注册/转换钩子。 -2. 删除 Core 对 `box.local.skills_root` 的配置 fallback。 -3. 新安装默认从历史 `./data/box/skills` 切到 `./data/skills`；届时 Box 部署只需只读访问 Core 明确下发的通用 artifact root。 +1. Core 暂时读取 `box.local.skills_root`，兼容尚未生成 `skills.root` 的持久化配置。 +2. 新安装默认暂时沿用历史 `./data/box/skills`，避免升级时搬迁已安装 Skill；下一大版本切换到 `./data/skills`。 ### 3.8 Security (`box/security.py`, 52 行) diff --git a/src/langbot/pkg/skill/repository.py b/src/langbot/pkg/skill/repository.py index 9f73ba6e9..86e0ba6ee 100644 --- a/src/langbot/pkg/skill/repository.py +++ b/src/langbot/pkg/skill/repository.py @@ -2,6 +2,7 @@ from __future__ import annotations import asyncio import os +import weakref from langbot_plugin.skill_store import ( SkillRevisionMismatchError, @@ -23,7 +24,14 @@ class SkillRepository: self._local_config = (config.get('box') or {}).get('local') or {} self._skills_config = config.get('skills') or {} self._store = SkillStore(self._skills_root()) - self._lock = asyncio.Lock() + self._locks: weakref.WeakValueDictionary[str, asyncio.Lock] = weakref.WeakValueDictionary() + + def _workspace_lock(self, namespace: str) -> asyncio.Lock: + lock = self._locks.get(namespace) + if lock is None: + lock = asyncio.Lock() + self._locks[namespace] = lock + return lock def _host_root(self) -> str: configured = str(self._local_config.get('host_root') or './data/box').strip() @@ -101,7 +109,7 @@ class SkillRepository: method = getattr(self._store.scoped(namespace), method_name) return method(*args, **kwargs) - async with self._lock: + async with self._workspace_lock(namespace): with blocking_work_scope(f'skill:{namespace}'): return await run_blocking_atomic(invoke) diff --git a/tests/unit_tests/test_skill_repository.py b/tests/unit_tests/test_skill_repository.py index 783dfa09a..9878878b6 100644 --- a/tests/unit_tests/test_skill_repository.py +++ b/tests/unit_tests/test_skill_repository.py @@ -34,7 +34,7 @@ def _repository(tmp_path) -> SkillRepository: 'local': { 'host_root': str(tmp_path / 'box'), }, - } + }, } ), ) @@ -47,6 +47,14 @@ def test_repository_prefers_standalone_skill_root(tmp_path): assert repository._store.root == str((tmp_path / 'skill-store').resolve()) +def test_repository_locks_are_workspace_scoped(tmp_path): + repository = _repository(tmp_path) + + first = repository._workspace_lock('workspace-a') + assert repository._workspace_lock('workspace-a') is first + assert repository._workspace_lock('workspace-b') is not first + + def test_repository_keeps_old_box_root_only_for_online_upgrade(tmp_path): app = SimpleNamespace( workspace_service=SimpleNamespace(get_execution_binding=_binding), @@ -89,7 +97,7 @@ async def test_repository_crud_and_reads_do_not_require_box(tmp_path): skill = await repository.get_skill(_CONTEXT, 'docs-only', snapshot=True) assert skill is not None - assert skill['revision'].startswith('sha256:') + assert skill['revision'].startswith('stat-v1:') assert [item['name'] for item in await repository.list_skills(_CONTEXT)] == ['docs-only'] listed = await repository.list_skill_resources( From 9b8d1b4034643caacdd15e0a0fd9e43acdd8fdb6 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:04:11 +0800 Subject: [PATCH 005/174] feat(skill): pin run revision bindings --- ARCHITECTURE.md | 13 +- .../pkg/api/http/controller/groups/skills.py | 14 +- src/langbot/pkg/api/http/service/skill.py | 43 ++- .../pkg/provider/tools/loaders/native.py | 74 ++--- .../pkg/provider/tools/loaders/skill.py | 94 +++++- .../provider/tools/loaders/skill_authoring.py | 20 +- src/langbot/pkg/skill/activation.py | 44 --- src/langbot/pkg/skill/repository.py | 66 +++- .../test_cloud_box_admission_integration.py | 20 +- tests/unit_tests/box/test_box_service.py | 79 +++-- tests/unit_tests/provider/test_skill_tools.py | 289 ++++++++++++------ tests/unit_tests/test_skill_repository.py | 130 +++++++- tests/unit_tests/test_skill_service.py | 35 ++- .../components/skill-form/SkillForm.tsx | 5 +- web/src/app/infra/entities/api/index.ts | 1 + web/src/app/infra/http/BackendClient.ts | 4 +- 16 files changed, 657 insertions(+), 274 deletions(-) delete mode 100644 src/langbot/pkg/skill/activation.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 3a4aaf42e..d49fdd2e6 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -199,7 +199,7 @@ In this repo: - `pkg/provider/tools/loaders/native.py` is the Core orchestration seam: the Skill loader supplies generic read-only mounts to Box execution. `mcp_stdio.py` and execution-backed tools depend on Box availability. -- `pkg/skill/repository.py` is the thin async/Workspace adapter over the Plugin SDK's execution-independent `SkillStore`; `skills.root` owns its location independently of Box. +- `pkg/skill/repository.py` is the thin async/Workspace adapter over the Plugin SDK's execution-independent `SkillStore`; `skills.root` owns its location independently of Box. Core's registry points to immutable content-addressed publications, and every update carries `base_revision` so concurrent writers fail instead of overwriting each other. - `pkg/skill/manager.py` caches the Core repository catalog for progressive disclosure. Activation and read-only resource tools do not require Box; script execution and Workspace mutation still do. Durable Box Workspace storage is shared across placement generations, but @@ -213,9 +213,14 @@ In `langbot-plugin-sdk`: - `src/langbot_plugin/box/server.py` implements `lbp box` and the WebSocket endpoints on `:5410`. - `src/langbot_plugin/box/runtime.py` owns sandbox sessions, generic read-only mounts, and managed processes. - `backend.py`, `nsjail_backend.py`, and `e2b_backend.py` implement sandbox backends. -- `src/langbot_plugin/skill_store.py` is consumed by Core, not Box. Core turns - selected package roots into generic read-only mounts; Box does not understand - Skill names, metadata, revisions, files, or CRUD. +- `src/langbot_plugin/skill_store.py` is consumed by Core, not Box. Activation + pins the first revision for a run; instructions, resource reads, restart + recovery, execution mounts, and revision-scoped dependency state all use that + exact publication. Core mounts only activated and authorized Skills. Published + trees are never agent write targets: authoring happens in Workspace drafts and + an explicit registration/update atomically publishes the next revision. Core + turns those package roots and content digests into generic read-only mounts; + Box does not understand Skill names, metadata, revisions, files, or CRUD. Skill storage uses `skills.root`. Box execution config lives under `box:`: `box.enabled`, `box.backend`, `box.runtime.endpoint`, and `box.local.*`. The old diff --git a/src/langbot/pkg/api/http/controller/groups/skills.py b/src/langbot/pkg/api/http/controller/groups/skills.py index 214230f0c..9d245b525 100644 --- a/src/langbot/pkg/api/http/controller/groups/skills.py +++ b/src/langbot/pkg/api/http/controller/groups/skills.py @@ -5,6 +5,7 @@ import quart from ...authz import Permission from ...context import RequestContext from .....operation_trace import service as settings_service +from .....skill.repository import SkillRevisionConflictError from .. import group @@ -76,6 +77,8 @@ class SkillsRouterGroup(group.RouterGroup): previous = None try: skill = await self.ap.skill_service.update_skill(request_context, skill_name, data) + except SkillRevisionConflictError as exc: + return self.http_status(409, -1, str(exc)) except ValueError as exc: return self.http_status(400, -1, str(exc)) @@ -115,6 +118,8 @@ class SkillsRouterGroup(group.RouterGroup): include_hidden=include_hidden, ) return self.success(data=result) + except SkillRevisionConflictError as exc: + return self.http_status(409, -1, str(exc)) except ValueError as exc: return self.http_status(400, -1, str(exc)) @@ -140,11 +145,18 @@ class SkillsRouterGroup(group.RouterGroup): async def write_skill_file(skill_name: str, path: str, request_context: RequestContext) -> quart.Response: data = await quart.request.json content = data.get('content', '') + base_revision = str(data.get('base_revision', '') or '').strip() or None if content is None: return self.http_status(400, -1, 'Missing required field: content') try: - result = await self.ap.skill_service.write_skill_file(request_context, skill_name, path, content) + result = await self.ap.skill_service.write_skill_file( + request_context, + skill_name, + path, + content, + base_revision=base_revision, + ) return self.success(data=result) except ValueError as exc: return self.http_status(400, -1, str(exc)) diff --git a/src/langbot/pkg/api/http/service/skill.py b/src/langbot/pkg/api/http/service/skill.py index fcd4e5313..bc613e7dc 100644 --- a/src/langbot/pkg/api/http/service/skill.py +++ b/src/langbot/pkg/api/http/service/skill.py @@ -103,15 +103,34 @@ class SkillService: await self._reload_skills(execution_context) return self._serialize_skill(created) - async def import_skill_directory(self, context: TenantContext, path: str, data: dict) -> dict: + async def import_skill_directory( + self, + context: TenantContext, + path: str, + data: dict, + *, + base_revision: str | None = None, + ) -> dict: execution_context = await self._execution_context(context) - created = await self._repository().import_skill_directory(execution_context, path, data) + created = await self._repository().import_skill_directory( + execution_context, + path, + data, + base_revision=base_revision, + ) await self._reload_skills(execution_context) return self._serialize_skill(created) async def update_skill(self, context: TenantContext, skill_name: str, data: dict) -> dict: execution_context = await self._execution_context(context) - updated = await self._repository().update_skill(execution_context, skill_name, data) + payload = dict(data) + base_revision = str(payload.pop('base_revision', '') or '').strip() or None + updated = await self._repository().update_skill( + execution_context, + skill_name, + payload, + base_revision=base_revision, + ) await self._reload_skills(execution_context) return self._serialize_skill(updated) @@ -142,9 +161,23 @@ class SkillService: execution_context = await self._execution_context(context) return await self._repository().read_skill_file(execution_context, skill_name, path) - async def write_skill_file(self, context: TenantContext, skill_name: str, path: str, content: str) -> dict: + async def write_skill_file( + self, + context: TenantContext, + skill_name: str, + path: str, + content: str, + *, + base_revision: str | None, + ) -> dict: execution_context = await self._execution_context(context) - result = await self._repository().write_skill_file(execution_context, skill_name, path, content) + result = await self._repository().write_skill_file( + execution_context, + skill_name, + path, + content, + base_revision=base_revision, + ) await self._reload_skills(execution_context) return result diff --git a/src/langbot/pkg/provider/tools/loaders/native.py b/src/langbot/pkg/provider/tools/loaders/native.py index 57d8e1f4c..1c6d6e345 100644 --- a/src/langbot/pkg/provider/tools/loaders/native.py +++ b/src/langbot/pkg/provider/tools/loaders/native.py @@ -340,11 +340,14 @@ class NativeToolLoader(loader.ToolLoader): if 'python_project' not in selected_skill: python_project = skill_loader.should_prepare_skill_python_env(package_root) if python_project: + revision_key = str(selected_skill.get('revision', '') or '').removeprefix('sha256:') + if not revision_key: + raise ValueError(f'Activated skill "{selected_skill_name}" has no pinned revision.') parameters = dict(parameters) parameters['command'] = skill_loader.wrap_skill_command_with_python_env( command, mount_path=skill_mount, - state_path=f'/workspace/.skill-envs/{selected_skill_name}', + state_path=f'/workspace/.skill-envs/{selected_skill_name}/{revision_key}', ) # All exec calls (with or without skills) go through the same container @@ -1119,25 +1122,27 @@ else: skill_request = self._resolve_skill_relative_path( query, path, - include_visible=True, + include_visible=False, include_activated=True, ) skill_repository = getattr(self.ap, 'skill_repository', None) if skill_request is not None and skill_repository is not None: selected_skill, relative = skill_request try: - result = await skill_repository.read_skill_file( + result = await skill_repository.read_skill_resource( self._execution_context(query), selected_skill['name'], relative, + expected_revision=selected_skill.get('revision'), ) return self._build_read_result_from_text(str(result.get('content', '')), parameters) except Exception: try: - result = await skill_repository.list_skill_files( + result = await skill_repository.list_skill_resources( self._execution_context(query), selected_skill['name'], relative, + expected_revision=selected_skill.get('revision'), ) entries = [entry['name'] for entry in result.get('entries', [])] return self._build_directory_result(entries) @@ -1147,7 +1152,7 @@ else: host_location = self._resolve_host_location( query, path, - include_visible=True, + include_visible=False, include_activated=True, ) if host_location is None: @@ -1161,22 +1166,22 @@ else: path = parameters['path'] content = parameters['content'] self.ap.logger.info(f'write tool invoked: query_id={query.query_id} path={path} length={len(content)}') - encoding, _mode = self._write_options(parameters) + self._write_options(parameters) skill_request = self._resolve_skill_relative_path( query, path, include_visible=False, include_activated=True, ) - skill_repository = getattr(self.ap, 'skill_repository', None) - if skill_request is not None and skill_repository is not None: - if encoding != 'text': - return {'ok': False, 'error': 'base64 writes to skill packages are not supported.'} - selected_skill, relative = skill_request - execution_context = self._execution_context(query) - await skill_repository.write_skill_file(execution_context, selected_skill['name'], relative, content) - await self.ap.skill_mgr.reload_skills(execution_context) - return {'ok': True, 'path': path} + if skill_request is not None: + return { + 'ok': False, + 'error': ( + 'Published Skill revisions are immutable. Copy the package to a writable ' + 'directory under /workspace/skill-drafts, edit it there, then call register_skill ' + 'with the activated revision as base_revision.' + ), + } host_location = self._resolve_host_location( query, @@ -1207,32 +1212,15 @@ else: include_visible=False, include_activated=True, ) - if skill_request is not None and getattr(self.ap, 'skill_repository', None) is not None: - selected_skill, relative = skill_request - try: - result = await self.ap.skill_repository.read_skill_file( - self._execution_context(query), - selected_skill['name'], - relative, - ) - except Exception: - return {'ok': False, 'error': f'File not found: {path}'} - content = result.get('content', '') - count = content.count(old_string) - if count == 0: - return {'ok': False, 'error': 'old_string not found in file.'} - if count > 1: - return {'ok': False, 'error': f'old_string matches {count} locations; provide a more unique string.'} - new_content = content.replace(old_string, new_string, 1) - execution_context = self._execution_context(query) - await self.ap.skill_repository.write_skill_file( - execution_context, - selected_skill['name'], - relative, - new_content, - ) - await self.ap.skill_mgr.reload_skills(execution_context) - return {'ok': True, 'path': path} + if skill_request is not None: + return { + 'ok': False, + 'error': ( + 'Published Skill revisions are immutable. Copy the package to a writable ' + 'directory under /workspace/skill-drafts, edit it there, then call register_skill ' + 'with the activated revision as base_revision.' + ), + } host_location = self._resolve_host_location( query, @@ -1532,7 +1520,7 @@ else: host_location = self._resolve_host_location( query, path, - include_visible=True, + include_visible=False, include_activated=True, ) if host_location is None: @@ -1554,7 +1542,7 @@ else: host_location = self._resolve_host_location( query, path, - include_visible=True, + include_visible=False, include_activated=True, ) if host_location is None: diff --git a/src/langbot/pkg/provider/tools/loaders/skill.py b/src/langbot/pkg/provider/tools/loaders/skill.py index adc016a73..9669b7982 100644 --- a/src/langbot/pkg/provider/tools/loaders/skill.py +++ b/src/langbot/pkg/provider/tools/loaders/skill.py @@ -74,24 +74,31 @@ def register_created_skill_visibility(query: pipeline_query.Query, skill_name: s bound_skills.append(skill_name) def build_execution_mounts(ap: app.Application, query: pipeline_query.Query) -> list[dict]: - """Translate visible Core-owned packages into generic read-only mounts.""" + """Mount only immutable revisions pinned by this run's activations.""" mounts: list[dict] = [] - for skill_name, skill_data in get_visible_skills(ap, query).items(): + for skill_name, skill_data in get_activated_skills(query).items(): package_root = str(skill_data.get('package_root', '') or '').strip() + manifest_path = str(skill_data.get('manifest_path', '') or '').strip() + revision = str(skill_data.get('revision', '') or '').strip() if not package_root: - continue + raise ValueError(f'Activated skill "{skill_name}" has no immutable package root.') + if not revision: + raise ValueError(f'Activated skill "{skill_name}" has no pinned revision.') if not os.path.isdir(package_root): - ap.logger.warning( - f'Skill "{skill_name}" package_root missing on the Core filesystem ' - f'({package_root}); skipping its execution mount. Reload the skill catalog.' + raise ValueError( + f'Activated skill "{skill_name}" pinned revision {revision} is unavailable; ' + 'the run cannot be recovered safely.' ) - continue + if not manifest_path or not os.path.isfile(manifest_path): + raise ValueError(f'Activated skill "{skill_name}" pinned revision {revision} has no publication manifest.') mounts.append( { 'host_path': package_root, 'mount_path': get_virtual_skill_mount_path(skill_name), 'mode': 'ro', + 'content_digest': revision, + 'manifest_path': manifest_path, } ) return mounts @@ -111,14 +118,18 @@ def get_activated_skill(query: pipeline_query.Query, skill_name: str) -> dict | return get_activated_skills(query).get(skill_name) -def register_activated_skill(query: pipeline_query.Query, skill_data: dict) -> None: +def register_activated_skill(query: pipeline_query.Query, skill_data: dict) -> dict: if query.variables is None: query.variables = {} activated = query.variables.setdefault(ACTIVATED_SKILLS_KEY, {}) skill_name = str(skill_data.get('name', '') or '').strip() - if skill_name and skill_name not in activated: - activated[skill_name] = skill_data + revision = str(skill_data.get('revision', '') or '').strip() + if not skill_name or not revision: + raise ValueError('Activated Skills require a name and immutable revision.') + if skill_name not in activated: + activated[skill_name] = dict(skill_data) + return activated[skill_name] def normalize_skill_names(value: typing.Any) -> list[str]: @@ -139,17 +150,68 @@ def get_activated_skill_names(query: pipeline_query.Query) -> list[str]: return normalize_skill_names(list(get_activated_skills(query).keys())) -def restore_activated_skills( +def get_activated_skill_bindings(query: pipeline_query.Query) -> list[dict[str, str]]: + """Return exact bindings suitable for persistence and restart recovery.""" + + return [ + {'name': name, 'revision': str(skill.get('revision', '') or '')} + for name, skill in get_activated_skills(query).items() + ] + + +def normalize_skill_bindings(value: typing.Any) -> list[dict[str, str]]: + if not isinstance(value, list): + raise ValueError('Activated Skill recovery requires revision bindings.') + bindings: list[dict[str, str]] = [] + seen: dict[str, str] = {} + for item in value: + if not isinstance(item, dict): + raise ValueError('Activated Skill recovery cannot use names without revisions.') + name = str(item.get('name', '') or '').strip() + revision = str(item.get('revision', '') or '').strip() + if not name or not revision: + raise ValueError('Each activated Skill binding requires name and revision.') + previous = seen.get(name) + if previous is not None and previous != revision: + raise ValueError(f'Conflicting pinned revisions supplied for Skill "{name}".') + if previous is None: + seen[name] = revision + bindings.append({'name': name, 'revision': revision}) + return bindings + + +async def restore_activated_skills( ap: app.Application, query: pipeline_query.Query, - skill_names: typing.Any, + skill_bindings: typing.Any, ) -> list[str]: - """Restore caller-provided names from the current visible skill set.""" + """Restore exact revisions or fail explicitly; never resolve latest by name.""" + + repository = getattr(ap, 'skill_repository', None) + if repository is None: + raise ValueError('Skill repository is unavailable during run recovery.') + context = ExecutionContext( + instance_uuid=str(getattr(query, 'instance_uuid', '') or ''), + workspace_uuid=str(getattr(query, 'workspace_uuid', '') or ''), + placement_generation=getattr(query, 'placement_generation', 0) or 0, + bot_uuid=getattr(query, 'bot_uuid', None), + pipeline_uuid=getattr(query, 'pipeline_uuid', None), + query_uuid=getattr(query, 'query_uuid', None), + ) + bound_names = get_bound_skill_names(query) restored: list[str] = [] - for skill_name in normalize_skill_names(skill_names): - skill_data = get_visible_skill(ap, query, skill_name) + for binding in normalize_skill_bindings(skill_bindings): + skill_name = binding['name'] + if bound_names is not None and skill_name not in bound_names: + raise ValueError(f'Skill "{skill_name}" is no longer authorized for this recoverable run.') + skill_data = await repository.get_skill( + context, + skill_name, + snapshot=True, + revision=binding['revision'], + ) if skill_data is None: - continue + raise ValueError(f'Skill "{skill_name}" pinned revision {binding["revision"]} is unavailable.') register_activated_skill(query, skill_data) restored.append(skill_name) return restored diff --git a/src/langbot/pkg/provider/tools/loaders/skill_authoring.py b/src/langbot/pkg/provider/tools/loaders/skill_authoring.py index dc0074d47..bbe566fa7 100644 --- a/src/langbot/pkg/provider/tools/loaders/skill_authoring.py +++ b/src/langbot/pkg/provider/tools/loaders/skill_authoring.py @@ -172,7 +172,8 @@ class SkillToolLoader(loader.ToolLoader): if skill_data is None: raise ValueError(f'Skill "{skill_name}" is no longer available; reload the skill catalog.') - skill_loader.register_activated_skill(query, skill_data) + # Keep the first revision pinned if the Skill is activated again in this run. + skill_data = skill_loader.register_activated_skill(query, skill_data) await skill_loader.persist_activated_skill(self.ap, query, skill_name) instructions = skill_data.get('instructions', '') @@ -289,8 +290,8 @@ class SkillToolLoader(loader.ToolLoader): if not skill_name: raise ValueError('skill name is required') - # Create the skill - created = await skill_service.import_skill_directory( + base_revision = str(parameters.get('base_revision', '') or '').strip() or None + published = await skill_service.import_skill_directory( execution_context, host_path, { @@ -299,6 +300,7 @@ class SkillToolLoader(loader.ToolLoader): 'description': str(parameters.get('description') or scanned.get('description', '')).strip(), 'instructions': str(parameters.get('instructions') or scanned.get('instructions', '')), }, + base_revision=base_revision, ) skill_loader.register_created_skill_visibility(query, skill_name) @@ -306,7 +308,8 @@ class SkillToolLoader(loader.ToolLoader): 'registered': True, 'skill_name': skill_name, 'source_path': sandbox_path, - 'skill': created, + 'revision': published.get('revision'), + 'skill': published, } def _resolve_workspace_directory( @@ -405,9 +408,10 @@ class SkillToolLoader(loader.ToolLoader): name=REGISTER_SKILL_TOOL_NAME, human_desc='Register a skill from sandbox', description=( - "Register a skill package from a directory under /workspace into LangBot's skill store. " - 'Use this after creating or preparing a skill in the sandbox with exec/read/write/edit. ' + "Publish a skill draft from a directory under /workspace into LangBot's skill store. " + 'Use this after creating or preparing the draft with exec/read/write/edit. ' 'The directory must contain a SKILL.md file. ' + 'Updating an existing Skill requires its current base_revision; conflicting updates are rejected. ' 'After registration, the skill can be activated with the activate tool.' ), parameters={ @@ -433,6 +437,10 @@ class SkillToolLoader(loader.ToolLoader): 'type': 'string', 'description': 'Optional instructions override.', }, + 'base_revision': { + 'type': 'string', + 'description': 'Required current revision when publishing an update; omit for a new Skill.', + }, }, 'required': ['path'], 'additionalProperties': False, diff --git a/src/langbot/pkg/skill/activation.py b/src/langbot/pkg/skill/activation.py deleted file mode 100644 index 27ce1b3da..000000000 --- a/src/langbot/pkg/skill/activation.py +++ /dev/null @@ -1,44 +0,0 @@ -from __future__ import annotations - -import typing - -from ..provider.tools.loaders import skill as skill_loader -from ..api.http.context import ExecutionContext - -if typing.TYPE_CHECKING: - from ..core import app - import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query - - -# Skill activation is now handled through Tool Call mechanism (activate tool). -# This file is kept for potential future extensions but the text marker -# detection mechanism has been removed. - - -def register_activated_skill( - ap: app.Application, - query: pipeline_query.Query, - skill_name: str, -) -> bool: - """Register an activated skill for sandbox mount path resolution. - - This is called by the activate tool when a skill is activated via Tool Call. - """ - skill_mgr = getattr(ap, 'skill_mgr', None) - if skill_mgr is None: - return False - - execution_context = ExecutionContext( - instance_uuid=str(getattr(query, 'instance_uuid', '') or ''), - workspace_uuid=str(getattr(query, 'workspace_uuid', '') or ''), - placement_generation=getattr(query, 'placement_generation', 0) or 0, - bot_uuid=getattr(query, 'bot_uuid', None), - pipeline_uuid=getattr(query, 'pipeline_uuid', None), - query_uuid=getattr(query, 'query_uuid', None), - ) - skill_data = skill_mgr.get_skill_by_name(execution_context, skill_name) - if skill_data is None: - return False - - skill_loader.register_activated_skill(query, skill_data) - return True diff --git a/src/langbot/pkg/skill/repository.py b/src/langbot/pkg/skill/repository.py index 86e0ba6ee..6aaaa0c95 100644 --- a/src/langbot/pkg/skill/repository.py +++ b/src/langbot/pkg/skill/repository.py @@ -5,7 +5,9 @@ import os import weakref from langbot_plugin.skill_store import ( + SkillRevisionConflictError, SkillRevisionMismatchError, + SkillRevisionNotFoundError, SkillStore, skill_namespace, ) @@ -116,13 +118,29 @@ class SkillRepository: async def list_skills(self, context: TenantContext) -> list[dict]: return await self._call(context, 'list_skills') - async def get_skill(self, context: TenantContext, name: str, *, snapshot: bool = False) -> dict | None: + async def get_skill( + self, + context: TenantContext, + name: str, + *, + snapshot: bool = False, + revision: str | None = None, + ) -> dict | None: + if revision is not None: + return await self._call(context, 'get_skill_snapshot', name, revision) return await self._call(context, 'get_skill_snapshot' if snapshot else 'get_skill', name) async def create_skill(self, context: TenantContext, skill: dict) -> dict: return await self._call(context, 'create_skill', skill) - async def import_skill_directory(self, context: TenantContext, path: str, skill: dict) -> dict: + async def import_skill_directory( + self, + context: TenantContext, + path: str, + skill: dict, + *, + base_revision: str | None = None, + ) -> dict: namespace = self._namespace(context) return await self._call( context, @@ -130,10 +148,24 @@ class SkillRepository: path, skill, source_root=self._workspace_root(namespace), + base_revision=base_revision, ) - async def update_skill(self, context: TenantContext, name: str, skill: dict) -> dict: - return await self._call(context, 'update_skill', name, skill) + async def update_skill( + self, + context: TenantContext, + name: str, + skill: dict, + *, + base_revision: str | None, + ) -> dict: + return await self._call( + context, + 'update_skill', + name, + skill, + base_revision=base_revision, + ) async def delete_skill(self, context: TenantContext, name: str) -> None: await self._call(context, 'delete_skill', name) @@ -194,8 +226,23 @@ class SkillRepository: expected_revision=expected_revision, ) - async def write_skill_file(self, context: TenantContext, name: str, path: str, content: str) -> dict: - return await self._call(context, 'write_skill_file', name, path, content) + async def write_skill_file( + self, + context: TenantContext, + name: str, + path: str, + content: str, + *, + base_revision: str | None, + ) -> dict: + return await self._call( + context, + 'write_skill_file', + name, + path, + content, + base_revision=base_revision, + ) async def preview_skill_zip(self, context: TenantContext, file_bytes: bytes, filename: str, **kwargs) -> list[dict]: return await self._call(context, 'preview_zip_upload', file_bytes=file_bytes, filename=filename, **kwargs) @@ -204,4 +251,9 @@ class SkillRepository: return await self._call(context, 'install_zip_upload', file_bytes=file_bytes, filename=filename, **kwargs) -__all__ = ['SkillRepository', 'SkillRevisionMismatchError'] +__all__ = [ + 'SkillRepository', + 'SkillRevisionConflictError', + 'SkillRevisionMismatchError', + 'SkillRevisionNotFoundError', +] diff --git a/tests/integration_tests/box/test_cloud_box_admission_integration.py b/tests/integration_tests/box/test_cloud_box_admission_integration.py index 78252c3c9..0d01bc1d9 100644 --- a/tests/integration_tests/box/test_cloud_box_admission_integration.py +++ b/tests/integration_tests/box/test_cloud_box_admission_integration.py @@ -368,13 +368,26 @@ async def test_cloud_core_skills_mount_generically_and_do_not_require_box_entitl 'instructions': 'Run scripts/main.py', }, ) - await repository.write_skill_file(first, 'runner', 'scripts/main.py', "print('ok')") - await repository.write_skill_file(first, 'runner', 'requirements.txt', 'requests==2.32.0\n') + script_revision = await repository.write_skill_file( + first, + 'runner', + 'scripts/main.py', + "print('ok')", + base_revision=own_skill['revision'], + ) + await repository.write_skill_file( + first, + 'runner', + 'requirements.txt', + 'requests==2.32.0\n', + base_revision=script_revision['revision'], + ) refreshed_skill = await repository.get_skill(first, 'runner') assert refreshed_skill is not None assert refreshed_skill['python_project'] is True await service.ap.skill_mgr.reload_skills(first) query = await _bound_query(service, first, 91) + skill_loader.register_activated_skill(query, refreshed_skill) await service.execute_tool( { 'command': 'python /workspace/.skills/runner/scripts/main.py', @@ -386,9 +399,10 @@ async def test_cloud_core_skills_mount_generically_and_do_not_require_box_entitl mounted_spec = backend.started_specs[-1] assert len(mounted_spec.extra_mounts) == 1 - assert mounted_spec.extra_mounts[0].host_path == own_skill['package_root'] + assert mounted_spec.extra_mounts[0].host_path == refreshed_skill['package_root'] assert mounted_spec.extra_mounts[0].mount_path == '/workspace/.skills/runner' assert mounted_spec.extra_mounts[0].mode.value == 'ro' + assert mounted_spec.extra_mounts[0].content_digest == refreshed_skill['revision'] assert await repository.get_skill(first, 'private') is None await repository.create_skill( diff --git a/tests/unit_tests/box/test_box_service.py b/tests/unit_tests/box/test_box_service.py index 943c035a0..610163c68 100644 --- a/tests/unit_tests/box/test_box_service.py +++ b/tests/unit_tests/box/test_box_service.py @@ -1802,29 +1802,36 @@ def test_disconnect_callback_does_not_schedule_without_running_event_loop(): class TestBuildSkillExecutionMounts: - """Robustness of skill mount construction against a stale skill cache. - - The three sandbox backends behave inconsistently when a skill's - package_root no longer exists on disk (nsjail aborts the whole sandbox - start, Docker silently auto-creates a root-owned empty directory, E2B - silently skips). Mount construction must filter these out up front so - the backend never sees a bad mount. - """ + """Execution materializes only revisions pinned by this run.""" def _make_app(self, logger, skills): app = make_app(logger) app.skill_mgr = SimpleNamespace(skills=skills, get_skills=Mock(return_value=skills)) return app - def test_skips_skill_with_missing_package_root(self): + def test_mounts_only_activated_revision(self): logger = Mock() with tempfile.TemporaryDirectory() as live_dir: + manifest_path = os.path.join(live_dir, 'manifest.json') + with open(manifest_path, 'w', encoding='utf-8') as file: + file.write('{}') skills = { - 'alive': {'name': 'alive', 'package_root': live_dir}, - 'ghost': {'name': 'ghost', 'package_root': '/nonexistent/path/should/never/exist'}, + 'alive': { + 'name': 'alive', + 'package_root': live_dir, + 'manifest_path': manifest_path, + 'revision': 'sha256:' + '1' * 64, + }, + 'visible-not-activated': { + 'name': 'visible-not-activated', + 'package_root': live_dir, + 'manifest_path': manifest_path, + 'revision': 'sha256:' + '2' * 64, + }, } app = self._make_app(logger, skills) query = make_query() + skill_loader.register_activated_skill(query, skills['alive']) mounts = skill_loader.build_execution_mounts(app, query) @@ -1833,42 +1840,56 @@ class TestBuildSkillExecutionMounts: 'host_path': live_dir, 'mount_path': '/workspace/.skills/alive', 'mode': 'ro', + 'content_digest': 'sha256:' + '1' * 64, + 'manifest_path': manifest_path, } ] - # Warning logged so operators can see what was dropped - assert any( - 'ghost' in str(call.args[0]) and 'package_root missing' in str(call.args[0]) - for call in logger.warning.call_args_list - ) - def test_rejects_missing_core_paths_when_filesystem_not_shared(self): - """Core owns package paths even when Box is a separate process.""" + def test_missing_pinned_revision_fails_instead_of_being_skipped(self): logger = Mock() skills = { - 'a': {'name': 'a', 'package_root': '/box/skills/a'}, - 'b': {'name': 'b', 'package_root': '/box/skills/b'}, + 'a': { + 'name': 'a', + 'package_root': '/box/skills/a', + 'manifest_path': '/box/skills/manifest.json', + 'revision': 'sha256:' + '1' * 64, + } } app = self._make_app(logger, skills) + query = make_query() + skill_loader.register_activated_skill(query, skills['a']) - mounts = skill_loader.build_execution_mounts(app, make_query()) + with pytest.raises(ValueError, match='cannot be recovered safely'): + skill_loader.build_execution_mounts(app, query) - assert mounts == [] - assert len(logger.warning.call_args_list) == 2 - - def test_skips_skill_with_empty_package_root(self): + def test_rejects_activated_skill_with_empty_package_root(self): logger = Mock() skills = { - 'no_root': {'name': 'no_root', 'package_root': ''}, - 'whitespace': {'name': 'whitespace', 'package_root': ' '}, + 'no_root': { + 'name': 'no_root', + 'package_root': '', + 'manifest_path': '', + 'revision': 'sha256:' + '1' * 64, + } } app = self._make_app(logger, skills) + query = make_query() + skill_loader.register_activated_skill(query, skills['no_root']) - assert skill_loader.build_execution_mounts(app, make_query()) == [] + with pytest.raises(ValueError, match='no immutable package root'): + skill_loader.build_execution_mounts(app, query) def test_empty_package_root_skipped_even_when_not_shared(self): """An empty package_root is always invalid regardless of topology.""" logger = Mock() - skills = {'no_root': {'name': 'no_root', 'package_root': ''}} + skills = { + 'no_root': { + 'name': 'no_root', + 'package_root': '', + 'manifest_path': '', + 'revision': 'sha256:' + '1' * 64, + } + } app = self._make_app(logger, skills) assert skill_loader.build_execution_mounts(app, make_query()) == [] diff --git a/tests/unit_tests/provider/test_skill_tools.py b/tests/unit_tests/provider/test_skill_tools.py index 663335149..7c25371a6 100644 --- a/tests/unit_tests/provider/test_skill_tools.py +++ b/tests/unit_tests/provider/test_skill_tools.py @@ -68,7 +68,9 @@ def _make_skill_data( 'description': kwargs.pop('description', f'Description of {name}'), 'instructions': instructions, 'package_root': package_root, + 'manifest_path': kwargs.pop('manifest_path', ''), 'entry_file': entry_file, + 'revision': kwargs.pop('revision', 'sha256:' + '1' * 64), **kwargs, } @@ -116,58 +118,6 @@ class TestSkillManagerCache: repository.list_skills.assert_awaited_once_with(_CONTEXT) -class TestSkillActivationHelper: - """Skill activation is now Tool-Call based. - - The legacy text-marker mechanism (``[ACTIVATE_SKILL: x]`` detection, - ``build_activation_prompt_for_skills``, ``remove_activation_marker``, - ``prepare_skill_activation``) has been removed. Activation now goes - through ``skill.activation.register_activated_skill``, invoked by the - ``activate`` Tool Call. - """ - - def test_register_activated_skill_records_known_skill(self): - from langbot.pkg.skill.activation import register_activated_skill - from langbot.pkg.provider.tools.loaders.skill import ACTIVATED_SKILLS_KEY - from langbot.pkg.skill.manager import SkillManager - - ap = _make_ap() - mgr = SkillManager(ap) - mgr._skills_by_scope[mgr._scope_key(_CONTEXT)] = { - 'primary': _make_skill_data(name='primary', instructions='Primary instructions'), - } - ap.skill_mgr = mgr - - query = _make_query() - - assert register_activated_skill(ap, query, 'primary') is True - assert set(query.variables[ACTIVATED_SKILLS_KEY].keys()) == {'primary'} - assert query.variables[ACTIVATED_SKILLS_KEY]['primary']['name'] == 'primary' - - def test_register_activated_skill_rejects_unknown_skill(self): - from langbot.pkg.skill.activation import register_activated_skill - from langbot.pkg.provider.tools.loaders.skill import ACTIVATED_SKILLS_KEY - from langbot.pkg.skill.manager import SkillManager - - ap = _make_ap() - mgr = SkillManager(ap) - mgr._skills_by_scope[mgr._scope_key(_CONTEXT)] = {'primary': _make_skill_data(name='primary')} - ap.skill_mgr = mgr - - query = _make_query() - - assert register_activated_skill(ap, query, 'missing') is False - assert ACTIVATED_SKILLS_KEY not in query.variables - - def test_register_activated_skill_without_skill_manager_returns_false(self): - from langbot.pkg.skill.activation import register_activated_skill - - ap = _make_ap() # no skill_mgr attribute - query = _make_query() - - assert register_activated_skill(ap, query, 'primary') is False - - class TestSkillPathHelpers: def test_get_visible_skills_filters_by_bound_names(self): from langbot.pkg.provider.tools.loaders.skill import PIPELINE_BOUND_SKILLS_KEY, get_visible_skills @@ -185,28 +135,49 @@ class TestSkillPathHelpers: assert list(result.keys()) == ['visible'] - def test_restore_activated_skills_uses_caller_provided_names_and_visibility(self): + @pytest.mark.asyncio + async def test_restore_activated_skills_uses_exact_revision_bindings(self): from langbot.pkg.provider.tools.loaders.skill import ( ACTIVATED_SKILLS_KEY, PIPELINE_BOUND_SKILLS_KEY, + get_activated_skill_bindings, get_activated_skill_names, restore_activated_skills, ) ap = _make_ap() - ap.skill_mgr = _make_skill_manager( - { - 'visible': _make_skill_data(name='visible'), - 'hidden': _make_skill_data(name='hidden'), - } + pinned = _make_skill_data(name='visible', revision='sha256:' + '2' * 64) + ap.skill_repository = SimpleNamespace( + get_skill=AsyncMock(return_value=pinned), ) query = _make_query(variables={PIPELINE_BOUND_SKILLS_KEY: ['visible']}) - restored = restore_activated_skills(ap, query, ['visible', 'hidden', 'visible', '']) + restored = await restore_activated_skills( + ap, + query, + [{'name': 'visible', 'revision': pinned['revision']}], + ) assert restored == ['visible'] assert list(query.variables[ACTIVATED_SKILLS_KEY].keys()) == ['visible'] assert get_activated_skill_names(query) == ['visible'] + assert get_activated_skill_bindings(query) == [{'name': 'visible', 'revision': pinned['revision']}] + ap.skill_repository.get_skill.assert_awaited_once_with( + _CONTEXT, + 'visible', + snapshot=True, + revision=pinned['revision'], + ) + + @pytest.mark.asyncio + async def test_restore_rejects_name_only_state(self): + from langbot.pkg.provider.tools.loaders.skill import restore_activated_skills + + ap = _make_ap() + ap.skill_repository = SimpleNamespace(get_skill=AsyncMock()) + + with pytest.raises(ValueError, match='cannot use names without revisions'): + await restore_activated_skills(ap, _make_query(), ['visible']) def test_resolve_virtual_skill_path_allows_visible_skill_reads(self): from langbot.pkg.provider.tools.loaders.skill import ( @@ -321,6 +292,37 @@ class TestSkillToolLoader: assert '' not in result['content'] assert set(query.variables[ACTIVATED_SKILLS_KEY].keys()) == {'demo'} + @pytest.mark.asyncio + async def test_reactivation_keeps_first_revision_and_instructions(self): + from langbot.pkg.provider.tools.loaders.skill_authoring import SkillToolLoader + + v1 = _make_skill_data( + name='demo', + instructions='version one', + revision='sha256:' + '1' * 64, + ) + v2 = _make_skill_data( + name='demo', + instructions='version two', + revision='sha256:' + '2' * 64, + ) + ap = _make_ap() + ap.skill_mgr = _make_skill_manager({'demo': v1}) + ap.skill_repository = SimpleNamespace( + get_skill=AsyncMock(side_effect=[v1, v2]), + ) + ap.box_service = SimpleNamespace(is_workspace_sandbox_available=AsyncMock(return_value=False)) + loader = SkillToolLoader(ap) + query = _make_query() + + first = await loader.invoke_tool('activate', {'skill_name': 'demo'}, query) + second = await loader.invoke_tool('activate', {'skill_name': 'demo'}, query) + + assert first['revision'] == v1['revision'] + assert second['revision'] == v1['revision'] + assert 'version one' in second['content'] + assert 'version two' not in second['content'] + @pytest.mark.asyncio async def test_activate_unknown_skill_raises(self): from langbot.pkg.provider.tools.loaders.skill_authoring import ( @@ -388,11 +390,55 @@ class TestSkillToolLoader: 'description': 'Imported from clone', 'instructions': 'Do work', }, + base_revision=None, ) assert result['registered'] is True assert result['skill_name'] == 'cloned-skill' assert result['source_path'] == '/workspace/repo' + @pytest.mark.asyncio + async def test_register_skill_forwards_base_revision_for_update(self): + from langbot.pkg.provider.tools.loaders.skill_authoring import ( + REGISTER_SKILL_TOOL_NAME, + SkillToolLoader, + ) + + with tempfile.TemporaryDirectory() as tmpdir: + draft = os.path.join(tmpdir, 'skill-drafts', 'demo') + os.makedirs(draft) + revision = 'sha256:' + '3' * 64 + ap = _make_ap() + ap.box_service = SimpleNamespace( + default_workspace=tmpdir, + available=True, + require_workspace_sandbox=AsyncMock(return_value=_CONTEXT), + ) + ap.skill_service = SimpleNamespace( + scan_directory_async=AsyncMock( + return_value={ + 'name': 'demo', + 'display_name': 'Demo', + 'description': 'Updated', + 'instructions': 'v2', + } + ), + import_skill_directory=AsyncMock( + return_value=_make_skill_data(name='demo', revision='sha256:' + '4' * 64) + ), + ) + + result = await SkillToolLoader(ap).invoke_tool( + REGISTER_SKILL_TOOL_NAME, + { + 'path': '/workspace/skill-drafts/demo', + 'base_revision': revision, + }, + _make_query(), + ) + + assert result['revision'] == 'sha256:' + '4' * 64 + assert ap.skill_service.import_skill_directory.await_args.kwargs['base_revision'] == revision + @pytest.mark.asyncio async def test_register_skill_rejects_workspace_escape(self): from langbot.pkg.provider.tools.loaders.skill_authoring import ( @@ -559,7 +605,7 @@ class TestNativeToolLoaderSkillPaths: @pytest.mark.asyncio async def test_read_visible_skill_file(self): from langbot.pkg.provider.tools.loaders.native import NativeToolLoader - from langbot.pkg.provider.tools.loaders.skill import PIPELINE_BOUND_SKILLS_KEY + from langbot.pkg.provider.tools.loaders.skill import PIPELINE_BOUND_SKILLS_KEY, register_activated_skill with tempfile.TemporaryDirectory() as tmpdir: skill_md = os.path.join(tmpdir, 'SKILL.md') @@ -575,20 +621,22 @@ class TestNativeToolLoaderSkillPaths: ap.skill_mgr = _make_skill_manager({'demo': _make_skill_data(name='demo', package_root=tmpdir)}) loader = NativeToolLoader(ap) + query = _make_bound_query(query_id='q1', variables={PIPELINE_BOUND_SKILLS_KEY: ['demo']}) + register_activated_skill(query, ap.skill_mgr.skills['demo']) if not loader._can_interpret_skill_host_paths(): # Windows lacks the secure descriptor-relative host file operations. - with pytest.raises(ValueError, match='owned by the Box Runtime'): + with pytest.raises(ValueError, match='Secure Core host file operations'): await loader.invoke_tool( 'read', {'path': '/workspace/.skills/demo/SKILL.md'}, - _make_bound_query(query_id='q1', variables={PIPELINE_BOUND_SKILLS_KEY: ['demo']}), + query, ) return result = await loader.invoke_tool( 'read', {'path': '/workspace/.skills/demo/SKILL.md'}, - _make_bound_query(query_id='q1', variables={PIPELINE_BOUND_SKILLS_KEY: ['demo']}), + query, ) assert result['ok'] is True @@ -598,7 +646,7 @@ class TestNativeToolLoaderSkillPaths: @pytest.mark.asyncio async def test_external_runtime_read_uses_core_skill_repository(self): from langbot.pkg.provider.tools.loaders.native import NativeToolLoader - from langbot.pkg.provider.tools.loaders.skill import PIPELINE_BOUND_SKILLS_KEY + from langbot.pkg.provider.tools.loaders.skill import PIPELINE_BOUND_SKILLS_KEY, register_activated_skill with tempfile.TemporaryDirectory() as tmpdir: with open(os.path.join(tmpdir, 'SKILL.md'), 'w', encoding='utf-8') as file_obj: @@ -610,7 +658,7 @@ class TestNativeToolLoaderSkillPaths: shares_filesystem_with_box=False, ) ap.skill_repository = SimpleNamespace( - read_skill_file=AsyncMock(return_value={'content': 'repository-content'}) + read_skill_resource=AsyncMock(return_value={'content': 'repository-content'}) ) ap.skill_mgr = _make_skill_manager({'demo': _make_skill_data(name='demo', package_root=tmpdir)}) loader = NativeToolLoader(ap) @@ -618,6 +666,7 @@ class TestNativeToolLoaderSkillPaths: query_id='q-external-read', variables={PIPELINE_BOUND_SKILLS_KEY: ['demo']}, ) + register_activated_skill(query, ap.skill_mgr.skills['demo']) result = await loader.invoke_tool( 'read', @@ -628,12 +677,17 @@ class TestNativeToolLoaderSkillPaths: assert result['ok'] is True assert result['content'] == 'repository-content' assert 'core-host-secret' not in repr(result) - ap.skill_repository.read_skill_file.assert_awaited_once_with(_CONTEXT, 'demo', 'SKILL.md') + ap.skill_repository.read_skill_resource.assert_awaited_once_with( + _CONTEXT, + 'demo', + 'SKILL.md', + expected_revision='sha256:' + '1' * 64, + ) @pytest.mark.asyncio async def test_core_owned_skill_path_does_not_depend_on_runtime_topology(self): from langbot.pkg.provider.tools.loaders.native import NativeToolLoader - from langbot.pkg.provider.tools.loaders.skill import PIPELINE_BOUND_SKILLS_KEY + from langbot.pkg.provider.tools.loaders.skill import PIPELINE_BOUND_SKILLS_KEY, register_activated_skill with tempfile.TemporaryDirectory() as tmpdir: with open(os.path.join(tmpdir, 'secret.txt'), 'w', encoding='utf-8') as file_obj: @@ -650,6 +704,7 @@ class TestNativeToolLoaderSkillPaths: query_id='q-external-no-protocol', variables={PIPELINE_BOUND_SKILLS_KEY: ['demo']}, ) + register_activated_skill(query, ap.skill_mgr.skills['demo']) result = await loader.invoke_tool( 'grep', @@ -669,13 +724,20 @@ class TestNativeToolLoaderSkillPaths: from langbot.pkg.provider.tools.loaders.skill import register_activated_skill with tempfile.TemporaryDirectory() as tmpdir: + manifest_path = os.path.join(tmpdir, 'manifest.json') + with open(manifest_path, 'w', encoding='utf-8') as file: + file.write('{}') ap = _make_ap() ap.box_service = SimpleNamespace( available=True, default_workspace=tmpdir, execute_tool=AsyncMock(return_value={'ok': True}), ) - skill_data = _make_skill_data(name='demo', package_root=tmpdir) + skill_data = _make_skill_data( + name='demo', + package_root=tmpdir, + manifest_path=manifest_path, + ) ap.skill_mgr = _make_skill_manager( {'demo': skill_data}, refresh_skill_from_disk=Mock(), @@ -706,40 +768,45 @@ class TestNativeToolLoaderSkillPaths: from langbot.pkg.provider.tools.loaders.native import NativeToolLoader from langbot.pkg.provider.tools.loaders.skill import register_activated_skill - ap = _make_ap() - ap.box_service = SimpleNamespace( - available=True, - shares_filesystem_with_box=False, - execute_tool=AsyncMock(return_value={'ok': True}), - ) - skill_data = _make_skill_data( - name='demo', - package_root='/box-runtime/skills/tenants/workspace/demo', - python_project=True, - ) - ap.skill_mgr = _make_skill_manager( - {'demo': skill_data}, - refresh_skill_from_disk=Mock(), - ) - loader = NativeToolLoader(ap) - query = _make_bound_query(query_id='q-external', launcher_type='person', launcher_id='123') - register_activated_skill(query, skill_data) + with tempfile.TemporaryDirectory() as tmpdir: + manifest_path = os.path.join(tmpdir, 'manifest.json') + with open(manifest_path, 'w', encoding='utf-8') as file: + file.write('{}') + ap = _make_ap() + ap.box_service = SimpleNamespace( + available=True, + shares_filesystem_with_box=False, + execute_tool=AsyncMock(return_value={'ok': True}), + ) + skill_data = _make_skill_data( + name='demo', + package_root=tmpdir, + manifest_path=manifest_path, + python_project=True, + ) + ap.skill_mgr = _make_skill_manager( + {'demo': skill_data}, + refresh_skill_from_disk=Mock(), + ) + loader = NativeToolLoader(ap) + query = _make_bound_query(query_id='q-external', launcher_type='person', launcher_id='123') + register_activated_skill(query, skill_data) - result = await loader.invoke_tool( - 'exec', - { - 'command': 'python /workspace/.skills/demo/scripts/run.py', - 'workdir': '/workspace/.skills/demo', - }, - query, - ) + result = await loader.invoke_tool( + 'exec', + { + 'command': 'python /workspace/.skills/demo/scripts/run.py', + 'workdir': '/workspace/.skills/demo', + }, + query, + ) assert result['ok'] is True tool_parameters = ap.box_service.execute_tool.await_args.args[0] wrapped = tool_parameters['command'] - assert '_LB_VENV_DIR="/workspace/.skill-envs/demo/.venv"' in wrapped + assert f'_LB_VENV_DIR="/workspace/.skill-envs/demo/{"1" * 64}/.venv"' in wrapped assert 'root = "/workspace/.skills/demo"' in wrapped - assert '/box-runtime/skills/tenants/workspace/demo' not in wrapped + assert tmpdir not in wrapped assert 'skill_name' not in ap.box_service.execute_tool.await_args.kwargs @pytest.mark.asyncio @@ -762,6 +829,36 @@ class TestNativeToolLoaderSkillPaths: query, ) + @pytest.mark.asyncio + @pytest.mark.parametrize('tool_name', ['write', 'edit']) + async def test_published_revision_rejects_direct_mutation(self, tool_name): + from langbot.pkg.provider.tools.loaders.native import NativeToolLoader + from langbot.pkg.provider.tools.loaders.skill import register_activated_skill + + with tempfile.TemporaryDirectory() as tmpdir: + ap = _make_ap() + ap.box_service = SimpleNamespace(available=True, default_workspace=tmpdir) + skill = _make_skill_data(name='demo', package_root=tmpdir) + ap.skill_mgr = _make_skill_manager({'demo': skill}) + loader = NativeToolLoader(ap) + query = _make_bound_query(query_id='immutable') + register_activated_skill(query, skill) + parameters = ( + {'path': '/workspace/.skills/demo/notes.txt', 'content': 'hi'} + if tool_name == 'write' + else { + 'path': '/workspace/.skills/demo/notes.txt', + 'old_string': 'old', + 'new_string': 'new', + } + ) + + result = await loader.invoke_tool(tool_name, parameters, query) + + assert result['ok'] is False + assert 'immutable' in result['error'] + assert '/workspace/skill-drafts' in result['error'] + @pytest.mark.asyncio async def test_native_skill_tools_require_runner_box_binding(): diff --git a/tests/unit_tests/test_skill_repository.py b/tests/unit_tests/test_skill_repository.py index 9878878b6..765ee6a92 100644 --- a/tests/unit_tests/test_skill_repository.py +++ b/tests/unit_tests/test_skill_repository.py @@ -1,9 +1,10 @@ from types import SimpleNamespace +from unittest.mock import Mock import pytest from langbot.pkg.api.http.context import ExecutionContext -from langbot.pkg.skill.repository import SkillRepository, SkillRevisionMismatchError +from langbot.pkg.skill.repository import SkillRepository, SkillRevisionConflictError _CONTEXT = ExecutionContext( @@ -79,7 +80,7 @@ def test_repository_keeps_old_box_root_only_for_online_upgrade(tmp_path): async def test_repository_crud_and_reads_do_not_require_box(tmp_path): repository = _repository(tmp_path) - await repository.create_skill( + created = await repository.create_skill( _CONTEXT, { 'name': 'docs-only', @@ -93,11 +94,12 @@ async def test_repository_crud_and_reads_do_not_require_box(tmp_path): 'docs-only', 'references/guide.md', '# Guide\n\nNo execution needed.', + base_revision=created['revision'], ) skill = await repository.get_skill(_CONTEXT, 'docs-only', snapshot=True) assert skill is not None - assert skill['revision'].startswith('stat-v1:') + assert skill['revision'].startswith('sha256:') assert [item['name'] for item in await repository.list_skills(_CONTEXT)] == ['docs-only'] listed = await repository.list_skill_resources( @@ -120,27 +122,47 @@ async def test_repository_crud_and_reads_do_not_require_box(tmp_path): @pytest.mark.asyncio -async def test_repository_rejects_traversal_and_stale_revision(tmp_path): +async def test_repository_rejects_traversal_and_conflicting_update(tmp_path): repository = _repository(tmp_path) - await repository.create_skill( + created = await repository.create_skill( _CONTEXT, {'name': 'safe', 'description': 'Safe', 'instructions': 'Use the reference.'}, ) - await repository.write_skill_file(_CONTEXT, 'safe', 'reference.md', 'first') + first = await repository.write_skill_file( + _CONTEXT, + 'safe', + 'reference.md', + 'first', + base_revision=created['revision'], + ) skill = await repository.get_skill(_CONTEXT, 'safe', snapshot=True) assert skill is not None with pytest.raises(ValueError, match='stay within'): await repository.read_skill_resource(_CONTEXT, 'safe', '../secret.txt') - await repository.write_skill_file(_CONTEXT, 'safe', 'reference.md', 'second') - with pytest.raises(SkillRevisionMismatchError, match='reactivate'): - await repository.read_skill_resource( + second = await repository.write_skill_file( + _CONTEXT, + 'safe', + 'reference.md', + 'second', + base_revision=first['revision'], + ) + pinned = await repository.read_skill_resource( + _CONTEXT, + 'safe', + 'reference.md', + expected_revision=skill['revision'], + ) + assert pinned['content'] == 'first' + with pytest.raises(SkillRevisionConflictError, match='changed since'): + await repository.update_skill( _CONTEXT, 'safe', - 'reference.md', - expected_revision=skill['revision'], + {'instructions': 'stale'}, + base_revision=first['revision'], ) + assert second['revision'] != first['revision'] @pytest.mark.asyncio @@ -207,3 +229,89 @@ async def test_repository_imports_only_from_the_fenced_workspace(tmp_path): (outside / 'SKILL.md').write_text('Outside', encoding='utf-8') with pytest.raises(ValueError, match='trusted source root'): await repository.scan_skill_directory(_CONTEXT, str(outside)) + + +@pytest.mark.asyncio +async def test_existing_run_keeps_v1_while_new_run_resolves_v2(tmp_path): + from langbot.pkg.provider.tools.loaders import skill as skill_loader + + repository = _repository(tmp_path) + created = await repository.create_skill( + _CONTEXT, + {'name': 'runner', 'instructions': 'Run scripts/main.py'}, + ) + v1_write = await repository.write_skill_file( + _CONTEXT, + 'runner', + 'scripts/main.py', + "print('v1')", + base_revision=created['revision'], + ) + v1 = await repository.get_skill(_CONTEXT, 'runner', snapshot=True) + old_run = SimpleNamespace(variables={}) + skill_loader.register_activated_skill(old_run, v1) + + await repository.write_skill_file( + _CONTEXT, + 'runner', + 'scripts/main.py', + "print('v2')", + base_revision=v1_write['revision'], + ) + v2 = await repository.get_skill(_CONTEXT, 'runner', snapshot=True) + new_run = SimpleNamespace(variables={}) + skill_loader.register_activated_skill(new_run, v2) + + old_resource = await repository.read_skill_resource( + _CONTEXT, + 'runner', + 'scripts/main.py', + expected_revision=v1['revision'], + ) + new_resource = await repository.read_skill_resource( + _CONTEXT, + 'runner', + 'scripts/main.py', + expected_revision=v2['revision'], + ) + app = SimpleNamespace(logger=Mock()) + old_mount = skill_loader.build_execution_mounts(app, old_run)[0] + new_mount = skill_loader.build_execution_mounts(app, new_run)[0] + + assert old_resource['content'] == "print('v1')" + assert new_resource['content'] == "print('v2')" + assert old_mount['host_path'] == v1['package_root'] + assert new_mount['host_path'] == v2['package_root'] + assert old_mount['content_digest'] == v1['revision'] + assert new_mount['content_digest'] == v2['revision'] + + +@pytest.mark.asyncio +async def test_deleted_skill_can_restore_exact_recoverable_run_revision(tmp_path): + from langbot.pkg.provider.tools.loaders import skill as skill_loader + + repository = _repository(tmp_path) + published = await repository.create_skill( + _CONTEXT, + {'name': 'recoverable', 'instructions': 'Pinned instructions'}, + ) + await repository.delete_skill(_CONTEXT, 'recoverable') + app = SimpleNamespace(skill_repository=repository) + query = SimpleNamespace( + variables={skill_loader.PIPELINE_BOUND_SKILLS_KEY: ['recoverable']}, + instance_uuid=_CONTEXT.instance_uuid, + workspace_uuid=_CONTEXT.workspace_uuid, + placement_generation=_CONTEXT.placement_generation, + bot_uuid=None, + pipeline_uuid=None, + query_uuid='recovered-query', + ) + + restored = await skill_loader.restore_activated_skills( + app, + query, + [{'name': 'recoverable', 'revision': published['revision']}], + ) + + assert restored == ['recoverable'] + assert skill_loader.get_activated_skill(query, 'recoverable')['revision'] == published['revision'] diff --git a/tests/unit_tests/test_skill_service.py b/tests/unit_tests/test_skill_service.py index 0b0ead5ab..a66e1472f 100644 --- a/tests/unit_tests/test_skill_service.py +++ b/tests/unit_tests/test_skill_service.py @@ -31,11 +31,11 @@ class TestSkillRepositoryBoundary: repository = SimpleNamespace( list_skills=AsyncMock(return_value=[{'name': 'x', 'instructions': 'Do work'}]), get_skill=AsyncMock(return_value={'name': 'x', 'instructions': 'Do work', 'revision': 'sha256:x'}), - create_skill=AsyncMock(return_value={'name': 'x', 'instructions': 'Do work'}), - update_skill=AsyncMock(return_value={'name': 'x', 'instructions': 'Updated'}), + create_skill=AsyncMock(return_value={'name': 'x', 'instructions': 'Do work', 'revision': 'sha256:v1'}), + update_skill=AsyncMock(return_value={'name': 'x', 'instructions': 'Updated', 'revision': 'sha256:v2'}), delete_skill=AsyncMock(), read_skill_file=AsyncMock(return_value={'path': 'a.txt', 'content': 'hello'}), - write_skill_file=AsyncMock(return_value={'path': 'a.txt'}), + write_skill_file=AsyncMock(return_value={'path': 'a.txt', 'revision': 'sha256:v3'}), ) return SimpleNamespace( skill_mgr=SimpleNamespace(reload_skills=AsyncMock()), @@ -62,12 +62,33 @@ class TestSkillRepositoryBoundary: service = SkillService(ap) await service.create_skill(_CONTEXT, {'name': 'x'}) - await service.update_skill(_CONTEXT, 'x', {'instructions': 'Updated'}) - await service.write_skill_file(_CONTEXT, 'x', 'a.txt', 'hello') + await service.update_skill( + _CONTEXT, + 'x', + {'instructions': 'Updated', 'base_revision': 'sha256:v1'}, + ) + await service.write_skill_file( + _CONTEXT, + 'x', + 'a.txt', + 'hello', + base_revision='sha256:v2', + ) ap.skill_repository.create_skill.assert_awaited_once_with(_CONTEXT, {'name': 'x'}) - ap.skill_repository.update_skill.assert_awaited_once_with(_CONTEXT, 'x', {'instructions': 'Updated'}) - ap.skill_repository.write_skill_file.assert_awaited_once_with(_CONTEXT, 'x', 'a.txt', 'hello') + ap.skill_repository.update_skill.assert_awaited_once_with( + _CONTEXT, + 'x', + {'instructions': 'Updated'}, + base_revision='sha256:v1', + ) + ap.skill_repository.write_skill_file.assert_awaited_once_with( + _CONTEXT, + 'x', + 'a.txt', + 'hello', + base_revision='sha256:v2', + ) @pytest.mark.asyncio async def test_get_skill_returns_repository_revision(self): diff --git a/web/src/app/home/skills/components/skill-form/SkillForm.tsx b/web/src/app/home/skills/components/skill-form/SkillForm.tsx index 8bba95314..8df14b163 100644 --- a/web/src/app/home/skills/components/skill-form/SkillForm.tsx +++ b/web/src/app/home/skills/components/skill-form/SkillForm.tsx @@ -825,7 +825,10 @@ export default function SkillForm({ try { if (initSkillName) { - const resp = await httpClient.updateSkill(initSkillName, baseSkillData); + const resp = await httpClient.updateSkill(initSkillName, { + ...baseSkillData, + base_revision: skill.revision || '', + }); toast.success(t('skills.saveSuccess')); onSkillUpdated(resp.skill.name); } else { diff --git a/web/src/app/infra/entities/api/index.ts b/web/src/app/infra/entities/api/index.ts index 301538bbb..664bc21f2 100644 --- a/web/src/app/infra/entities/api/index.ts +++ b/web/src/app/infra/entities/api/index.ts @@ -936,6 +936,7 @@ export interface Skill { description: string; instructions?: string; package_root?: string; + revision?: string; is_builtin?: boolean; created_at?: string; updated_at?: string; diff --git a/web/src/app/infra/http/BackendClient.ts b/web/src/app/infra/http/BackendClient.ts index 71812449f..d17327cf8 100644 --- a/web/src/app/infra/http/BackendClient.ts +++ b/web/src/app/infra/http/BackendClient.ts @@ -2263,7 +2263,7 @@ export class BackendClient extends BaseHttpClient { public updateSkill( name: string, - skill: Partial, + skill: Partial & { base_revision: string }, ): Promise { return this.put(`/api/v1/skills/${name}`, skill); } @@ -2327,6 +2327,7 @@ export class BackendClient extends BaseHttpClient { skillName: string, filePath: string, content: string, + baseRevision: string, ): Promise<{ skill: { name: string }; path: string; @@ -2334,6 +2335,7 @@ export class BackendClient extends BaseHttpClient { }> { return this.put(`/api/v1/skills/${skillName}/files/${filePath}`, { content, + base_revision: baseRevision, }); } } From 38a7c10d04550ac80aee83c58a3528e157e4a569 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Sat, 12 Sep 2026 13:21:45 +0800 Subject: [PATCH 006/174] build(deps): pin paired skill sdk revision --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index cf7ebbf9e..c944d1771 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ dependencies = [ "langchain-text-splitters>=1.1.2", "chromadb>=1.0.0,<2.0.0", "qdrant-client (>=1.15.1,<2.0.0)", - "langbot-plugin==0.6.20", + "langbot-plugin @ git+https://github.com/langbot-app/langbot-plugin-sdk.git@05a5ff3f62fa774b1dbf0c96b80cdc4d16dbf250", "asyncpg>=0.30.0", "line-bot-sdk>=3.19.0", "matrix-nio>=0.25.2", From c56fa0ba2e45e6be1f1fa05fa3a8d8f1d1435eee Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Mon, 28 Sep 2026 10:54:31 +0800 Subject: [PATCH 007/174] fix(skill): reconcile rebase with runner contracts --- docs/review/box-architecture.md | 2 + src/langbot/pkg/agent/runner/orchestrator.py | 11 ++--- .../pkg/api/http/service/maintenance.py | 4 ++ src/langbot/pkg/box/runner.py | 3 -- .../pkg/provider/tools/loaders/native.py | 2 +- .../pkg/provider/tools/loaders/skill.py | 37 ++++++++++------ src/langbot/pkg/provider/tools/toolmgr.py | 10 +++-- src/langbot/pkg/skill/repository.py | 3 ++ .../test_cloud_box_admission_integration.py | 34 +++++++++++++- .../agent/test_orchestrator_integration.py | 44 ++++++++++++++++--- .../api/service/test_maintenance_service.py | 8 +++- tests/unit_tests/box/test_box_service.py | 1 - tests/unit_tests/box/test_runner_box.py | 1 - tests/unit_tests/provider/test_skill_tools.py | 27 ++++++++++++ 14 files changed, 152 insertions(+), 35 deletions(-) diff --git a/docs/review/box-architecture.md b/docs/review/box-architecture.md index 5cbae1566..64adf4627 100644 --- a/docs/review/box-architecture.md +++ b/docs/review/box-architecture.md @@ -582,6 +582,8 @@ volumes: > Core 的 SkillRepository 是 `langbot_plugin.skill_store.SkillStore` 的异步 Workspace 适配层。默认 `skills.root` 保持原 `data/box/skills/tenants/...` 布局，升级时无需移动已安装 Skill；旧 `box.local.skills_root` 仅作为在线升级 fallback，并将在下一大版本删除。Box 只消费 Core 下发的通用只读 mount。 +自定义 `skills.root` 若位于 Box 的 `allowed_mount_roots` 之外，Skill 仍可管理和读取，但脚本执行的只读挂载会被拒绝。执行这些 Skill 时，需将该路径加入允许的挂载根；分离部署还需让 Core 和 Box Runtime 在相同路径看见这份目录。 + ### Session scope Pipeline 与 Runner 配置不再暴露 sandbox session 模板。Host 将当前平台会话/事件 scope 规范化后哈希成固定长度的 `lb-box-`；相同 scope 稳定复用，不同 scope 隔离，缺少 identity 时拒绝执行。SDK/Python 与 scoped MCP bridge 的工具调用遵守同一规则。详见 [box-session-scope.md](./box-session-scope.md)。 diff --git a/src/langbot/pkg/agent/runner/orchestrator.py b/src/langbot/pkg/agent/runner/orchestrator.py index c2cf5f9cb..de6327a40 100644 --- a/src/langbot/pkg/agent/runner/orchestrator.py +++ b/src/langbot/pkg/agent/runner/orchestrator.py @@ -150,10 +150,15 @@ class AgentRunOrchestrator: resources=resources, ) + authorized_skill_names = [ + str(skill['skill_name']) for skill in resources.get('skills', []) if skill.get('skill_name') + ] + prepare_execution_query(execution_query, event, authorized_skill_names) + session_query_id = None if adapter_context: if execution_query is not None: - skill_loader.restore_activated_skills_from_state( + await skill_loader.restore_activated_skills_from_state( self.ap, execution_query, context.get('state', {}), @@ -164,10 +169,6 @@ class AgentRunOrchestrator: if 'params' in adapter_context: context['adapter']['extra']['params'] = adapter_context['params'] - authorized_skill_names = [ - str(skill['skill_name']) for skill in resources.get('skills', []) if skill.get('skill_name') - ] - prepare_execution_query(execution_query, event, authorized_skill_names) context['variables'] = { key: value for key, value in (execution_query.variables or {}).items() diff --git a/src/langbot/pkg/api/http/service/maintenance.py b/src/langbot/pkg/api/http/service/maintenance.py index cdaf32091..4fe3cc642 100644 --- a/src/langbot/pkg/api/http/service/maintenance.py +++ b/src/langbot/pkg/api/http/service/maintenance.py @@ -148,6 +148,10 @@ class MaintenanceService: scoped_storage_path = Path('data/storage') / self.ap.storage_mgr.scoped_prefix(context) roots = [('storage', scoped_storage_path)] + skill_repository = getattr(self.ap, 'skill_repository', None) + if skill_repository is not None: + roots.append(('skills', Path(skill_repository.storage_root(context)))) + sections = await asyncio.to_thread(self._collect_sections, roots) runtime_processes = await self._runtime_storage_processes( context, diff --git a/src/langbot/pkg/box/runner.py b/src/langbot/pkg/box/runner.py index fa1b8aab6..6be620f88 100644 --- a/src/langbot/pkg/box/runner.py +++ b/src/langbot/pkg/box/runner.py @@ -152,8 +152,6 @@ class RunnerBoxService: if set(request.options) - {'image'}: raise BoxValidationError('Only the Box image can be specified; resource limits are Host-owned') spec = {'session_id': session_id, **request.options} - if query is not None: - spec['extra_mounts'] = self.box.build_skill_extra_mounts(query) result = await self.box.create_session(context, spec) return self.public_session(result) @@ -174,7 +172,6 @@ class RunnerBoxService: return self.binding_info(current) fields = ('image', 'network', 'cpus', 'memory_mb', 'pids_limit', 'read_only_rootfs', 'persistent') spec = {key: item[key] for key in fields if key in item} - spec['extra_mounts'] = self.box.build_skill_extra_mounts(query) binding = RunBoxBinding(run_id, box_id, spec, run_id) object.__setattr__(query, '_box_binding', binding) return self.binding_info(binding) diff --git a/src/langbot/pkg/provider/tools/loaders/native.py b/src/langbot/pkg/provider/tools/loaders/native.py index 1c6d6e345..74b067320 100644 --- a/src/langbot/pkg/provider/tools/loaders/native.py +++ b/src/langbot/pkg/provider/tools/loaders/native.py @@ -352,7 +352,7 @@ class NativeToolLoader(loader.ToolLoader): # All exec calls (with or without skills) go through the same container # via execute_tool. Skills are mounted at /workspace/.skills/{name}/ - # via extra_mounts built by BoxService. + # via read-only mounts composed by the Core skill loader. result = await self.ap.box_service.execute_tool( parameters, query, diff --git a/src/langbot/pkg/provider/tools/loaders/skill.py b/src/langbot/pkg/provider/tools/loaders/skill.py index 9669b7982..7ddaab86d 100644 --- a/src/langbot/pkg/provider/tools/loaders/skill.py +++ b/src/langbot/pkg/provider/tools/loaders/skill.py @@ -73,6 +73,7 @@ def register_created_skill_visibility(query: pipeline_query.Query, skill_name: s if isinstance(bound_skills, list) and skill_name not in bound_skills: bound_skills.append(skill_name) + def build_execution_mounts(ap: app.Application, query: pipeline_query.Query) -> list[dict]: """Mount only immutable revisions pinned by this run's activations.""" @@ -217,20 +218,30 @@ async def restore_activated_skills( return restored -def restore_activated_skills_from_state( +async def restore_activated_skills_from_state( ap: app.Application, query: pipeline_query.Query, state: dict[str, dict[str, typing.Any]], ) -> list[str]: - """Restore persisted activated skill names into Query variables. - - The state value stores names only. Full skill metadata is rebuilt from the - current pipeline-visible skill cache so removed or unbound skills remain - unavailable to native exec/write/edit. - """ + """Restore the exact activated Skill revisions stored by the Host.""" conversation_state = state.get('conversation', {}) if isinstance(state, dict) else {} - skill_names = normalize_skill_names(conversation_state.get(ACTIVATED_SKILL_NAMES_STATE_KEY)) - return restore_activated_skills(ap, query, skill_names) + if not isinstance(conversation_state, dict): + raise ValueError('Activated Skill conversation state must be an object.') + if ACTIVATED_SKILL_NAMES_STATE_KEY not in conversation_state: + return [] + bindings = conversation_state[ACTIVATED_SKILL_NAMES_STATE_KEY] + if isinstance(bindings, list) and bindings and all(isinstance(item, str) for item in bindings): + logger = getattr(ap, 'logger', None) + if logger is not None: + workspace_uuid = getattr(query, 'workspace_uuid', None) or getattr( + getattr(query, '_execution_context', None), 'workspace_uuid', 'unknown' + ) + logger.warning( + f'Legacy name-only activated Skill state in Workspace {workspace_uuid}; ' + 'reactivate Skills to pin their revisions.' + ) + return [] + return await restore_activated_skills(ap, query, bindings) async def persist_activated_skill( @@ -238,10 +249,10 @@ async def persist_activated_skill( query: pipeline_query.Query, skill_name: str, ) -> None: - """Persist activated skill names into host-owned conversation state. + """Persist activated Skill revision bindings into Host-owned conversation state. - ``activate`` runs host-side. This writes the run's current activated skill - names to the conversation-scope ``host.activated_skills`` snapshot so a later + ``activate`` runs host-side. This writes the run's pinned Skill revisions + to the conversation-scope ``host.activated_skills`` snapshot so a later run can restore them via ``restore_activated_skills_from_state``. Host writes here and a runner ``state.updated`` to the same key follow last-write-wins. @@ -273,7 +284,7 @@ async def persist_activated_skill( await store.state_set( scope_key=conversation_scope_key, state_key=ACTIVATED_SKILL_NAMES_STATE_KEY, - value=get_activated_skill_names(query), + value=get_activated_skill_bindings(query), runner_id=str(session.get('runner_id', '') or ''), binding_identity=str(state_context.get('binding_identity', 'unknown') or 'unknown'), scope='conversation', diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index fe6f8dfd4..e3fb26519 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -376,7 +376,8 @@ class ToolManager: if source in {'builtin', 'native'}: return await self.native_tool_loader.get_tool(name) if source == 'skill': - return await self.skill_tool_loader.get_tool(name) + sandbox_available = await self._workspace_sandbox_available(context) + return await self.skill_tool_loader.get_tool(name, sandbox_available=sandbox_available) if source == 'plugin': if not source_id: return None @@ -512,11 +513,11 @@ class ToolManager: telemetry_source = 'native' exists = await loader.has_tool(name) elif source == 'skill': - if not sandbox_available: + if not sandbox_available and self.skill_tool_loader.is_sandbox_tool(name): raise ToolNotFoundError(name) loader = self.skill_tool_loader telemetry_source = 'skill' - exists = await loader.has_tool(name) + exists = await loader.has_tool(name, sandbox_available=sandbox_available) elif source == 'plugin' and source_id: loader = self.plugin_tool_loader telemetry_source = 'plugin' @@ -537,6 +538,9 @@ class ToolManager: if not exists: raise ToolNotFoundError(name) + if source == 'skill' and isinstance(getattr(query, 'variables', None), dict): + query.variables['_skill_execution_available'] = sandbox_available + async def invoke_selected_tool() -> typing.Any: if source == 'mcp': return await loader.invoke_tool( diff --git a/src/langbot/pkg/skill/repository.py b/src/langbot/pkg/skill/repository.py index 6aaaa0c95..0d1de38dc 100644 --- a/src/langbot/pkg/skill/repository.py +++ b/src/langbot/pkg/skill/repository.py @@ -103,6 +103,9 @@ class SkillRepository: def _workspace_root(self, namespace: str) -> str: return os.path.join(self._default_workspace(), 'tenants', namespace) + def storage_root(self, context: TenantContext) -> str: + return self._store.scoped(self._namespace(context)).root + async def _call(self, context: TenantContext, method_name: str, *args, **kwargs): execution_context = await self._validated_execution_context(context) namespace = self._namespace(execution_context) diff --git a/tests/integration_tests/box/test_cloud_box_admission_integration.py b/tests/integration_tests/box/test_cloud_box_admission_integration.py index 0d01bc1d9..4be9ccfc6 100644 --- a/tests/integration_tests/box/test_cloud_box_admission_integration.py +++ b/tests/integration_tests/box/test_cloud_box_admission_integration.py @@ -411,7 +411,39 @@ async def test_cloud_core_skills_mount_generically_and_do_not_require_box_entitl ) assert [skill['name'] for skill in await repository.list_skills(ineligible)] == ['docs-only'] with pytest.raises(EntitlementUnavailableError): - await service.execute_tool({'command': 'true'}, _query(ineligible, 92)) + await RunnerBoxService(service).acquire(ineligible, {'reuse_key': 'global'}, _query(ineligible, 92)) + finally: + server_task.cancel() + client_task.cancel() + await runtime.shutdown() + + +@pytest.mark.asyncio +async def test_activated_skill_mount_changes_recreate_shared_box_session(tmp_path): + service, runtime, backend, entitlements, server_task, client_task = await _stack(tmp_path) + context = _context('workspace-a') + entitlements.snapshots[context.workspace_uuid] = _snapshot(context.workspace_uuid) + try: + first_query = await _bound_query(service, context, 101) + await service.execute_tool({'command': 'echo before activation'}, first_query) + assert len(backend.started_specs) == 1 + + skill = await service.ap.skill_repository.create_skill( + context, {'name': 'demo', 'instructions': 'Run a script'} + ) + second_query = await _bound_query(service, context, 102) + skill_loader.register_activated_skill(second_query, skill) + await service.execute_tool( + {'command': 'echo with skill'}, + second_query, + read_only_mounts=skill_loader.build_execution_mounts(service.ap, second_query), + ) + assert len(backend.started_specs) == 2 + assert backend.stopped_sessions == [backend.started_specs[0].session_id] + + await service.execute_tool({'command': 'echo first query again'}, first_query) + assert len(backend.started_specs) == 3 + assert backend.stopped_sessions == [backend.started_specs[0].session_id] * 2 finally: server_task.cancel() client_task.cancel() diff --git a/tests/unit_tests/agent/test_orchestrator_integration.py b/tests/unit_tests/agent/test_orchestrator_integration.py index 1c5f2ea66..955e41ccd 100644 --- a/tests/unit_tests/agent/test_orchestrator_integration.py +++ b/tests/unit_tests/agent/test_orchestrator_integration.py @@ -1007,7 +1007,7 @@ class TestQueryEntrySessionQueryId: @pytest.mark.asyncio async def test_no_query_id_for_pure_event_first_flow(self, clean_agent_state): - """Pure event-first flow has query_id=None in session.""" + """Pure event-first flow restores authorized Skill revisions without a Query.""" from langbot.pkg.agent.runner.host_models import ( AgentEventEnvelope, AgentBinding, @@ -1018,6 +1018,8 @@ class TestQueryEntrySessionQueryId: ) from langbot_plugin.api.entities.builtin.runner.input import AgentInput from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext + from langbot.pkg.agent.runner.persistent_state_store import get_persistent_state_store + from langbot.pkg.provider.tools.loaders.skill import ACTIVATED_SKILL_NAMES_STATE_KEY, ACTIVATED_SKILLS_KEY db_engine = clean_agent_state descriptor = make_descriptor() @@ -1030,6 +1032,10 @@ class TestQueryEntrySessionQueryId: ] ) ap = FakeApplication(plugin_connector, db_engine) + revision = 'sha256:' + '1' * 64 + ap.skill_repository = types.SimpleNamespace( + get_skill=AsyncMock(return_value={'name': 'demo', 'revision': revision}) + ) async def build_resource_context(execution_query): from langbot.pkg.provider.tools.loaders.mcp import ( @@ -1085,10 +1091,21 @@ class TestQueryEntrySessionQueryId: 'mcp-resource-agent-read-enabled': True, }, resource_policy=ResourcePolicy(), - state_policy=StatePolicy(enable_state=False, state_scopes=[]), + state_policy=StatePolicy(enable_state=True, state_scopes=['conversation']), delivery_policy=DeliveryPolicy(enable_streaming=True, enable_reply=True), ) + success, error = await get_persistent_state_store(db_engine).apply_update_from_event( + event.model_copy(update={'workspace_id': TEST_CONTEXT.workspace_uuid}), + binding, + descriptor, + 'conversation', + ACTIVATED_SKILL_NAMES_STATE_KEY, + [{'name': 'demo', 'revision': revision}], + None, + ) + assert success is True and error is None + messages = [ message async for message in orchestrator.run( @@ -1114,6 +1131,7 @@ class TestQueryEntrySessionQueryId: assert execution_query.message_event.type == event.event_type assert '_host_box_scope' not in execution_query.variables assert execution_query.variables['_pipeline_bound_skills'] == ['demo', 'hidden'] + assert execution_query.variables[ACTIVATED_SKILLS_KEY]['demo']['revision'] == revision assert execution_query.variables['_pipeline_mcp_resource_attachments'][0]['server_uuid'] == 'srv-1' assert execution_query.variables['_pipeline_mcp_resource_agent_read_enabled'] is True assert 'MCP resource context selected by LangBot host:' in plugin_connector.contexts[0]['input']['text'] @@ -1344,8 +1362,9 @@ class TestQueryEntryAdapterHostCapabilities: assert snapshot['conversation']['external.test_key'] == 'test_value' @pytest.mark.asyncio - async def test_run_from_query_restores_activated_skills_from_state(self, clean_agent_state): - """Persisted activated skill names are restored into the next Query run.""" + @pytest.mark.parametrize('legacy_state', [False, True]) + async def test_run_from_query_restores_activated_skills_from_state(self, clean_agent_state, legacy_state): + """Pinned revisions restore; old name-only state requires reactivation.""" from langbot.pkg.agent.runner.persistent_state_store import get_persistent_state_store from langbot.pkg.provider.tools.loaders.skill import ( ACTIVATED_SKILL_NAMES_STATE_KEY, @@ -1363,6 +1382,10 @@ class TestQueryEntryAdapterHostCapabilities: ] ) ap = FakeApplication(plugin_connector, db_engine) + revision = 'sha256:' + '1' * 64 + ap.skill_repository = types.SimpleNamespace( + get_skill=AsyncMock(return_value={'name': 'demo', 'revision': revision}) + ) orchestrator = AgentRunOrchestrator(ap, FakeRegistry(descriptor)) query = make_query() @@ -1376,7 +1399,7 @@ class TestQueryEntryAdapterHostCapabilities: descriptor, 'conversation', ACTIVATED_SKILL_NAMES_STATE_KEY, - ['demo'], + ['demo'] if legacy_state else [{'name': 'demo', 'revision': revision}], None, ) assert success is True @@ -1385,7 +1408,16 @@ class TestQueryEntryAdapterHostCapabilities: messages = [message async for message in orchestrator.run_from_query(query)] assert len(messages) == 1 - assert query.variables[ACTIVATED_SKILLS_KEY]['demo']['name'] == 'demo' + if legacy_state: + assert ACTIVATED_SKILLS_KEY not in query.variables + assert ap.skill_repository.get_skill.await_count == 0 + assert ( + len([warning for warning in ap.logger.warnings if 'Legacy name-only activated Skill state' in warning]) + == 1 + ) + else: + assert query.variables[ACTIVATED_SKILLS_KEY]['demo']['name'] == 'demo' + assert query.variables[ACTIVATED_SKILLS_KEY]['demo']['revision'] == revision @pytest.mark.asyncio async def test_event_log_and_transcript_written(self, clean_agent_state): diff --git a/tests/unit_tests/api/service/test_maintenance_service.py b/tests/unit_tests/api/service/test_maintenance_service.py index b4b00e6fb..16f27aa38 100644 --- a/tests/unit_tests/api/service/test_maintenance_service.py +++ b/tests/unit_tests/api/service/test_maintenance_service.py @@ -383,6 +383,7 @@ class TestMaintenanceServiceGetStorageAnalysis: handler=plugin_handler, ), box_service=box_service, + skill_repository=SimpleNamespace(storage_root=Mock(return_value='/core/skills/tenants/test')), ) service = MaintenanceService(ap) service._path_size = Mock(return_value=10) @@ -394,12 +395,17 @@ class TestMaintenanceServiceGetStorageAnalysis: result = await service.get_storage_analysis(TEST_CONTEXT) - assert result['total_size_bytes'] == 115 + assert result['total_size_bytes'] == 125 + assert ( + next(section for section in result['sections'] if section['key'] == 'skills')['path'] + == '/core/skills/tenants/test' + ) assert result['processes'][1]['directories'][0]['key'] == 'artifacts' assert result['processes'][2]['directories'][1]['key'] == 'mcp' assert result['processes'][2]['managed_processes'] == 2 plugin_handler.get_storage_analysis.assert_awaited_once_with() box_service.get_storage_analysis.assert_awaited_once_with(TEST_CONTEXT) + ap.skill_repository.storage_root.assert_called_once_with(TEST_CONTEXT) async def test_get_storage_analysis_postgresql(self): """Handles PostgreSQL database type.""" diff --git a/tests/unit_tests/box/test_box_service.py b/tests/unit_tests/box/test_box_service.py index 610163c68..4ae7e3517 100644 --- a/tests/unit_tests/box/test_box_service.py +++ b/tests/unit_tests/box/test_box_service.py @@ -585,7 +585,6 @@ async def test_box_service_reconnect_restores_workspace_and_runs_cleanup( connector.reconnect.assert_awaited_once() service._ensure_default_workspace.assert_called_once() service._purge_attachment_dirs.assert_awaited_once() - app.skill_mgr.reload_skills.assert_not_awaited() assert service.available is True diff --git a/tests/unit_tests/box/test_runner_box.py b/tests/unit_tests/box/test_runner_box.py index ad49482f5..1f462d2cc 100644 --- a/tests/unit_tests/box/test_runner_box.py +++ b/tests/unit_tests/box/test_runner_box.py @@ -24,7 +24,6 @@ def service(): require_workspace_sandbox=AsyncMock(side_effect=lambda context: context), _action_context=lambda context: context, client=SimpleNamespace(get_sessions=AsyncMock(side_effect=lambda **kw: list(sessions.values()))), - build_skill_extra_mounts=lambda query: [], ) return RunnerBoxService(box) diff --git a/tests/unit_tests/provider/test_skill_tools.py b/tests/unit_tests/provider/test_skill_tools.py index 7c25371a6..7396c00ed 100644 --- a/tests/unit_tests/provider/test_skill_tools.py +++ b/tests/unit_tests/provider/test_skill_tools.py @@ -261,6 +261,33 @@ class TestSkillToolLoader: handled by SkillService via the HTTP API / web UI instead. """ + @pytest.mark.asyncio + async def test_frozen_skill_source_runs_activation_without_box(self): + from langbot.pkg.provider.tools.loaders.skill_authoring import SkillToolLoader + from langbot.pkg.provider.tools.toolmgr import ToolManager + + skill = _make_skill_data(name='demo', instructions='Read this first') + ap = _make_ap() + ap.skill_mgr = _make_skill_manager({'demo': skill}) + ap.skill_repository = SimpleNamespace(get_skill=AsyncMock(return_value=skill)) + ap.box_service = SimpleNamespace(is_workspace_sandbox_available=AsyncMock(return_value=False)) + manager = ToolManager(ap) + manager.skill_tool_loader = SkillToolLoader(ap) + query = _make_query(_execution_context=_CONTEXT) + source_ref = {'source': 'skill', 'source_id': None} + + result = await manager.execute_func_call('activate', {'skill_name': 'demo'}, query, source_ref) + + assert result['activated'] is True + assert result['revision'] == skill['revision'] + assert result['capabilities']['execution_available'] is False + assert query.variables['_skill_execution_available'] is False + assert await manager.get_tool_by_source(_CONTEXT, 'activate', source_ref) is not None + assert await manager.get_tool_by_source(_CONTEXT, 'register_skill', source_ref) is None + + ap.box_service.is_workspace_sandbox_available.return_value = True + assert await manager.get_tool_by_source(_CONTEXT, 'register_skill', source_ref) is not None + @pytest.mark.asyncio async def test_activate_returns_instructions_and_registers_skill(self): from langbot.pkg.provider.tools.loaders.skill_authoring import ( From 84b2c1f86d3dea8c2f96d1d31dddf6dc8274cd24 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:06:04 +0800 Subject: [PATCH 008/174] build(deps): pin validated skill sdk head --- pyproject.toml | 2 +- uv.lock | 10 +++------- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index c944d1771..31d15a7d7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -71,7 +71,7 @@ dependencies = [ "langchain-text-splitters>=1.1.2", "chromadb>=1.0.0,<2.0.0", "qdrant-client (>=1.15.1,<2.0.0)", - "langbot-plugin @ git+https://github.com/langbot-app/langbot-plugin-sdk.git@05a5ff3f62fa774b1dbf0c96b80cdc4d16dbf250", + "langbot-plugin @ git+https://github.com/langbot-app/langbot-plugin-sdk.git@661b1475091000c94595016d3a4914ff3a1ad81d", "asyncpg>=0.30.0", "line-bot-sdk>=3.19.0", "matrix-nio>=0.25.2", diff --git a/uv.lock b/uv.lock index 9274f34cd..2788ba272 100644 --- a/uv.lock +++ b/uv.lock @@ -2185,7 +2185,7 @@ requires-dist = [ { name = "gewechat-client", specifier = ">=0.1.5" }, { name = "html2text", specifier = ">=2024.2.26" }, { name = "httpx", extras = ["socks"], specifier = ">=0.28.1" }, - { name = "langbot-plugin", specifier = "==0.6.20" }, + { name = "langbot-plugin", git = "https://github.com/langbot-app/langbot-plugin-sdk.git?rev=661b1475091000c94595016d3a4914ff3a1ad81d" }, { name = "langchain", specifier = ">=1.3.9" }, { name = "langchain-core", specifier = ">=1.3.3" }, { name = "langchain-text-splitters", specifier = ">=1.1.2" }, @@ -2255,8 +2255,8 @@ dev = [ [[package]] name = "langbot-plugin" -version = "0.6.20" -source = { registry = "https://pypi.org/simple" } +version = "0.6.21" +source = { git = "https://github.com/langbot-app/langbot-plugin-sdk.git?rev=661b1475091000c94595016d3a4914ff3a1ad81d#661b1475091000c94595016d3a4914ff3a1ad81d" } dependencies = [ { name = "aiofiles" }, { name = "aiohttp" }, @@ -2276,10 +2276,6 @@ dependencies = [ { name = "watchdog" }, { name = "websockets" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/dd/bb/08bfa37c29a9b6823678e0b837adddacab162b982b36737e0b5e7bea6c40/langbot_plugin-0.6.20.tar.gz", hash = "sha256:86b3803fb83e67379db4a6cf166dec67a8233fbfbc1dbbf744b2b765d0615879", size = 663741, upload-time = "2026-09-27T08:56:36.473Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/28/f537c212f67afd3820bb56920b9b29783b68096fb6b2ca1a388d268dd27c/langbot_plugin-0.6.20-py3-none-any.whl", hash = "sha256:b110878d880b24ac739b26b3eb158067fb4d0be8b55ba3baf97d2092da920f10", size = 429929, upload-time = "2026-09-27T08:56:34.995Z" }, -] [[package]] name = "langchain" From 3987051da3ce708d32dcdc1fe8927b65070be42d Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Mon, 28 Sep 2026 11:10:37 +0800 Subject: [PATCH 009/174] test(skill): align native exec assertion with mount API --- tests/unit_tests/plugin/test_handler_actions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit_tests/plugin/test_handler_actions.py b/tests/unit_tests/plugin/test_handler_actions.py index 105cd6125..bec6c9e19 100644 --- a/tests/unit_tests/plugin/test_handler_actions.py +++ b/tests/unit_tests/plugin/test_handler_actions.py @@ -1630,7 +1630,7 @@ class TestAgentRunProxyActions: app.box_service.execute_tool.assert_awaited_once_with( {'command': 'pwd'}, query, - skill_name=None, + read_only_mounts=[], ) @pytest.mark.asyncio From 4482e8d8768d7f6765b58e9d51177bc3f1d78227 Mon Sep 17 00:00:00 2001 From: Hyu Date: Wed, 7 Oct 2026 09:41:41 +0800 Subject: [PATCH 010/174] feat(telemetry)!: upload one record per execution node and one closure per chain Core no longer assembles a whole trace locally. Every workflow node is uploaded the moment it completes as its own `execution_node` record, and each execution closes with exactly one `execution_chain` record; all records of one execution carry the same `event_id`, so Space groups them back into a single chain. - every execution starts at an inbound event: event-less Pipeline lanes and WebUI/API debug runs synthesize their virtual inbound event at the boundary and are marked debug=true, so a Runner is never a trace origin - sampling is decided once per event from a stable hash of the event id (the previous `int(id[:8], 16)` raised on every `platform:`-prefixed id and so dropped every successful inbound-event chain) and is reported on each record as sample{mode,denominator}; the default mode is now `all` - the local trace buffer, the 120s TTL, the per-process trace cap and the shared open-node mirror are gone, which also removes the truncated-chain upload and the re-buffered-closed-trace leak they caused - a broken chain is always uploaded, and its reason falls back to the first failed node so the chain explains itself - flush keeps records until the sender accepts them and counts what it drops; telemetry health replaces the unused send-task queue - heartbeat reports bound Agent / plugin-processor / event-binding counts - bound Pipeline and Agent resources (plugins, MCP servers, knowledge bases, tools with their Host source, skills, models and the Runner schema selectors that picked them) are reported on the chain record - failure text is bounded and redacted instead of passing exception text through - the dead per-query feature counters and their call sites are removed Wire contract: schema 2 (`execution_node` / `execution_chain`). --- src/langbot/pkg/agent/runner/config_schema.py | 59 ++ src/langbot/pkg/agent/runner/orchestrator.py | 17 + src/langbot/pkg/api/http/service/agent.py | 53 +- src/langbot/pkg/box/service.py | 3 - src/langbot/pkg/core/app.py | 10 +- src/langbot/pkg/pipeline/pipelinemgr.py | 71 ++- src/langbot/pkg/pipeline/preproc/preproc.py | 24 + src/langbot/pkg/provider/tools/loaders/mcp.py | 3 - src/langbot/pkg/provider/tools/toolmgr.py | 7 - src/langbot/pkg/telemetry/execution.py | 588 ++++++++++-------- src/langbot/pkg/telemetry/features.py | 102 --- src/langbot/pkg/telemetry/heartbeat.py | 78 +++ src/langbot/pkg/telemetry/resources.py | 244 ++++++++ src/langbot/pkg/telemetry/telemetry.py | 103 ++- src/langbot/pkg/telemetry/trace.py | 264 ++++---- src/langbot/templates/config.yaml | 12 +- .../platform/test_botmgr_execution_trace.py | 77 ++- .../unit_tests/provider/test_mcp_resources.py | 5 - .../telemetry/test_cloud_sampling.py | 7 +- tests/unit_tests/telemetry/test_features.py | 92 --- .../telemetry/test_pipeline_lane.py | 114 ++-- tests/unit_tests/telemetry/test_telemetry.py | 51 -- tests/unit_tests/telemetry/test_trace.py | 579 ++++++++++------- tests/unit_tests/test_telemetry.py | 29 - 24 files changed, 1532 insertions(+), 1060 deletions(-) delete mode 100644 src/langbot/pkg/telemetry/features.py create mode 100644 src/langbot/pkg/telemetry/resources.py delete mode 100644 tests/unit_tests/telemetry/test_features.py delete mode 100644 tests/unit_tests/test_telemetry.py diff --git a/src/langbot/pkg/agent/runner/config_schema.py b/src/langbot/pkg/agent/runner/config_schema.py index aa7d13f7e..1fad3193f 100644 --- a/src/langbot/pkg/agent/runner/config_schema.py +++ b/src/langbot/pkg/agent/runner/config_schema.py @@ -16,6 +16,65 @@ KB_SELECTOR_TYPES = {'knowledge-base-multi-selector'} PROMPT_EDITOR_TYPES = {'prompt-editor'} NONE_SENTINELS = {'', '__none__', '__none'} +# Selector field types that pick a Host resource. The Runner config form is +# plugin-declared and open-ended (DynamicFormItemSchema allows extra types), so +# this is a registry, not a fixed list: a new selector type is one line here and +# is then reported by telemetry without touching the extraction call sites. +RESOURCE_SELECTOR_TYPES: dict[str, str] = { + 'knowledge-base-multi-selector': 'knowledge_bases', + 'tool-multi-selector': 'tools', + 'mcp-server-multi-selector': 'mcp_servers', + 'skill-multi-selector': 'skills', +} + + +def declared_resource_selectors(descriptor: RunnerDescriptor | None) -> dict[str, list[str]]: + """Map each selected resource kind to the schema field names that select it. + + Only the descriptor is read, so this reports what a Runner *can* select even + when the current configuration leaves the field empty. + """ + declared: dict[str, list[str]] = {} + if descriptor is None: + return declared + for item in descriptor.config_schema or []: + if not isinstance(item, dict): + continue + kind = RESOURCE_SELECTOR_TYPES.get(normalize_schema_item_type(item.get('type'))) + field_name = item.get('name') + if not kind or not isinstance(field_name, str) or not field_name: + continue + declared.setdefault(kind, []) + if field_name not in declared[kind]: + declared[kind].append(field_name) + return declared + + +def extract_selected_resources( + descriptor: RunnerDescriptor | None, + runner_config: dict[str, typing.Any] | None, +) -> dict[str, list[str]]: + """Extract the resource ids selected through schema-declared selector fields. + + Field names are never hard-coded: they come from the Runner's own config + schema, so a form that renames or adds a selector keeps working. + """ + selected: dict[str, list[str]] = {} + if not isinstance(runner_config, dict): + return selected + for kind, field_names in declared_resource_selectors(descriptor).items(): + values: list[str] = [] + for field_name in field_names: + value = runner_config.get(field_name) + if isinstance(value, str): + if value not in NONE_SENTINELS: + values.append(value) + elif isinstance(value, (list, tuple)): + values.extend(item for item in value if isinstance(item, str) and item not in NONE_SENTINELS) + if values: + selected[kind] = list(dict.fromkeys(values)) + return selected + def normalize_schema_item_type(item_type: typing.Any) -> typing.Any: """Normalize legacy/frontend DynamicForm aliases to protocol field types.""" diff --git a/src/langbot/pkg/agent/runner/orchestrator.py b/src/langbot/pkg/agent/runner/orchestrator.py index c7c9443eb..b04ec8394 100644 --- a/src/langbot/pkg/agent/runner/orchestrator.py +++ b/src/langbot/pkg/agent/runner/orchestrator.py @@ -37,6 +37,7 @@ from .run_journal import AgentRunJournal from .session_registry import AgentRunSessionRegistry, get_session_registry from .state_scope import build_state_context from ...provider.tools.loaders import skill as skill_loader +from ...telemetry import resources as telemetry_resources from ...telemetry import trace as trace_mod from ...telemetry.execution import bind_trace as bind_execution_trace from ...telemetry.execution import close_trace as close_execution_trace @@ -142,6 +143,22 @@ class AgentRunOrchestrator: binding=binding, descriptor=descriptor, ) + # Telemetry: report what this run was granted (never what it used). The + # materialized resource list is authoritative for tools/skills/models; + # the Runner's own config form is authoritative for its selectors. + try: + from . import config_schema as runner_config_schema + + capabilities = getattr(descriptor, 'capabilities', None) + telemetry_resources.note_agent( + runner_id=descriptor.id, + resources=resources, + selectors=runner_config_schema.extract_selected_resources(descriptor, binding.runner_config), + capabilities=capabilities.model_dump() if hasattr(capabilities, 'model_dump') else None, + tools_all=getattr(binding.resource_policy, 'allow_all_tools', None), + ) + except Exception: + pass context = await self.context_builder.build_context_from_event( event=execution_event, diff --git a/src/langbot/pkg/api/http/service/agent.py b/src/langbot/pkg/api/http/service/agent.py index 260b1f2df..5cfcf116d 100644 --- a/src/langbot/pkg/api/http/service/agent.py +++ b/src/langbot/pkg/api/http/service/agent.py @@ -348,25 +348,42 @@ class AgentService: execution_events.append(visible_result) final_text = '' - async for output in self.ap.agent_run_orchestrator.run( - event, - binding, - adapter_context={ - '_query': execution_query, - '_execution_context': execution_context, - '_result_observer': observe_result, - }, + # A debug run has no platform event: the synthetic event envelope is the + # execution's inbound boundary, so the chain is opened here with the real + # event id and marked as a debug WebUI/API execution. The Runner inside + # then joins this chain instead of becoming a trace origin of its own. + from ....telemetry.execution import ingress as execution_ingress + + principal_type = str(getattr(getattr(context, 'principal', None), 'principal_type', '') or '') + origin = 'api' if principal_type == 'api_key' else 'webui' + with execution_ingress( + self.ap, + 'event_done', + execution_context, + event_id, + debug=True, + origin=origin, + synthetic_event=event_type, ): - output_text = self._provider_output_to_text(output) - if output_text: - final_text = output_text - output_items.append( - { - 'kind': output.__class__.__name__, - 'role': str(getattr(output, 'role', '') or ''), - 'text': output_text, - } - ) + async for output in self.ap.agent_run_orchestrator.run( + event, + binding, + adapter_context={ + '_query': execution_query, + '_execution_context': execution_context, + '_result_observer': observe_result, + }, + ): + output_text = self._provider_output_to_text(output) + if output_text: + final_text = output_text + output_items.append( + { + 'kind': output.__class__.__name__, + 'role': str(getattr(output, 'role', '') or ''), + 'text': output_text, + } + ) return { 'event_id': event_id, diff --git a/src/langbot/pkg/box/service.py b/src/langbot/pkg/box/service.py index 4f51b15aa..a18ebd914 100644 --- a/src/langbot/pkg/box/service.py +++ b/src/langbot/pkg/box/service.py @@ -20,7 +20,6 @@ from langbot_plugin.box.security import BOX_SHARED_WORKSPACE_PROBE_PREFIX from .admission import SandboxAdmissionController, require_cloud_admission_policy from .connector import BoxRuntimeConnector, _get_box_config from . import secure_fs -from ..telemetry import features as telemetry_features from ..utils import httpclient from ..api.http.context import ExecutionContext from ..api.http.service.tenant import TenantContext, require_workspace_uuid @@ -636,7 +635,6 @@ class BoxService: f'query_id={query.query_id} ' f'summary={json.dumps(self._summarize_result(result), ensure_ascii=False)}' ) - telemetry_features.increment(query, 'sandbox', 'execs') return self._serialize_result(result) def resolve_box_session_id(self, query: pipeline_query.Query) -> str: @@ -2011,7 +2009,6 @@ class BoxService: # ── Observability ───────────────────────────────────────────────── def _record_error(self, exc: Exception, query: pipeline_query.Query): - telemetry_features.increment(query, 'sandbox', 'errors') self._recent_errors.append( { 'timestamp': _dt.datetime.now(_UTC).isoformat(), diff --git a/src/langbot/pkg/core/app.py b/src/langbot/pkg/core/app.py index 7da4a0a15..cf3ecd13a 100644 --- a/src/langbot/pkg/core/app.py +++ b/src/langbot/pkg/core/app.py @@ -309,6 +309,14 @@ class Application: if runtime_ops_reporter is not None: runtime_ops_stats = runtime_ops_reporter.get_stats() + telemetry_stats = {} + telemetry_health = getattr(self.telemetry, 'health', None) + if callable(telemetry_health): + try: + telemetry_stats = telemetry_health() + except Exception: + telemetry_stats = {} + return { 'asyncio_tasks': asyncio_tasks, 'event_loop': self.event_loop_monitor.snapshot(), @@ -320,7 +328,7 @@ class Application: 'models': model_stats, 'runtimes': runtime_stats, 'runtime_ops': runtime_ops_stats, - 'telemetry_tasks': len(getattr(self.telemetry, 'send_tasks', ())), + 'telemetry': telemetry_stats, } async def initialize(self): diff --git a/src/langbot/pkg/pipeline/pipelinemgr.py b/src/langbot/pkg/pipeline/pipelinemgr.py index c98b94c8a..ad45b4c50 100644 --- a/src/langbot/pkg/pipeline/pipelinemgr.py +++ b/src/langbot/pkg/pipeline/pipelinemgr.py @@ -194,6 +194,19 @@ class RuntimePipeline: query.variables['_pipeline_bound_mcp_servers'] = self.bound_mcp_servers query.variables['_pipeline_mcp_resource_attachments'] = self.mcp_resource_attachments query.variables['_pipeline_mcp_resource_agent_read_enabled'] = self.mcp_resource_agent_read_enabled + # Telemetry: the bindings this lane runs with. None means "all enabled", + # which the *_all flags make explicit in the chain record. + try: + from ..telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline( + plugins=self.bound_plugins, + plugins_all=self.enable_all_plugins, + mcp_servers=self.bound_mcp_servers, + mcp_all=self.enable_all_mcp_servers, + ) + except Exception: + pass # Record query start for monitoring try: @@ -370,7 +383,38 @@ class RuntimePipeline: i += 1 + @staticmethod + def _synthetic_origin(query: pipeline_query.Query) -> str: + """Origin of a lane that starts without an inbound platform event.""" + try: + variables = getattr(query, 'variables', None) + declared = str(variables.get('_telemetry_origin') or '').strip() if isinstance(variables, dict) else '' + except Exception: + declared = '' + if declared in ('webui', 'api'): + return declared + adapter = getattr(query, 'adapter', None) + if adapter is not None and adapter.__class__.__name__ == 'WebSocketAdapter': + # WebChat and the CLI diagnostic surface both speak through the + # WebSocket proxy adapter; the CLI marks itself in its variables. + try: + variables = getattr(query, 'variables', None) + if isinstance(variables, dict) and '_cli_run_status' in variables: + return 'api' + except Exception: + pass + return 'webui' + return 'api' + + @staticmethod + def _synthetic_event_type(query: pipeline_query.Query) -> str: + """Event type of the virtual inbound event a synthetic lane stands for.""" + event = getattr(query, 'message_event', None) + event_type = str(getattr(event, 'type', None) or '').strip() + return event_type[:160] or 'message.received' + async def process_query(self, query: pipeline_query.Query): + from ..telemetry import trace as trace_mod from ..telemetry.execution import ingress from ..telemetry.execution import record as record_execution from ..telemetry.platform import processing_mode @@ -380,10 +424,21 @@ class RuntimePipeline: # The Workspace-scoped opaque query uuid is the execution identity Space # shows for this lane; it is a stable UUID for every pooled query. execution_id = str(getattr(query, 'query_uuid', '') or '').strip() or str(query.query_id) + # A lane with no inbound platform event in flight (WebChat, WebUI debug, + # HTTP pipeline run) synthesizes its virtual inbound event here, so every + # execution starts at an event boundary and the Runner is never the trace + # origin. A lane running under a platform route nests inside that ingress. + synthesizing = trace_mod.current() is None try: - # Callers without a platform event (Webchat, HTTP API) still get one - # trace for the whole Pipeline lane; nested calls reuse the trace. - with ingress(self.ap, 'pipeline_done', getattr(self, 'execution_context', None), execution_id=execution_id): + with ingress( + self.ap, + 'pipeline_done', + getattr(self, 'execution_context', None), + execution_id=execution_id, + debug=synthesizing, + origin=self._synthetic_origin(query) if synthesizing else 'platform', + synthetic_event=self._synthetic_event_type(query) if synthesizing else '', + ): # The lane is its own workflow step: acknowledgements and replies it # sends directly, and the runner it drives, hang under this node. # When the lane runs under a platform route the node nests there. @@ -576,14 +631,8 @@ class RuntimePipeline: state = current_trace() # The runner orchestrator already lands its own failed ``runner/execute`` # node when the break happens inside a runner; in that case only carry the - # reason over instead of repeating the node on the trace. - already_recorded = bool( - state is not None - and any( - stage.get('family') == 'runner' and stage.get('outcome') not in ('success', 'skipped') - for stage in state.stages - ) - ) + # reason over instead of repeating the node on the chain. + already_recorded = bool(state is not None and state.runner_failed) if not already_recorded: # The lane failure is its own workflow step, hanging under whatever # step routed into it (or a root when nothing did). diff --git a/src/langbot/pkg/pipeline/preproc/preproc.py b/src/langbot/pkg/pipeline/preproc/preproc.py index 67db3b1a4..2a0878170 100644 --- a/src/langbot/pkg/pipeline/preproc/preproc.py +++ b/src/langbot/pkg/pipeline/preproc/preproc.py @@ -108,6 +108,12 @@ class PreProcessor(stage.PipelineStage): ) catalog = ResourcePolicyProjector.filter_tools(catalog, policy) ToolManager.bind_query_tool_sources(query, catalog) + try: + from ...telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline(tools=catalog, tools_all=policy.allow_all_tools) + except Exception: + pass return ToolManager.tools_from_catalog(catalog) def _runner_accepts_multimodal_input(self, descriptor: RunnerDescriptor | None) -> bool: @@ -393,6 +399,18 @@ class PreProcessor(stage.PipelineStage): descriptor, runner_config, ) + # Telemetry: the knowledge bases this lane may retrieve from, plus the + # schema-declared selector fields that selected them. + try: + from ...telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline( + knowledge_bases=query.variables['_knowledge_base_uuids'], + runner_id=descriptor.id if descriptor is not None else '', + selectors=config_schema.extract_selected_resources(descriptor, runner_config), + ) + except Exception: + pass # Emit PromptPreProcessing before the runner receives the query. @@ -427,4 +445,10 @@ class PreProcessor(stage.PipelineStage): bound_skills = extensions_prefs['skills'] query.variables['_pipeline_bound_skills'] = bound_skills + try: + from ...telemetry import resources as telemetry_resources + + telemetry_resources.note_pipeline(skills=bound_skills) + except Exception: + pass return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) diff --git a/src/langbot/pkg/provider/tools/loaders/mcp.py b/src/langbot/pkg/provider/tools/loaders/mcp.py index 48501292b..7b5a83afd 100644 --- a/src/langbot/pkg/provider/tools/loaders/mcp.py +++ b/src/langbot/pkg/provider/tools/loaders/mcp.py @@ -1464,9 +1464,6 @@ class RuntimeMCPSession: if query is None: return try: - from langbot.pkg.telemetry import features as telemetry_features - - telemetry_features.increment(query, 'mcp_resource_reads', envelope.get('source') or 'unknown') query.variables.setdefault(MCP_RESOURCE_TRACE_QUERY_KEY, []).append( { 'server_name': envelope.get('server_name'), diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index b09dff9d8..26329dad0 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -480,8 +480,6 @@ class ToolManager: query: pipeline_query.Query, source_ref: ToolSourceRef | None = None, ) -> typing.Any: - from langbot.pkg.telemetry import features as telemetry_features - source_ref = source_ref or self.get_query_tool_source(query, name) if source_ref is not None: execution_context = get_query_execution_context(query) @@ -534,7 +532,6 @@ class ToolManager: return await loader.invoke_tool(name, parameters, query, source_id=source_id) return await loader.invoke_tool(name, parameters, query) - telemetry_features.increment(query, 'tool_calls', telemetry_source) return await self._invoke_tool_with_monitoring( source=telemetry_source, name=name, @@ -547,7 +544,6 @@ class ToolManager: await self._bind_plugin_workspace(execution_context) sandbox_available = await self._workspace_sandbox_available(execution_context) if sandbox_available and await self.native_tool_loader.has_tool(name): - telemetry_features.increment(query, 'tool_calls', 'native') return await self._invoke_tool_with_monitoring( source='native', name=name, @@ -556,7 +552,6 @@ class ToolManager: invoke=lambda: self.native_tool_loader.invoke_tool(name, parameters, query), ) if await self.plugin_tool_loader.has_tool(name): - telemetry_features.increment(query, 'tool_calls', 'plugin') return await self._invoke_tool_with_monitoring( source='plugin', name=name, @@ -565,7 +560,6 @@ class ToolManager: invoke=lambda: self.plugin_tool_loader.invoke_tool(name, parameters, query), ) if await self.mcp_tool_loader.has_tool(execution_context, name): - telemetry_features.increment(query, 'tool_calls', 'mcp') return await self._invoke_tool_with_monitoring( source='mcp', name=name, @@ -574,7 +568,6 @@ class ToolManager: invoke=lambda: self.mcp_tool_loader.invoke_tool(name, parameters, query), ) if sandbox_available and await self.skill_tool_loader.has_tool(name): - telemetry_features.increment(query, 'tool_calls', 'skill') return await self._invoke_tool_with_monitoring( source='skill', name=name, diff --git a/src/langbot/pkg/telemetry/execution.py b/src/langbot/pkg/telemetry/execution.py index da0111f47..1e5a16a1c 100644 --- a/src/langbot/pkg/telemetry/execution.py +++ b/src/langbot/pkg/telemetry/execution.py @@ -1,18 +1,16 @@ -"""Content-free execution records for the existing telemetry sender. +"""Point-wise execution records for the existing telemetry sender. -This module reports observations only. Coverage catalogs and acceptance rules -belong to Space. +One execution is reported as a chain of independent records that all carry the +same ``event_id``: one ``execution_node`` per workflow node, emitted the moment +that node completes, plus exactly one ``execution_chain`` emitted when the +execution closes. The sender never assembles a trace locally, so there is no +per-trace buffer, no TTL and no per-process trace cap; memory is bounded by the +outbound record buffer, the flush batch size and the small cross-task registry. -One complete execution is one record: the execution identity (Pipeline query, -inbound platform event or Agent run), its bounded ordered stages and its -terminal outcome. Nothing is aggregated or counted across executions — every -record carries the id of the execution it belongs to, so Space can fetch the -whole chain by primary identity. Records are buffered whole and flushed as -``{"records": [...]}`` on the configured cadence and on shutdown; an in-flight -trace is never flushed half-written. - -Memory is bounded in stages per trace, buffered traces, buffered records, the -cross-task execution registry and flush batch size. +Sampling is decided exactly once per event, at ingress, from a stable hash of +the execution identity, so every record of one execution carries the same +decision. A chain that broke is always emitted even when sampling excluded it, +which is why a chain may arrive with zero or partial nodes. """ from __future__ import annotations @@ -21,29 +19,48 @@ import asyncio import contextlib import contextvars import json +import re import time +import zlib from datetime import datetime, timezone from . import trace as trace_mod from .identity import workspace_identity +from .resources import snapshot as resources_snapshot +from .resources import begin as resources_begin +from .resources import end as resources_end from .trace import TraceState MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'}) OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'}) +FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'}) +ORIGINS = frozenset({'platform', 'webui', 'api'}) +CLOSED_BY = frozenset({'event_done', 'pipeline_done', 'runner_done', 'timeout'}) +CHAIN_FAILURES = frozenset({'failed', 'cancelled', 'timeout'}) -# Trace bounds: stages per trace, traces buffered per process, trace lifetime. -MAX_TRACES = 64 -TRACE_TTL_SECONDS = 120 +SCHEMA = 2 TRACE_MODES = frozenset({'off', 'failures', 'sampled', 'all'}) -DEFAULT_TRACE_MODE = 'sampled' +DEFAULT_TRACE_MODE = 'all' DEFAULT_TRACE_SAMPLE = 20 -# Record bounds: buffered records, one flush batch, flush cadence, registry. +# Sampling bounds: outbound record buffer, one flush batch, flush cadence. MAX_BUFFERED_RECORDS = 512 MAX_RECORDS_PER_FLUSH = 128 MAX_FLUSH_BYTES = 200 * 1024 DEFAULT_FLUSH_SECONDS = 180 +MAX_FLUSH_ATTEMPTS = 3 +FLUSH_RETRY_BACKOFF_SECONDS = 0.2 + +# Cross-task registry: execution id -> owning execution, bounded and time-limited. MAX_REGISTRY_KEYS = 256 +REGISTRY_TTL_SECONDS = 300 + +# Failure details are operator-facing: keep the failure class and a short +# machine-ish prefix, never the free text a user or a model produced. +ERROR_LIMIT = 200 +ERROR_PREFIX_CHARS = 120 +ERROR_PREFIX_TOKENS = 12 +_ERROR_CLASS = re.compile(r'[A-Za-z_][\w.]*') # Set by cross-task call sites (plugin/RPC actions) that run without the # ingress context but know which execution they belong to. @@ -68,16 +85,36 @@ def current_execution_id() -> str: return _execution_id.get() -def _bounded_error(value, limit: int = 400) -> str: - """Bound and de-fang one failure detail before it leaves the process. +def _short_prefix(text: str) -> str: + """Keep at most a short machine-ish prefix of one failure message.""" + if len(text) > ERROR_PREFIX_CHARS: + text = text[:ERROR_PREFIX_CHARS] + parts = text.split(' ') + if len(parts) > ERROR_PREFIX_TOKENS: + text = ' '.join(parts[:ERROR_PREFIX_TOKENS]) + return text - Error text is operator-facing: keep it single-line, printable and short so a - noisy failure cannot smuggle control characters or bloat the payload. + +def _bounded_error(value, limit: int = ERROR_LIMIT) -> str: + """Bound and redact one failure detail before it leaves the process. + + The failure class and a short machine-ish prefix survive; control characters + are stripped, whitespace is collapsed and the result is capped, so a noisy + failure cannot smuggle user or model content into the payload. """ if value is None: return '' text = ''.join(ch if ch.isprintable() else ' ' for ch in str(value)) text = ' '.join(text.split()) + if not text: + return '' + head, separator, rest = text.partition(':') + head = head.strip() + if separator and len(head) <= 80 and _ERROR_CLASS.fullmatch(head): + prefix = _short_prefix(rest.strip()) + text = f'{head}: {prefix}' if prefix else head + else: + text = _short_prefix(text) return text[:limit] @@ -91,18 +128,21 @@ def _duration_ms(started_at: str, ended_at: str) -> int: class ExecutionCounters: + """Point-wise execution reporting: one record per node, one per closure.""" + def __init__(self, manager): self.manager = manager self.records: list[dict] = [] self.task: asyncio.Task | None = None + # Exposed health: records dropped or accepted by the sender. self.dropped = 0 - self.traces: dict[str, TraceState] = {} - self.trace_deadlines: dict[str, float] = {} - # Execution identity -> in-flight trace, for stages recorded by tasks + self.sent = 0 + # Execution identity -> owning execution, for nodes recorded by tasks # that no longer share the ingress context (plugin/RPC platform calls). + # It resolves identity and the sampling decision only; node payloads are + # never accumulated here. self.registry: dict[str, TraceState] = {} self.registry_deadlines: dict[str, float] = {} - self.dropped_traces = 0 # ------------------------------------------------------------------ config @@ -130,6 +170,63 @@ class ExecutionCounters: seconds = DEFAULT_FLUSH_SECONDS return seconds if 1 <= seconds <= 86400 else DEFAULT_FLUSH_SECONDS + def health(self) -> dict: + """Real telemetry health for the instance liveness snapshot.""" + return { + 'records_buffered': len(self.records), + 'records_sent': self.sent, + 'records_dropped': self.dropped, + 'executions_in_flight': len({id(state) for state in self.registry.values()}), + } + + # ----------------------------------------------------------------- sampling + + def _sampling_decision(self, event_id: str) -> tuple[str, int, bool]: + """Decide once, deterministically, whether this event is reported. + + The identity is hashed instead of being parsed, so every id shape + (``platform:...``, uuids, debug ids) samples the same way. + """ + mode = self.trace_mode() + if mode == 'sampled': + denominator = self.trace_sample() + try: + sampled_in = zlib.crc32(str(event_id).encode('utf-8')) % denominator == 0 + except Exception: + sampled_in = False + return mode, denominator, sampled_in + # 'off' reports nothing, 'failures' keeps nodes out but still reports a + # broken chain, 'all' reports everything; denominator is 1 for all three. + return mode, 1, mode == 'all' + + def configure( + self, + state: TraceState, + context=None, + *, + debug: bool = False, + origin: str = 'platform', + identity=None, + ) -> None: + """Bind the execution identity, the sampling decision and the flags.""" + if state.configured: + return + state.configured = True + state.debug = bool(debug) + state.origin = origin if origin in ORIGINS else 'platform' + mode, denominator, sampled_in = self._sampling_decision(state.event_id) + state.mode = mode + state.denominator = denominator + # Debug chains are always emitted (unless reporting is off entirely). + state.emit_nodes = mode != 'off' and (state.debug or sampled_in) + if identity is not None: + state.identity = dict(identity) + elif context is not None: + try: + state.identity = workspace_identity(context) + except Exception: + pass + # --------------------------------------------------------------- recording def record( @@ -148,102 +245,96 @@ class ExecutionCounters: node: str = '', ): try: - cfg = self.manager.telemetry_config - if not cfg or cfg.get('disable_telemetry', False) or not cfg.get('url'): + if not self._enabled(): return - if family not in {'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'}: - return - if mode not in MODES or outcome not in OUTCOMES: + if family not in FAMILIES or mode not in MODES or outcome not in OUTCOMES: return # Only code-defined identifiers are accepted; never pass user values. if any(not isinstance(v, str) or len(v) > 160 for v in (operation, adapter, runner, node)): return state = trace_mod.current() if state is None: - # Cross-task work has no inherited trace; resolve the owning - # execution through the registry instead. + # Cross-task work has no inherited context; resolve the owning + # execution through the registry. It has no open step here, so + # its node becomes a root of that execution instead of being + # mirrored from another task's stack. lookup = str(execution_id or '').strip() or current_execution_id() if lookup: - state = self.registry.get(lookup) - if state is None: + state = self._resolve(lookup) + if state is None or state.closed: # Every observation belongs to exactly one execution chain. return + self.configure(state) try: identity = workspace_identity(context) except Exception: # A cross-task caller may only carry the Workspace id; reuse the - # identity already bound to the owning trace. + # identity already bound to the owning execution. identity = state.identity or {} if not identity.get('instance_id') or not identity.get('workspace_uuid'): return - self._record_trace_stage( - state, - identity=identity, - family=family, - operation=operation, - mode=mode, - adapter=adapter, - runner=runner, - outcome=outcome, - synthetic=synthetic, - error=_bounded_error(error), - node=node, + state.identity = identity + + ended_at = datetime.now(timezone.utc).isoformat() + node_id, seq, started_at, parent = trace_mod.resolve_node(state, str(node or ''), ended_at) + root = parent == '' and not state.root_seen + if root: + state.root_seen = True + bounded_error = _bounded_error(error) + state.note_node(outcome=outcome, adapter=adapter, runner=runner, family=family, error=bounded_error) + if not state.emit_nodes: + # Sampling excluded this execution: the node is never buffered, + # only counted so the terminal chain can explain the gap. + state.dropped_nodes += 1 + return + self._enqueue_record( + { + 'event_type': 'execution_node', + 'schema': SCHEMA, + **self._common_fields(state, ended_at), + 'node_id': node_id, + 'parent_node_id': parent, + 'root': root, + 'seq': seq, + 'family': family, + 'operation': operation, + 'mode': mode, + 'adapter': adapter, + 'runner': runner, + 'outcome': outcome, + 'error': bounded_error, + 'route_ref': trace_mod.current_route(), + 'run_id': trace_mod.current_run(), + 'synthetic': bool(synthetic), + 'started_at': started_at, + 'ended_at': ended_at, + } ) - self._ensure_loop() except Exception: # Observability must never change execution behavior. return - def _record_trace_stage( - self, - state: TraceState, - *, - identity, - family, - operation, - mode, - adapter, - runner, - outcome, - synthetic, - error: str = '', - node: str = '', - ) -> None: - if state.abandoned: - return - if trace_mod.current() is state: - # Recorded by the task that owns the step: its step stack is exact. - # A step's own record sits under its enclosing step, never itself. - parent = trace_mod.current_parent() - if node and parent == node: - parent = trace_mod.enclosing_parent() - else: - # A cross-task observation (plugin/RPC) cannot see the step - # contextvar; attach it to the innermost step the owning task has - # open on the trace. - parent = state.open_nodes[-1] if state.open_nodes else '' - if state.trace_id not in self.traces: - # Never buffer traces this configuration would discard anyway. - if self.trace_mode() == 'off' or len(self.traces) >= MAX_TRACES: - self.dropped += 1 - state.abandoned = True - self._unregister_state(state) - return - self.traces[state.trace_id] = state - self.trace_deadlines[state.trace_id] = time.monotonic() + TRACE_TTL_SECONDS - state.identity = identity - state.append( - family=family, - operation=operation, - mode=mode, - adapter=adapter, - runner=runner, - outcome=outcome, - synthetic=bool(synthetic), - error=error, - node=node, - parent=parent, - ) + def _common_fields(self, state: TraceState, timestamp: str) -> dict: + identity = state.identity + from ..utils import constants + + return { + 'event_id': state.event_id, + 'instance_id': identity['instance_id'], + 'workspace_uuid': identity['workspace_uuid'], + 'version': constants.semantic_version, + 'edition': constants.edition, + 'timestamp': timestamp, + 'debug': bool(state.debug), + 'sample': {'mode': state.mode, 'denominator': state.denominator}, + } + + def _enabled(self) -> bool: + try: + cfg = self.manager.telemetry_config + except Exception: + return False + return bool(cfg) and not cfg.get('disable_telemetry', False) and bool(cfg.get('url')) def _ensure_loop(self) -> None: if self.task is None or self.task.done(): @@ -252,20 +343,27 @@ class ExecutionCounters: # --------------------------------------------------------------- registry def register_trace(self, state: TraceState, execution_id) -> None: - """Alias one more execution identity onto an in-flight trace.""" + """Alias one more execution identity onto an in-flight execution.""" try: exec_id = str(execution_id or '').strip() - if not exec_id or state.abandoned: + if not exec_id: return + self.configure(state) + now = time.monotonic() + self._sweep_registry(now) if exec_id not in self.registry and len(self.registry) >= MAX_REGISTRY_KEYS: oldest = next(iter(self.registry)) self.registry.pop(oldest, None) self.registry_deadlines.pop(oldest, None) self.registry[exec_id] = state - self.registry_deadlines[exec_id] = time.monotonic() + TRACE_TTL_SECONDS + self.registry_deadlines[exec_id] = now + REGISTRY_TTL_SECONDS except Exception: return + def _resolve(self, execution_id: str) -> TraceState | None: + self._sweep_registry(time.monotonic()) + return self.registry.get(execution_id) + def _unregister_state(self, state: TraceState) -> None: for exec_id in [key for key, value in self.registry.items() if value is state]: self.registry.pop(exec_id, None) @@ -277,26 +375,68 @@ class ExecutionCounters: self.registry.pop(exec_id, None) self.registry_deadlines.pop(exec_id, None) - # ------------------------------------------------------------------ traces + # ------------------------------------------------------------------ chains def close_trace(self, state: TraceState, reason: str = 'event_done', error: str = '') -> None: - """Buffer one complete execution record when it matches the sample.""" + """Emit the one terminal ``execution_chain`` record of this execution.""" try: if state.closed: return state.closed = True - registered = self.traces.pop(state.trace_id, None) is not None - self.trace_deadlines.pop(state.trace_id, None) self._unregister_state(state) - if not registered and not state.failure_reason: + if not self._enabled(): + # Nothing can be uploaded; never grow the buffer for a record + # the sender would refuse to send. return - if (not state.stages and not state.failure_reason) or not self._trace_emitted(state): + self.configure(state) + if not self._chain_emitted(state): return - record = self._build_record(state, reason, error) - if record is not None: - self._enqueue_record(record) + self._enqueue_record(self._build_chain_record(state, reason, error)) except Exception: return + finally: + # The snapshot was read while building the chain record; stop + # collecting so a late sibling task cannot leak into the next chain. + resources_end() + + def _chain_emitted(self, state: TraceState) -> bool: + if state.mode == 'off': + return False + if state.emit_nodes or state.debug: + return True + # A chain that broke is always uploaded, even when sampling excluded it. + return bool(state.failure_reason) or state.outcome() in CHAIN_FAILURES + + def _build_chain_record(self, state: TraceState, reason: str, error: str = '') -> dict: + ended_at = datetime.now(timezone.utc).isoformat() + record = { + 'event_type': 'execution_chain', + 'schema': SCHEMA, + **self._common_fields(state, ended_at), + 'closed_by': reason if reason in CLOSED_BY else 'event_done', + 'outcome': state.outcome(), + # First failure detail in this chain, so Space can show why it broke. + 'error': _bounded_error(error) + or _bounded_error(state.failure_reason) + or _bounded_error(state.first_error) + or '', + 'duration_ms': _duration_ms(state.started_at, ended_at), + 'stage_count': state.stage_count, + 'dropped_nodes': state.dropped_nodes, + 'origin': state.origin, + 'model_name': state.model_name or '', + 'adapter': state.adapter or state.node_adapter, + 'runner': state.runner or state.node_runner, + 'runner_category': state.runner_category or '', + 'pipeline_plugins': state.pipeline_plugins, + } + # Bound resources of this execution, when any lane reported them. The + # key is omitted entirely when nothing was noted, so the payload stays + # byte-identical for executions with no bindings. + resources = resources_snapshot() + if resources: + record['resources'] = resources + return record def _enqueue_record(self, record: dict) -> None: if len(self.records) >= MAX_BUFFERED_RECORDS: @@ -305,137 +445,52 @@ class ExecutionCounters: self.records.append(record) self._ensure_loop() - def _trace_emitted(self, state: TraceState) -> bool: - mode = self.trace_mode() - if mode == 'off': - return False - # A chain that failed must always be uploaded, even when no stage was - # recorded before it broke: the reason is the whole point of the trace. - if state.failure_reason: - return True - if mode == 'all': - return True - # A failed, cancelled or timed-out stage is always worth keeping. - if any(stage['outcome'] not in ('success', 'skipped') for stage in state.stages): - return True - # WebUI debug runs are deliberately traced; they stay flagged synthetic. - if state.synthetic: - return True - if mode == 'failures': - return False - try: - return int(state.trace_id[:8], 16) % self.trace_sample() == 0 - except ValueError: - return False - - def _build_record(self, state: TraceState, reason: str, error: str = '') -> dict | None: - from ..utils import constants - - identity = state.identity - if not identity.get('instance_id') or not identity.get('workspace_uuid'): - return None - ended_at = datetime.now(timezone.utc).isoformat() - stages = state.stages - observations = [] - for stage in stages: - observation = { - 'seq': stage['seq'], - 'family': stage['family'], - 'operation': stage['operation'], - 'mode': stage['mode'], - 'adapter': stage['adapter'], - 'runner': stage['runner'], - 'outcome': stage['outcome'], - 'synthetic': stage['synthetic'], - 'error': stage['error'], - 'route_ref': stage['route_ref'], - 'run_id': stage['run_id'], - 'first_seen': stage['first_seen'], - 'last_seen': stage['last_seen'], - 'trace_id': state.trace_id, - } - # Workflow position is optional: records outside any step omit both - # keys entirely, so their payloads stay byte-identical. - if stage.get('node'): - observation['node'] = stage['node'] - if stage.get('parent'): - observation['parent'] = stage['parent'] - observations.append(observation) - return { - 'event_type': 'feature_execution', - # The execution identity: one record == one chain == one Space row. - 'query_id': state.trace_id, - 'instance_id': identity['instance_id'], - 'workspace_uuid': identity['workspace_uuid'], - 'version': constants.semantic_version, - 'edition': constants.edition, - 'timestamp': ended_at, - 'trusted': True, - # First failure detail in this chain, so Space can show why it broke. - 'error': _bounded_error(error) - or state.failure_reason - or next((stage['error'] for stage in stages if stage['error']), ''), - # Legacy-compatible columns Space already maps; best effort. - 'duration_ms': _duration_ms(state.started_at, ended_at), - 'model_name': state.model_name or '', - 'adapter': state.adapter or next((stage['adapter'] for stage in stages if stage['adapter']), ''), - 'runner': state.runner or next((stage['runner'] for stage in stages if stage['runner']), ''), - 'runner_category': state.runner_category or '', - 'pipeline_plugins': state.pipeline_plugins, - 'features': { - 'schema': 1, - 'observations': observations, - 'trace': { - 'closed_by': reason, - 'started_at': state.started_at, - 'ended_at': ended_at, - 'route_ref': next((stage['route_ref'] for stage in stages if stage['route_ref']), ''), - 'run_id': next((stage['run_id'] for stage in stages if stage['run_id']), ''), - 'outcome': state.outcome(), - 'dropped_stages': state.dropped_stages, - }, - }, - } - - def _sweep_traces(self, now: float) -> None: - for trace_id in [key for key, deadline in self.trace_deadlines.items() if deadline <= now]: - state = self.traces.get(trace_id) - if state is not None: - self.close_trace(state, 'ttl') - # ----------------------------------------------------------------- flushing async def _loop(self): interval = self.flush_seconds() - last_flush = time.monotonic() - while self.records or self.traces or self.registry: - # Traces expire on a short TTL; records keep their own cadence. - await asyncio.sleep(1 if (self.traces or self.registry) else interval) - now = time.monotonic() - self._sweep_traces(now) - self._sweep_registry(now) - if self.records and now - last_flush >= interval: - last_flush = now + try: + # The outbound loop exists only to flush the record buffer. + while self.records: + await asyncio.sleep(interval) await self.flush() - self.task = None + finally: + self.task = None - async def flush(self): + async def flush(self) -> bool: + """Upload the buffered prefix; records stay buffered until accepted.""" if not self.records: - return + return True batch: list[dict] = [] size = 0 - while self.records and len(batch) < MAX_RECORDS_PER_FLUSH: - encoded = len(json.dumps(self.records[0], ensure_ascii=False, default=str)) + for record in self.records: + if len(batch) >= MAX_RECORDS_PER_FLUSH: + break + encoded = len(json.dumps(record, ensure_ascii=False, default=str)) if batch and size + encoded > MAX_FLUSH_BYTES: break - batch.append(self.records.pop(0)) + batch.append(record) size += encoded if not batch: - return + return True payload = {'records': batch} - if not await self.manager.send(payload): - await asyncio.sleep(1) - await self.manager.send(payload) + for attempt in range(MAX_FLUSH_ATTEMPTS): + try: + if await self.manager.send(payload): + # Peek-then-send: nothing is removed until the sender + # accepted the exact batch it was given. + del self.records[: len(batch)] + self.sent += len(batch) + return True + except asyncio.CancelledError: + raise + except Exception: + pass + if attempt + 1 < MAX_FLUSH_ATTEMPTS: + await asyncio.sleep(FLUSH_RETRY_BACKOFF_SECONDS * (2**attempt)) + # The batch stays buffered for the next cycle; a permanently unaccepted + # record is dropped by the buffer bound above and counted there. + return False async def shutdown(self): if self.task is not None: @@ -451,12 +506,8 @@ class ExecutionCounters: break except (Exception, asyncio.CancelledError): pass + self.dropped += len(self.records) self.records.clear() - # Executions still in flight cannot be completed during shutdown. - self.dropped += len(self.traces) - self.dropped_traces += len(self.traces) - self.traces.clear() - self.trace_deadlines.clear() self.registry.clear() self.registry_deadlines.clear() @@ -477,7 +528,7 @@ def record(ap, context, **observation): def close_trace(ap, state, reason: str = 'event_done', error: str = '') -> None: - """Best-effort trace close usable with optional telemetry and test doubles.""" + """Best-effort chain close usable with optional telemetry and test doubles.""" try: counters = _execution_counters(ap) if counters is not None: @@ -487,8 +538,10 @@ def close_trace(ap, state, reason: str = 'event_done', error: str = '') -> None: def bind_trace(ap, execution_id: str | None = None) -> trace_mod.TraceBinding: - """Bind an execution trace and alias its identity for cross-task stages.""" + """Bind an execution and alias its identity for cross-task nodes.""" binding = trace_mod.bind(execution_id) + if binding.created: + resources_begin() if execution_id: try: counters = _execution_counters(ap) @@ -500,36 +553,61 @@ def bind_trace(ap, execution_id: str | None = None) -> trace_mod.TraceBinding: @contextlib.contextmanager -def ingress(ap, reason: str = 'event_done', context=None, execution_id: str | None = None): - """Trace one execution boundary; only the owner of the trace closes it. +def ingress( + ap, + reason: str = 'event_done', + context=None, + execution_id: str | None = None, + *, + debug: bool = False, + origin: str = 'platform', + synthetic_event: str = '', +): + """Open one execution boundary; only the owner of the chain closes it. Nested boundaries (an event routed into a Pipeline, a Runner invoked from a - dispatch) reuse the in-flight trace instead of starting a second one, and - only alias their own execution id onto it. + dispatch) reuse the in-flight chain instead of starting a second one, and + only alias their own execution id onto it. ``debug``/``origin`` apply when + this boundary starts the chain: a WebUI/API run without a real platform + event synthesizes its virtual inbound event here, so the chain has a real + identity and a node for the event it stands for. """ binding = trace_mod.bind(execution_id) - if binding.created and context is not None: - try: - binding.state.identity = workspace_identity(context) - except Exception: - pass - if execution_id and not binding.state.abandoned: - try: - counters = _execution_counters(ap) - if counters is not None: - counters.register_trace(binding.state, execution_id) - except Exception: - pass + state = binding.state + if binding.created: + resources_begin() + counters = _execution_counters(ap) + if counters is not None: + if binding.created: + counters.configure(state, context, debug=debug, origin=origin) + if execution_id: + counters.register_trace(state, execution_id) + scope = None + if binding.created and synthetic_event: + scope = trace_mod.stage_scope() + node = scope.__enter__() + record( + ap, + context, + family='platform_event', + operation=str(synthetic_event)[:160], + outcome='success', + synthetic=True, + node=node, + ) failure = '' try: yield binding except BaseException as exc: - # The boundary itself failed: keep the reason on the trace so the uploaded - # payload explains the failure instead of reporting a bare outcome. + # The boundary itself failed: keep the reason on the chain so the + # uploaded payload explains the failure instead of reporting a bare + # outcome, even when sampling had excluded this execution. if binding.created: failure = f'{type(exc).__name__}: {exc}' - binding.state.mark_failure(failure) + state.mark_failure(failure) raise finally: + if scope is not None: + scope.__exit__(None, None, None) if trace_mod.unbind_root(binding): - close_trace(ap, binding.state, reason, failure) + close_trace(ap, state, reason, failure) diff --git a/src/langbot/pkg/telemetry/features.py b/src/langbot/pkg/telemetry/features.py deleted file mode 100644 index ab66f9884..000000000 --- a/src/langbot/pkg/telemetry/features.py +++ /dev/null @@ -1,102 +0,0 @@ -"""Per-query telemetry feature counters. - -Collects anonymous, content-free usage signals (tool call counts, knowledge -base usage, sandbox executions, ...) into ``query.variables`` during query -processing. The chat handler reads the accumulated dict when building the -telemetry payload and ships it as the ``features`` JSON object. - -Every helper here is defensive: telemetry must NEVER break the pipeline, so -all mutations are wrapped and failures are silently ignored. -""" - -from __future__ import annotations - -import typing - -if typing.TYPE_CHECKING: - import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query - - -FEATURES_KEY = '_telemetry_features' - - -def get_features(query: pipeline_query.Query) -> dict: - """Return the mutable features dict for this query, creating it if needed.""" - try: - return query.variables.setdefault(FEATURES_KEY, {}) - except Exception: - return {} - - -def increment(query: pipeline_query.Query, group: str, key: str | None = None, amount: int = 1) -> None: - """Increment a counter. - - ``increment(q, 'sandbox', 'execs')`` -> features['sandbox']['execs'] += 1 - ``increment(q, 'tool_call_rounds')`` -> features['tool_call_rounds'] += 1 - """ - try: - features = get_features(query) - if key is None: - features[group] = int(features.get(group, 0)) + amount - else: - nested = features.setdefault(group, {}) - if isinstance(nested, dict): - nested[key] = int(nested.get(key, 0)) + amount - except Exception: - pass - - -def set_value(query: pipeline_query.Query, group: str, value: typing.Any) -> None: - """Set a feature value (overwrites).""" - try: - get_features(query)[group] = value - except Exception: - pass - - -def collect_features(query: pipeline_query.Query) -> dict: - """Build the final ``features`` object for the telemetry payload. - - Combines the counters accumulated during processing with end-of-query - snapshots (activated skills, bound MCP servers). Returns a plain dict - that must be JSON-serializable; non-serializable values are dropped. - """ - features: dict = {} - try: - accumulated = query.variables.get(FEATURES_KEY) - if isinstance(accumulated, dict): - features.update(accumulated) - except Exception: - pass - - # Activated skills (names only, registered by the activate tool) - try: - activated = query.variables.get('_activated_skills', {}) - if isinstance(activated, dict) and activated: - features['activated_skills'] = sorted(activated.keys()) - except Exception: - pass - - # MCP servers bound to the pipeline (names only; None means "all enabled") - try: - bound_mcp = query.variables.get('_pipeline_bound_mcp_servers', None) - if bound_mcp is not None: - features['mcp_servers'] = list(bound_mcp) - except Exception: - pass - - # Drop anything that is not JSON-serializable - import json - - try: - json.dumps(features) - return features - except Exception: - safe: dict = {} - for k, v in features.items(): - try: - json.dumps({k: v}) - safe[k] = v - except Exception: - continue - return safe diff --git a/src/langbot/pkg/telemetry/heartbeat.py b/src/langbot/pkg/telemetry/heartbeat.py index 2499c7e06..7deb20400 100644 --- a/src/langbot/pkg/telemetry/heartbeat.py +++ b/src/langbot/pkg/telemetry/heartbeat.py @@ -13,6 +13,7 @@ credentials. from __future__ import annotations import asyncio +import time import typing from datetime import datetime, timezone @@ -36,10 +37,66 @@ class WorkspaceResourceSnapshot(typing.TypedDict): mcp_server_count: int extension_count: int skill_count: int + agent_count: int + event_processor_count: int + plugin_processor_count: int + event_binding_count: int adapters: list[str] execution_generation: int +# Agent rows are configuration, not runtime registries, so a Workspace count +# needs one scoped query. The heartbeat runs every five minutes on Cloud; a +# one-hour cache keeps the cost at one bounded query per Workspace per hour +# instead of one per cycle. Unavailable counts stay -1, never a fake zero. +_AGENT_COUNT_CACHE_TTL_SECONDS = 3600 +_agent_count_cache: dict[str, tuple[float, dict[str, int]]] = {} + + +async def _workspace_agent_kind_counts(ap: core_app.Application, workspace_uuid: str) -> dict[str, int]: + """Count one Workspace's Agents per ``kind``; empty when unavailable.""" + if not workspace_uuid: + return {} + now = time.monotonic() + cached = _agent_count_cache.get(workspace_uuid) + if cached is not None and cached[0] > now: + return dict(cached[1]) + counts: dict[str, int] = {} + try: + from ..entity.persistence import agent as persistence_agent + + statement = sqlalchemy.select(persistence_agent.Agent.kind, sqlalchemy.func.count()).group_by( + persistence_agent.Agent.kind + ) + tenant_uow = getattr(ap.persistence_mgr, 'tenant_uow', None) + if callable(tenant_uow): + async with tenant_uow(workspace_uuid): + result = await ap.persistence_mgr.execute_async(statement) + else: + result = await ap.persistence_mgr.execute_async(statement) + for kind, count in result.all(): + counts[str(kind or 'agent')] = int(count or 0) + except Exception: + counts = {} + _agent_count_cache[workspace_uuid] = (now + _AGENT_COUNT_CACHE_TTL_SECONDS, dict(counts)) + return counts + + +def _bot_processor_counts(bots) -> tuple[int, int]: + """Count plugin-processor subscriptions and event bindings across bots.""" + processors = 0 + bindings = 0 + for bot in bots: + entity = getattr(bot, 'bot_entity', bot) + declared_processors = getattr(entity, 'plugin_processors', None) + if isinstance(declared_processors, list): + processors += len(declared_processors) + declared_bindings = getattr(entity, 'event_bindings', None) + if isinstance(declared_bindings, list): + bindings += len(declared_bindings) + return processors, bindings + + async def _count( ap: core_app.Application, table, @@ -81,6 +138,10 @@ async def _cloud_workspace_resource_counts(ap: core_app.Application, bindings) - 'mcp_server_count': 0, 'extension_count': 0, 'skill_count': 0, + 'agent_count': -1, + 'event_processor_count': -1, + 'plugin_processor_count': 0, + 'event_binding_count': 0, 'adapters': [], 'execution_generation': binding.placement_generation, } @@ -91,6 +152,9 @@ async def _cloud_workspace_resource_counts(ap: core_app.Application, bindings) - for key, bot in getattr(ap.platform_mgr, '_bots_by_key', {}).items(): if len(key) >= 2 and key[1] in counts: counts[key[1]]['bot_count'] += 1 + processors, bindings = _bot_processor_counts([bot]) + counts[key[1]]['plugin_processor_count'] += processors + counts[key[1]]['event_binding_count'] += bindings adapter = getattr(bot, 'adapter', None) if adapter is not None and getattr(bot, 'enable', False): adapter_sets[key[1]].add(adapter.__class__.__name__) @@ -113,6 +177,10 @@ async def _cloud_workspace_resource_counts(ap: core_app.Application, bindings) - for workspace_uuid, resource in counts.items(): resource['extension_count'] = resource['plugin_count'] + resource['mcp_server_count'] resource['adapters'] = sorted(adapter_sets[workspace_uuid]) + agent_counts = await _workspace_agent_kind_counts(ap, workspace_uuid) + if agent_counts: + resource['agent_count'] = int(agent_counts.get('agent', 0)) + resource['event_processor_count'] = int(agent_counts.get('event_processor', 0)) return list(counts.values()) @@ -163,6 +231,9 @@ async def build_heartbeat_payload( features['bot_count'] = len(platform_mgr.bots) adapters = sorted({bot.adapter.__class__.__name__ for bot in enabled_bots if getattr(bot, 'adapter', None)}) features['adapters'] = adapters + processors, bindings = _bot_processor_counts(platform_mgr.bots) + features['plugin_processor_count'] = processors + features['event_binding_count'] = bindings except Exception: pass @@ -206,6 +277,13 @@ async def build_heartbeat_payload( except Exception: pass + # Agent processors: configuration rows, so one scoped count per Workspace. + # The Cloud snapshot supplies these already and overrides below. + if workspace_resource is None: + agent_counts = await _workspace_agent_kind_counts(ap, workspace_uuid) + features['agent_count'] = int(agent_counts.get('agent', 0)) if agent_counts else -1 + features['event_processor_count'] = int(agent_counts.get('event_processor', 0)) if agent_counts else -1 + if workspace_resource is not None: features.update({key: value for key, value in workspace_resource.items() if key != 'workspace_uuid'}) diff --git a/src/langbot/pkg/telemetry/resources.py b/src/langbot/pkg/telemetry/resources.py new file mode 100644 index 000000000..5164e090e --- /dev/null +++ b/src/langbot/pkg/telemetry/resources.py @@ -0,0 +1,244 @@ +"""Bound-resource reporting for one execution. + +Answers "what was this execution given", never "what did it use": the ids and +names of the Plugins, MCP servers, knowledge bases, tools, skills and models +that were bound or granted to a Pipeline lane or an Agent run. Nothing here is +user content - only code-defined identifiers (uuids, plugin names, tool names) +and booleans. + +Two sources feed one execution-scoped snapshot: + +* the Host policy actually granted to the run (``ResourcePolicy`` / + materialized ``AgentResources``), which is authoritative for tools, skills and + platform APIs; +* the Runner's own configuration form (``DynamicFormItemSchema``), which is + where knowledge bases and models are selected. Selector field types are read + through :mod:`langbot.pkg.agent.runner.config_schema`, so a Runner that + declares a new selector type is picked up without a change here. + +The snapshot is attached to the terminal ``execution_chain`` record. It is +bounded twice - per list and in total - and it is best effort: telemetry must +never change execution behaviour, so every entry point swallows failures. +""" + +from __future__ import annotations + +import contextlib +import contextvars +import json +import typing + +MAX_ITEMS_PER_LIST = 64 +MAX_NAME_CHARS = 160 +MAX_RESOURCES_BYTES = 8 * 1024 + +_resources: contextvars.ContextVar[dict[str, typing.Any] | None] = contextvars.ContextVar( + 'telemetry_resources', default=None +) + + +def begin() -> None: + """Start collecting for one execution (called when a chain is opened).""" + _resources.set({}) + + +def end() -> None: + """Stop collecting; the snapshot has already been read.""" + _resources.set(None) + + +def _bucket(section: str) -> dict[str, typing.Any] | None: + root = _resources.get() + if root is None: + return None + bucket = root.get(section) + if not isinstance(bucket, dict): + bucket = {} + root[section] = bucket + return bucket + + +def _names(values: typing.Any) -> list[str]: + """Bound one list of code-defined identifiers, preserving source order.""" + if not isinstance(values, (list, tuple)): + return [] + names: list[str] = [] + seen: set[str] = set() + for value in values: + name = '' + if isinstance(value, str): + name = value + elif isinstance(value, dict): + name = str(value.get('name') or value.get('kb_id') or value.get('tool_name') or '') + if not name: + continue + name = name[:MAX_NAME_CHARS] + if name in seen: + continue + seen.add(name) + names.append(name) + if len(names) >= MAX_ITEMS_PER_LIST: + break + return names + + +def _tool_refs(values: typing.Any) -> list[dict[str, str]]: + """Bound one list of tool grants, keeping their Host source.""" + if not isinstance(values, (list, tuple)): + return [] + refs: list[dict[str, str]] = [] + seen: set[str] = set() + for value in values: + if isinstance(value, str): + name, source, source_id = value, '', '' + elif isinstance(value, dict): + name = str(value.get('tool_name') or value.get('name') or '') + source = str(value.get('tool_type') or value.get('source') or '') + raw_id = value.get('source_id') + source_id = raw_id if isinstance(raw_id, str) else '' + else: + continue + name = name[:MAX_NAME_CHARS] + if not name or name in seen: + continue + seen.add(name) + ref = {'name': name} + if source: + ref['source'] = source[:64] + if source_id: + ref['source_id'] = source_id[:MAX_NAME_CHARS] + refs.append(ref) + if len(refs) >= MAX_ITEMS_PER_LIST: + break + return refs + + +def _set(bucket: dict[str, typing.Any] | None, key: str, value: typing.Any) -> None: + if bucket is None or value in (None, [], {}, ''): + return + bucket[key] = value + + +def note_pipeline( + *, + plugins: typing.Any = None, + plugins_all: typing.Any = None, + mcp_servers: typing.Any = None, + mcp_all: typing.Any = None, + knowledge_bases: typing.Any = None, + skills: typing.Any = None, + tools: typing.Any = None, + tools_all: typing.Any = None, + runner_id: str = '', + selectors: typing.Any = None, +) -> None: + """Record the resources one Pipeline lane is bound to. + + ``plugins``/``mcp_servers`` accept ``None`` to mean "all enabled", matching + the binding model; the ``*_all`` flags make that explicit in the payload. + """ + try: + bucket = _bucket('pipeline') + if bucket is None: + return + _set(bucket, 'plugins', _names(plugins)) + _set(bucket, 'mcp_servers', _names(mcp_servers)) + _set(bucket, 'knowledge_bases', _names(knowledge_bases)) + _set(bucket, 'skills', _names(skills)) + _set(bucket, 'tools', _tool_refs(tools)) + _set(bucket, 'runner_id', str(runner_id or '')[:MAX_NAME_CHARS]) + if isinstance(selectors, dict): + cleaned = { + str(kind)[:64]: _names(fields) + for kind, fields in selectors.items() + if isinstance(fields, (list, tuple)) and fields + } + _set(bucket, 'selectors', cleaned) + if plugins_all is not None: + bucket['plugins_all'] = bool(plugins_all) + if mcp_all is not None: + bucket['mcp_all'] = bool(mcp_all) + if tools_all is not None: + bucket['tools_all'] = bool(tools_all) + except Exception: + return + + +def note_agent( + *, + runner_id: str = '', + resources: typing.Any = None, + selectors: typing.Any = None, + capabilities: typing.Any = None, + tools_all: typing.Any = None, +) -> None: + """Record the resources one Agent run was granted. + + ``resources`` is the materialized ``AgentResources`` mapping the Host + actually handed to the Runner (tools carry their source), ``selectors`` + names the Runner config-form fields that selected a resource, and + ``capabilities`` records which resource categories the Runner declared. + """ + try: + bucket = _bucket('agent') + if bucket is None: + return + _set(bucket, 'runner_id', str(runner_id or '')[:MAX_NAME_CHARS]) + if isinstance(resources, dict): + _set(bucket, 'knowledge_bases', _names(resources.get('knowledge_bases'))) + _set(bucket, 'skills', _names(resources.get('skills'))) + _set(bucket, 'models', _names(resources.get('models'))) + _set(bucket, 'tools', _tool_refs(resources.get('tools'))) + _set(bucket, 'platform_apis', _names(resources.get('platform_tools'))) + if isinstance(selectors, dict): + cleaned = { + str(kind)[:64]: _names(fields) + for kind, fields in selectors.items() + if isinstance(fields, (list, tuple)) and fields + } + _set(bucket, 'selectors', cleaned) + if isinstance(capabilities, dict): + declared = {key: bool(value) for key, value in capabilities.items() if value} + _set(bucket, 'capabilities', declared) + if tools_all is not None: + bucket['tools_all'] = bool(tools_all) + except Exception: + return + + +def snapshot() -> dict[str, typing.Any] | None: + """Return the bounded snapshot of this execution; None when nothing was noted.""" + try: + root = _resources.get() + if not isinstance(root, dict) or not root: + return None + + def encoded_size(value: typing.Any) -> int: + return len(json.dumps(value, ensure_ascii=False, default=str).encode('utf-8')) + + if encoded_size(root) <= MAX_RESOURCES_BYTES: + return root + # Over budget: shed the widest detail (per-tool grants) first, then fall + # back to a marker, so an oversized binding can never block the chain + # record or bloat the upload. + reduced = json.loads(json.dumps(root, ensure_ascii=False, default=str)) + for section in ('pipeline', 'agent'): + bucket = reduced.get(section) + if isinstance(bucket, dict): + bucket.pop('tools', None) + reduced['truncated'] = True + if encoded_size(reduced) <= MAX_RESOURCES_BYTES: + return reduced + return {'truncated': True} + except Exception: + return None + + +@contextlib.contextmanager +def collected() -> typing.Iterator[None]: + """Bind a collection scope, for callers outside an execution boundary.""" + token = _resources.set({}) + try: + yield + finally: + _resources.reset(token) diff --git a/src/langbot/pkg/telemetry/telemetry.py b/src/langbot/pkg/telemetry/telemetry.py index c36b35e3c..9188a5c54 100644 --- a/src/langbot/pkg/telemetry/telemetry.py +++ b/src/langbot/pkg/telemetry/telemetry.py @@ -12,9 +12,6 @@ from ..utils import httpclient from .execution import ExecutionCounters -_MAX_INFLIGHT_TELEMETRY_TASKS = 8 - - class TelemetryManager: """TelemetryManager handles sending telemetry for a given application instance. @@ -27,36 +24,18 @@ class TelemetryManager: self.ap = ap self.telemetry_config: dict[str, typing.Any] = {} - self.send_tasks: list[asyncio.Task] = [] self._client: httpx.AsyncClient | None = None self.execution = ExecutionCounters(self) async def initialize(self): self.telemetry_config = self.ap.instance_config.data.get('space', {}) - async def start_send_task(self, payload: dict): - self.send_tasks = [task for task in self.send_tasks if not task.done()] - if len(self.send_tasks) >= _MAX_INFLIGHT_TELEMETRY_TASKS: - self.ap.logger.debug('Telemetry queue is full; dropping best-effort event') - return - task = asyncio.create_task(self.send(payload)) - self.send_tasks.append(task) - task.add_done_callback(self._send_task_done) - - def _send_task_done(self, task: asyncio.Task) -> None: - try: - self.send_tasks.remove(task) - except ValueError: - pass + def health(self) -> dict: + """Telemetry health for the instance liveness snapshot.""" + return self.execution.health() async def shutdown(self) -> None: await self.execution.shutdown() - tasks = list(self.send_tasks) - for task in tasks: - task.cancel() - if tasks: - await asyncio.gather(*tasks, return_exceptions=True) - self.send_tasks.clear() if self._client is not None: await self._client.aclose() self._client = None @@ -95,45 +74,53 @@ class TelemetryManager: url = server.rstrip('/') + '/api/v1/telemetry' try: - # Sanitize payload so string fields are strings and not nulls + # Sanitize payload so string fields are strings and not nulls. + # A record batch is an envelope of whole execution records; it + # carries no top-level record fields, so only a single-record + # payload gets the legacy defaults. sanitized = dict(payload) - if 'query_id' in sanitized: - try: - sanitized['query_id'] = '' if sanitized['query_id'] is None else str(sanitized['query_id']) - except Exception: - sanitized['query_id'] = str(sanitized.get('query_id', '')) + if isinstance(sanitized.get('records'), list): + sanitized['records'] = [item for item in sanitized['records'] if isinstance(item, dict)] + else: + if 'query_id' in sanitized: + try: + sanitized['query_id'] = ( + '' if sanitized['query_id'] is None else str(sanitized['query_id']) + ) + except Exception: + sanitized['query_id'] = str(sanitized.get('query_id', '')) - for sfield in ( - 'adapter', - 'runner', - 'runner_category', - 'model_name', - 'version', - 'edition', - 'error', - 'timestamp', - 'event_type', - ): - if sfield not in sanitized: - continue - v = sanitized.get(sfield) - sanitized[sfield] = '' if v is None else str(v) + for sfield in ( + 'adapter', + 'runner', + 'runner_category', + 'model_name', + 'version', + 'edition', + 'error', + 'timestamp', + 'event_type', + ): + if sfield not in sanitized: + continue + v = sanitized.get(sfield) + sanitized[sfield] = '' if v is None else str(v) - # event_type defaults to 'query' for backward compatibility - if not sanitized.get('event_type'): - sanitized['event_type'] = 'query' + # event_type defaults to 'query' for backward compatibility + if not sanitized.get('event_type'): + sanitized['event_type'] = 'query' - # features must be a JSON object - if 'features' in sanitized and not isinstance(sanitized['features'], dict): - sanitized['features'] = {} + # features must be a JSON object + if 'features' in sanitized and not isinstance(sanitized['features'], dict): + sanitized['features'] = {} - if 'duration_ms' in sanitized: - try: - sanitized['duration_ms'] = ( - int(sanitized['duration_ms']) if sanitized['duration_ms'] is not None else 0 - ) - except Exception: - sanitized['duration_ms'] = 0 + if 'duration_ms' in sanitized: + try: + sanitized['duration_ms'] = ( + int(sanitized['duration_ms']) if sanitized['duration_ms'] is not None else 0 + ) + except Exception: + sanitized['duration_ms'] = 0 async with self._client_context() as client: try: diff --git a/src/langbot/pkg/telemetry/trace.py b/src/langbot/pkg/telemetry/trace.py index 445bb470e..fb4d18031 100644 --- a/src/langbot/pkg/telemetry/trace.py +++ b/src/langbot/pkg/telemetry/trace.py @@ -1,22 +1,22 @@ -"""Content-free per-execution traces. +"""Per-execution telemetry context for the v2 execution chain. -One trace identity is one execution identity: the id a Pipeline query, inbound -platform event or Agent run is known by, so a stored chain can be joined with -the Space view of that same execution. Stage records appended under it describe -how that one execution was routed and processed, using only code-defined -identifiers. Nothing here is aggregated: a trace is a bounded, ordered sequence -for one execution. +One execution identity (a Pipeline query, inbound platform event or Agent run) +is one chain. The chain is not assembled here: every workflow node is emitted +by ``telemetry.execution`` the moment it completes, carrying its own identity +and the identity of the execution it belongs to. This module only holds the +small execution-scoped context the emitter needs: -Three bounds keep memory independent of traffic volume: +* the execution state (identity, the once-per-event sampling decision, the + execution-scoped display fields); +* the routing/run identity pinned around the block that produced a node; +* the open-step stack, which lives in a ContextVar holding *one task's own + immutable tuple*. Child tasks inherit a snapshot at creation; no shared + mutable stack and no cross-task mirror exists, so sibling tasks can never + corrupt each other's parent links. -* ``MAX_STAGES`` stages per trace (further stages are counted, not kept); -* ``MAX_TRACES`` traces in flight, enforced by the sender that owns the buffer; -* a wall-clock TTL, enforced by the sender's sweep. - -The identity itself lives in a ContextVar so that asynchronous work spawned -while handling one execution inherits it, mirroring ``telemetry.platform``. -Cross-task stages (plugin/RPC work with no inherited context) are resolved by -the owner's execution-id registry instead. +Cross-task callers (plugin/RPC work with no inherited context) resolve their +execution through the owner's execution-id registry instead, which only +carries the execution state - never node payloads. """ from __future__ import annotations @@ -27,64 +27,79 @@ import typing from datetime import datetime, timezone from uuid import uuid4 -MAX_STAGES = 32 - _current: contextvars.ContextVar['TraceState | None'] = contextvars.ContextVar('telemetry_trace', default=None) _route: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_route', default='') _run: contextvars.ContextVar[str] = contextvars.ContextVar('telemetry_trace_run', default='') -# Open workflow step nodes, innermost last. Records emitted inside a step carry -# the innermost node as their ``parent``; the step's own record carries the one -# below it (its enclosing step), never itself. -_node: contextvars.ContextVar[tuple[str, ...]] = contextvars.ContextVar('telemetry_trace_node', default=()) +# Open workflow step frames of the current task, innermost last. The tuple is +# never mutated in place: opening a step publishes a new tuple on this task's +# context only, and closing it restores the previous one. +_node: contextvars.ContextVar[tuple['NodeFrame', ...]] = contextvars.ContextVar('telemetry_trace_node', default=()) def _now() -> str: return datetime.now(timezone.utc).isoformat() +class NodeFrame(typing.NamedTuple): + """One workflow step open in the task that opened it.""" + + node_id: str + seq: int + started_at: str + + class TraceState: - """Bounded stage buffer for exactly one execution.""" + """The small per-execution context; node payloads never accumulate here.""" __slots__ = ( - 'trace_id', - 'execution_id', - 'started_at', - 'stages', - 'dropped_stages', - 'synthetic', - 'sequence', - 'node_sequence', - 'open_nodes', + 'event_id', 'identity', - 'abandoned', + 'configured', + 'mode', + 'denominator', + 'emit_nodes', + 'debug', + 'origin', + 'started_at', + 'node_seq', + 'root_seen', + 'stage_count', + 'dropped_nodes', + 'last_outcome', + 'first_error', 'failure_reason', + 'runner_failed', 'closed', 'adapter', 'runner', 'runner_category', 'model_name', 'pipeline_plugins', + 'node_adapter', + 'node_runner', ) def __init__(self, execution_id: str | None = None) -> None: - # The execution identity is the trace identity so a stored chain can be - # joined to the Space view of that same Pipeline query / event / run. - self.execution_id = str(execution_id).strip() if execution_id else '' - self.trace_id = self.execution_id or str(uuid4()) - self.started_at = _now() - self.stages: list[dict[str, typing.Any]] = [] - self.dropped_stages = 0 - self.synthetic = False - self.sequence = 0 - # Node ids number the workflow steps of this one execution ("n1", "n2"...). - self.node_sequence = 0 - # Mirror of the open step stack, readable from any task: cross-task - # observations (plugin/RPC) resolve their parent through this. - self.open_nodes: tuple[str, ...] = () + # The execution identity every record of this chain carries. + self.event_id = str(execution_id).strip() if execution_id else str(uuid4()) self.identity: dict[str, str] = {} - # Set when the sender refused to buffer this trace: stop appending. - self.abandoned = False + self.configured = False + # Sampling, decided exactly once per event (at ingress or first touch). + self.mode = 'all' + self.denominator = 1 + self.emit_nodes = False + self.debug = False + self.origin = 'platform' + self.started_at = _now() + self.node_seq = 0 + self.root_seen = False + self.stage_count = 0 + self.dropped_nodes = 0 + self.last_outcome = '' + self.first_error = '' self.failure_reason = '' + # Set when a runner node already reported a non-success outcome. + self.runner_failed = False self.closed = False # Execution-scoped record fields attached by the owning lane when known. self.adapter = '' @@ -92,54 +107,40 @@ class TraceState: self.runner_category = '' self.model_name = '' self.pipeline_plugins: typing.Any = None + # First adapter/runner seen on a node, used for the chain record. + self.node_adapter = '' + self.node_runner = '' - def append( + def allocate_seq(self) -> int: + """Mint the next chain-local node order.""" + seq = self.node_seq + self.node_seq += 1 + return seq + + def note_node( self, *, - family: str, - operation: str, - mode: str, - adapter: str, - runner: str, outcome: str, - synthetic: bool, + adapter: str = '', + runner: str = '', + family: str = '', error: str = '', - node: str = '', - parent: str = '', ) -> None: - if len(self.stages) >= MAX_STAGES: - self.dropped_stages += 1 - return - seen = _now() - entry = { - 'family': family, - 'operation': operation, - 'mode': mode, - 'adapter': adapter, - 'runner': runner, - 'outcome': outcome, - 'synthetic': bool(synthetic), - 'seq': self.sequence, - 'route_ref': _route.get(), - 'run_id': _run.get(), - 'first_seen': seen, - 'last_seen': seen, - 'error': error, - } - # Only workflow steps carry these; legacy records stay byte-identical. - if node: - entry['node'] = node - if parent: - entry['parent'] = parent - self.stages.append(entry) - self.sequence += 1 - if synthetic: - self.synthetic = True - - def allocate_node(self) -> str: - """Mint the next step-node id for this execution ("n1", "n2", ...).""" - self.node_sequence += 1 - return f'n{self.node_sequence}' + """Remember only what the terminal chain record needs from a node.""" + self.stage_count += 1 + self.last_outcome = outcome or 'unknown' + if adapter and not self.node_adapter: + self.node_adapter = adapter + if runner and not self.node_runner: + self.node_runner = runner + if family == 'runner' and outcome not in ('success', 'skipped'): + # The lane records its own failed runner step only when the runner + # orchestrator has not already landed one. + self.runner_failed = True + if outcome in ('failed', 'cancelled', 'timeout') and error and not self.first_error: + # The chain must explain itself even when the break was reported as + # a failed node rather than as an exception at the boundary. + self.first_error = error def mark_failure(self, reason: str) -> None: """Remember why this chain broke so the payload can explain itself.""" @@ -147,18 +148,10 @@ class TraceState: self.failure_reason = reason def outcome(self) -> str: - """Terminal outcome of this chain, for space-side display. - - A chain that carries a failure reason never reports ``success``: the - reason is the point of the trace. - """ - if self.stages: - last = self.stages[-1]['outcome'] - else: - last = 'unknown' - if self.failure_reason and last in ('success', 'skipped', 'unknown'): + """Terminal outcome of this chain; a broken chain never reports success.""" + if self.failure_reason and self.last_outcome in ('', 'success', 'skipped', 'unknown'): return 'failed' - return last + return self.last_outcome or 'unknown' class TraceBinding(typing.NamedTuple): @@ -168,9 +161,9 @@ class TraceBinding(typing.NamedTuple): def bind(execution_id: str | None = None) -> TraceBinding: - """Start a trace unless one is already in flight in this context. + """Start a chain unless one is already in flight in this context. - The execution id becomes the trace id only when this call starts the trace; + The execution id becomes the chain identity only when this call starts it; a nested boundary keeps the enclosing execution's identity. """ existing = _current.get() @@ -181,7 +174,7 @@ def bind(execution_id: str | None = None) -> TraceBinding: def unbind(binding: TraceBinding) -> bool: - """Detach this binding. Returns True when this caller owns the trace.""" + """Detach this binding. Returns True when this caller owns the chain.""" if binding.token is not None: try: _current.reset(binding.token) @@ -192,7 +185,7 @@ def unbind(binding: TraceBinding) -> bool: def unbind_root(binding: TraceBinding) -> bool: - """Detach only when this caller started the trace, else leave it in place.""" + """Detach only when this caller started the chain, else leave it in place.""" if not binding.created: return False return unbind(binding) @@ -202,39 +195,64 @@ def current() -> TraceState | None: return _current.get() +def node_stack() -> tuple[NodeFrame, ...]: + """This task's own open-step frames, innermost last.""" + return _node.get() + + def current_parent() -> str: """The step node that records emitted right now attach to ('' when none).""" stack = _node.get() - return stack[-1] if stack else '' + return stack[-1].node_id if stack else '' def enclosing_parent() -> str: """The step enclosing the innermost open one; '' when it is a root step.""" stack = _node.get() - return stack[-2] if len(stack) > 1 else '' + return stack[-2].node_id if len(stack) > 1 else '' + + +def resolve_node( + state: TraceState, + node: str, + fallback_started_at: str, +) -> tuple[str, int, str, str]: + """Resolve one record to ``(node_id, seq, started_at, parent_node_id)``. + + A record stamped with a ``node`` the calling task has open reuses that + step's identity, order and start time; anything else (a plain observation, + or a node opened by another task) is a fresh node under this task's + innermost open step. + """ + stack = _node.get() + if node: + for index in range(len(stack) - 1, -1, -1): + frame = stack[index] + if frame.node_id == node: + parent = stack[index - 1].node_id if index > 0 else '' + return frame.node_id, frame.seq, frame.started_at, parent + parent = stack[-1].node_id if stack else '' + return node or uuid4().hex, state.allocate_seq(), fallback_started_at, parent @contextlib.contextmanager def stage_scope() -> typing.Iterator[str]: """Open one workflow step node for the duration of the block. - Allocates a node id unique within the execution, publishes it as the current - parent so that nested observations recorded inside the block attach to it, - and yields that id so the step's own records can be stamped with ``node=``. - Outside any trace nothing is allocated and the empty id is yielded. + Allocates a globally unique node id and the chain-local order, publishes the + frame on this task's own stack so nested observations attach to it as their + parent, and yields the node id so the step's own record can be stamped with + ``node=``. Outside any execution nothing is allocated and '' is yielded. """ state = _current.get() if state is None: yield '' return - node = state.allocate_node() - previous_open = state.open_nodes - state.open_nodes = previous_open + (node,) - token = _node.set(_node.get() + (node,)) + frame = NodeFrame(uuid4().hex, state.allocate_seq(), _now()) + token = _node.set(_node.get() + (frame,)) try: - yield node + yield frame.node_id finally: - state.open_nodes = previous_open try: _node.reset(token) except (ValueError, RuntimeError): @@ -258,7 +276,7 @@ def reset_run(token: typing.Any) -> None: @contextlib.contextmanager def scope(*, route_ref: str | None = None) -> typing.Iterator[None]: - """Pin routing identity for stages recorded inside this block.""" + """Pin routing identity for nodes recorded inside this block.""" if not route_ref: yield return @@ -270,3 +288,11 @@ def scope(*, route_ref: str | None = None) -> typing.Iterator[None]: _route.reset(token) except (ValueError, RuntimeError): pass + + +def current_route() -> str: + return _route.get() + + +def current_run() -> str: + return _run.get() diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index 9e5a0f7ac..f8752314f 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -444,11 +444,13 @@ space: # heartbeat and execution reporting; restart after changing this setting. disable_telemetry: false # Per-event execution traces: off | failures | sampled | all. - # 'sampled' keeps every failed run plus every N-th successful one - # (N is execution_trace_sample) and always keeps WebUI debug runs. - execution_trace: sampled + # Every execution is reported by default: one record per workflow node plus + # one terminal chain record. 'failures' keeps only broken chains, 'sampled' + # keeps broken chains plus every N-th successful one (N is + # execution_trace_sample), and 'off' reports no executions at all. + execution_trace: all execution_trace_sample: 20 # How often buffered execution records are uploaded, in seconds. - # Records of complete executions are batched as {"records": [...]}; a - # still-running execution is never uploaded half-written. + # Complete execution records are batched as {"records": [...]}; records are + # removed from the buffer only once the sender accepted them. telemetry_flush_seconds: 180 diff --git a/tests/unit_tests/platform/test_botmgr_execution_trace.py b/tests/unit_tests/platform/test_botmgr_execution_trace.py index 75605aa04..8fbad1463 100644 --- a/tests/unit_tests/platform/test_botmgr_execution_trace.py +++ b/tests/unit_tests/platform/test_botmgr_execution_trace.py @@ -1,4 +1,4 @@ -"""RuntimeBot ingress tracing: one inbound event owns one execution trace.""" +"""RuntimeBot ingress tracing: one inbound event owns one v2 execution chain.""" from __future__ import annotations @@ -28,9 +28,6 @@ class FakeTelemetryManager: self.sent.append(payload) return True - def start_send_task(self, payload: dict) -> None: - self.sent.append(payload) - def make_bot(event_bindings: list[dict], config=None, agent=None): from langbot.pkg.platform.botmgr import RuntimeBot @@ -76,7 +73,7 @@ class FakeAdapter: async def flushed_records(counters, manager) -> list[dict]: - """Flush buffered telemetry and return everything the pipeline emitted. + """Flush buffered telemetry and return everything the event emitted. Records are batched and uploaded by ``ExecutionCounters.flush()``; there is no per-trace direct send any more. @@ -87,34 +84,48 @@ async def flushed_records(counters, manager) -> list[dict]: return [record for payload in manager.sent for record in payload.get('records', [])] +def execution_nodes(records: list[dict]) -> list[dict]: + return [record for record in records if record.get('event_type') == 'execution_node'] + + +def execution_chains(records: list[dict]) -> list[dict]: + return [record for record in records if record.get('event_type') == 'execution_chain'] + + @pytest.mark.asyncio -async def test_route_miss_emits_one_closed_trace(): +async def test_route_miss_emits_one_closed_chain(): bot, manager, counters = make_bot([]) await bot._handle_platform_event(message_received_event(), FakeAdapter()) records = await flushed_records(counters, manager) assert len(manager.sent) == 1 - trace_records = [item for item in records if item.get('event_type') == 'feature_execution'] - assert len(trace_records) == 1 - payload = trace_records[0] - assert payload['event_type'] == 'feature_execution' + chains = execution_chains(records) + assert len(chains) == 1 + chain = chains[0] + assert chain['schema'] == 2 # The execution identity is deterministic: one inbound event, one chain. - assert payload['query_id'] == 'platform:bot-1:message-1' - assert payload['instance_id'] == 'instance-test' - assert payload['workspace_uuid'] == 'workspace-test' - features = payload['features'] - assert features['schema'] == 1 - assert [(row['family'], row['operation'], row['outcome']) for row in features['observations']] == [ + assert chain['event_id'] == 'platform:bot-1:message-1' + assert chain['instance_id'] == 'instance-test' + assert chain['workspace_uuid'] == 'workspace-test' + assert chain['closed_by'] == 'event_done' + assert chain['stage_count'] == 2 + + rows = execution_nodes(records) + assert [(row['family'], row['operation'], row['outcome']) for row in rows] == [ ('event_route', 'message.received', 'skipped'), ('platform_event', 'message.received', 'success'), ] - assert [row['seq'] for row in features['observations']] == [0, 1] - assert all(row['trace_id'] == payload['query_id'] for row in features['observations']) - assert all(row['adapter'] == 'FakeAdapter' for row in features['observations']) - assert features['trace']['closed_by'] == 'event_done' - # The trace is closed and deregistered: nothing dangles after the event. - assert counters.traces == {} + assert {row['seq'] for row in rows} == {0, 1} + assert all(row['event_id'] == chain['event_id'] for row in rows) + assert all(row['adapter'] == 'FakeAdapter' for row in rows) + # The inbound event is the root step; the routing decision nests under it. + root = next(row for row in rows if row['root']) + route = next(row for row in rows if row['family'] == 'event_route') + assert (root['family'], root['parent_node_id']) == ('platform_event', '') + assert route['parent_node_id'] == root['node_id'] + # The chain is closed and deregistered: nothing dangles after the event. + assert counters.registry == {} @pytest.mark.asyncio @@ -133,13 +144,13 @@ async def test_unavailable_route_target_carries_route_identity(): await bot._handle_platform_event(message_received_event(), FakeAdapter()) - stages = (await flushed_records(counters, manager))[0]['features']['observations'] - assert [(stage['family'], stage['outcome'], stage['mode']) for stage in stages] == [ + nodes = execution_nodes(await flushed_records(counters, manager)) + assert [(node['family'], node['outcome'], node['mode']) for node in nodes] == [ ('event_route', 'failed', 'agent'), ('platform_event', 'success', 'none'), ] - assert stages[0]['route_ref'] == 'agent:agent-1' - assert stages[1]['route_ref'] == '' + assert nodes[0]['route_ref'] == 'agent:agent-1' + assert nodes[1]['route_ref'] == '' @pytest.mark.asyncio @@ -157,10 +168,10 @@ async def test_discarded_route_is_visible_without_user_values(): await bot._handle_platform_event(SimpleNamespace(type='platform.member.joined'), FakeAdapter()) - stages = (await flushed_records(counters, manager))[0]['features']['observations'] - assert [stage['outcome'] for stage in stages] == ['skipped', 'success'] - assert stages[0]['operation'] == 'platform.member.joined' - assert stages[0]['mode'] == 'none' + nodes = execution_nodes(await flushed_records(counters, manager)) + assert [node['outcome'] for node in nodes] == ['skipped', 'success'] + assert nodes[0]['operation'] == 'platform.member.joined' + assert nodes[0]['mode'] == 'none' @pytest.mark.asyncio @@ -171,7 +182,7 @@ async def test_telemetry_opt_out_emits_nothing(): assert manager.sent == [] assert counters.records == [] - assert counters.traces == {} + assert counters.registry == {} @pytest.mark.asyncio @@ -185,13 +196,13 @@ async def test_route_trace_scope_is_restored_after_dispatch(): assert trace_mod.current() is None buffered = list(counters.records) - # A later record outside the ingress must not join the closed trace, and an + # A later record outside the ingress must not join the closed chain, and an # observation without an owning chain is not buffered at all. from langbot.pkg.telemetry.execution import record record(bot.ap, TEST_CONTEXT, family='platform_api', operation='send_message', outcome='success') assert counters.records == buffered - assert counters.traces == {} + assert counters.registry == {} @pytest.mark.asyncio diff --git a/tests/unit_tests/provider/test_mcp_resources.py b/tests/unit_tests/provider/test_mcp_resources.py index 0ba164e9a..97b136d56 100644 --- a/tests/unit_tests/provider/test_mcp_resources.py +++ b/tests/unit_tests/provider/test_mcp_resources.py @@ -24,7 +24,6 @@ from langbot.pkg.provider.tools.loaders.mcp import ( MCPToolCallTimeoutError, RuntimeMCPSession, ) -from langbot.pkg.telemetry import features as telemetry_features from langbot.pkg.workspace.errors import WorkspaceGenerationMismatchError @@ -298,10 +297,6 @@ async def test_read_resource_envelope_truncates_caches_and_records_trace(): traces = query.variables[MCP_RESOURCE_TRACE_QUERY_KEY] assert [trace['source'] for trace in traces] == ['ui_preview', 'agent_tool'] assert traces[1]['cache_hit'] is True - assert query.variables[telemetry_features.FEATURES_KEY]['mcp_resource_reads'] == { - 'ui_preview': 1, - 'agent_tool': 1, - } @pytest.mark.asyncio diff --git a/tests/unit_tests/telemetry/test_cloud_sampling.py b/tests/unit_tests/telemetry/test_cloud_sampling.py index b1a604489..ef4de5e2e 100644 --- a/tests/unit_tests/telemetry/test_cloud_sampling.py +++ b/tests/unit_tests/telemetry/test_cloud_sampling.py @@ -1,6 +1,7 @@ from types import SimpleNamespace from langbot.pkg.telemetry.execution import ExecutionCounters +from langbot.pkg.telemetry.trace import TraceState from langbot.pkg.utils import constants @@ -8,8 +9,10 @@ def test_cloud_counts_are_not_sampled(monkeypatch): monkeypatch.setattr(constants, 'edition', 'cloud') counters = ExecutionCounters(SimpleNamespace(telemetry_config={'execution_trace': 'sampled'})) assert counters.trace_mode() == 'all' - state = SimpleNamespace(failure_reason='', stages=[], synthetic=False, trace_id='00000001') - assert counters._trace_emitted(state) + state = TraceState('00000001') + counters.configure(state) + assert state.emit_nodes is True + assert counters._chain_emitted(state) is True def test_community_retains_sampling(monkeypatch): diff --git a/tests/unit_tests/telemetry/test_features.py b/tests/unit_tests/telemetry/test_features.py deleted file mode 100644 index 6e19d0828..000000000 --- a/tests/unit_tests/telemetry/test_features.py +++ /dev/null @@ -1,92 +0,0 @@ -"""Unit tests for telemetry feature counters (pkg/telemetry/features.py).""" - -from __future__ import annotations - -from importlib import import_module - - -def get_features_module(): - return import_module('langbot.pkg.telemetry.features') - - -class FakeQuery: - def __init__(self): - self.variables = {} - - -class TestIncrement: - def test_increment_nested_counter(self): - features = get_features_module() - q = FakeQuery() - features.increment(q, 'tool_calls', 'native') - features.increment(q, 'tool_calls', 'native') - features.increment(q, 'tool_calls', 'mcp') - assert q.variables[features.FEATURES_KEY]['tool_calls'] == {'native': 2, 'mcp': 1} - - def test_increment_flat_counter(self): - features = get_features_module() - q = FakeQuery() - features.increment(q, 'something') - features.increment(q, 'something', amount=2) - assert q.variables[features.FEATURES_KEY]['something'] == 3 - - def test_increment_never_raises_on_broken_query(self): - features = get_features_module() - - class Broken: - @property - def variables(self): - raise RuntimeError('boom') - - # Must not raise - features.increment(Broken(), 'tool_calls', 'native') - - def test_set_value(self): - features = get_features_module() - q = FakeQuery() - features.set_value(q, 'tool_call_rounds', 5) - assert q.variables[features.FEATURES_KEY]['tool_call_rounds'] == 5 - - -class TestCollectFeatures: - def test_collect_empty(self): - features = get_features_module() - q = FakeQuery() - assert features.collect_features(q) == {} - - def test_collect_combines_counters_and_snapshots(self): - features = get_features_module() - q = FakeQuery() - features.increment(q, 'sandbox', 'execs') - features.set_value(q, 'kb', {'kb_count': 2, 'engine_plugins': ['builtin'], 'retrieved_entries': 7}) - q.variables['_activated_skills'] = {'pdf-tools': {}, 'a-skill': {}} - q.variables['_pipeline_bound_mcp_servers'] = ['srv1', 'srv2'] - - result = features.collect_features(q) - assert result['sandbox'] == {'execs': 1} - assert result['kb']['kb_count'] == 2 - assert result['activated_skills'] == ['a-skill', 'pdf-tools'] # sorted - assert result['mcp_servers'] == ['srv1', 'srv2'] - - def test_collect_omits_mcp_when_all_enabled(self): - """None means 'all enabled' and is not reported.""" - features = get_features_module() - q = FakeQuery() - q.variables['_pipeline_bound_mcp_servers'] = None - assert 'mcp_servers' not in features.collect_features(q) - - def test_collect_drops_non_json_serializable(self): - features = get_features_module() - q = FakeQuery() - features.set_value(q, 'good', 1) - features.set_value(q, 'bad', object()) - result = features.collect_features(q) - assert result == {'good': 1} - - def test_collect_is_json_serializable(self): - import json - - features = get_features_module() - q = FakeQuery() - features.increment(q, 'tool_calls', 'skill') - json.dumps(features.collect_features(q)) diff --git a/tests/unit_tests/telemetry/test_pipeline_lane.py b/tests/unit_tests/telemetry/test_pipeline_lane.py index f091f2bb8..4b3e8625e 100644 --- a/tests/unit_tests/telemetry/test_pipeline_lane.py +++ b/tests/unit_tests/telemetry/test_pipeline_lane.py @@ -1,4 +1,4 @@ -"""The Pipeline lane must land its failure on the owning execution trace.""" +"""The Pipeline lane must land its nodes on one v2 execution chain.""" from __future__ import annotations @@ -43,7 +43,7 @@ def build_pipeline(pipelinemgr, ap): return pipeline -def test_lane_failure_marks_the_owned_trace_without_any_prior_stage(): +def test_lane_failure_marks_the_owned_chain_without_any_prior_stage(): pipelinemgr, execution = get_modules() manager = FakeManager() counters = execution.ExecutionCounters(manager) @@ -54,15 +54,16 @@ def test_lane_failure_marks_the_owned_trace_without_any_prior_stage(): with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-7'): pipeline._record_lane_failure(query, 'sandbox exited 127') - assert len(counters.records) == 1 - record = counters.records[0] - assert record['query_id'] == 'query-7' - assert record['error'] == 'sandbox exited 127' - assert record['features']['trace']['outcome'] == 'failed' - observations = record['features']['observations'] - assert [(row['family'], row['operation'], row['outcome']) for row in observations] == [ - ('runner', 'execute', 'failed') - ] + rows = [record for record in counters.records if record['event_type'] == 'execution_node'] + assert len(rows) == 1 + node = rows[0] + assert node['event_id'] == 'query-7' + assert node['error'] == 'sandbox exited 127' + assert (node['family'], node['operation'], node['outcome']) == ('runner', 'execute', 'failed') + chain = [record for record in counters.records if record['event_type'] == 'execution_chain'][0] + assert chain['event_id'] == 'query-7' + assert chain['outcome'] == 'failed' + assert chain['error'] == 'sandbox exited 127' def test_lane_failure_does_not_duplicate_the_runner_node_the_orchestrator_recorded(): @@ -87,13 +88,13 @@ def test_lane_failure_does_not_duplicate_the_runner_node_the_orchestrator_record ) pipeline._record_lane_failure(query, 'No authorized model for local-agent') - observations = [ - (row['family'], row['operation'], row['outcome']) - for record in counters.records - for row in record['features']['observations'] + rows = [record for record in counters.records if record['event_type'] == 'execution_node'] + assert [(record['family'], record['operation'], record['outcome']) for record in rows] == [ + ('runner', 'execute', 'failed') ] - assert observations == [('runner', 'execute', 'failed')] - assert counters.records[-1]['features']['trace']['outcome'] == 'failed' + chain = [record for record in counters.records if record['event_type'] == 'execution_chain'][-1] + assert chain['outcome'] == 'failed' + assert chain['error'] == 'No authorized model for local-agent' def build_counters(): @@ -118,11 +119,14 @@ def build_query(**overrides): return query +def lane_nodes(counters): + return [record for record in counters.records if record['event_type'] == 'execution_node'] + + def lane_shapes(counters): return [ - (row['family'], row['operation'], row.get('node'), row.get('parent')) - for record in counters.records - for row in record['features']['observations'] + (record['family'], record['operation'], record['node_id'], record['parent_node_id']) + for record in lane_nodes(counters) ] @@ -138,7 +142,7 @@ async def stop_counters(counters): counters.task = None -def test_lane_step_owns_the_apis_and_runner_it_calls(): +async def test_lane_step_owns_the_apis_and_runner_it_calls(): pipelinemgr, execution = get_modules() trace = get_trace() counters, ap = build_counters() @@ -168,21 +172,28 @@ def test_lane_step_owns_the_apis_and_runner_it_calls(): await lane.process_query(query) await stop_counters(counters) - asyncio.run(scenario()) + await scenario() shapes = lane_shapes(counters) - lane_node = shapes[-1][2] - assert lane_node == 'n1' - assert shapes == [ - ('platform_api', 'send_message', None, lane_node), - ('runner', 'execute', seen['run'], lane_node), - ('platform_api', 'reply_message', None, lane_node), - # The lane step is recorded last; a standalone lane is a root. - ('pipeline', 'run', lane_node, None), + # The synthetic inbound event roots the chain; the lane step hangs under it. + assert [shape[0:2] for shape in shapes] == [ + ('platform_event', 'message.received'), + ('platform_api', 'send_message'), + ('runner', 'execute'), + ('platform_api', 'reply_message'), + ('pipeline', 'run'), ] + event_node = shapes[0][2] + lane_node = shapes[-1][2] + assert shapes[0][3] == '' + assert shapes[1][3] == lane_node + assert shapes[2] == ('runner', 'execute', seen['run'], lane_node) + assert shapes[3][3] == lane_node + # The lane step is recorded last and nests under the synthetic event node. + assert shapes[4] == ('pipeline', 'run', lane_node, event_node) -def test_lane_node_nests_under_the_platform_route_that_called_it(): +async def test_lane_node_nests_under_the_platform_route_that_called_it(): pipelinemgr, execution = get_modules() trace = get_trace() counters, ap = build_counters() @@ -197,7 +208,7 @@ def test_lane_node_nests_under_the_platform_route_that_called_it(): lane._process_query = fake_process async def scenario(): - # A platform event owns the trace; the route step wraps the Pipeline lane. + # A platform event owns the chain; the route step wraps the Pipeline lane. with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-routed'): with trace.stage_scope() as event_node: execution.record( @@ -213,19 +224,21 @@ def test_lane_node_nests_under_the_platform_route_that_called_it(): await stop_counters(counters) return event_node, route_node - event_node, route_node = asyncio.run(scenario()) + event_node, route_node = await scenario() - assert (event_node, route_node) == ('n1', 'n2') - assert lane_shapes(counters) == [ - ('platform_event', 'message.received', event_node, None), - ('event_route', 'message.received', route_node, event_node), - # The API is called by the lane, so it hangs on the lane node. - ('platform_api', 'send_message', None, 'n3'), - ('pipeline', 'run', 'n3', route_node), - ] + assert event_node != route_node + shapes = lane_shapes(counters) + assert shapes[0] == ('platform_event', 'message.received', event_node, '') + assert shapes[1] == ('event_route', 'message.received', route_node, event_node) + lane_node = shapes[-1][2] + # The API is called by the lane, so it hangs on the lane node. + assert shapes[2][0:2] == ('platform_api', 'send_message') + assert shapes[2][3] == lane_node and shapes[2][2] != lane_node + # The lane nests under the route step that called it. + assert shapes[3] == ('pipeline', 'run', lane_node, route_node) -def test_broken_lane_records_a_failed_step_with_its_reason(): +async def test_broken_lane_records_a_failed_step_with_its_reason(): pipelinemgr, execution = get_modules() counters, ap = build_counters() lane = build_lane(pipelinemgr, ap) @@ -242,16 +255,19 @@ def test_broken_lane_records_a_failed_step_with_its_reason(): await stop_counters(counters) with pytest.raises(RuntimeError, match='lane exploded'): - asyncio.run(scenario()) + await scenario() - family, operation, node, parent = lane_shapes(counters)[-1] - row = counters.records[0]['features']['observations'][-1] - assert (family, operation, node, parent) == ('pipeline', 'run', 'n1', None) + shapes = lane_shapes(counters) + family, operation, node, parent = shapes[-1] + row = lane_nodes(counters)[-1] + # The lane step nests under the synthetic inbound event that roots the chain. + assert (family, operation) == ('pipeline', 'run') + assert shapes[0] == ('platform_event', 'message.received', parent, '') assert row['outcome'] == 'failed' assert row['error'] == 'lane exploded' -def test_lane_reported_failure_marks_the_step_failed(): +async def test_lane_reported_failure_marks_the_step_failed(): pipelinemgr, execution = get_modules() counters, ap = build_counters() lane = build_lane(pipelinemgr, ap) @@ -266,7 +282,7 @@ def test_lane_reported_failure_marks_the_step_failed(): await lane.process_query(query) await stop_counters(counters) - asyncio.run(scenario()) + await scenario() - row = counters.records[0]['features']['observations'][-1] + row = lane_nodes(counters)[-1] assert (row['family'], row['operation'], row['outcome']) == ('pipeline', 'run', 'failed') diff --git a/tests/unit_tests/telemetry/test_telemetry.py b/tests/unit_tests/telemetry/test_telemetry.py index 004009a68..87699a18d 100644 --- a/tests/unit_tests/telemetry/test_telemetry.py +++ b/tests/unit_tests/telemetry/test_telemetry.py @@ -6,7 +6,6 @@ Tests cover: - Early return conditions (disabled, empty config, no server) - URL construction (with actual URL verification) - HTTP request success/failure scenarios -- Source code bug: send_tasks should be instance variable """ from __future__ import annotations @@ -629,53 +628,3 @@ class TestAuthenticatedWorkspaceReporter: 'Authorization': 'Bearer refreshed-workspace-owner-token' } - -class TestStartSendTask: - """Tests for start_send_task() method.""" - - @pytest.mark.asyncio - async def test_start_send_task_creates_task(self): - """Test that start_send_task creates an asyncio task.""" - telemetry = get_telemetry_module() - mock_app = Mock() - mock_app.logger = Mock() - mock_app.instance_config = Mock() - mock_app.instance_config.data = {} - - manager = telemetry.TelemetryManager(mock_app) - manager.telemetry_config = {} - - await manager.start_send_task({'query_id': 'test'}) - - # Task should be added to send_tasks list - assert len(manager.send_tasks) >= 1 - - # Clean up the task - for task in manager.send_tasks: - if not task.done(): - task.cancel() - manager.send_tasks.clear() - - @pytest.mark.asyncio - async def test_start_send_task_multiple_tasks(self): - """Test that multiple tasks are tracked.""" - telemetry = get_telemetry_module() - mock_app = Mock() - mock_app.logger = Mock() - mock_app.instance_config = Mock() - mock_app.instance_config.data = {} - - manager = telemetry.TelemetryManager(mock_app) - manager.telemetry_config = {} - - await manager.start_send_task({'query_id': 'test1'}) - await manager.start_send_task({'query_id': 'test2'}) - await manager.start_send_task({'query_id': 'test3'}) - - assert len(manager.send_tasks) >= 3 - - # Clean up - for task in manager.send_tasks: - if not task.done(): - task.cancel() - manager.send_tasks.clear() diff --git a/tests/unit_tests/telemetry/test_trace.py b/tests/unit_tests/telemetry/test_trace.py index 283cdac65..95b9f06f1 100644 --- a/tests/unit_tests/telemetry/test_trace.py +++ b/tests/unit_tests/telemetry/test_trace.py @@ -1,4 +1,10 @@ -"""Unit tests for per-execution execution traces (pkg/telemetry/trace.py + execution.py).""" +"""Unit tests for per-execution execution chains (pkg/telemetry/trace.py + execution.py). + +One execution is one chain of independent records: one ``execution_node`` per +workflow node, emitted the moment the node completes, plus exactly one +``execution_chain`` when the execution closes. Every record of a chain carries +the same ``event_id``. +""" from __future__ import annotations @@ -31,8 +37,21 @@ class FakeManager: self.sent.append(payload) return True - def start_send_task(self, payload: dict) -> None: + +class RejectingManager(FakeManager): + """Stand-in whose sender refuses every batch until told to accept them.""" + + def __init__(self, config=None): + super().__init__(config) + self.rejecting = True + self.attempts = 0 + + async def send(self, payload: dict) -> bool: + self.attempts += 1 + if self.rejecting: + return False self.sent.append(payload) + return True def make_counters(config=None): @@ -56,6 +75,14 @@ def records_sent(manager): return [record for payload in manager.sent for record in payload.get('records', [])] +def nodes(counters): + return [record for record in counters.records if record.get('event_type') == 'execution_node'] + + +def chains(counters): + return [record for record in counters.records if record.get('event_type') == 'execution_chain'] + + CONTEXT = types.SimpleNamespace(instance_uuid='instance-1', workspace_uuid='workspace-1') STAGE = { @@ -66,6 +93,35 @@ STAGE = { 'outcome': 'success', } +NODE_KEYS = { + 'event_type', + 'schema', + 'event_id', + 'instance_id', + 'workspace_uuid', + 'version', + 'edition', + 'timestamp', + 'debug', + 'sample', + 'node_id', + 'parent_node_id', + 'root', + 'seq', + 'family', + 'operation', + 'mode', + 'adapter', + 'runner', + 'outcome', + 'error', + 'route_ref', + 'run_id', + 'synthetic', + 'started_at', + 'ended_at', +} + class TestTraceIdentity: def test_nested_bind_reuses_the_in_flight_trace(self): @@ -93,13 +149,12 @@ class TestTraceIdentity: # A repeated unbind of the same binding must never raise. assert trace.unbind_root(outer) is True - def test_trace_id_is_the_execution_id(self): + def test_event_id_is_the_execution_id(self): trace, _ = get_modules() binding = trace.bind('exec-1234') try: assert binding.created is True - assert binding.state.trace_id == 'exec-1234' - assert binding.state.execution_id == 'exec-1234' + assert binding.state.event_id == 'exec-1234' finally: trace.unbind_root(binding) @@ -109,19 +164,20 @@ class TestTraceIdentity: inner = trace.bind('query-9') try: assert inner.created is False - assert inner.state.trace_id == 'event-1' + assert inner.state.event_id == 'event-1' finally: trace.unbind_root(inner) trace.unbind_root(outer) def test_route_and_run_are_only_pinned_inside_the_block(self): trace, _ = get_modules() + _, counters = make_counters(trace_config()) binding = trace.bind() token = trace.set_run('run-1') try: with trace.scope(route_ref='agent:agent-1'): - inside = trace.current() - inside.append( + counters.record( + CONTEXT, family='runner', operation='execute', mode='agent', @@ -130,8 +186,8 @@ class TestTraceIdentity: outcome='success', synthetic=False, ) - outside = trace.current() - outside.append( + counters.record( + CONTEXT, family='platform_api', operation='send_message', mode='agent', @@ -143,7 +199,7 @@ class TestTraceIdentity: finally: trace.reset_run(token) trace.unbind_root(binding) - assert [(s['route_ref'], s['run_id']) for s in binding.state.stages] == [ + assert [(row['route_ref'], row['run_id']) for row in nodes(counters)] == [ ('agent:agent-1', 'run-1'), ('', 'run-1'), ] @@ -152,65 +208,42 @@ class TestTraceIdentity: class TestTraceRecording: def test_stage_carries_sequence_identity_and_timestamps(self): trace, _ = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = trace.bind() try: counters.record(CONTEXT, **STAGE) counters.record(CONTEXT, **{**STAGE, 'family': 'runner', 'operation': 'execute', 'mode': 'pipeline'}) + counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - stages = binding.state.stages - assert [stage['seq'] for stage in stages] == [0, 1] - assert stages[0]['first_seen'] == stages[0]['last_seen'] - assert binding.state.identity == {'instance_id': 'instance-1', 'workspace_uuid': 'workspace-1'} - assert binding.state.outcome() == 'success' - - def test_stage_buffer_is_bounded_and_counts_overflow(self): - trace, _ = get_modules() - _, counters = make_counters() - binding = trace.bind() - try: - for _ in range(trace.MAX_STAGES + 5): - counters.record(CONTEXT, **STAGE) - finally: - trace.unbind_root(binding) - assert len(binding.state.stages) == trace.MAX_STAGES - assert binding.state.dropped_stages == 5 + rows = nodes(counters) + assert [row['seq'] for row in rows] == [0, 1] + # A plain observation starts and ends at the same instant. + assert rows[0]['started_at'] == rows[0]['ended_at'] + assert {(row['instance_id'], row['workspace_uuid']) for row in rows} == {('instance-1', 'workspace-1')} + chain = chains(counters)[0] + assert chain['outcome'] == 'success' + assert chain['stage_count'] == 2 def test_oversized_identifier_never_becomes_a_stage(self): trace, _ = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = trace.bind() try: counters.record(CONTEXT, **{**STAGE, 'operation': 'x' * 200}) counters.record(CONTEXT, **{**STAGE, 'family': 'not-a-family'}) finally: trace.unbind_root(binding) - assert binding.state.stages == [] - - def test_trace_limit_abandons_further_traces(self): - trace, execution = get_modules() - _, counters = make_counters() - bindings = [] - for _ in range(execution.MAX_TRACES + 1): - binding = trace.bind() - bindings.append(binding) - counters.record(CONTEXT, **STAGE) - trace.unbind_root(binding) - assert trace.current() is None - assert len(counters.traces) == execution.MAX_TRACES - assert counters.dropped == 1 - assert bindings[-1].state.abandoned is True - assert bindings[-1].state.stages == [] + assert nodes(counters) == [] def test_observation_without_an_owning_execution_is_dropped(self): - trace, _ = get_modules() + _, _ = get_modules() _, counters = make_counters() - # Clustering is gone: an observation that belongs to no execution is - # discarded instead of being aggregated into a synthetic bucket. + # An observation that belongs to no execution is discarded instead of + # being aggregated into a synthetic bucket. counters.record(CONTEXT, **STAGE) assert counters.records == [] - assert counters.traces == {} + assert counters.registry == {} class TestTraceNodeScope: @@ -238,11 +271,13 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - step_row, api_row = binding.state.stages - assert step_row['node'] == step - assert 'parent' not in step_row # a root step carries no parent - assert api_row['parent'] == step - assert 'node' not in api_row # observations are not step nodes + step_row, api_row = nodes(counters) + assert step_row['node_id'] == step + assert step_row['parent_node_id'] == '' # a root step carries no parent + assert step_row['root'] is True + assert api_row['parent_node_id'] == step + assert api_row['root'] is False + assert api_row['node_id'] != step # observations are not step nodes def test_nested_step_records_its_enclosing_step_as_parent(self): trace, _ = get_modules() @@ -257,11 +292,11 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - outer_row, inner_row, api_row = binding.state.stages - assert (outer_row['node'], outer_row.get('parent')) == (outer, None) + outer_row, inner_row, api_row = nodes(counters) + assert (outer_row['node_id'], outer_row['parent_node_id']) == (outer, '') # The step's own record sits under its enclosing step, never under itself. - assert (inner_row['node'], inner_row.get('parent')) == (inner, outer) - assert api_row.get('parent') == inner and 'node' not in api_row + assert (inner_row['node_id'], inner_row['parent_node_id']) == (inner, outer) + assert api_row['parent_node_id'] == inner and api_row['node_id'] != inner def test_sequential_steps_get_distinct_ids_and_roots_have_no_parent(self): trace, _ = get_modules() @@ -275,13 +310,16 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - assert (first, second) == ('n1', 'n2') - assert [row['node'] for row in binding.state.stages] == [first, second] - assert all('parent' not in row for row in binding.state.stages) + assert first and second and first != second + rows = nodes(counters) + assert [row['node_id'] for row in rows] == [first, second] + assert [row['seq'] for row in rows] == [0, 1] + assert rows[0]['root'] is True + assert all(row['parent_node_id'] == '' for row in rows) - def test_cross_task_observation_attaches_to_the_open_step(self): + def test_cross_task_observation_joins_the_owning_execution(self): trace, execution = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = execution.bind_trace(make_ap(counters), 'exec-cross') try: with trace.stage_scope() as lane: @@ -290,7 +328,7 @@ class TestTraceNodeScope: ) # A plugin/RPC observation arrives on a task that never had the # trace context; it is resolved through the execution registry and - # must still hang off the step the owning task has open. + # still lands on the owning chain. foreign = contextvars.Context() foreign.run(execution.set_execution_id, 'exec-cross') foreign.run( @@ -304,13 +342,16 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - rows = binding.state.stages - assert rows[0]['node'] == lane and 'parent' not in rows[0] - assert rows[1].get('parent') == lane and 'node' not in rows[1] + lane_row, foreign_row = nodes(counters) + assert lane_row['node_id'] == lane and lane_row['parent_node_id'] == '' and lane_row['root'] is True + # The cross-task node joins the same execution, but never mirrors another + # task's open-step stack: it is a root of the chain. + assert foreign_row['event_id'] == 'exec-cross' + assert foreign_row['parent_node_id'] == '' def test_cross_task_observation_without_an_open_step_has_no_parent(self): trace, execution = get_modules() - _, counters = make_counters() + _, counters = make_counters(trace_config()) binding = execution.bind_trace(make_ap(counters), 'exec-bare') try: foreign = contextvars.Context() @@ -326,8 +367,8 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - row = binding.state.stages[0] - assert 'node' not in row and 'parent' not in row + row = nodes(counters)[0] + assert row['parent_node_id'] == '' and row['root'] is True def test_scope_outside_any_trace_is_inert(self): trace, _ = get_modules() @@ -335,9 +376,9 @@ class TestTraceNodeScope: assert node == '' assert trace.current_parent() == '' - def test_records_outside_any_scope_serialize_without_node_keys(self): - trace, execution = get_modules() - manager, counters = make_counters(trace_config()) + def test_records_outside_any_scope_are_root_nodes(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) binding = trace.bind('exec-legacy') try: counters.record(CONTEXT, **STAGE) @@ -345,29 +386,16 @@ class TestTraceNodeScope: finally: trace.unbind_root(binding) - stage = binding.state.stages[0] - assert set(stage) == { - 'family', - 'operation', - 'mode', - 'adapter', - 'runner', - 'outcome', - 'synthetic', - 'seq', - 'route_ref', - 'run_id', - 'first_seen', - 'last_seen', - 'error', - } - observation = counters.records[0]['features']['observations'][0] - assert 'node' not in json.dumps(observation, sort_keys=True) - assert 'parent' not in json.dumps(observation, sort_keys=True) + node = nodes(counters)[0] + assert set(node) == NODE_KEYS + assert node['node_id'] and node['parent_node_id'] == '' and node['root'] is True + assert node['seq'] == 0 + # Node placement lives on node records only; the chain carries none. + assert 'node_id' not in json.dumps(chains(counters)[0], sort_keys=True) class TestTraceDelivery: - def test_close_buffers_one_record_per_trace(self): + def test_close_emits_one_terminal_chain_record_per_execution(self): trace, _ = get_modules() manager, counters = make_counters(trace_config()) binding = trace.bind('exec-1') @@ -383,49 +411,55 @@ class TestTraceDelivery: # Closing buffers; nothing is sent before the flush cadence. assert manager.sent == [] - assert len(counters.records) == 1 - assert counters.traces == {} - - record = counters.records[0] - assert record['event_type'] == 'feature_execution' - assert record['query_id'] == 'exec-1' + assert len(nodes(counters)) == 2 + chain_rows = chains(counters) + assert len(chain_rows) == 1 + record = chain_rows[0] + assert record['schema'] == 2 + assert record['event_id'] == 'exec-1' assert record['instance_id'] == 'instance-1' assert record['workspace_uuid'] == 'workspace-1' - assert record['trusted'] is True - features = record['features'] - assert features['schema'] == 1 - assert 'count' not in json.dumps(record) - assert [row['seq'] for row in features['observations']] == [0, 1] - assert [row['trace_id'] for row in features['observations']] == ['exec-1'] * 2 - assert features['trace'] == { - 'closed_by': 'event_done', - 'started_at': binding.state.started_at, - 'ended_at': features['trace']['ended_at'], - 'route_ref': '', - 'run_id': '', - 'outcome': 'success', - 'dropped_stages': 0, - } + assert record['closed_by'] == 'event_done' + assert record['outcome'] == 'success' + assert record['stage_count'] == 2 + assert isinstance(record['duration_ms'], int) and record['duration_ms'] >= 0 + assert 'query_id' not in record and 'features' not in record def test_sampling_decision_is_deterministic_and_keeps_failures(self): trace, _ = get_modules() _, counters = make_counters(trace_config(execution_trace='sampled', execution_trace_sample=2)) - binding = trace.bind() - try: - binding.state.trace_id = '00000001-0000-4000-8000-000000000000' - counters.record(CONTEXT, **STAGE) - assert counters._trace_emitted(binding.state) is False - binding.state.trace_id = '00000000-0000-4000-8000-000000000000' - assert counters._trace_emitted(binding.state) is True - binding.state.trace_id = '00000001-0000-4000-8000-000000000001' - counters.record(CONTEXT, **{**STAGE, 'outcome': 'timeout'}) - assert counters._trace_emitted(binding.state) is True - finally: - trace.unbind_root(binding) + in_id = next(i for i in (f'sample-{n}' for n in range(64)) if counters._sampling_decision(i)[2]) + out_id = next(i for i in (f'sample-{n}' for n in range(64)) if not counters._sampling_decision(i)[2]) + # Same id, same answer: the decision is a pure function of the identity. + assert counters._sampling_decision(in_id) == counters._sampling_decision(in_id) - def test_failures_mode_keeps_failed_and_debug_traces_only(self): + excluded = trace.bind(out_id) + counters.record(CONTEXT, **STAGE) + assert excluded.state.emit_nodes is False + assert counters._chain_emitted(excluded.state) is False + counters.close_trace(excluded.state, 'event_done') + trace.unbind_root(excluded) + assert counters.records == [] + + included = trace.bind(in_id) + counters.record(CONTEXT, **STAGE) + assert included.state.emit_nodes is True + counters.close_trace(included.state, 'event_done') + trace.unbind_root(included) + assert [record['event_type'] for record in counters.records] == ['execution_node', 'execution_chain'] + + counters.records.clear() + broken = trace.bind(out_id) + counters.record(CONTEXT, **{**STAGE, 'outcome': 'timeout'}) + assert counters._chain_emitted(broken.state) is True + counters.close_trace(broken.state, 'event_done') + trace.unbind_root(broken) + assert [record['event_type'] for record in counters.records] == ['execution_chain'] + assert counters.records[0]['outcome'] == 'timeout' + + def test_failures_mode_keeps_failed_and_debug_chains_only(self): trace, _ = get_modules() - _, counters = make_counters(trace_config(execution_trace='failures', execution_trace_sample=1000000)) + _, counters = make_counters(trace_config(execution_trace='failures')) failed = trace.bind('exec-failed') counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) @@ -433,7 +467,8 @@ class TestTraceDelivery: trace.unbind_root(failed) debug = trace.bind('exec-debug') - counters.record(CONTEXT, **{**STAGE, 'synthetic': True}) + counters.configure(debug.state, debug=True) + counters.record(CONTEXT, **STAGE) counters.close_trace(debug.state, 'event_done') trace.unbind_root(debug) @@ -442,9 +477,12 @@ class TestTraceDelivery: counters.close_trace(sampled_out.state, 'event_done') trace.unbind_root(sampled_out) - assert [record['query_id'] for record in counters.records] == ['exec-failed', 'exec-debug'] - assert counters.records[0]['features']['observations'][0]['outcome'] == 'failed' - assert counters.records[1]['features']['observations'][0]['synthetic'] is True + assert [record['event_id'] for record in chains(counters)] == ['exec-failed', 'exec-debug'] + assert chains(counters)[0]['outcome'] == 'failed' + # Its node was sampled out, but the broken chain still explains itself. + assert chains(counters)[0]['dropped_nodes'] == 1 + assert chains(counters)[1]['debug'] is True + assert [record['event_id'] for record in nodes(counters)] == ['exec-debug'] def test_off_mode_and_opt_out_buffer_nothing(self): trace, _ = get_modules() @@ -469,24 +507,9 @@ class TestTraceDelivery: trace.unbind_root(binding) assert disabled_manager.sent == [] assert disabled.records == [] - assert disabled.traces == {} + assert disabled.registry == {} - def test_ttl_sweep_closes_stale_trace(self): - trace, _ = get_modules() - manager, counters = make_counters(trace_config()) - binding = trace.bind('exec-ttl') - try: - counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - counters.trace_deadlines[binding.state.trace_id] = 0 - counters._sweep_traces(time.monotonic()) - finally: - trace.unbind_root(binding) - assert counters.records[0]['features']['trace']['closed_by'] == 'ttl' - assert counters.traces == {} - assert counters.trace_deadlines == {} - assert manager.sent == [] - - async def test_shutdown_drops_open_traces(self): + async def test_shutdown_does_not_claim_an_unfinished_execution(self): trace, _ = get_modules() manager, counters = make_counters(trace_config()) binding = trace.bind('exec-open') @@ -495,26 +518,12 @@ class TestTraceDelivery: await counters.shutdown() finally: trace.unbind_root(binding) - # Shutdown must not claim a trace that never completed. - assert records_sent(manager) == [] + # The already-emitted node is flushed, but shutdown never fabricates a + # terminal chain for an execution that never completed. + assert [record['event_type'] for record in records_sent(manager)] == ['execution_node'] assert counters.records == [] - assert counters.traces == {} + assert chains(counters) == [] assert counters.registry == {} - assert counters.dropped_traces == 1 - - async def test_loop_sweeps_and_flushes_expired_trace(self): - trace, _ = get_modules() - manager, counters = make_counters(trace_config(telemetry_flush_seconds=1)) - binding = trace.bind('exec-ttl-loop') - try: - counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - counters.trace_deadlines[binding.state.trace_id] = 0 - await asyncio.sleep(1.3) - assert [record['features']['trace']['closed_by'] for record in records_sent(manager)] == ['ttl'] - assert counters.traces == {} - finally: - await counters.shutdown() - trace.unbind_root(binding) async def test_flush_sends_one_batch_of_complete_records(self): _, execution = get_modules() @@ -523,14 +532,20 @@ class TestTraceDelivery: for index in range(2): with execution.ingress(ap, 'event_done', CONTEXT, execution_id=f'exec-{index}'): counters.record(CONTEXT, **STAGE) - assert len(counters.records) == 2 + assert len(counters.records) == 4 await counters.flush() assert len(manager.sent) == 1 payload = manager.sent[0] assert set(payload) == {'records'} - assert [record['query_id'] for record in payload['records']] == ['exec-0', 'exec-1'] - assert 'count' not in json.dumps(payload) + assert [record['event_type'] for record in payload['records']] == [ + 'execution_node', + 'execution_chain', + 'execution_node', + 'execution_chain', + ] + assert [record['event_id'] for record in payload['records']] == ['exec-0', 'exec-0', 'exec-1', 'exec-1'] + await counters.shutdown() async def test_flush_interval_is_honoured(self): _, execution = get_modules() @@ -541,7 +556,9 @@ class TestTraceDelivery: assert counters.flush_seconds() == 1 assert manager.sent == [] await asyncio.sleep(1.3) - assert [record['query_id'] for record in records_sent(manager)] == ['exec-timed'] + sent = records_sent(manager) + assert [record['event_type'] for record in sent] == ['execution_node', 'execution_chain'] + assert {record['event_id'] for record in sent} == {'exec-timed'} await counters.shutdown() def test_flush_interval_config_is_bounded(self): @@ -563,11 +580,10 @@ class TestTraceDelivery: for index in range(5): binding = trace.bind(f'exec-{index}') counters.record(CONTEXT, **STAGE) - counters.close_trace(binding.state, 'event_done') trace.unbind_root(binding) finally: execution.MAX_BUFFERED_RECORDS = original - assert [record['query_id'] for record in counters.records] == ['exec-2', 'exec-3', 'exec-4'] + assert [record['event_id'] for record in nodes(counters)] == ['exec-2', 'exec-3', 'exec-4'] assert counters.dropped == 2 @@ -579,9 +595,9 @@ class TestIngress: with execution.ingress(ap, 'event_done', execution_id='exec-outer'): with execution.ingress(ap, 'pipeline_done', execution_id='exec-inner'): counters.record(CONTEXT, **{**STAGE, 'outcome': 'failed'}) - assert len(counters.records) == 1 - assert counters.records[0]['query_id'] == 'exec-outer' - assert counters.records[0]['features']['trace']['closed_by'] == 'event_done' + assert [record['event_type'] for record in counters.records] == ['execution_node', 'execution_chain'] + assert all(record['event_id'] == 'exec-outer' for record in counters.records) + assert chains(counters)[0]['closed_by'] == 'event_done' assert manager.sent == [] def test_ingress_without_telemetry_is_a_no_op(self): @@ -602,7 +618,7 @@ class TestIngress: class TestCrossTaskAttribution: - def test_registry_joins_stages_from_a_task_without_a_trace(self): + def test_registry_joins_nodes_from_a_task_without_a_trace(self): trace, execution = get_modules() _, counters = make_counters(trace_config()) ap = make_ap(counters) @@ -615,23 +631,24 @@ class TestCrossTaskAttribution: counters.record( CONTEXT, **{**STAGE, 'family': 'platform_api', 'operation': 'send_message'}, execution_id='run-1' ) - assert len(binding.state.stages) == 1 - assert binding.state.stages[0]['operation'] == 'send_message' + rows = nodes(counters) + assert len(rows) == 1 + assert rows[0]['operation'] == 'send_message' and rows[0]['event_id'] == 'run-1' counters.close_trace(binding.state, 'event_done') - assert counters.records[0]['query_id'] == 'run-1' + assert chains(counters)[0]['event_id'] == 'run-1' def test_registry_aliases_a_nested_execution(self): _, execution = get_modules() _, counters = make_counters(trace_config()) ap = make_ap(counters) - with execution.ingress(ap, 'event_done', CONTEXT, execution_id='event-1') as binding: + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='event-1'): with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='query-1'): pass assert set(counters.registry) == {'event-1', 'query-1'} counters.record(CONTEXT, **STAGE, execution_id='query-1') - assert counters.records[0]['query_id'] == 'event-1' - assert len(binding.state.stages) == 1 + rows = nodes(counters) + assert len(rows) == 1 and rows[0]['event_id'] == 'event-1' def test_contextvar_execution_id_is_used_when_no_kwarg(self): trace, execution = get_modules() @@ -644,7 +661,8 @@ class TestCrossTaskAttribution: counters.record(CONTEXT, **STAGE) finally: execution.reset_execution_id(token) - assert len(binding.state.stages) == 1 + rows = nodes(counters) + assert len(rows) == 1 and rows[0]['event_id'] == 'run-2' def test_registry_alias_is_swept_when_never_closed(self): trace, execution = get_modules() @@ -667,8 +685,10 @@ class TestTraceDispatch: with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-async'): counters.record(CONTEXT, **STAGE) await counters.flush() - assert [record['query_id'] for record in records_sent(manager)] == ['exec-async'] - assert manager.sent[0]['records'][0]['event_type'] == 'feature_execution' + sent = records_sent(manager) + assert [record['event_type'] for record in sent] == ['execution_node', 'execution_chain'] + assert all(record['event_id'] == 'exec-async' for record in sent) + await counters.shutdown() class AsyncSendManager(FakeManager): @@ -689,9 +709,9 @@ class TestTraceFailureDetail: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - record = counters.records[0] - assert record['error'] == 'boom: sandbox exited 127' - assert record['features']['observations'][0]['error'] == 'boom: sandbox exited 127' + assert nodes(counters)[0]['error'] == 'boom: sandbox exited 127' + # The detail lives on the node; the chain carries the terminal outcome. + assert chains(counters)[0]['outcome'] == 'failed' def test_successful_stage_reports_no_error(self): trace, _ = get_modules() @@ -702,9 +722,8 @@ class TestTraceFailureDetail: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - record = counters.records[0] - assert record['error'] == '' - assert record['features']['observations'][0]['error'] == '' + assert nodes(counters)[0]['error'] == '' + assert chains(counters)[0]['error'] == '' def test_error_detail_is_bounded_and_single_line(self): trace, _ = get_modules() @@ -715,8 +734,8 @@ class TestTraceFailureDetail: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - detail = counters.records[0]['error'] - assert len(detail) <= 400 + detail = chains(counters)[0]['error'] + assert len(detail) <= 200 assert '\n' not in detail and '\t' not in detail @@ -733,10 +752,10 @@ class TestIngressFailureDetail: except RuntimeError: raised = True assert raised is True - record = counters.records[0] - assert record['error'].startswith('RuntimeError: dispatch exploded') - assert record['features']['trace']['outcome'] == 'failed' - assert record['features']['trace']['closed_by'] == 'event_done' + chain = chains(counters)[0] + assert chain['error'].startswith('RuntimeError: dispatch exploded') + assert chain['outcome'] == 'failed' + assert chain['closed_by'] == 'event_done' def test_successful_ingress_still_reports_success(self): _, execution = get_modules() @@ -744,27 +763,29 @@ class TestIngressFailureDetail: ap = make_ap(counters) with execution.ingress(ap, 'event_done', execution_id='exec-fine'): counters.record(CONTEXT, **STAGE) - record = counters.records[0] - assert record['error'] == '' - assert record['features']['trace']['outcome'] == 'success' + chain = chains(counters)[0] + assert chain['error'] == '' + assert chain['outcome'] == 'success' class TestFailureOnlyTrace: def test_failure_without_stages_still_uploads(self): _, execution = get_modules() - # sampled-out config: only failures are guaranteed to be kept - _, counters = make_counters({'url': 'https://space.example.test', 'execution_trace_sample': 1}) + # failures mode: nodes are not reported, but a broken chain always is. + _, counters = make_counters({'url': 'https://space.example.test', 'execution_trace': 'failures'}) ap = make_ap(counters) try: with execution.ingress(ap, 'pipeline_done', CONTEXT, execution_id='exec-empty'): raise RuntimeError('pipeline exploded before any stage') except RuntimeError: pass - assert len(counters.records) == 1 - record = counters.records[0] + rows = chains(counters) + assert len(rows) == 1 + record = rows[0] assert record['error'].startswith('RuntimeError: pipeline exploded') - assert record['features']['trace']['outcome'] == 'failed' - assert record['features']['observations'] == [] + assert record['outcome'] == 'failed' + assert record['stage_count'] == 0 + assert nodes(counters) == [] def test_cancelled_and_timeout_chains_report_their_outcome(self): trace, _ = get_modules() @@ -776,15 +797,14 @@ class TestFailureOnlyTrace: counters.close_trace(binding.state, 'runner_done') finally: trace.unbind_root(binding) - assert len(counters.records) == 1 - record = counters.records.pop(0) - assert record['query_id'] == f'exec-{outcome}' - assert record['features']['trace']['outcome'] == outcome - assert record['error'] == reason + record = chains(counters)[-1] + assert record['event_id'] == f'exec-{outcome}' + assert record['outcome'] == outcome + assert nodes(counters)[-1]['error'] == reason class TestLegacyRecordFields: - def test_record_carries_legacy_top_level_columns(self): + def test_chain_carries_execution_scoped_columns(self): trace, _ = get_modules() _, counters = make_counters(trace_config()) binding = trace.bind('exec-legacy') @@ -800,14 +820,13 @@ class TestLegacyRecordFields: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - record = counters.records[0] + record = chains(counters)[0] assert record['model_name'] == 'gpt-4o' assert record['pipeline_plugins'] == ['plugin:a'] assert record['runner_category'] == 'cloud' assert record['adapter'] == 'AiocqhttpAdapter' assert record['runner'] == 'runner-1' assert record['duration_ms'] >= 1000 - assert record['timestamp'] == record['features']['trace']['ended_at'] def test_legacy_string_fields_are_never_null(self): trace, _ = get_modules() @@ -818,9 +837,135 @@ class TestLegacyRecordFields: counters.close_trace(binding.state, 'event_done') finally: trace.unbind_root(binding) - record = counters.records[0] + record = chains(counters)[0] assert record['model_name'] == '' assert record['runner'] == '' assert record['runner_category'] == '' assert isinstance(record['duration_ms'], int) and record['duration_ms'] >= 0 assert record['pipeline_plugins'] is None + + +class TestNoTraceTtl: + def test_nodes_more_than_two_minutes_apart_still_emit(self): + trace, _ = get_modules() + _, counters = make_counters(trace_config()) + binding = trace.bind('exec-long') + try: + counters.record(CONTEXT, **STAGE) + # A gap far beyond the old 120s TTL must not sweep the chain: there is + # no local trace deadline any more, so both nodes and the terminal + # chain are still emitted. + binding.state.started_at = (datetime.now(timezone.utc) - timedelta(seconds=200)).isoformat() + counters.record(CONTEXT, **{**STAGE, 'operation': 'still.alive'}) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + rows = nodes(counters) + assert [record['seq'] for record in rows] == [0, 1] + assert [record['operation'] for record in rows] == ['message.received', 'still.alive'] + chain = chains(counters)[0] + assert chain['stage_count'] == 2 + assert chain['duration_ms'] >= 120_000 + + +class TestFlushDelivery: + async def test_rejected_batch_stays_buffered_and_is_retried(self): + trace, execution = get_modules() + manager = RejectingManager(trace_config()) + counters = execution.ExecutionCounters(manager) + binding = trace.bind('exec-retry') + try: + counters.record(CONTEXT, **STAGE) + assert await counters.flush() is False + assert manager.attempts == execution.MAX_FLUSH_ATTEMPTS + # The record was never lost: it is still buffered and unsent. + assert len(counters.records) == 1 and manager.sent == [] + manager.rejecting = False + assert await counters.flush() is True + finally: + trace.unbind_root(binding) + assert counters.records == [] + assert [record['event_id'] for record in records_sent(manager)] == ['exec-retry'] + await counters.shutdown() + + async def test_health_reports_buffered_sent_and_dropped(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + binding = execution.bind_trace(ap, 'exec-health') + try: + counters.record(CONTEXT, **STAGE) + health = counters.health() + assert health['records_buffered'] == 1 + assert health['records_sent'] == 0 + assert health['records_dropped'] == 0 + assert health['executions_in_flight'] == 1 + await counters.flush() + health = counters.health() + assert health['records_buffered'] == 0 + assert health['records_sent'] == 1 + finally: + trace.unbind_root(binding) + + # Overflowing the outbound buffer is surfaced as dropped records. + original = execution.MAX_BUFFERED_RECORDS + execution.MAX_BUFFERED_RECORDS = 1 + try: + for index in range(3): + inner = trace.bind(f'exec-drop-{index}') + counters.record(CONTEXT, **STAGE) + trace.unbind_root(inner) + finally: + execution.MAX_BUFFERED_RECORDS = original + health = counters.health() + assert health['records_dropped'] == 2 + assert health['records_buffered'] == 1 + await counters.shutdown() + + +class TestSiblingTaskIsolation: + async def test_sibling_stage_scopes_do_not_corrupt_parent_links(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + ap = make_ap(counters) + opened = {} + + async def work(name): + with trace.stage_scope() as node: + opened[name] = node + counters.record(CONTEXT, **{**STAGE, 'operation': name, 'node': node}) + + async def scenario(): + with execution.ingress(ap, 'event_done', CONTEXT, execution_id='exec-siblings'): + await asyncio.gather(work('alpha'), work('beta')) + snapshot = list(counters.records) + await counters.shutdown() + return snapshot + + records = await scenario() + rows = {record['operation']: record for record in records if record['event_type'] == 'execution_node'} + assert opened['alpha'] != opened['beta'] + # Each sibling's step is its own root: neither becomes the other's parent. + assert rows['alpha']['node_id'] == opened['alpha'] + assert rows['beta']['node_id'] == opened['beta'] + assert rows['alpha']['parent_node_id'] == '' + assert rows['beta']['parent_node_id'] == '' + assert [record['event_type'] for record in records].count('execution_chain') == 1 + + +class TestSamplingShape: + def test_platform_prefixed_event_id_is_emitted_at_default_config(self): + trace, _ = get_modules() + _, counters = make_counters() # no execution_trace override -> default mode + assert counters.trace_mode() == 'all' + # The identity is hashed, not parsed: a platform-prefixed id samples the + # same way any other id shape does. + assert counters._sampling_decision('platform:bot-1:message-1') == ('all', 1, True) + binding = trace.bind('platform:bot-1:message-1') + try: + counters.record(CONTEXT, **STAGE) + counters.close_trace(binding.state, 'event_done') + finally: + trace.unbind_root(binding) + assert [record['event_type'] for record in counters.records] == ['execution_node', 'execution_chain'] + assert all(record['event_id'] == 'platform:bot-1:message-1' for record in counters.records) diff --git a/tests/unit_tests/test_telemetry.py b/tests/unit_tests/test_telemetry.py deleted file mode 100644 index ed3994923..000000000 --- a/tests/unit_tests/test_telemetry.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from types import SimpleNamespace -import asyncio - -import pytest - -from langbot.pkg.telemetry.telemetry import TelemetryManager - - -@pytest.mark.asyncio -async def test_send_tasks_are_scoped_to_manager_instance(monkeypatch): - async def fake_send(self, payload): - return payload - - monkeypatch.setattr(TelemetryManager, 'send', fake_send) - - first = TelemetryManager(SimpleNamespace()) - second = TelemetryManager(SimpleNamespace()) - - assert first.send_tasks is not second.send_tasks - - await first.start_send_task({'event': 'first'}) - task = first.send_tasks[0] - await task - await asyncio.sleep(0) - - assert first.send_tasks == [] - assert second.send_tasks == [] From de886edc1b60be23b9b0fac4c37296cb81dca16b Mon Sep 17 00:00:00 2001 From: Hyu Date: Wed, 7 Oct 2026 10:20:08 +0800 Subject: [PATCH 011/174] style(telemetry): apply ruff format to the telemetry manager The single-line assignment fits the configured line length; ruff format rejects the wrapped form. --- src/langbot/pkg/telemetry/telemetry.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/langbot/pkg/telemetry/telemetry.py b/src/langbot/pkg/telemetry/telemetry.py index 9188a5c54..5c9a41d22 100644 --- a/src/langbot/pkg/telemetry/telemetry.py +++ b/src/langbot/pkg/telemetry/telemetry.py @@ -84,9 +84,7 @@ class TelemetryManager: else: if 'query_id' in sanitized: try: - sanitized['query_id'] = ( - '' if sanitized['query_id'] is None else str(sanitized['query_id']) - ) + sanitized['query_id'] = '' if sanitized['query_id'] is None else str(sanitized['query_id']) except Exception: sanitized['query_id'] = str(sanitized.get('query_id', '')) From 9f76037e6427372d66dd18acd531bc3cccf02e60 Mon Sep 17 00:00:00 2001 From: Hyu Date: Wed, 7 Oct 2026 22:36:10 +0800 Subject: [PATCH 012/174] feat(monitoring): unify execution views and trace tool calls --- src/langbot/pkg/agent/runner/orchestrator.py | 15 + .../api/http/controller/groups/monitoring.py | 43 +- .../pkg/api/http/service/monitoring.py | 598 ++++++++++++++ src/langbot/pkg/pipeline/monitoring_helper.py | 97 +++ src/langbot/pkg/provider/tools/toolmgr.py | 74 ++ src/langbot/pkg/telemetry/execution.py | 8 +- src/langbot/pkg/telemetry/trace.py | 37 +- .../api/service/test_monitoring_executions.py | 350 ++++++++ .../unit_tests/provider/test_tool_manager.py | 123 +++ tests/unit_tests/telemetry/test_trace.py | 49 +- .../components/ConversationTurnList.tsx | 8 +- .../monitoring/components/ExportDropdown.tsx | 22 +- .../monitoring/components/FeedbackCard.tsx | 123 +-- .../monitoring/components/FeedbackList.tsx | 102 ++- .../components/MessageContentRenderer.tsx | 47 +- .../components/MessageDetailsCard.tsx | 30 +- .../home/monitoring/components/TabState.tsx | 50 ++ .../monitoring/components/TokenMonitoring.tsx | 285 +++---- .../executions/ExecutionDetailSheet.tsx | 361 +++++++++ .../components/executions/ExecutionTable.tsx | 169 ++++ .../components/filters/MonitoringFilters.tsx | 125 ++- .../overview-cards/ExecutionOverviewCards.tsx | 197 +++++ .../components/overview-cards/MetricCard.tsx | 138 ++-- .../overview-cards/OverviewCards.tsx | 100 --- .../overview-cards/SystemStatusCards.tsx | 45 +- .../overview-cards/TrafficChart.tsx | 22 +- .../home/monitoring/hooks/useExecutions.ts | 147 ++++ .../monitoring/hooks/useMonitoringData.ts | 49 +- web/src/app/home/monitoring/page.tsx | 752 +++++++++++------- .../app/home/monitoring/utils/callLinks.ts | 100 +++ .../app/home/monitoring/utils/dateUtils.ts | 45 ++ web/src/app/infra/entities/api/index.ts | 17 + .../entities/api/monitoring-executions.ts | 136 ++++ web/src/app/infra/http/BackendClient.ts | 65 ++ web/src/components/ui/table.tsx | 11 +- web/src/i18n/locales/en-US.ts | 77 ++ web/src/i18n/locales/zh-Hans.ts | 76 ++ web/src/i18n/locales/zh-Hant.ts | 76 ++ web/tests/e2e/fixtures/langbot-api.ts | 157 ++++ web/tests/e2e/home-smoke.spec.ts | 9 +- web/tests/e2e/monitoring-turns.spec.ts | 4 + web/tests/unit/monitoring-call-links.test.mjs | 131 +++ 42 files changed, 4181 insertions(+), 889 deletions(-) create mode 100644 tests/unit_tests/api/service/test_monitoring_executions.py create mode 100644 web/src/app/home/monitoring/components/TabState.tsx create mode 100644 web/src/app/home/monitoring/components/executions/ExecutionDetailSheet.tsx create mode 100644 web/src/app/home/monitoring/components/executions/ExecutionTable.tsx create mode 100644 web/src/app/home/monitoring/components/overview-cards/ExecutionOverviewCards.tsx delete mode 100644 web/src/app/home/monitoring/components/overview-cards/OverviewCards.tsx create mode 100644 web/src/app/home/monitoring/hooks/useExecutions.ts create mode 100644 web/src/app/home/monitoring/utils/callLinks.ts create mode 100644 web/src/app/infra/entities/api/monitoring-executions.ts create mode 100644 web/tests/unit/monitoring-call-links.test.mjs diff --git a/src/langbot/pkg/agent/runner/orchestrator.py b/src/langbot/pkg/agent/runner/orchestrator.py index b04ec8394..9d3db00d2 100644 --- a/src/langbot/pkg/agent/runner/orchestrator.py +++ b/src/langbot/pkg/agent/runner/orchestrator.py @@ -194,6 +194,21 @@ class AgentRunOrchestrator: state_context = build_state_context(event, binding, descriptor) run_id = context['run_id'] + + # Monitoring attributes every call to the processor that ran it and to the + # record that owns it; a Runner has to claim both itself, since no + # Pipeline did. Runs before the Runner is invoked and before the run + # context is handed to it, so its own variables stay private. + from ...pipeline import monitoring_helper + + await monitoring_helper.prepare_monitoring_identity( + self.ap, + execution_query, + processor_type=binding.processor_type, + processor_id=binding.processor_id, + runner_id=binding.runner_id, + execution_record_id=run_id, + ) context['context']['available_apis']['reply_stream'] = hasattr(PluginToRuntimeAction, 'REPLY_STREAM') and any( tool.get('tool_name') == 'event_reply' and tool.get('tool_type') == 'platform' for tool in resources.get('tools', []) diff --git a/src/langbot/pkg/api/http/controller/groups/monitoring.py b/src/langbot/pkg/api/http/controller/groups/monitoring.py index 01a901743..1d9656761 100644 --- a/src/langbot/pkg/api/http/controller/groups/monitoring.py +++ b/src/langbot/pkg/api/http/controller/groups/monitoring.py @@ -50,6 +50,47 @@ class MonitoringRouterGroup(group.RouterGroup): return self.success(data=metrics) + @self.route('/executions', methods=['GET'], permission=Permission.RESOURCE_VIEW) + async def get_executions(request_context: RequestContext) -> str: + """Unified execution list across agent runs and pipeline queries.""" + bot_ids = quart.request.args.getlist('botId') + pipeline_ids = quart.request.args.getlist('pipelineId') + agent_ids = quart.request.args.getlist('agentId') + statuses = quart.request.args.getlist('status') + source = quart.request.args.get('source', 'all') + mode = quart.request.args.get('mode', 'all') + start_time = parse_iso_datetime(quart.request.args.get('startTime')) + end_time = parse_iso_datetime(quart.request.args.get('endTime')) + + result = await self.ap.monitoring_service.get_executions( + request_context, + bot_ids=bot_ids if bot_ids else None, + pipeline_ids=pipeline_ids if pipeline_ids else None, + agent_ids=agent_ids if agent_ids else None, + statuses=statuses if statuses else None, + source=source, + mode=mode, + start_time=start_time, + end_time=end_time, + limit=quart.request.args.get('limit', 50), + offset=quart.request.args.get('offset', 0), + ) + + return self.success(data=result) + + @self.route( + '/executions//', + methods=['GET'], + permission=Permission.RESOURCE_VIEW, + ) + async def get_execution_detail(source: str, execution_id: str, request_context: RequestContext) -> str: + """Full trace of a single execution (agent run or pipeline query).""" + try: + detail = await self.ap.monitoring_service.get_execution_detail(request_context, source, execution_id) + except ValueError as exc: + return self.fail(404, str(exc)) + return self.success(data=detail) + @self.route('/token-statistics', methods=['GET'], permission=Permission.RESOURCE_VIEW) async def get_token_statistics(request_context: RequestContext) -> str: """Get detailed token usage statistics (summary, per-model, timeseries).""" @@ -639,7 +680,7 @@ class MonitoringRouterGroup(group.RouterGroup): 'platform', ] else: - return self.error(message=f'Invalid export type: {export_type}', code=400) + return self.fail(400, f'Invalid export type: {export_type}') # Generate CSV content with UTF-8 BOM for Excel compatibility import io diff --git a/src/langbot/pkg/api/http/service/monitoring.py b/src/langbot/pkg/api/http/service/monitoring.py index 0bae983a6..bb7407248 100644 --- a/src/langbot/pkg/api/http/service/monitoring.py +++ b/src/langbot/pkg/api/http/service/monitoring.py @@ -10,7 +10,10 @@ from sqlalchemy.dialects import postgresql as postgresql_dialect from sqlalchemy.dialects import sqlite as sqlite_dialect from ....core import app +from ....entity.persistence import agent as persistence_agent from ....entity.persistence import monitoring as persistence_monitoring +from ....entity.persistence import agent_run as persistence_agent_run +from ....entity.persistence import agent_interaction as persistence_agent_interaction from ..authz import WorkspaceRequiredError from ..context import ExecutionContext from .tenant import TenantContext, require_workspace_uuid @@ -29,6 +32,76 @@ _HARD_MAX_MONITORING_OFFSET = 10000000 _DEFAULT_CLEANUP_BATCHES_PER_TABLE = 4 _HARD_MAX_CLEANUP_BATCHES_PER_TABLE = 100 +# Canonical execution status groups shared by the agent ledger and pipeline query +# records so both object types can be summarised with one vocabulary. +_AGENT_STATUS_GROUP = { + 'completed': 'completed', + 'failed': 'failed', + 'timeout': 'failed', + 'cancelled': 'cancelled', + 'created': 'queued', + 'queued': 'queued', + 'claimed': 'running', + 'running': 'running', +} +_PIPELINE_STATUS_GROUP = { + 'success': 'completed', + 'error': 'failed', + 'pending': 'running', + 'discarded': 'ignored', + 'not_matched': 'ignored', +} + + +def _status_group(status: str, source: str) -> str: + table = _AGENT_STATUS_GROUP if source == 'agent' else _PIPELINE_STATUS_GROUP + return table.get(status, status) + + +def _epoch_ms(value: datetime.datetime | None) -> int | None: + """Epoch milliseconds for a DB timestamp, treating naive values as UTC.""" + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=datetime.timezone.utc) + return int(round(value.timestamp() * 1000)) + + +def _percentile_ms(values: list[float], fraction: float) -> int | None: + """Nearest-rank percentile (ms) over a non-empty, prefix-sorted list.""" + if not values: + return None + rank = max(1, min(len(values), int(round(fraction * len(values))))) + return int(round(values[rank - 1])) + + +def _collect_text(node, out: list[str], depth: int = 0) -> None: + """Best-effort extraction of human text from a serialised message chain.""" + if depth > 6: + return + if isinstance(node, dict): + for key, value in node.items(): + if key == 'text' and isinstance(value, str): + out.append(value) + elif isinstance(value, (dict, list)): + _collect_text(value, out, depth + 1) + elif isinstance(node, list): + for item in node: + _collect_text(item, out, depth + 1) + + +def _message_preview(raw: str | None, limit: int = 160) -> str: + if not raw: + return '' + try: + parsed = json.loads(raw) + except (TypeError, ValueError): + return raw[:limit] + parts: list[str] = [] + _collect_text(parsed, parts) + text = ' '.join(part.strip() for part in parts if part.strip()) + return (text or raw)[:limit] + def _normalize_user_id(value: str | int | None) -> str | None: """Convert numeric platform IDs before binding a VARCHAR with asyncpg. @@ -885,6 +958,10 @@ class MonitoringService: active_sessions_result = await self.ap.persistence_mgr.execute_async(active_session_query) active_sessions = active_sessions_result.scalar() or 0 + # Execution-first KPI values (runs + pipeline queries, latency, token + # coverage) are served by `get_executions`, which applies the execution + # view filters. This endpoint stays the message/traffic rollup so the + # dashboard does not compute the same rollup twice per page load. return { 'total_messages': total_messages, 'llm_calls': llm_calls, @@ -894,6 +971,527 @@ class MonitoringService: 'active_sessions': active_sessions, } + # ========== Execution view ========== + + def _agent_run_conditions( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + agent_ids: list[str] | None = None, + statuses: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + debug: bool | None = None, + ) -> list: + AgentRun = persistence_agent_run.AgentRun + conditions = [AgentRun.workspace_id == workspace_uuid] + if bot_ids: + conditions.append(AgentRun.bot_id.in_(bot_ids)) + if agent_ids: + conditions.append(AgentRun.agent_id.in_(agent_ids)) + if statuses is not None: + conditions.append(AgentRun.status.in_(statuses) if statuses else sqlalchemy.false()) + if start_time: + conditions.append(AgentRun.created_at >= start_time) + if end_time: + conditions.append(AgentRun.created_at <= end_time) + if debug is not None: + debug_condition = sqlalchemy.or_( + AgentRun.binding_id.like('debug:%'), + AgentRun.conversation_id.like('debug:%'), + ) + conditions.append(debug_condition if debug else sqlalchemy.not_(debug_condition)) + return conditions + + def _pipeline_execution_conditions( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + statuses: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + ) -> list: + MonitoringMessage = persistence_monitoring.MonitoringMessage + conditions = [ + MonitoringMessage.workspace_uuid == workspace_uuid, + # Assistant replies are outputs of a query, not an execution of their own. + sqlalchemy.or_(MonitoringMessage.role.is_(None), MonitoringMessage.role != 'assistant'), + ] + if bot_ids: + conditions.append(MonitoringMessage.bot_id.in_(bot_ids)) + if pipeline_ids: + conditions.append(MonitoringMessage.pipeline_id.in_(pipeline_ids)) + if statuses is not None: + conditions.append(MonitoringMessage.status.in_(statuses) if statuses else sqlalchemy.false()) + if start_time: + conditions.append(MonitoringMessage.timestamp >= start_time) + if end_time: + conditions.append(MonitoringMessage.timestamp <= end_time) + return conditions + + @staticmethod + def _raw_agent_statuses(groups: list[str] | None) -> list[str] | None: + """Raw agent statuses for a group filter; empty means "no group applies".""" + if not groups: + return None + raw: list[str] = [] + for group in groups: + raw.extend(status for status, mapped in _AGENT_STATUS_GROUP.items() if mapped == group) + return raw + + @staticmethod + def _raw_pipeline_statuses(groups: list[str] | None) -> list[str] | None: + """Raw pipeline statuses for a group filter; empty means "no group applies".""" + if not groups: + return None + raw: list[str] = [] + for group in groups: + raw.extend(status for status, mapped in _PIPELINE_STATUS_GROUP.items() if mapped == group) + return raw + + @staticmethod + def _model_rows(result) -> list: + """Rows of one ``select(Model)`` executed through ``execute_async``. + + The Core keeps the connection result contract, where ``select(Model)`` + yields a *flat* row of the model's columns (``row.uuid``, ``row.status``) + rather than a one-item ORM row. ``.scalars()`` would therefore return the + first column - the id - instead of the row, so the rows are taken as-is. + """ + return list(result.all()) + + @staticmethod + def _deserialize_json(raw: str | None) -> dict: + if not raw: + return {} + try: + value = json.loads(raw) + except (TypeError, ValueError): + return {} + return value if isinstance(value, dict) else {} + + async def _agent_names(self, workspace_uuid: str, rows: list) -> dict[str, str]: + """The processor name of every Agent run in one page, in one query. + + A run stores the Agent it executed as a uuid; the list has to name the + processor, so the names are resolved for the whole page at once. + """ + agent_ids = {row.agent_id for row in rows if row.agent_id} + if not agent_ids: + return {} + Agent = persistence_agent.Agent + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(Agent.uuid, Agent.name).where( + sqlalchemy.and_( + Agent.workspace_uuid == workspace_uuid, + Agent.uuid.in_(agent_ids), + ) + ) + ) + return {row.uuid: row.name for row in self._model_rows(result)} + + def _serialize_agent_execution(self, row, agent_names: dict[str, str] | None = None) -> dict: + metadata = self._deserialize_json(row.metadata_json) + duration_ms = None + if row.started_at and row.finished_at: + duration_ms = int(round((row.finished_at - row.started_at).total_seconds() * 1000)) + created_at_ms = _epoch_ms(row.created_at) + started_at_ms = _epoch_ms(row.started_at) + finished_at_ms = _epoch_ms(row.finished_at) + is_debug = str(row.binding_id or '').startswith('debug:') or str(row.conversation_id or '').startswith('debug:') + return { + 'source': 'agent', + 'id': row.run_id, + 'event_id': row.event_id, + 'status': row.status, + 'status_group': _status_group(row.status, 'agent'), + 'title': metadata.get('event_type') or row.queue_name or row.runner_id, + 'target_kind': 'agent' if row.agent_id else 'processor', + 'target_id': row.agent_id or row.binding_id, + 'target_name': metadata.get('agent_name') + or metadata.get('target_name') + or (agent_names or {}).get(row.agent_id or ''), + 'bot_id': row.bot_id, + 'bot_name': metadata.get('bot_name'), + 'pipeline_id': None, + 'pipeline_name': None, + 'runner_id': row.runner_id, + 'conversation_id': row.conversation_id, + 'session_id': row.conversation_id, + 'created_at_ms': created_at_ms, + 'started_at_ms': started_at_ms, + 'finished_at_ms': finished_at_ms, + 'duration_ms': duration_ms, + 'usage': self._deserialize_json(row.usage_json) or None, + 'cost': self._deserialize_json(row.cost_json) or None, + 'queue_name': row.queue_name, + 'debug': is_debug, + 'has_error': row.status in {'failed', 'timeout'}, + } + + def _serialize_pipeline_execution(self, row) -> dict: + created_at_ms = _epoch_ms(row.timestamp) + return { + 'source': 'pipeline', + 'id': row.id, + 'event_id': None, + 'status': row.status, + 'status_group': _status_group(row.status, 'pipeline'), + 'title': _message_preview(row.message_content), + 'target_kind': 'pipeline', + 'target_id': row.pipeline_id, + 'target_name': row.pipeline_name, + 'bot_id': row.bot_id, + 'bot_name': row.bot_name, + 'pipeline_id': row.pipeline_id, + 'pipeline_name': row.pipeline_name, + 'runner_id': row.runner_name, + 'conversation_id': None, + 'session_id': row.session_id, + 'created_at_ms': created_at_ms, + 'started_at_ms': None, + 'finished_at_ms': None, + 'duration_ms': None, + 'usage': None, + 'cost': None, + 'queue_name': None, + 'debug': False, + 'has_error': row.status == 'error', + 'platform': row.platform, + 'user_id': row.user_id, + 'user_name': row.user_name, + } + + async def _get_token_coverage( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + ) -> dict: + LLMCall = persistence_monitoring.MonitoringLLMCall + conditions = [LLMCall.workspace_uuid == workspace_uuid] + if bot_ids: + conditions.append(LLMCall.bot_id.in_(bot_ids)) + if pipeline_ids: + conditions.append(LLMCall.pipeline_id.in_(pipeline_ids)) + if start_time: + conditions.append(LLMCall.timestamp >= start_time) + if end_time: + conditions.append(LLMCall.timestamp <= end_time) + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select( + sqlalchemy.func.count(LLMCall.id), + sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.total_tokens), 0), + sqlalchemy.func.sum(sqlalchemy.case((LLMCall.total_tokens > 0, 1), else_=0)), + ).where(sqlalchemy.and_(*conditions)) + ) + row = result.first() + calls, total, calls_with_usage = row if row else (0, 0, 0) + return { + 'total_tokens': int(total or 0), + 'calls': calls or 0, + 'calls_with_usage': calls_with_usage or 0, + } + + async def _get_execution_summary( + self, + workspace_uuid: str, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + ) -> dict: + AgentRun = persistence_agent_run.AgentRun + run_conditions = self._agent_run_conditions( + workspace_uuid, bot_ids=bot_ids, start_time=start_time, end_time=end_time + ) + run_status_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(AgentRun.status, sqlalchemy.func.count(AgentRun.id)) + .where(sqlalchemy.and_(*run_conditions)) + .group_by(AgentRun.status) + ) + agent_counts = {row[0]: row[1] for row in run_status_result.all()} + agent_total = sum(agent_counts.values()) + + debug_count_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(AgentRun.id)).where( + sqlalchemy.and_( + *run_conditions, + sqlalchemy.or_( + AgentRun.binding_id.like('debug:%'), + AgentRun.conversation_id.like('debug:%'), + ), + ) + ) + ) + debug_count = debug_count_result.scalar() or 0 + + # Percentiles are computed in Python, so the sample must stay bounded: + # take the most recent completed runs of the window and label the count + # so the UI can state how many runs the numbers rest on. + timing_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(AgentRun.started_at, AgentRun.finished_at) + .where( + sqlalchemy.and_( + *run_conditions, + AgentRun.status == 'completed', + AgentRun.started_at.is_not(None), + AgentRun.finished_at.is_not(None), + ) + ) + .order_by(AgentRun.created_at.desc()) + .limit(self._detail_limit()) + ) + durations = sorted((row[1] - row[0]).total_seconds() * 1000 for row in timing_result.all() if row[0] and row[1]) + + pipeline_conditions = self._pipeline_execution_conditions( + workspace_uuid, bot_ids=bot_ids, pipeline_ids=pipeline_ids, start_time=start_time, end_time=end_time + ) + pipeline_status_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select( + persistence_monitoring.MonitoringMessage.status, + sqlalchemy.func.count(persistence_monitoring.MonitoringMessage.id), + ) + .where(sqlalchemy.and_(*pipeline_conditions)) + .group_by(persistence_monitoring.MonitoringMessage.status) + ) + pipeline_counts = {row[0]: row[1] for row in pipeline_status_result.all()} + pipeline_total = sum(pipeline_counts.values()) + + # Pending interactions are gated by the same bot/window scope as the runs + # so "waiting input" cannot drift from the rest of the card. + waiting_conditions = [ + persistence_agent_interaction.AgentInteraction.workspace_id == workspace_uuid, + persistence_agent_interaction.AgentInteraction.status == 'pending', + ] + if bot_ids: + waiting_conditions.append(persistence_agent_interaction.AgentInteraction.bot_id.in_(bot_ids)) + if start_time: + waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at >= start_time) + if end_time: + waiting_conditions.append(persistence_agent_interaction.AgentInteraction.created_at <= end_time) + waiting_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(persistence_agent_interaction.AgentInteraction.id)).where( + sqlalchemy.and_(*waiting_conditions) + ) + ) + waiting = waiting_result.scalar() or 0 + + by_status: dict[str, int] = {} + for status, count in agent_counts.items(): + group = _status_group(status, 'agent') + by_status[group] = by_status.get(group, 0) + count + for status, count in pipeline_counts.items(): + group = _status_group(status, 'pipeline') + by_status[group] = by_status.get(group, 0) + count + + completed = by_status.get('completed', 0) + failed = by_status.get('failed', 0) + denominator = completed + failed + total = agent_total + pipeline_total + return { + 'total': total, + 'by_source': {'agent': agent_total, 'pipeline': pipeline_total}, + 'by_status': by_status, + 'completed': completed, + 'failed': failed, + 'running': by_status.get('running', 0), + 'queued': by_status.get('queued', 0), + 'cancelled': by_status.get('cancelled', 0), + 'ignored': by_status.get('ignored', 0), + 'waiting': waiting, + 'success_rate': round(completed / denominator * 100, 2) if denominator > 0 else None, + 'denominator': denominator, + 'p50_duration_ms': _percentile_ms(durations, 0.5), + 'p95_duration_ms': _percentile_ms(durations, 0.95), + 'duration_sample': len(durations), + 'debug': debug_count, + 'real': total - debug_count, + } + + async def get_executions( + self, + context: TenantContext, + *, + bot_ids: list[str] | None = None, + pipeline_ids: list[str] | None = None, + agent_ids: list[str] | None = None, + statuses: list[str] | None = None, + source: str = 'all', + mode: str = 'all', + start_time: datetime.datetime | None = None, + end_time: datetime.datetime | None = None, + limit: int = 50, + offset: int = 0, + ) -> dict: + """List executions from both object types (agent runs and pipeline queries).""" + workspace_uuid = require_workspace_uuid(context) + limit, offset = self.normalize_page_window(limit, offset) + fetch_limit = min(limit + offset, self._configured_query_limit('page_rows', 1000, 5000)) + if source not in {'all', 'agent', 'pipeline'}: + source = 'all' + if mode not in {'all', 'real', 'debug'}: + mode = 'all' + debug = None if mode == 'all' else (mode == 'debug') + + include_agent = source in {'all', 'agent'} and (not pipeline_ids or bool(agent_ids)) + include_pipeline = source in {'all', 'pipeline'} and (not agent_ids or bool(pipeline_ids)) + + items: list[dict] = [] + total = 0 + + if include_agent: + run_conditions = self._agent_run_conditions( + workspace_uuid, + bot_ids=bot_ids, + agent_ids=agent_ids, + statuses=self._raw_agent_statuses(statuses), + start_time=start_time, + end_time=end_time, + debug=debug, + ) + AgentRun = persistence_agent_run.AgentRun + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(AgentRun) + .where(sqlalchemy.and_(*run_conditions)) + .order_by(AgentRun.created_at.desc(), AgentRun.id.desc()) + .limit(fetch_limit) + ) + rows = self._model_rows(result) + count_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(AgentRun.id)).where(sqlalchemy.and_(*run_conditions)) + ) + total += count_result.scalar() or 0 + agent_names = await self._agent_names(workspace_uuid, rows) + items.extend(self._serialize_agent_execution(row, agent_names=agent_names) for row in rows) + + if include_pipeline and (debug is None or debug is False): + message_conditions = self._pipeline_execution_conditions( + workspace_uuid, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + statuses=self._raw_pipeline_statuses(statuses), + start_time=start_time, + end_time=end_time, + ) + MonitoringMessage = persistence_monitoring.MonitoringMessage + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(MonitoringMessage) + .where(sqlalchemy.and_(*message_conditions)) + .order_by(MonitoringMessage.timestamp.desc()) + .limit(fetch_limit) + ) + rows = self._model_rows(result) + count_result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(sqlalchemy.func.count(MonitoringMessage.id)).where( + sqlalchemy.and_(*message_conditions) + ) + ) + total += count_result.scalar() or 0 + items.extend(self._serialize_pipeline_execution(row) for row in rows) + + items.sort(key=lambda item: item.get('created_at_ms') or 0, reverse=True) + page = items[offset : offset + limit] + return { + 'items': page, + 'total': total, + 'limit': limit, + 'offset': offset, + 'has_more': total > offset + len(page), + 'summary': { + 'executions': await self._get_execution_summary( + workspace_uuid, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + start_time=start_time, + end_time=end_time, + ), + 'tokens': await self._get_token_coverage( + workspace_uuid, + bot_ids=bot_ids, + pipeline_ids=pipeline_ids, + start_time=start_time, + end_time=end_time, + ), + }, + } + + async def get_execution_detail( + self, + context: TenantContext, + source: str, + execution_id: str, + ) -> dict: + """Full trace of a single execution, keyed by object type.""" + workspace_uuid = require_workspace_uuid(context) + + if source == 'agent': + from ....agent.runner.run_ledger_store import RunLedgerStore + + store = RunLedgerStore(self.ap.persistence_mgr.get_db_engine()) + run = await store.get_run(execution_id) + if run is None or run.get('workspace_id') != workspace_uuid: + raise ValueError('Execution not found') + items, next_cursor, _, has_more = await store.page_run_events(run_id=execution_id, limit=200) + return { + 'source': 'agent', + 'run': run, + 'events': items, + 'has_more': has_more, + 'next_cursor': next_cursor, + } + + if source == 'pipeline': + MonitoringMessage = persistence_monitoring.MonitoringMessage + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(MonitoringMessage).where( + sqlalchemy.and_( + MonitoringMessage.id == execution_id, + MonitoringMessage.workspace_uuid == workspace_uuid, + ) + ) + ) + rows = self._model_rows(result) + if not rows: + raise ValueError('Execution not found') + serialized = self.ap.persistence_mgr.serialize_model(MonitoringMessage, rows[0]) + + async def _related(model, order_column): + related = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(model) + .where( + sqlalchemy.and_( + model.workspace_uuid == workspace_uuid, + model.message_id == execution_id, + ) + ) + .order_by(order_column.asc()) + ) + return [self.ap.persistence_mgr.serialize_model(model, row) for row in self._model_rows(related)] + + return { + 'source': 'pipeline', + 'message': serialized, + 'llm_calls': await _related( + persistence_monitoring.MonitoringLLMCall, persistence_monitoring.MonitoringLLMCall.timestamp + ), + 'tool_calls': await _related( + persistence_monitoring.MonitoringToolCall, persistence_monitoring.MonitoringToolCall.timestamp + ), + 'errors': await _related( + persistence_monitoring.MonitoringError, persistence_monitoring.MonitoringError.timestamp + ), + } + + raise ValueError('Unknown execution source') + async def get_token_statistics( self, context: TenantContext, diff --git a/src/langbot/pkg/pipeline/monitoring_helper.py b/src/langbot/pkg/pipeline/monitoring_helper.py index 15ae38f47..c4c7fc8f1 100644 --- a/src/langbot/pkg/pipeline/monitoring_helper.py +++ b/src/langbot/pkg/pipeline/monitoring_helper.py @@ -18,6 +18,103 @@ if typing.TYPE_CHECKING: from .pool import get_query_execution_context +async def resolve_processor_name( + ap: app.Application, + workspace_uuid: str, + processor_type: str, + processor_id: str | None, + runner_id: str = '', +) -> str: + """Display name of the processor kind that handled a run. + + Monitoring attributed every call to a Pipeline, so an Agent or a plugin event + processor recorded "Unknown" for the entity that actually ran. Each kind is + resolved from the table that owns it, falling back to the identity the + binding carried. + """ + import sqlalchemy + + from ..entity.persistence import agent as persistence_agent + from ..entity.persistence import pipeline as persistence_pipeline + + identifier = str(processor_id or '').strip() + if identifier: + try: + model = persistence_pipeline.LegacyPipeline if processor_type == 'pipeline' else persistence_agent.Agent + result = await ap.persistence_mgr.execute_async( + sqlalchemy.select(model.name).where( + sqlalchemy.and_( + model.uuid == identifier, + model.workspace_uuid == workspace_uuid, + ) + ) + ) + name = str(result.scalar() or '').strip() + if name: + return name + except Exception as e: + ap.logger.warning(f'Failed to resolve monitoring processor name: {e}') + # A deleted Agent or an unregistered plugin processor still names the + # component that ran, never a bare "Unknown". + return identifier or runner_id or processor_type + + +async def prepare_monitoring_identity( + ap: app.Application, + query: pipeline_query.Query, + *, + processor_type: str, + processor_id: str | None, + runner_id: str = '', + execution_record_id: str = '', +) -> None: + """Pin the monitoring identity of a runner-owned execution. + + A Pipeline pins these variables when it starts a query; a Runner started by + an Agent or a plugin event processor has to pin them itself, otherwise every + LLM/tool call it makes is attributed to "Unknown". An enclosing Pipeline + already named the run, so its label is never overwritten. + + ``_monitoring_message_id`` is the execution record the calls belong to: the + Pipeline's message, or the Agent run's id. Both are the id the executions + list shows, which is what lets a call jump back to its record. + """ + try: + variables = getattr(query, 'variables', None) + if not isinstance(variables, dict): + return + if variables.get('_monitoring_pipeline_name'): + return + + bot_name = 'WebChat' + bot_uuid = str(getattr(query, 'bot_uuid', '') or '') + workspace_uuid = str(getattr(query, 'workspace_uuid', '') or '') + if bot_uuid and workspace_uuid: + try: + bot = await ap.bot_service.get_bot( + workspace_uuid, + bot_uuid, + include_secret=False, + ) + if bot: + bot_name = bot.get('name', bot_name) + except Exception as e: + ap.logger.warning(f'Failed to resolve monitoring bot name: {e}') + + variables['_monitoring_bot_name'] = bot_name + variables['_monitoring_pipeline_name'] = await resolve_processor_name( + ap, + workspace_uuid, + processor_type, + processor_id, + runner_id, + ) + if execution_record_id: + variables['_monitoring_message_id'] = execution_record_id + except Exception as e: + ap.logger.error(f'Failed to prepare monitoring identity: {e}') + + class MonitoringHelper: """Helper class for monitoring operations""" diff --git a/src/langbot/pkg/provider/tools/toolmgr.py b/src/langbot/pkg/provider/tools/toolmgr.py index 26329dad0..97e554f42 100644 --- a/src/langbot/pkg/provider/tools/toolmgr.py +++ b/src/langbot/pkg/provider/tools/toolmgr.py @@ -2,6 +2,7 @@ from __future__ import annotations import typing import time +import asyncio import inspect from typing import TYPE_CHECKING @@ -448,6 +449,19 @@ class ToolManager: start_time = time.perf_counter() try: result = await invoke() + except asyncio.CancelledError: + duration_ms = int((time.perf_counter() - start_time) * 1000) + await self._record_tool_call( + name=name, + source=source, + parameters=parameters, + query=query, + duration_ms=duration_ms, + status='error', + error_message='cancelled', + ) + self._trace_tool_call(source=source, name=name, query=query, outcome='cancelled', error='cancelled') + raise except Exception as e: duration_ms = int((time.perf_counter() - start_time) * 1000) await self._record_tool_call( @@ -459,6 +473,14 @@ class ToolManager: status='error', error_message=str(e), ) + outcome = 'timeout' if isinstance(e, TimeoutError) else 'failed' + self._trace_tool_call( + source=source, + name=name, + query=query, + outcome=outcome, + error=str(e) or type(e).__name__, + ) raise duration_ms = int((time.perf_counter() - start_time) * 1000) @@ -471,8 +493,60 @@ class ToolManager: status='success', result=result, ) + self._trace_tool_call(source=source, name=name, query=query, outcome='success') return result + def _trace_tool_call( + self, + *, + source: str, + name: str, + query: pipeline_query.Query, + outcome: str, + error: str = '', + ) -> None: + """Report one tool invocation as its own step of the execution chain. + + Every tool call a Run makes - sandbox, skill, plugin or MCP - is the + observable action of that Run, so the chain shows it under the step that + issued it instead of collapsing the Run into a single opaque node. + + An Agent's tool call is executed for the Runtime's RPC task, which never + inherited the chain context. The Run id the caller already carries is the + alias that task's record resolves the owning execution through. + """ + # Imported here: telemetry pulls the application, and the tool manager is + # constructed while the application is still assembling. + from ...telemetry.execution import record as record_execution + from ...telemetry.platform import processing_mode + + try: + record_execution( + self.ap, + get_query_execution_context(query), + execution_id=self._trace_execution_id(query), + family='tool', + operation=name, + adapter=source, + mode=processing_mode.get(), + outcome=outcome, + error=error, + ) + except Exception as e: + self.ap.logger.warning(f'Failed to record tool call trace: {e}') + + @staticmethod + def _trace_execution_id(query: pipeline_query.Query) -> str: + """The execution chain this call belongs to, when the caller knows it. + + ``''`` lets the emitter fall back to the task's own trace context, which + is what a Pipeline-lane tool call has. + """ + session = getattr(query, '_agent_run_session', None) + if not isinstance(session, dict): + return '' + return str(session.get('run_id') or '').strip() + async def execute_func_call( self, name: str, diff --git a/src/langbot/pkg/telemetry/execution.py b/src/langbot/pkg/telemetry/execution.py index 1e5a16a1c..8a72288f2 100644 --- a/src/langbot/pkg/telemetry/execution.py +++ b/src/langbot/pkg/telemetry/execution.py @@ -33,7 +33,7 @@ from .trace import TraceState MODES = frozenset({'pipeline', 'agent', 'event_processor', 'none'}) OUTCOMES = frozenset({'success', 'failed', 'cancelled', 'timeout', 'skipped', 'unknown'}) -FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api'}) +FAMILIES = frozenset({'platform_event', 'event_route', 'pipeline', 'runner', 'platform_api', 'tool'}) ORIGINS = frozenset({'platform', 'webui', 'api'}) CLOSED_BY = frozenset({'event_done', 'pipeline_done', 'runner_done', 'timeout'}) CHAIN_FAILURES = frozenset({'failed', 'cancelled', 'timeout'}) @@ -255,9 +255,9 @@ class ExecutionCounters: state = trace_mod.current() if state is None: # Cross-task work has no inherited context; resolve the owning - # execution through the registry. It has no open step here, so - # its node becomes a root of that execution instead of being - # mirrored from another task's stack. + # execution through the registry. It owns no step here, so the + # record attaches to the innermost step the chain has open - + # the step that caused this work. lookup = str(execution_id or '').strip() or current_execution_id() if lookup: state = self._resolve(lookup) diff --git a/src/langbot/pkg/telemetry/trace.py b/src/langbot/pkg/telemetry/trace.py index fb4d18031..6ae8da2d1 100644 --- a/src/langbot/pkg/telemetry/trace.py +++ b/src/langbot/pkg/telemetry/trace.py @@ -11,12 +11,17 @@ small execution-scoped context the emitter needs: * the routing/run identity pinned around the block that produced a node; * the open-step stack, which lives in a ContextVar holding *one task's own immutable tuple*. Child tasks inherit a snapshot at creation; no shared - mutable stack and no cross-task mirror exists, so sibling tasks can never - corrupt each other's parent links. + mutable stack exists, so sibling tasks can never corrupt each other's parent + links. Cross-task callers (plugin/RPC work with no inherited context) resolve their execution through the owner's execution-id registry instead, which only -carries the execution state - never node payloads. +carries the execution state - never node payloads. Such a caller owns no step +of its own, yet its work is still caused by a step of the chain: the sandbox +tool a Runner invoked, the platform API a plugin called back over RPC. The +execution state therefore also keeps the ordered set of steps any task +currently has open, and a record emitted with no open step of its own attaches +to the innermost one of those instead of becoming a root of the chain. """ from __future__ import annotations @@ -77,6 +82,7 @@ class TraceState: 'pipeline_plugins', 'node_adapter', 'node_runner', + 'open_frames', ) def __init__(self, execution_id: str | None = None) -> None: @@ -110,6 +116,10 @@ class TraceState: # First adapter/runner seen on a node, used for the chain record. self.node_adapter = '' self.node_runner = '' + # Steps currently open anywhere in this chain, in the order they were + # opened. A record emitted by a task that owns no step of its own + # attaches to the innermost entry here. + self.open_frames: dict[str, NodeFrame] = {} def allocate_seq(self) -> int: """Mint the next chain-local node order.""" @@ -117,6 +127,17 @@ class TraceState: self.node_seq += 1 return seq + def open_lane(self) -> str: + """The innermost step open in this chain, whichever task opened it.""" + frames = list(self.open_frames) + return frames[-1] if frames else '' + + def open_step(self, frame: NodeFrame) -> None: + self.open_frames[frame.node_id] = frame + + def close_step(self, frame: NodeFrame) -> None: + self.open_frames.pop(frame.node_id, None) + def note_node( self, *, @@ -221,8 +242,10 @@ def resolve_node( A record stamped with a ``node`` the calling task has open reuses that step's identity, order and start time; anything else (a plain observation, - or a node opened by another task) is a fresh node under this task's - innermost open step. + or a node opened by another task) is a fresh node under the innermost step + open in this task. A task that inherited no step at all - plugin/RPC work + that only knows the execution - attaches to the innermost step the chain + has open, which is the step that caused the work. """ stack = _node.get() if node: @@ -231,7 +254,7 @@ def resolve_node( if frame.node_id == node: parent = stack[index - 1].node_id if index > 0 else '' return frame.node_id, frame.seq, frame.started_at, parent - parent = stack[-1].node_id if stack else '' + parent = stack[-1].node_id if stack else state.open_lane() return node or uuid4().hex, state.allocate_seq(), fallback_started_at, parent @@ -250,9 +273,11 @@ def stage_scope() -> typing.Iterator[str]: return frame = NodeFrame(uuid4().hex, state.allocate_seq(), _now()) token = _node.set(_node.get() + (frame,)) + state.open_step(frame) try: yield frame.node_id finally: + state.close_step(frame) try: _node.reset(token) except (ValueError, RuntimeError): diff --git a/tests/unit_tests/api/service/test_monitoring_executions.py b/tests/unit_tests/api/service/test_monitoring_executions.py new file mode 100644 index 000000000..cd8d767ec --- /dev/null +++ b/tests/unit_tests/api/service/test_monitoring_executions.py @@ -0,0 +1,350 @@ +"""Execution-view tests for the monitoring service. + +Covers the unified agent-run + pipeline-query execution list, its rollup +summary, and single-execution traces. +""" + +from __future__ import annotations + +import datetime +from types import SimpleNamespace + +import pytest +from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine +from sqlalchemy.orm import sessionmaker + +from langbot.pkg.api.http.service.monitoring import MonitoringService +from langbot.pkg.entity.persistence import agent_interaction as persistence_interaction +from langbot.pkg.entity.persistence import agent_run as persistence_agent_run +from langbot.pkg.entity.persistence import monitoring as persistence_monitoring +from langbot.pkg.entity.persistence import workspace as persistence_workspace # noqa: F401 +from langbot.pkg.entity.persistence.base import Base + +WORKSPACE = 'workspace-1' + + +class FakePersistenceManager: + """Runs statements against a real in-memory SQLite engine.""" + + def __init__(self, engine): + self._engine = engine + self._factory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + def get_db_engine(self): + return self._engine + + async def execute_async(self, statement): + # Match PersistenceManager's connection-level result shape. + async with self._engine.connect() as connection: + return await connection.execute(statement) + + @staticmethod + def serialize_model(model, data): + out = {} + for column in model.__table__.columns: + value = getattr(data, column.name) + out[column.name] = value.isoformat() if isinstance(value, datetime.datetime) else value + return out + + +@pytest.fixture +async def engine(tmp_path): + engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "monitoring_executions.db"}') + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield engine + await engine.dispose() + + +@pytest.fixture +def service(engine): + ap = SimpleNamespace( + persistence_mgr=FakePersistenceManager(engine), + instance_config=None, + ) + return MonitoringService(ap) + + +def _dt(hour: int, minute: int = 0, second: int = 0) -> datetime.datetime: + return datetime.datetime(2026, 1, 1, hour, minute, second) + + +async def _seed(engine) -> None: + factory = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + async with factory() as session: + session.add_all( + [ + persistence_agent_run.AgentRun( + run_id='run-done', + event_id='evt-1', + agent_id='agent-1', + binding_id='agent_agent-1_default', + runner_id='runner-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + status='completed', + created_at=_dt(10), + started_at=_dt(10, 0, 1), + finished_at=_dt(10, 0, 3), + usage_json='{"total_tokens": 42}', + ), + persistence_agent_run.AgentRun( + run_id='run-failed', + binding_id='agent_agent-1_default', + runner_id='runner-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + status='failed', + created_at=_dt(10, 5), + ), + persistence_agent_run.AgentRun( + run_id='run-debug', + event_id='debug:agent-1:e1', + agent_id='agent-1', + binding_id='debug:agent-1:runner-1', + runner_id='runner-1', + workspace_id=WORKSPACE, + bot_id='bot-1', + status='completed', + created_at=_dt(10, 6), + started_at=_dt(10, 6, 0), + finished_at=_dt(10, 6, 1), + ), + persistence_agent_run.AgentRunEvent( + run_id='run-done', + sequence=1, + type='run.completed', + data_json='{"message": {"role": "assistant", "content": "hi"}}', + ), + persistence_interaction.AgentInteraction( + interaction_id='int-1', + run_id='run-done', + binding_id='agent_agent-1_default', + runner_id='runner-1', + processor_type='agent', + processor_id='agent-1', + workspace_id=WORKSPACE, + status='pending', + request_json='{}', + callback_token_hash='hash-1', + ), + ] + ) + for message in [ + dict( + id='msg-user', + timestamp=_dt(10, 2), + session_id='session-1', + message_content='{"chain": [{"type": "Plain", "text": "hello there"}]}', + status='success', + role='user', + ), + dict( + id='msg-assistant', + timestamp=_dt(10, 3), + session_id='session-1', + message_content='{"chain": [{"type": "Plain", "text": "reply"}]}', + status='success', + role='assistant', + ), + dict( + id='msg-discarded', + timestamp=_dt(10, 4), + session_id='session-2', + message_content='{"chain": [{"type": "Plain", "text": "ignored"}]}', + status='discarded', + role='user', + ), + ]: + session.add( + persistence_monitoring.MonitoringMessage( + workspace_uuid=WORKSPACE, + bot_id='bot-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + level='info', + **message, + ) + ) + session.add_all( + [ + persistence_monitoring.MonitoringLLMCall( + id='llm-1', + workspace_uuid=WORKSPACE, + timestamp=_dt(10, 2, 30), + model_name='gpt', + input_tokens=10, + output_tokens=20, + total_tokens=30, + duration=100, + status='success', + bot_id='bot-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + session_id='session-1', + message_id='msg-user', + ), + persistence_monitoring.MonitoringLLMCall( + id='llm-2', + workspace_uuid=WORKSPACE, + timestamp=_dt(10, 4, 30), + model_name='gpt', + input_tokens=0, + output_tokens=0, + total_tokens=0, + duration=50, + status='success', + bot_id='bot-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + session_id='session-2', + message_id='msg-discarded', + ), + ] + ) + session.add( + persistence_monitoring.MonitoringSession( + workspace_uuid=WORKSPACE, + bot_id='bot-1', + session_id='session-1', + bot_name='Bot One', + pipeline_id='pipeline-1', + pipeline_name='Pipeline One', + message_count=2, + start_time=_dt(10, 2), + last_activity=_dt(10, 5), + is_active=True, + ) + ) + await session.commit() + + +@pytest.mark.asyncio +async def test_executions_merge_and_order_both_sources(engine, service): + await _seed(engine) + + result = await service.get_executions(WORKSPACE, limit=50) + + assert result['total'] == 5 + ids = [(item['source'], item['id']) for item in result['items']] + # Assistant replies are outputs, not executions. + assert ('pipeline', 'msg-assistant') not in ids + # Newest first across both object types. + assert ids == [ + ('agent', 'run-debug'), + ('agent', 'run-failed'), + ('pipeline', 'msg-discarded'), + ('pipeline', 'msg-user'), + ('agent', 'run-done'), + ] + + by_id = {item['id']: item for item in result['items']} + assert result['summary']['executions']['total'] == 5 + assert result['summary']['executions']['denominator'] == 4 + assert result['summary']['tokens']['calls'] == 2 + assert result['summary']['tokens']['calls_with_usage'] == 1 + assert by_id['run-done']['status_group'] == 'completed' + assert by_id['run-failed']['status_group'] == 'failed' + assert by_id['run-debug']['debug'] is True + assert by_id['run-done']['debug'] is False + assert by_id['run-done']['usage'] == {'total_tokens': 42} + assert by_id['run-done']['duration_ms'] == 2000 + assert by_id['msg-user']['status_group'] == 'completed' + assert by_id['msg-discarded']['status_group'] == 'ignored' + assert by_id['msg-user']['title'] == 'hello there' + assert by_id['msg-user']['target_id'] == 'pipeline-1' + + +@pytest.mark.asyncio +async def test_executions_source_status_and_paging(engine, service): + await _seed(engine) + + only_agents = await service.get_executions(WORKSPACE, source='agent') + assert {item['source'] for item in only_agents['items']} == {'agent'} + assert only_agents['total'] == 3 + + pipeline_only = await service.get_executions(WORKSPACE, source='pipeline') + assert {item['source'] for item in pipeline_only['items']} == {'pipeline'} + assert pipeline_only['total'] == 2 + + # A single-object target selection narrows the union to that source. + by_pipeline = await service.get_executions(WORKSPACE, pipeline_ids=['pipeline-1']) + assert {item['source'] for item in by_pipeline['items']} == {'pipeline'} + + failed = await service.get_executions(WORKSPACE, statuses=['failed']) + assert [item['id'] for item in failed['items']] == ['run-failed'] + + # `ignored` only exists for pipeline queries: it must not widen the agent + # filter back to "every status". + ignored = await service.get_executions(WORKSPACE, statuses=['ignored']) + assert [item['id'] for item in ignored['items']] == ['msg-discarded'] + + debug_only = await service.get_executions(WORKSPACE, mode='debug') + assert [item['id'] for item in debug_only['items']] == ['run-debug'] + + first_page = await service.get_executions(WORKSPACE, limit=2) + assert len(first_page['items']) == 2 + assert first_page['has_more'] is True + second_page = await service.get_executions(WORKSPACE, limit=2, offset=2) + assert first_page['items'][0]['id'] != second_page['items'][0]['id'] + + +@pytest.mark.asyncio +async def test_execution_summary_uses_explicit_denominator(engine, service): + await _seed(engine) + + summary = (await service.get_executions(WORKSPACE))['summary'] + executions = summary['executions'] + + # completed = run-done + run-debug + msg-user; failed = run-failed. + assert executions['by_source'] == {'agent': 3, 'pipeline': 2} + assert executions['completed'] == 3 + assert executions['failed'] == 1 + assert executions['ignored'] == 1 + assert executions['denominator'] == 4 + assert executions['success_rate'] == 75.0 + assert executions['debug'] == 1 + assert executions['real'] == 4 + assert executions['waiting'] == 1 + assert executions['p50_duration_ms'] == 1000 + assert executions['p95_duration_ms'] == 2000 + # Only run-done and run-debug are completed runs with a start and a finish. + assert executions['duration_sample'] == 2 + + # Token coverage is explicit: llm-2 succeeded without usage. + assert summary['tokens'] == {'total_tokens': 30, 'calls': 2, 'calls_with_usage': 1} + + +@pytest.mark.asyncio +async def test_overview_metrics_keep_message_rollup(engine, service): + await _seed(engine) + + overview = await service.get_overview_metrics(WORKSPACE) + + assert overview['total_messages'] == 3 + assert overview['llm_calls'] == 2 + assert overview['model_calls'] == 2 + assert overview['active_sessions'] == 1 + + +@pytest.mark.asyncio +async def test_execution_detail_for_agent_and_pipeline(engine, service): + await _seed(engine) + + agent_detail = await service.get_execution_detail(WORKSPACE, 'agent', 'run-done') + assert agent_detail['run']['run_id'] == 'run-done' + assert agent_detail['events'][0]['type'] == 'run.completed' + + pipeline_detail = await service.get_execution_detail(WORKSPACE, 'pipeline', 'msg-user') + assert pipeline_detail['message']['id'] == 'msg-user' + assert [call['id'] for call in pipeline_detail['llm_calls']] == ['llm-1'] + assert pipeline_detail['tool_calls'] == [] + + with pytest.raises(ValueError): + await service.get_execution_detail(WORKSPACE, 'agent', 'missing-run') + + with pytest.raises(ValueError): + await service.get_execution_detail(WORKSPACE, 'pipeline', 'missing-message') diff --git a/tests/unit_tests/provider/test_tool_manager.py b/tests/unit_tests/provider/test_tool_manager.py index d404400a4..0e81ae5e2 100644 --- a/tests/unit_tests/provider/test_tool_manager.py +++ b/tests/unit_tests/provider/test_tool_manager.py @@ -7,6 +7,8 @@ Tests cover: from __future__ import annotations +import asyncio +import contextvars from importlib import import_module from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -380,6 +382,127 @@ class TestToolManagerExecuteFuncCall: assert 'MCP resource agent reads are disabled' in record.await_args.kwargs['error_message'] +class _RecordingTelemetrySender: + """Telemetry sender stand-in that keeps every payload it was handed.""" + + def __init__(self): + self.telemetry_config = {'url': 'https://space.example.test', 'execution_trace': 'all'} + self.sent: list[dict] = [] + + async def send(self, payload: dict) -> bool: + self.sent.append(payload) + return True + + +class TestToolCallTracing: + """Every tool call a Run makes is its own step of the execution chain.""" + + @pytest.fixture + def traced_app(self): + execution = import_module('langbot.pkg.telemetry.execution') + counters = execution.ExecutionCounters(_RecordingTelemetrySender()) + app = Mock() + app.logger = Mock() + app.telemetry = SimpleNamespace(execution=counters) + # Tool-call monitoring is asserted by the tests above; this path only + # needs the execution chain. + app.monitoring_service = None + + def _inert_loader(): + loader = Mock() + loader.has_tool = AsyncMock(return_value=False) + loader.invoke_tool = AsyncMock(return_value=None) + return loader + + app._native_loader = _inert_loader() + app._plugin_loader = _inert_loader() + app._mcp_loader = _inert_loader() + app._skill_loader = _inert_loader() + return app, counters + + @pytest.fixture + def sample_query(self): + query = Mock(spec=pipeline_query.Query) + query._execution_context = _CONTEXT + return query + + @staticmethod + def _manager(app): + """ToolManager wired to app-local inert loaders.""" + manager = get_toolmgr_module().ToolManager(app) + manager.native_tool_loader = app._native_loader + manager.plugin_tool_loader = app._plugin_loader + manager.mcp_tool_loader = app._mcp_loader + manager.skill_tool_loader = app._skill_loader + return manager + + @staticmethod + def _tool_nodes(counters): + return [record for record in counters.records if record['event_type'] == 'execution_node'] + + @pytest.mark.asyncio + async def test_a_tool_call_becomes_a_node_under_the_step_that_issued_it(self, traced_app, sample_query): + trace = import_module('langbot.pkg.telemetry.trace') + execution = import_module('langbot.pkg.telemetry.execution') + app, counters = traced_app + manager = self._manager(app) + + binding = execution.bind_trace(app, 'exec-tool') + # The Runtime's RPC task executes the call with no inherited trace + # context; the Run id on the query is what resolves the owning chain. + sample_query._agent_run_session = {'run_id': 'exec-tool'} + try: + with trace.stage_scope() as lane: + foreign = contextvars.Context() + task = asyncio.get_running_loop().create_task( + manager._invoke_tool_with_monitoring( + source='native', + name='exec', + parameters={'command': 'ls'}, + query=sample_query, + invoke=AsyncMock(return_value='sandbox output'), + ), + context=foreign, + ) + assert await task == 'sandbox output' + finally: + trace.unbind_root(binding) + + rows = self._tool_nodes(counters) + assert len(rows) == 1 + assert rows[0]['family'] == 'tool' + assert rows[0]['operation'] == 'exec' + assert rows[0]['adapter'] == 'native' + assert rows[0]['outcome'] == 'success' + assert rows[0]['parent_node_id'] == lane + + @pytest.mark.asyncio + async def test_a_failed_tool_call_carries_its_reason(self, traced_app, sample_query): + execution = import_module('langbot.pkg.telemetry.execution') + trace = import_module('langbot.pkg.telemetry.trace') + app, counters = traced_app + manager = self._manager(app) + + binding = execution.bind_trace(app, 'exec-tool-failed') + try: + with pytest.raises(RuntimeError): + await manager._invoke_tool_with_monitoring( + source='mcp', + name='read_resource', + parameters={}, + query=sample_query, + invoke=AsyncMock(side_effect=RuntimeError('server unreachable')), + ) + finally: + trace.unbind_root(binding) + + rows = self._tool_nodes(counters) + assert len(rows) == 1 + assert rows[0]['family'] == 'tool' + assert rows[0]['outcome'] == 'failed' + assert rows[0]['error'] == 'server unreachable' + + class TestToolManagerSourceResolution: @pytest.mark.asyncio @pytest.mark.parametrize('source', ['mcp', 'plugin']) diff --git a/tests/unit_tests/telemetry/test_trace.py b/tests/unit_tests/telemetry/test_trace.py index 95b9f06f1..381b0ee6b 100644 --- a/tests/unit_tests/telemetry/test_trace.py +++ b/tests/unit_tests/telemetry/test_trace.py @@ -344,10 +344,53 @@ class TestTraceNodeScope: lane_row, foreign_row = nodes(counters) assert lane_row['node_id'] == lane and lane_row['parent_node_id'] == '' and lane_row['root'] is True - # The cross-task node joins the same execution, but never mirrors another - # task's open-step stack: it is a root of the chain. + # The cross-task node joins the same execution and hangs off the step the + # chain had open - the step that caused the work - instead of mirroring + # any task's own stack. assert foreign_row['event_id'] == 'exec-cross' - assert foreign_row['parent_node_id'] == '' + assert foreign_row['parent_node_id'] == lane + assert foreign_row['root'] is False + + def test_cross_task_observation_attaches_to_the_innermost_open_step(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + binding = execution.bind_trace(make_ap(counters), 'exec-nested') + try: + with trace.stage_scope() as outer: + counters.record(CONTEXT, **{**STAGE, 'node': outer}) + with trace.stage_scope() as inner: + counters.record(CONTEXT, **{**STAGE, 'node': inner}) + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-nested') + foreign.run(counters.record, CONTEXT, **{**STAGE, 'family': 'tool', 'operation': 'exec'}) + finally: + trace.unbind_root(binding) + + outer_row, inner_row, tool_row = nodes(counters) + assert outer_row['parent_node_id'] == '' + assert inner_row['parent_node_id'] == outer + # The lane that caused the cross-task work is the innermost one open. + assert tool_row['parent_node_id'] == inner + + def test_cross_task_observation_after_the_lane_closed_is_a_root(self): + trace, execution = get_modules() + _, counters = make_counters(trace_config()) + binding = execution.bind_trace(make_ap(counters), 'exec-late') + try: + with trace.stage_scope() as lane: + counters.record(CONTEXT, **{**STAGE, 'node': lane}) + # A late callback arrives after the owning step returned: nothing is + # open anymore, so the node is a root of the chain rather than a + # child of a finished step. + foreign = contextvars.Context() + foreign.run(execution.set_execution_id, 'exec-late') + foreign.run(counters.record, CONTEXT, family='platform_api', operation='reply', mode='pipeline') + finally: + trace.unbind_root(binding) + + lane_row, late_row = nodes(counters) + assert late_row['parent_node_id'] == '' and late_row['root'] is False + assert lane_row['root'] is True def test_cross_task_observation_without_an_open_step_has_no_parent(self): trace, execution = get_modules() diff --git a/web/src/app/home/monitoring/components/ConversationTurnList.tsx b/web/src/app/home/monitoring/components/ConversationTurnList.tsx index afcaa1388..a1e61b03e 100644 --- a/web/src/app/home/monitoring/components/ConversationTurnList.tsx +++ b/web/src/app/home/monitoring/components/ConversationTurnList.tsx @@ -12,6 +12,7 @@ import { Wrench, } from 'lucide-react'; import { cn } from '@/lib/utils'; +import { Button } from '@/components/ui/button'; import { MessageContentRenderer } from './MessageContentRenderer'; import { ConversationTurn, @@ -507,10 +508,11 @@ export function ConversationTurnList({ key={call.id} className="border-t border-border/70 py-2 first:border-t-0 first:pt-0 last:pb-0" > - + {hasToolDetails && expandedToolCall && (
{exporting ? ( <> - - - - + {t('monitoring.export.exporting')} ) : ( diff --git a/web/src/app/home/monitoring/components/FeedbackCard.tsx b/web/src/app/home/monitoring/components/FeedbackCard.tsx index c2e93e52b..b5882b7f1 100644 --- a/web/src/app/home/monitoring/components/FeedbackCard.tsx +++ b/web/src/app/home/monitoring/components/FeedbackCard.tsx @@ -9,6 +9,7 @@ import { Heart, Smile, } from 'lucide-react'; +import { MetricCard } from './overview-cards/MetricCard'; interface FeedbackCardProps { title: string; @@ -23,6 +24,18 @@ interface FeedbackCardProps { loading?: boolean; } +// The variant only tints the icon tile; the card surface stays neutral so the +// feedback row matches the token-monitoring cards. +const variantAccents: Record< + NonNullable, + string +> = { + default: '#64748b', + success: '#10b981', + warning: '#f59e0b', + danger: '#ef4444', +}; + export function FeedbackCard({ title, value, @@ -32,89 +45,37 @@ export function FeedbackCard({ variant = 'default', loading = false, }: FeedbackCardProps) { - const variantStyles = { - default: 'bg-white dark:bg-[#2a2a2e] border-gray-200 dark:border-gray-700', - success: - 'bg-green-50 dark:bg-green-900/20 border-green-200 dark:border-green-800', - warning: - 'bg-yellow-50 dark:bg-yellow-900/20 border-yellow-200 dark:border-yellow-800', - danger: 'bg-red-50 dark:bg-red-900/20 border-red-200 dark:border-red-800', - }; - - const iconStyles = { - default: 'text-gray-500 dark:text-gray-400', - success: 'text-green-500 dark:text-green-400', - warning: 'text-yellow-500 dark:text-yellow-400', - danger: 'text-red-500 dark:text-red-400', - }; - const trendStyles = { up: 'text-green-500', - down: 'text-red-500', - neutral: 'text-gray-500', + down: 'text-destructive', + neutral: 'text-muted-foreground', }; - if (loading) { - return ( -
-
-
-
-
-
-
-
-
-
- ); - } - return ( -
-
-
-

- {title} -

-

- {value} -

- {subtitle && ( -

- {subtitle} -

- )} - {trend && ( -
- {trend.direction === 'up' && ( - - )} - {trend.direction === 'down' && ( - - )} - {trend.direction === 'neutral' && ( - - )} - - {trend.value > 0 ? '+' : ''} - {trend.value}% - -
- )} -
+ {trend && (
- {icon} + {trend.direction === 'up' && } + {trend.direction === 'down' && ( + + )} + {trend.direction === 'neutral' && } + + {trend.value > 0 ? '+' : ''} + {trend.value}% +
-
-
+ )} + ); } @@ -135,35 +96,37 @@ export function FeedbackStatsCards({ stats, loading }: FeedbackStatsProps) { { title: t('monitoring.feedback.totalFeedback'), value: stats?.totalFeedback ?? 0, - icon: , + icon: , variant: 'default' as const, }, { title: t('monitoring.feedback.totalLikes'), value: stats?.totalLikes ?? 0, - icon: , + icon: , variant: 'success' as const, }, { title: t('monitoring.feedback.totalDislikes'), value: stats?.totalDislikes ?? 0, - icon: , + icon: , variant: 'danger' as const, }, { title: t('monitoring.feedback.satisfactionRate'), value: stats ? `${stats.satisfactionRate}%` : '0%', - icon: , + icon: , + // The rate is the one feedback metric that is good or bad on its own, so + // its tile keeps encoding the verdict (as it did before the restyle). variant: (stats && stats.satisfactionRate >= 80 ? 'success' : stats && stats.satisfactionRate >= 50 ? 'warning' - : 'danger') as 'default' | 'success' | 'warning' | 'danger', + : 'danger') as 'success' | 'warning' | 'danger', }, ]; return ( -
+
{cards.map((card, index) => ( - -
- ); + return ; } if (!feedback || feedback.length === 0) { return ( -
- -

- {t('monitoring.feedback.noFeedback')} -

-

- {t('monitoring.feedback.noFeedbackDescription')} -

-
+ } + title={t('monitoring.feedback.noFeedback')} + hint={t('monitoring.feedback.noFeedbackDescription')} + /> ); } return (
{feedback.map((item) => ( -
{item.botName && ( <> - → - + → + {item.botName} )} {item.platform && ( - + {item.platform} - + )} {item.streamId && onViewMessage && ( - ))} -
+ { + if (value === 'hour' || value === 'day') setBucket(value); + }} + > + + {(['hour', 'day'] as const).map((b) => ( + + {t(`monitoring.tokens.bucket.${b}`)} + + ))} + +
@@ -372,93 +334,86 @@ export default function TokenMonitoring({
-
+ {/* Per-model breakdown */} -
+

{t('monitoring.tokens.byModel')}

-
- - - - - - - - - - - - - - {by_model.map((m) => { - const share = - summary.total_tokens > 0 - ? (m.total_tokens / summary.total_tokens) * 100 - : 0; - return ( - - - - - - - - - - ); - })} - -
- {t('monitoring.tokens.model')} - - {t('monitoring.tokens.calls')} - - {t('monitoring.tokens.inputTokens')} - - {t('monitoring.tokens.outputTokens')} - - {t('monitoring.tokens.totalTokens')} - - {t('monitoring.tokens.avgPerCall')} - - {t('monitoring.tokens.avgLatency')} -
-
- {m.model_name} -
-
-
-
-
- {m.calls} - {m.error_calls > 0 && ( - - {' '} - ({m.error_calls}✕) - - )} - - {formatNumber(m.input_tokens)} - - {formatNumber(m.output_tokens)} - - {formatNumber(m.total_tokens)} - - {formatNumber(m.avg_tokens_per_call)} - - {m.avg_duration_ms}ms -
-
-
+ + + + {t('monitoring.tokens.model')} + + {t('monitoring.tokens.calls')} + + + {t('monitoring.tokens.inputTokens')} + + + {t('monitoring.tokens.outputTokens')} + + + {t('monitoring.tokens.totalTokens')} + + + {t('monitoring.tokens.avgPerCall')} + + + {t('monitoring.tokens.avgLatency')} + + + + + {by_model.map((m) => { + const share = + summary.total_tokens > 0 + ? (m.total_tokens / summary.total_tokens) * 100 + : 0; + return ( + + +
+ {m.model_name} +
+
+
+
+ + + {m.calls} + {m.error_calls > 0 && ( + + {' '} + ({m.error_calls}✕) + + )} + + + {formatNumber(m.input_tokens)} + + + {formatNumber(m.output_tokens)} + + + {formatNumber(m.total_tokens)} + + + {formatNumber(m.avg_tokens_per_call)} + + + {m.avg_duration_ms}ms + + + ); + })} + +
+
); } diff --git a/web/src/app/home/monitoring/components/executions/ExecutionDetailSheet.tsx b/web/src/app/home/monitoring/components/executions/ExecutionDetailSheet.tsx new file mode 100644 index 000000000..f0ad3c947 --- /dev/null +++ b/web/src/app/home/monitoring/components/executions/ExecutionDetailSheet.tsx @@ -0,0 +1,361 @@ +import { useEffect, useRef, useState } from 'react'; +import { useTranslation } from 'react-i18next'; +import { Link } from 'react-router-dom'; +import { ExternalLink } from 'lucide-react'; +import { + Sheet, + SheetContent, + SheetDescription, + SheetHeader, + SheetTitle, +} from '@/components/ui/sheet'; +import { Badge } from '@/components/ui/badge'; +import { Button } from '@/components/ui/button'; +import { Skeleton } from '@/components/ui/skeleton'; +import { ScrollArea } from '@/components/ui/scroll-area'; +import { Separator } from '@/components/ui/separator'; +import { backendClient } from '@/app/infra/http'; +import type { + ExecutionCallRow, + ExecutionDetail, + ExecutionRow, +} from '@/app/infra/entities/api/monitoring-executions'; +import AgentExecutionTrace from '@/app/home/agents/components/AgentExecutionTrace'; +import { formatRunDuration } from '@/app/home/agents/components/processor-run-timing'; +import { formatDateTime } from '../../utils/dateUtils'; +import { + executionKindLabel, + executionProcessorLabel, + executionStatusLabel, + executionStatusVariant, + executionTargetLabel, +} from './ExecutionTable'; + +interface ExecutionDetailSheetProps { + row: ExecutionRow | null; + onClose: () => void; +} + +/** Walk parsed JSON, collecting every `"text"` value (OpenAI-style content). */ +function collectText(value: unknown): string { + if (typeof value === 'string') return value; + if (Array.isArray(value)) { + return value.map(collectText).filter(Boolean).join(' '); + } + if (value && typeof value === 'object') { + const record = value as Record; + if (typeof record.text === 'string') return record.text; + return Object.values(record).map(collectText).filter(Boolean).join(' '); + } + return ''; +} + +function messagePreview(raw: string | null | undefined): string { + if (!raw) return ''; + try { + return collectText(JSON.parse(raw)).trim() || raw; + } catch { + return raw; + } +} + +function callTitle(call: ExecutionCallRow, index: number): string { + const candidates = [ + call.model_name, + call.model, + call.name, + call.tool_name, + call.function, + call.error_type, + ]; + for (const candidate of candidates) { + if (typeof candidate === 'string' && candidate) return candidate; + } + return typeof call.id === 'string' && call.id ? call.id : `#${index + 1}`; +} + +export default function ExecutionDetailSheet({ + row, + onClose, +}: ExecutionDetailSheetProps) { + const { t } = useTranslation(); + const [detail, setDetail] = useState(null); + const [loading, setLoading] = useState(false); + const [error, setError] = useState(false); + const requestRef = useRef(0); + + useEffect(() => { + if (!row) { + setDetail(null); + setError(false); + setLoading(false); + return; + } + const requestId = requestRef.current + 1; + requestRef.current = requestId; + setLoading(true); + setError(false); + setDetail(null); + backendClient + .getExecutionDetail(row.source, row.id) + .then((result) => { + if (requestRef.current !== requestId) return; + setDetail(result); + }) + .catch(() => { + if (requestRef.current !== requestId) return; + setError(true); + }) + .finally(() => { + if (requestRef.current === requestId) setLoading(false); + }); + }, [row]); + + const isAgent = + detail != null ? detail.source === 'agent' : row?.source === 'agent'; + + const metadata: { label: string; value: string }[] = []; + if (row) { + metadata.push({ + label: t('monitoring.execution.detail.executionId'), + value: row.id, + }); + if (row.event_id) { + metadata.push({ + label: t('monitoring.execution.detail.eventId'), + value: row.event_id, + }); + } + if (row.runner_id) { + metadata.push({ + label: t('monitoring.execution.detail.runner'), + value: row.runner_id, + }); + } + if (row.queue_name) { + metadata.push({ + label: t('monitoring.execution.detail.queue'), + value: row.queue_name, + }); + } + if (row.created_at_ms != null) { + metadata.push({ + label: t('monitoring.execution.detail.created'), + value: formatDateTime(new Date(row.created_at_ms)), + }); + } + if (row.duration_ms != null) { + metadata.push({ + label: t('monitoring.execution.detail.duration'), + value: formatRunDuration(row.duration_ms), + }); + } + if (isAgent && row.usage?.total_tokens != null) { + metadata.push({ + label: t('monitoring.execution.detail.usage'), + value: row.usage.total_tokens.toLocaleString(), + }); + } + } + + const objectHref = row + ? row.target_kind === 'agent' && row.target_id + ? `/home/agents?id=${encodeURIComponent(row.target_id)}` + : row.target_kind === 'pipeline' && row.target_id + ? `/home/pipelines?id=${encodeURIComponent(row.target_id)}` + : null + : null; + + const renderCalls = (title: string, calls: ExecutionCallRow[]) => ( +
+

{title}

+ {calls.length === 0 ? ( +

+ {t('monitoring.execution.detail.noEvents')} +

+ ) : ( +
+ {calls.map((call, index) => { + const duration = + typeof call.duration === 'number' ? call.duration : null; + return ( +
+
+
{callTitle(call, index)}
+ {call.error_message && ( +
+ {call.error_message} +
+ )} +
+
+ {call.status && ( + + {call.status} + + )} + + {duration == null ? '—' : formatRunDuration(duration)} + +
+
+ ); + })} +
+ )} +
+ ); + + return ( + { + if (!open) onClose(); + }} + > + + + {t('monitoring.execution.detail.a11yDescription')} + + +
+ {row && ( + <> + + + {executionProcessorLabel(row)} + +

+ {t('monitoring.execution.triggeredBy', { + trigger: executionTargetLabel(row, t), + })} +

+
+ + {executionKindLabel(row, t)} + + + {executionStatusLabel(row, t)} + + {row.debug && ( + + {t('monitoring.execution.debug')} + + )} + {objectHref && ( + + )} +
+ + +
+ {metadata.map((item) => ( +
+
+ {item.label} +
+
+ {item.value} +
+
+ ))} +
+ + + + {isAgent ? ( +
+

+ {t('monitoring.execution.detail.timeline')} +

+ {loading && ( +
+ + +
+ )} + {!loading && error && ( +

+ {t('monitoring.execution.detail.loadError')} +

+ )} + {!loading && + !error && + detail?.source === 'agent' && + (detail.events.length > 0 ? ( + + ) : ( +

+ {t('monitoring.execution.detail.noEvents')} +

+ ))} +
+ ) : ( +
+ {loading && ( +
+ + +
+ )} + {!loading && error && ( +

+ {t('monitoring.execution.detail.loadError')} +

+ )} + {!loading && !error && detail?.source === 'pipeline' && ( + <> +
+

+ {t('monitoring.execution.detail.query')} +

+

+ {messagePreview(detail.message.message_content) || + '—'} +

+
+ {renderCalls( + t('monitoring.execution.detail.modelCalls'), + detail.llm_calls, + )} + {renderCalls( + t('monitoring.execution.detail.toolCalls'), + detail.tool_calls, + )} + {renderCalls( + t('monitoring.execution.detail.errors'), + detail.errors, + )} + + )} +
+ )} + + )} +
+ + + + ); +} diff --git a/web/src/app/home/monitoring/components/executions/ExecutionTable.tsx b/web/src/app/home/monitoring/components/executions/ExecutionTable.tsx new file mode 100644 index 000000000..901adef5d --- /dev/null +++ b/web/src/app/home/monitoring/components/executions/ExecutionTable.tsx @@ -0,0 +1,169 @@ +import React from 'react'; +import { useTranslation } from 'react-i18next'; +import type { TFunction } from 'i18next'; +import { + Table, + TableBody, + TableCell, + TableHead, + TableHeader, + TableRow, +} from '@/components/ui/table'; +import { Badge } from '@/components/ui/badge'; +import type { ExecutionRow } from '@/app/infra/entities/api/monitoring-executions'; +import { eventPatternLabel } from '@/app/home/components/event-patterns/event-pattern-groups'; +import { formatRunDuration } from '@/app/home/agents/components/processor-run-timing'; +import { formatDateTime } from '../../utils/dateUtils'; + +interface ExecutionTableProps { + rows: ExecutionRow[]; + selectedId?: string | null; + onSelect: (row: ExecutionRow) => void; +} + +const STATUS_GROUPS = [ + 'completed', + 'failed', + 'running', + 'queued', + 'cancelled', + 'ignored', +]; + +export function executionStatusLabel(row: ExecutionRow, t: TFunction): string { + if (STATUS_GROUPS.includes(row.status_group)) { + return t(`monitoring.execution.status.${row.status_group}`); + } + return row.status; +} + +export function executionStatusVariant( + statusGroup: string, +): 'default' | 'secondary' | 'destructive' | 'outline' { + if (statusGroup === 'failed') return 'destructive'; + if (statusGroup === 'ignored' || statusGroup === 'queued') return 'secondary'; + return 'outline'; +} + +const KIND_GROUPS = ['agent', 'pipeline', 'processor']; + +export function executionKindLabel(row: ExecutionRow, t: TFunction): string { + if (KIND_GROUPS.includes(row.target_kind)) { + return t(`monitoring.execution.kind.${row.target_kind}`); + } + return row.target_kind; +} + +export function executionTargetLabel(row: ExecutionRow, t: TFunction): string { + if (row.target_kind === 'agent') { + return eventPatternLabel(row.title, t); + } + const title = row.title ?? ''; + return title.length > 48 ? `${title.slice(0, 48)}…` : title; +} + +// The processor the row ran: the Agent or Pipeline that handled it. A run whose +// Agent was deleted still names the Runner that executed it. +export function executionProcessorLabel(row: ExecutionRow): string { + return row.target_name || row.runner_id || row.target_id || '—'; +} + +export default function ExecutionTable({ + rows, + selectedId, + onSelect, +}: ExecutionTableProps) { + const { t } = useTranslation(); + + return ( + // The page hands this table a fixed-height region: the table's own scroll + // container takes that height so the header stays sticky while rows scroll + // under it. +
+ + + + {t('monitoring.execution.columns.time')} + {t('monitoring.execution.columns.target')} + {t('monitoring.execution.columns.status')} + + {t('monitoring.execution.columns.duration')} + + + {t('monitoring.execution.columns.tokens')} + + + + + {rows.map((row) => { + const selected = selectedId === row.id; + return ( + onSelect(row)} + onKeyDown={( + event: React.KeyboardEvent, + ) => { + if (event.key === 'Enter' || event.key === ' ') { + event.preventDefault(); + onSelect(row); + } + }} + > + + {row.created_at_ms == null + ? '—' + : formatDateTime(new Date(row.created_at_ms))} + + +
+
+ + {executionKindLabel(row, t)} + + + {executionProcessorLabel(row)} + +
+ + {t('monitoring.execution.triggeredBy', { + trigger: executionTargetLabel(row, t), + })} + +
+ + +
+ + {executionStatusLabel(row, t)} + + {row.debug && ( + + {t('monitoring.execution.debug')} + + )} +
+ + + {row.duration_ms == null + ? '—' + : formatRunDuration(row.duration_ms)} + + + {row.usage?.total_tokens != null + ? row.usage.total_tokens.toLocaleString() + : '—'} + + + ); + })} + +
+
+ ); +} diff --git a/web/src/app/home/monitoring/components/filters/MonitoringFilters.tsx b/web/src/app/home/monitoring/components/filters/MonitoringFilters.tsx index 6cf204e14..2470451f6 100644 --- a/web/src/app/home/monitoring/components/filters/MonitoringFilters.tsx +++ b/web/src/app/home/monitoring/components/filters/MonitoringFilters.tsx @@ -7,8 +7,13 @@ import { SelectTrigger, SelectValue, } from '@/components/ui/select'; +import { Label } from '@/components/ui/label'; import { backendClient } from '@/app/infra/http'; import { TimeRangeOption } from '../../types/monitoring'; +import type { + ExecutionModeFilter, + ExecutionSourceFilter, +} from '@/app/infra/entities/api/monitoring-executions'; interface MonitoringFiltersProps { selectedBots: string[]; @@ -17,8 +22,23 @@ interface MonitoringFiltersProps { onBotsChange: (bots: string[]) => void; onPipelinesChange: (pipelines: string[]) => void; onTimeRangeChange: (timeRange: TimeRangeOption) => void; + source?: ExecutionSourceFilter; + onSourceChange?: (value: ExecutionSourceFilter) => void; + mode?: ExecutionModeFilter; + onModeChange?: (value: ExecutionModeFilter) => void; + statusGroup?: string; + onStatusGroupChange?: (value: string) => void; } +const STATUS_GROUP_OPTIONS = [ + 'completed', + 'failed', + 'running', + 'queued', + 'cancelled', + 'ignored', +]; + interface Bot { uuid: string; name: string; @@ -36,6 +56,12 @@ export default function MonitoringFilters({ onBotsChange, onPipelinesChange, onTimeRangeChange, + source, + onSourceChange, + mode, + onModeChange, + statusGroup, + onStatusGroupChange, }: MonitoringFiltersProps) { const { t } = useTranslation(); const [bots, setBots] = useState([]); @@ -110,11 +136,98 @@ export default function MonitoringFilters({ return (
+ {/* Source Filter */} + {onSourceChange && ( +
+ + +
+ )} + + {/* Mode Filter */} + {onModeChange && ( +
+ + +
+ )} + + {/* Status Filter */} + {onStatusGroupChange && ( +
+ + +
+ )} + {/* Bot Filter */}
- - - onSourceChange(value as ExecutionSourceFilter) - } +
+ {/* Bot Filter */} +
+ + +
+ + {/* Processor Filter */} +
+ + -
- )} + ))} + + +
{/* Mode Filter */} {onModeChange && ( -
-