feat(agent-runner): enforce 4.x host-owned execution

This commit is contained in:
huanghuoguoguo
2026-07-12 20:36:32 +08:00
parent e6384aae5d
commit 99d9c227f9
171 changed files with 6958 additions and 5385 deletions
+4 -3
View File
@@ -1,4 +1,5 @@
"""Agent runner subsystem for LangBot."""
from __future__ import annotations
from .runner.descriptor import AgentRunnerDescriptor
@@ -15,7 +16,7 @@ from .runner.context_builder import AgentRunContextBuilder
from .runner.resource_builder import AgentResourceBuilder
from .runner.result_normalizer import AgentResultNormalizer
from .runner.orchestrator import AgentRunOrchestrator
from .runner.config_migration import ConfigMigration
from .runner.config_resolver import RunnerConfigResolver
__all__ = [
'AgentRunnerDescriptor',
@@ -33,5 +34,5 @@ __all__ = [
'AgentResourceBuilder',
'AgentResultNormalizer',
'AgentRunOrchestrator',
'ConfigMigration',
]
'RunnerConfigResolver',
]
+2 -2
View File
@@ -16,7 +16,7 @@ from .context_builder import AgentRunContextBuilder
from .resource_builder import AgentResourceBuilder
from .result_normalizer import AgentResultNormalizer
from .orchestrator import AgentRunOrchestrator
from .config_migration import ConfigMigration
from .config_resolver import RunnerConfigResolver
from .default_config import AgentRunnerDefaultConfigService
from .binding_resolver import AgentBindingResolver, AgentBindingResolutionError
from .session_registry import (
@@ -49,7 +49,7 @@ __all__ = [
'AgentResourceBuilder',
'AgentResultNormalizer',
'AgentRunOrchestrator',
'ConfigMigration',
'RunnerConfigResolver',
'AgentRunnerDefaultConfigService',
'AgentBindingResolver',
'AgentBindingResolutionError',
@@ -1,64 +0,0 @@
"""Helpers for the current AgentRunner config shape."""
from __future__ import annotations
import typing
class ConfigMigration:
"""Configuration helpers for the current AgentRunner shape.
Responsibilities:
- Resolve runner ID from ai.runner.id
- Extract Agent/runner config from ai.runner_config
- Read current conversation expiry settings
"""
@staticmethod
def resolve_runner_id(pipeline_config: dict[str, typing.Any]) -> str | None:
"""Resolve runner ID from current configuration.
Args:
pipeline_config: Current configuration container
Returns:
Runner ID string, or None if not configured
"""
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
runner = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
runner_id = runner.get('id') if isinstance(runner, dict) else None
return runner_id if isinstance(runner_id, str) and runner_id else None
@staticmethod
def resolve_runner_config(
pipeline_config: dict[str, typing.Any],
runner_id: str,
) -> dict[str, typing.Any]:
"""Resolve Agent/runner configuration from the current container.
Args:
pipeline_config: Current configuration container
runner_id: Resolved runner ID
Returns:
Runner configuration dict (empty if not found)
"""
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
runner_configs = ai_config.get('runner_config', {}) if isinstance(ai_config, dict) else {}
runner_config = runner_configs.get(runner_id, {}) if isinstance(runner_configs, dict) else {}
return runner_config if isinstance(runner_config, dict) else {}
@staticmethod
def get_expire_time(pipeline_config: dict[str, typing.Any]) -> int:
"""Get conversation expire time from configuration.
Args:
pipeline_config: Current configuration container
Returns:
Expire time in seconds (0 means no expiry)
"""
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
runner = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
expire_time = runner.get('expire-time', 0) if isinstance(runner, dict) else 0
return expire_time if isinstance(expire_time, int) else 0
@@ -0,0 +1,199 @@
"""Resolve the current AgentRunner configuration shape."""
from __future__ import annotations
import typing
HOST_SECURITY_BOOLEAN_FIELDS = (
'enable-all-tools',
'mcp-resource-agent-read-enabled',
)
class RunnerConfigResolver:
"""Configuration helpers for the current AgentRunner shape.
Responsibilities:
- Resolve runner ID from ai.runner.id
- Extract Agent/runner config from ai.runner_config
- Read current conversation expiry settings
- Validate the persisted 4.x Agent binding container
"""
@staticmethod
def validate_agent_config(config: typing.Any) -> dict[str, typing.Any]:
"""Validate and return the persisted 4.x Agent config container."""
if not isinstance(config, dict):
raise ValueError('Agent config must be an object')
runner = config.get('runner')
if not isinstance(runner, dict):
raise ValueError("Agent config field 'runner' must be an object")
runner_id = runner.get('id')
if not isinstance(runner_id, str):
raise ValueError("Agent config field 'runner.id' must be a string")
runner_configs = config.get('runner_config')
if not isinstance(runner_configs, dict):
raise ValueError("Agent config field 'runner_config' must be an object")
for configured_runner_id, runner_config in runner_configs.items():
if not isinstance(configured_runner_id, str) or not configured_runner_id:
raise ValueError("Agent config field 'runner_config' must use non-empty string runner IDs")
if not isinstance(runner_config, dict):
raise ValueError(f'Agent runner_config[{configured_runner_id!r}] must be an object')
if runner_id and runner_id not in runner_configs:
raise ValueError(f'Agent runner_config is missing selected runner {runner_id!r}')
if runner_id:
RunnerConfigResolver.validate_runner_security_fields(
runner_configs[runner_id],
context=f'Agent runner_config[{runner_id!r}]',
)
return config
@staticmethod
def validate_runner_security_fields(
runner_config: dict[str, typing.Any],
*,
context: str = 'Runner config',
) -> dict[str, typing.Any]:
"""Reject malformed Host-owned security toggles instead of enabling them."""
for field_name in HOST_SECURITY_BOOLEAN_FIELDS:
if field_name in runner_config and not isinstance(runner_config[field_name], bool):
raise ValueError(f'{context} field {field_name!r} must be a boolean')
RunnerConfigResolver.validate_mcp_resource_attachments(
runner_config.get('mcp-resources'),
context=context,
field_name='mcp-resources',
)
return runner_config
@staticmethod
def validate_mcp_resource_attachments(
resources: typing.Any,
*,
context: str,
field_name: str,
) -> typing.Any:
"""Validate explicit attachment enable flags shared by Agent and Pipeline inputs."""
if not isinstance(resources, list):
return resources
for index, resource in enumerate(resources):
if not isinstance(resource, dict) or 'enabled' not in resource:
continue
if not isinstance(resource['enabled'], bool):
raise ValueError(f"{context} field '{field_name}[{index}].enabled' must be a boolean")
return resources
@classmethod
def validate_pipeline_config(cls, config: typing.Any) -> dict[str, typing.Any]:
"""Validate the selected Runner container in a current 4.x Pipeline config."""
if not isinstance(config, dict):
raise ValueError('Pipeline config must be an object')
ai_config = config.get('ai')
if ai_config is None:
return config
if not isinstance(ai_config, dict):
raise ValueError("Pipeline config field 'ai' must be an object")
runner = ai_config.get('runner')
if runner is None:
return config
if not isinstance(runner, dict):
raise ValueError("Pipeline config field 'ai.runner' must be an object")
runner_id = runner.get('id')
if not isinstance(runner_id, str):
raise ValueError("Pipeline config field 'ai.runner.id' must be a string")
if not runner_id:
return config
runner_configs = ai_config.get('runner_config')
if not isinstance(runner_configs, dict):
raise ValueError("Pipeline config field 'ai.runner_config' must be an object")
if runner_id not in runner_configs:
raise ValueError(f'Pipeline runner_config is missing selected runner {runner_id!r}')
selected_config = runner_configs[runner_id]
if not isinstance(selected_config, dict):
raise ValueError(f'Pipeline runner_config[{runner_id!r}] must be an object')
cls.validate_runner_security_fields(
selected_config,
context=f'Pipeline runner_config[{runner_id!r}]',
)
return config
@staticmethod
def resolve_agent_runner_id(config: dict[str, typing.Any]) -> str | None:
"""Resolve a runner ID from a validated persisted Agent config."""
runner = config.get('runner', {})
runner_id = runner.get('id') if isinstance(runner, dict) else None
return runner_id if isinstance(runner_id, str) and runner_id else None
@classmethod
def resolve_agent_runner_config(
cls,
config: typing.Any,
) -> tuple[dict[str, typing.Any], str | None, dict[str, typing.Any]]:
"""Validate an Agent config and return its selected runner configuration."""
validated = cls.validate_agent_config(config)
runner_id = cls.resolve_agent_runner_id(validated)
runner_configs = typing.cast(dict[str, typing.Any], validated['runner_config'])
runner_config = runner_configs.get(runner_id, {}) if runner_id else {}
return validated, runner_id, typing.cast(dict[str, typing.Any], runner_config)
@staticmethod
def resolve_runner_id(pipeline_config: dict[str, typing.Any]) -> str | None:
"""Resolve runner ID from current configuration.
Args:
pipeline_config: Current configuration container
Returns:
Runner ID string, or None if not configured
"""
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
runner = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
runner_id = runner.get('id') if isinstance(runner, dict) else None
return runner_id if isinstance(runner_id, str) and runner_id else None
@staticmethod
def resolve_runner_config(
pipeline_config: dict[str, typing.Any],
runner_id: str,
) -> dict[str, typing.Any]:
"""Resolve Agent/runner configuration from the current container.
Args:
pipeline_config: Current configuration container
runner_id: Resolved runner ID
Returns:
Runner configuration dict (empty if not found)
"""
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
runner_configs = ai_config.get('runner_config', {}) if isinstance(ai_config, dict) else {}
runner_config = runner_configs.get(runner_id, {}) if isinstance(runner_configs, dict) else {}
return runner_config if isinstance(runner_config, dict) else {}
@staticmethod
def get_expire_time(pipeline_config: dict[str, typing.Any]) -> int:
"""Get conversation expire time from configuration.
Args:
pipeline_config: Current configuration container
Returns:
Expire time in seconds (0 means no expiry)
"""
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
runner = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
expire_time = runner.get('expire-time', 0) if isinstance(runner, dict) else 0
return expire_time if isinstance(expire_time, int) else 0
@@ -75,6 +75,9 @@ class ToolResource(typing.TypedDict):
tool_type: str | None
description: str | None
operations: list[str]
parameters: dict[str, typing.Any] | None
source: typing.NotRequired[str]
source_id: typing.NotRequired[str | None]
class KnowledgeBaseResource(typing.TypedDict):
@@ -7,7 +7,7 @@ import sqlalchemy
from ...core import app
from ...entity.persistence import pipeline as persistence_pipeline
from . import config_schema
from .config_migration import ConfigMigration
from .config_resolver import RunnerConfigResolver
class AgentRunnerDefaultConfigService:
@@ -53,7 +53,7 @@ class AgentRunnerDefaultConfigService:
if not isinstance(pipeline_config, dict):
return False
runner_id = ConfigMigration.resolve_runner_id(pipeline_config)
runner_id = RunnerConfigResolver.resolve_runner_id(pipeline_config)
if not runner_id:
return False
@@ -0,0 +1,319 @@
"""Host-only Query compatibility views for AgentRunner tool execution."""
from __future__ import annotations
import copy
import hashlib
import json
import typing
from langbot_plugin.api.entities.builtin.pipeline import query as pipeline_query
from langbot_plugin.api.entities.builtin.platform import events as platform_events
from langbot_plugin.api.entities.builtin.platform import message as platform_message
from langbot_plugin.api.entities.builtin.provider import message as provider_message
from langbot_plugin.api.entities.builtin.provider import session as provider_session
from ...utils import constants
from .host_models import AgentEventEnvelope
HOST_BOX_SCOPE_VARIABLE = '_host_box_scope'
AUTHORIZED_SKILLS_VARIABLE = '_pipeline_bound_skills'
MCP_RESOURCE_ATTACHMENTS_VARIABLE = '_pipeline_mcp_resource_attachments'
MCP_RESOURCE_AGENT_READ_ENABLED_VARIABLE = '_pipeline_mcp_resource_agent_read_enabled'
def project_mcp_resource_config(
query: pipeline_query.Query,
runner_config: dict[str, typing.Any],
) -> dict[str, typing.Any]:
"""Attach standard MCP resource settings to a Host execution Query."""
variables = getattr(query, 'variables', None)
if not isinstance(variables, dict):
variables = {}
query.variables = variables
attachments = runner_config.get('mcp-resources', [])
variables.setdefault(
MCP_RESOURCE_ATTACHMENTS_VARIABLE,
list(attachments) if isinstance(attachments, list) else [],
)
variables.setdefault(
MCP_RESOURCE_AGENT_READ_ENABLED_VARIABLE,
runner_config.get('mcp-resource-agent-read-enabled', True) is True,
)
return variables
async def build_mcp_resource_context_addition(
ap: typing.Any,
query: pipeline_query.Query,
) -> str:
"""Build model-facing MCP context without mutating the canonical input."""
tool_mgr = getattr(ap, 'tool_mgr', None)
if tool_mgr is None:
return ''
mcp_loader = getattr(tool_mgr, '__dict__', {}).get('mcp_tool_loader')
if mcp_loader is None:
return ''
resource_context = await mcp_loader.build_resource_context_for_query(query)
if not resource_context:
return ''
addition = (
'\n\nMCP resource context selected by LangBot host:\n'
f'{resource_context}\n\n'
'Use this context as read-only reference material. If it conflicts with the user message, '
'ask for clarification before taking external actions.'
)
return addition
def append_mcp_resource_context_to_event(event: AgentEventEnvelope, addition: str) -> None:
"""Append pinned context only to the run-scoped execution input."""
if not addition:
return
has_text = event.input.text is not None
has_structured_content = bool(event.input.contents)
if has_text:
event.input.text += addition
if has_structured_content or not has_text:
if not _append_text_to_content(event.input.contents, addition):
event.input.contents.append(provider_message.ContentElement.from_text(addition.strip()))
def _append_text_to_content(content: typing.Any, addition: str) -> bool:
if isinstance(content, str):
return False
if not isinstance(content, list):
return False
for content_element in content:
if getattr(content_element, 'type', None) == 'text':
content_element.text = (content_element.text or '') + addition
return True
content.append(provider_message.ContentElement.from_text(addition.strip()))
return True
def prepare_execution_query(
query: pipeline_query.Query,
event: AgentEventEnvelope,
authorized_skill_names: list[str],
) -> pipeline_query.Query:
"""Attach Host-owned execution metadata without changing Query identity."""
variables = prepare_box_scope(query, event, preserve_existing=True)
variables[AUTHORIZED_SKILLS_VARIABLE] = list(dict.fromkeys(authorized_skill_names))
return query
def prepare_box_scope(
query: pipeline_query.Query,
event: AgentEventEnvelope,
*,
preserve_existing: bool = False,
) -> dict[str, typing.Any]:
"""Attach the Host Box scope before any Query-side file staging."""
variables = getattr(query, 'variables', None)
if not isinstance(variables, dict):
variables = {}
query.variables = variables
existing_scope = variables.get(HOST_BOX_SCOPE_VARIABLE)
if preserve_existing and isinstance(existing_scope, str) and existing_scope.strip():
return variables
variables[HOST_BOX_SCOPE_VARIABLE] = build_host_box_scope(event, query=query)
return variables
def build_execution_query(
event: AgentEventEnvelope,
authorized_skill_names: list[str],
) -> pipeline_query.Query:
"""Build the minimum Query view required by Host-owned tool loaders."""
launcher_type, launcher_id, sender_id = _resolve_session_identity(event)
message_chain = _build_message_chain(event)
message_event = platform_events.MessageEvent(
type=event.source_event_type or event.event_type,
message_chain=message_chain,
time=float(event.event_time) if event.event_time is not None else None,
)
session = provider_session.Session(
launcher_type=launcher_type,
launcher_id=launcher_id,
sender_id=sender_id,
)
contents = copy.deepcopy(event.input.contents)
user_content: str | list[provider_message.ContentElement]
if contents:
user_content = contents
else:
user_content = event.input.text or ''
query = pipeline_query.Query(
query_id=_synthetic_query_id(event.event_id),
launcher_type=launcher_type,
launcher_id=launcher_id,
sender_id=sender_id,
message_event=message_event,
message_chain=message_chain,
bot_uuid=event.bot_id,
pipeline_uuid=None,
pipeline_config=None,
session=session,
messages=[],
user_message=provider_message.Message(role='user', content=user_content),
variables={},
resp_messages=[],
)
query.variables[HOST_BOX_SCOPE_VARIABLE] = build_host_box_scope(event)
query.variables[AUTHORIZED_SKILLS_VARIABLE] = list(dict.fromkeys(authorized_skill_names))
return query
def build_host_box_scope(
event: AgentEventEnvelope,
*,
query: pipeline_query.Query | None = None,
) -> str | None:
"""Return a stable Host scope for a Query session or event."""
target_type, target_id = _resolve_box_target(event, query)
if target_type is None or target_id is None:
return None
capabilities = event.delivery.platform_capabilities or {}
adapter_identity = None
if query is not None:
adapter = getattr(query, 'adapter', None)
if adapter is not None:
adapter_identity = adapter.__class__.__name__
adapter_identity = _first_present(
adapter_identity,
capabilities.get('adapter'),
capabilities.get('source'),
event.source if query is None else None,
)
return json.dumps(
{
'instance_id': _nonempty(constants.instance_id),
'workspace_id': _nonempty(event.workspace_id),
'bot_id': _nonempty(event.bot_id),
'platform_adapter': adapter_identity,
'target_type': target_type,
'target_id': target_id,
'thread_id': _nonempty(event.thread_id),
},
ensure_ascii=False,
sort_keys=True,
separators=(',', ':'),
)
def _resolve_session_identity(
event: AgentEventEnvelope,
) -> tuple[provider_session.LauncherTypes, str, str]:
subject_data = event.subject.data if event.subject is not None else {}
reply_target = event.delivery.reply_target or {}
launcher_type_value = _first_present(
reply_target.get('target_type'),
subject_data.get('launcher_type'),
event.data.get('launcher_type'),
reply_target.get('launcher_type'),
)
if launcher_type_value == provider_session.LauncherTypes.GROUP.value:
launcher_type_value = provider_session.LauncherTypes.GROUP.value
else:
launcher_type_value = provider_session.LauncherTypes.PERSON.value
launcher_id = _first_nonempty(
reply_target.get('target_id'),
subject_data.get('launcher_id'),
event.data.get('launcher_id'),
reply_target.get('launcher_id'),
event.conversation_id,
event.subject.subject_id if event.subject is not None else None,
event.actor.actor_id if event.actor is not None else None,
event.event_id,
)
sender_id = _first_nonempty(
subject_data.get('sender_id'),
event.data.get('sender_id'),
event.actor.actor_id if event.actor is not None else None,
launcher_id,
)
return provider_session.LauncherTypes(launcher_type_value), launcher_id, sender_id
def _resolve_box_target(
event: AgentEventEnvelope,
query: pipeline_query.Query | None,
) -> tuple[str | None, str | None]:
if query is not None:
launcher_type = getattr(query, 'launcher_type', None)
if hasattr(launcher_type, 'value'):
launcher_type = launcher_type.value
launcher_id = getattr(query, 'launcher_id', None)
normalized_type = _nonempty(launcher_type)
normalized_id = _nonempty(launcher_id)
if normalized_type is not None and normalized_id is not None:
return normalized_type, normalized_id
reply_target = event.delivery.reply_target or {}
target_type = _first_present(
reply_target.get('target_type'),
reply_target.get('launcher_type'),
)
target_id = _first_present(
reply_target.get('target_id'),
reply_target.get('launcher_id'),
)
if target_type is not None and target_id is not None:
return target_type, target_id
conversation_id = _nonempty(event.conversation_id)
if conversation_id is not None:
return 'conversation', conversation_id
event_id = _nonempty(event.event_id)
if event_id is not None:
return 'event', event_id
return None, None
def _build_message_chain(event: AgentEventEnvelope) -> platform_message.MessageChain:
text = event.input.to_text()
if not text:
return platform_message.MessageChain([])
return platform_message.MessageChain([platform_message.Plain(text=text)])
def _synthetic_query_id(event_id: str) -> int:
digest = hashlib.sha256(event_id.encode('utf-8')).hexdigest()
return int(digest[:15], 16) or 1
def _first_nonempty(*values: typing.Any) -> str:
normalized = _first_present(*values)
if normalized is not None:
return normalized
raise ValueError('Agent event does not contain a usable execution identity')
def _first_present(*values: typing.Any) -> str | None:
for value in values:
normalized = _nonempty(value)
if normalized is not None:
return normalized
return None
def _nonempty(value: typing.Any) -> str | None:
if value is None:
return None
normalized = str(value).strip()
return normalized or None
+12 -5
View File
@@ -2,6 +2,7 @@
These are Host-internal models, not exposed to SDK.
"""
from __future__ import annotations
import typing
@@ -73,7 +74,7 @@ class AgentEventEnvelope(pydantic.BaseModel):
class BindingScope(pydantic.BaseModel):
"""Scope for agent binding."""
scope_type: typing.Literal["agent", "bot", "workspace", "global"] = "agent"
scope_type: typing.Literal['agent', 'bot', 'workspace', 'global'] = 'agent'
"""Scope type."""
scope_id: str | None = None
@@ -92,6 +93,12 @@ class ResourcePolicy(pydantic.BaseModel):
allowed_tool_names: list[str] | None = None
"""Additional tool name grants. None means no additional tool grants."""
allowed_tool_sources: dict[str, dict[str, str | None]] | None = None
"""Host-resolved implementation identity for each allowed tool name."""
allow_all_tools: bool = False
"""Whether all tools visible to the current Host scope are granted."""
allowed_kb_uuids: list[str] | None = None
"""Additional knowledge base UUID grants. None means no additional KB grants."""
@@ -114,8 +121,8 @@ class StatePolicy(pydantic.BaseModel):
enable_state: bool = True
"""Whether host-owned state is enabled."""
state_scopes: list[typing.Literal["conversation", "actor", "subject", "runner"]] = (
pydantic.Field(default_factory=lambda: ["conversation", "actor"])
state_scopes: list[typing.Literal['conversation', 'actor', 'subject', 'runner']] = pydantic.Field(
default_factory=lambda: ['conversation', 'actor']
)
"""Enabled state scopes."""
@@ -162,7 +169,7 @@ class AgentConfig(pydantic.BaseModel):
delivery_policy: DeliveryPolicy = pydantic.Field(default_factory=DeliveryPolicy)
"""Delivery policy for this Agent."""
event_types: list[str] = pydantic.Field(default_factory=lambda: ["message.received"])
event_types: list[str] = pydantic.Field(default_factory=lambda: ['message.received'])
"""Event types this Agent handles."""
enabled: bool = True
@@ -185,7 +192,7 @@ class AgentBinding(pydantic.BaseModel):
scope: BindingScope = pydantic.Field(default_factory=BindingScope)
"""Binding scope."""
event_types: list[str] = pydantic.Field(default_factory=lambda: ["message.received"])
event_types: list[str] = pydantic.Field(default_factory=lambda: ['message.received'])
"""Event types this binding handles."""
runner_id: str
+39 -6
View File
@@ -12,6 +12,14 @@ from ...core import app
from .binding_resolver import AgentBindingResolver
from .context_builder import AgentRunContextBuilder, AgentRunContextPayload
from .descriptor import AgentRunnerDescriptor
from .execution_context import (
append_mcp_resource_context_to_event,
build_mcp_resource_context_addition,
build_execution_query,
prepare_box_scope,
prepare_execution_query,
project_mcp_resource_config,
)
from .host_models import AgentBinding, AgentEventEnvelope
from .invoker import AgentRunnerInvoker
from .query_bridge import QueryRunBridge
@@ -73,6 +81,17 @@ class AgentRunOrchestrator:
runner_id = binding.runner_id
descriptor = await self.registry.get(runner_id, bound_plugins)
execution_query = adapter_context.get('_query') if adapter_context else None
if execution_query is None:
execution_query = build_execution_query(event, [])
project_mcp_resource_config(execution_query, binding.runner_config)
execution_event = event
resource_addition = await build_mcp_resource_context_addition(self.ap, execution_query)
if resource_addition:
execution_event = event.model_copy(deep=True)
append_mcp_resource_context_to_event(execution_event, resource_addition)
resources = await self.resource_builder.build_resources_from_binding(
event=event,
binding=binding,
@@ -80,7 +99,7 @@ class AgentRunOrchestrator:
)
context = await self.context_builder.build_context_from_event(
event=event,
event=execution_event,
binding=binding,
descriptor=descriptor,
resources=resources,
@@ -88,19 +107,23 @@ class AgentRunOrchestrator:
session_query_id = None
if adapter_context:
query = adapter_context.get('_query')
if query is not None:
if execution_query is not None:
skill_loader.restore_activated_skills_from_state(
self.ap,
query,
execution_query,
context.get('state', {}),
)
session_query_id = adapter_context.get('query_id')
if query is not None or session_query_id is not None:
if execution_query is not None or session_query_id is not None:
context['context']['available_apis']['prompt_get'] = True
if 'params' in adapter_context:
context['adapter']['extra']['params'] = adapter_context['params']
authorized_skill_names = [
str(skill['skill_name']) for skill in resources.get('skills', []) if skill.get('skill_name')
]
prepare_execution_query(execution_query, event, authorized_skill_names)
state_context = build_state_context(event, binding, descriptor)
run_id = context['run_id']
available_apis = context.get('context', {}).get('available_apis')
@@ -152,6 +175,7 @@ class AgentRunOrchestrator:
'state_scopes': list(binding.state_policy.state_scopes),
},
state_context=state_context,
execution_query=execution_query,
)
event_log_id = await self.journal.write_event_log(
@@ -309,6 +333,9 @@ class AgentRunOrchestrator:
adapter_context = dict(plan.adapter_context)
adapter_context['_query'] = query
# Inbound files and subsequent runner tools must share one Host scope.
prepare_box_scope(query, plan.event)
# Materialize inbound attachments into sandbox before running
await self._materialize_inbound_attachments(query, plan.event)
@@ -392,7 +419,13 @@ class AgentRunOrchestrator:
if target_run_id is None:
return False
steering_item = self._build_steering_item(event, target_run_id, descriptor.id)
execution_event = event
resource_addition = await build_mcp_resource_context_addition(self.ap, query)
if resource_addition:
execution_event = event.model_copy(deep=True)
append_mcp_resource_context_to_event(execution_event, resource_addition)
steering_item = self._build_steering_item(execution_event, target_run_id, descriptor.id)
if not await self._session_registry.enqueue_steering(target_run_id, steering_item):
return False
+3 -3
View File
@@ -8,7 +8,7 @@ import typing
from langbot_plugin.api.entities.builtin.pipeline import query as pipeline_query
from .binding_resolver import AgentBindingResolver
from .config_migration import ConfigMigration
from .config_resolver import RunnerConfigResolver
from .errors import RunnerNotFoundError
from .host_models import AgentBinding, AgentEventEnvelope
from .query_entry_adapter import QueryEntryAdapter
@@ -34,7 +34,7 @@ class QueryRunBridge:
def build_plan(self, query: pipeline_query.Query) -> QueryRunPlan:
"""Build an event-first run plan from a Pipeline Query."""
runner_id = ConfigMigration.resolve_runner_id(query.pipeline_config)
runner_id = RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
if not runner_id:
raise RunnerNotFoundError('no runner configured')
@@ -53,4 +53,4 @@ class QueryRunBridge:
def resolve_runner_id_for_telemetry(self, query: pipeline_query.Query) -> str | None:
"""Resolve runner ID for telemetry/logging without full execution."""
return ConfigMigration.resolve_runner_id(query.pipeline_config)
return RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
@@ -3,6 +3,7 @@
This adapter bridges the current Query entry point with the event-first
Protocol v1 architecture without exposing Query internals to runners.
"""
from __future__ import annotations
import hashlib
@@ -23,12 +24,13 @@ from langbot_plugin.api.entities.builtin.agent_runner.delivery import DeliveryCo
from .host_models import (
AgentConfig,
AgentEventEnvelope,
ResourcePolicy,
StatePolicy,
DeliveryPolicy,
)
from .config_migration import ConfigMigration
from .config_resolver import RunnerConfigResolver
from .resource_policy import ResourcePolicyProjector
from . import events as runner_events
from ...provider.tools.toolmgr import TOOL_SOURCE_REFS_QUERY_KEY
class QueryEntryAdapter:
@@ -83,7 +85,7 @@ class QueryEntryAdapter:
event_id=event.event_id or str(query.query_id),
event_type=event.event_type or runner_events.MESSAGE_RECEIVED,
event_time=event.event_time,
source="host_adapter",
source='host_adapter',
source_event_type=event.source_event_type,
bot_id=query.bot_uuid,
workspace_id=None, # Not available in Query
@@ -105,21 +107,22 @@ class QueryEntryAdapter:
) -> AgentConfig:
"""Project the current Pipeline config container into target Agent config."""
pipeline_config = query.pipeline_config or {}
runner_config = ConfigMigration.resolve_runner_config(pipeline_config, runner_id)
runner_config = RunnerConfigResolver.resolve_runner_config(pipeline_config, runner_id)
agent_id = getattr(query, 'pipeline_uuid', None)
# Build resource policy from current config
resource_policy = ResourcePolicy(
allowed_model_uuids=cls._extract_allowed_models(query),
allowed_tool_names=cls._extract_allowed_tools(query),
allowed_kb_uuids=cls._extract_allowed_kbs(query),
allowed_skill_names=cls._extract_allowed_skills(query),
resource_policy = ResourcePolicyProjector.from_runner_config(
runner_config,
resolved_model_uuids=cls._extract_allowed_models(query),
resolved_tool_names=cls._extract_allowed_tools(query),
resolved_tool_sources=cls._extract_allowed_tool_sources(query),
resolved_kb_uuids=cls._extract_allowed_kbs(query),
resolved_skill_names=cls._extract_allowed_skills(query),
)
# Build state policy
state_policy = StatePolicy(
enable_state=True,
state_scopes=["conversation", "actor", "subject", "runner"],
state_scopes=['conversation', 'actor', 'subject', 'runner'],
)
# Build delivery policy
@@ -181,10 +184,7 @@ class QueryEntryAdapter:
if isinstance(value, (list, tuple)):
return all(cls.is_json_serializable(item) for item in value)
if isinstance(value, dict):
return all(
isinstance(k, str) and cls.is_json_serializable(v)
for k, v in value.items()
)
return all(isinstance(k, str) and cls.is_json_serializable(v) for k, v in value.items())
return False
# Private helper methods
@@ -228,7 +228,7 @@ class QueryEntryAdapter:
event_id=cls._build_scoped_event_id(query, source_event_id, event_time),
event_type=runner_events.MESSAGE_RECEIVED,
event_time=event_time,
source="host_adapter",
source='host_adapter',
source_event_type=source_event_type,
data=event_data,
)
@@ -345,7 +345,7 @@ class QueryEntryAdapter:
actor_name = sender.get_name() if sender and hasattr(sender, 'get_name') else None
return ActorContext(
actor_type="user",
actor_type='user',
actor_id=str(actor_id) if actor_id is not None else None,
actor_name=actor_name,
metadata={},
@@ -371,13 +371,13 @@ class QueryEntryAdapter:
launcher_type_value = getattr(launcher_type, 'value', launcher_type)
return SubjectContext(
subject_type="message",
subject_type='message',
subject_id=str(message_id or query_id or ''),
data={
"launcher_type": launcher_type_value,
"launcher_id": getattr(query, 'launcher_id', None),
"sender_id": str(getattr(query, 'sender_id', '')) if getattr(query, 'sender_id', None) else None,
"bot_uuid": getattr(query, 'bot_uuid', None),
'launcher_type': launcher_type_value,
'launcher_id': getattr(query, 'launcher_id', None),
'sender_id': str(getattr(query, 'sender_id', '')) if getattr(query, 'sender_id', None) else None,
'bot_uuid': getattr(query, 'bot_uuid', None),
},
)
@@ -467,31 +467,39 @@ class QueryEntryAdapter:
if elem_type == 'image_url':
image_url = elem.get('image_url') or {}
add_attachment({
'type': 'image',
'source': 'url',
'url': image_url.get('url') if isinstance(image_url, dict) else str(image_url),
})
add_attachment(
{
'type': 'image',
'source': 'url',
'url': image_url.get('url') if isinstance(image_url, dict) else str(image_url),
}
)
elif elem_type == 'image_base64':
add_attachment({
'type': 'image',
'source': 'base64',
'content': elem.get('image_base64'),
})
add_attachment(
{
'type': 'image',
'source': 'base64',
'content': elem.get('image_base64'),
}
)
elif elem_type == 'file_url':
add_attachment({
'type': 'file',
'source': 'url',
'url': elem.get('file_url'),
'name': elem.get('file_name'),
})
add_attachment(
{
'type': 'file',
'source': 'url',
'url': elem.get('file_url'),
'name': elem.get('file_name'),
}
)
elif elem_type == 'file_base64':
add_attachment({
'type': 'file',
'source': 'base64',
'content': elem.get('file_base64'),
'name': elem.get('file_name'),
})
add_attachment(
{
'type': 'file',
'source': 'base64',
'content': elem.get('file_base64'),
'name': elem.get('file_name'),
}
)
message_chain = getattr(query, 'message_chain', None)
if message_chain:
@@ -505,30 +513,36 @@ class QueryEntryAdapter:
image_id = component.image_id or None
image_url = component.url or None
image_base64 = component.base64 or None
add_attachment({
'type': 'image',
'source': 'message_chain',
'id': image_id,
'url': image_url,
'content': image_base64,
})
add_attachment(
{
'type': 'image',
'source': 'message_chain',
'id': image_id,
'url': image_url,
'content': image_base64,
}
)
elif isinstance(component, platform_message.File):
add_attachment({
'type': 'file',
'source': 'message_chain',
'id': component.id or None,
'name': component.name or None,
'url': component.url or None,
'content': component.base64 or None,
})
add_attachment(
{
'type': 'file',
'source': 'message_chain',
'id': component.id or None,
'name': component.name or None,
'url': component.url or None,
'content': component.base64 or None,
}
)
elif isinstance(component, platform_message.Voice):
add_attachment({
'type': 'voice',
'source': 'message_chain',
'id': component.voice_id or None,
'url': component.url or None,
'content': component.base64 or None,
})
add_attachment(
{
'type': 'voice',
'source': 'message_chain',
'id': component.voice_id or None,
'url': component.url or None,
'content': component.base64 or None,
}
)
return attachments
@@ -557,9 +571,9 @@ class QueryEntryAdapter:
"""Build DeliveryContext from Query."""
message_chain = getattr(query, 'message_chain', None)
return DeliveryContext(
surface="platform",
surface='platform',
reply_target={
"message_id": getattr(message_chain, 'message_id', None),
'message_id': getattr(message_chain, 'message_id', None),
},
supports_streaming=True,
supports_edit=False,
@@ -601,22 +615,24 @@ class QueryEntryAdapter:
) -> list[str] | None:
"""Extract allowed tool names from query."""
use_funcs = getattr(query, 'use_funcs', None)
if not use_funcs:
if use_funcs is None:
return None
try:
tool_names = []
for func in use_funcs:
if isinstance(func, dict):
name = func.get('name')
elif hasattr(func, 'name'):
name = func.name
else:
continue
if name:
tool_names.append(name)
return tool_names if tool_names else None
return ResourcePolicyProjector.extract_tool_names(use_funcs)
except (TypeError, AttributeError):
return []
@classmethod
def _extract_allowed_tool_sources(
cls,
query: pipeline_query.Query,
) -> dict[str, typing.Any] | None:
"""Extract the Host-frozen implementation for each allowed tool."""
variables = getattr(query, 'variables', None)
if not isinstance(variables, dict):
return None
refs = variables.get(TOOL_SOURCE_REFS_QUERY_KEY)
return refs if isinstance(refs, dict) else None
@classmethod
def _extract_allowed_kbs(
@@ -627,10 +643,9 @@ class QueryEntryAdapter:
variables = getattr(query, 'variables', None)
if not variables:
return None
kb_uuids = variables.get('_knowledge_base_uuids')
if kb_uuids:
return kb_uuids
return None
if '_knowledge_base_uuids' not in variables:
return None
return ResourcePolicyProjector.normalize_names(variables.get('_knowledge_base_uuids'))
@classmethod
def _extract_allowed_skills(
+108 -41
View File
@@ -1,4 +1,5 @@
"""Agent resource builder for constructing authorized resources."""
from __future__ import annotations
import typing
@@ -15,6 +16,9 @@ from .context_builder import (
)
from . import config_schema
from .host_models import AgentEventEnvelope, AgentBinding
from .resource_policy import ResourcePolicyProjector
from ...provider.tools.loaders.mcp import MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE
from ...provider.tools.toolmgr import ToolSourceRef
class AgentResourceBuilder:
@@ -66,18 +70,12 @@ class AgentResourceBuilder:
manifest_perms = descriptor.permissions
# Build each resource category
models = await self._build_models_from_binding(
manifest_perms, resource_policy, descriptor, runner_config
)
tools = await self._build_tools_from_binding(
manifest_perms, resource_policy, descriptor
)
models = await self._build_models_from_binding(manifest_perms, resource_policy, descriptor, runner_config)
tools = await self._build_tools_from_binding(manifest_perms, resource_policy, descriptor, runner_config)
knowledge_bases = await self._build_knowledge_bases_from_binding(
manifest_perms, resource_policy, descriptor, runner_config
)
skills = self._build_skills_from_binding(
resource_policy, descriptor
)
skills = self._build_skills_from_binding(resource_policy, descriptor)
storage = self._build_storage_from_binding(manifest_perms, binding)
return {
@@ -133,6 +131,7 @@ class AgentResourceBuilder:
manifest_perms: typing.Any,
resource_policy: typing.Any,
descriptor: AgentRunnerDescriptor,
runner_config: dict[str, typing.Any],
) -> list[ToolResource]:
"""Build tools list from binding."""
tools: list[ToolResource] = []
@@ -143,8 +142,43 @@ class AgentResourceBuilder:
if not config_schema.uses_host_tools(descriptor):
return tools
# Get tool names from resource policy
allowed_names = resource_policy.allowed_tool_names
allowed_sources = resource_policy.allowed_tool_sources
if resource_policy.allow_all_tools or allowed_sources is None:
get_catalog = getattr(getattr(self.ap, 'tool_mgr', None), 'get_resolved_tool_catalog', None)
if get_catalog is None:
return tools
try:
catalog = await get_catalog(
include_skill_authoring=True,
include_mcp_resource_tools=True,
)
except Exception as e:
self.ap.logger.warning(f'Failed to resolve visible Host tools: {e}')
return tools
if not resource_policy.allow_all_tools:
selected_names = set(allowed_names or [])
catalog = [item for item in catalog if item.get('name') in selected_names]
allowed_names = ResourcePolicyProjector.extract_tool_names(catalog)
allowed_sources = {
item['name']: ref for item in catalog if (ref := self._catalog_source_ref(item)) is not None
}
if runner_config.get('mcp-resource-agent-read-enabled', True) is not True:
denied_resource_tools = {MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE}
allowed_names = [
name
for name in (allowed_names or [])
if not (
name in denied_resource_tools
and allowed_sources
and (source_ref := allowed_sources.get(name)) is not None
and source_ref['source'] == 'mcp'
and source_ref.get('source_id') is None
)
]
tool_operations = [operation for operation in ('detail', 'call') if operation in tool_perms]
# Prefill full tool schema (best-effort) so runners can build LLM tool
@@ -153,20 +187,39 @@ class AgentResourceBuilder:
get_tool_schema = getattr(getattr(self.ap, 'tool_mgr', None), 'get_tool_schema', None)
if allowed_names:
for tool_name in allowed_names:
source_ref = allowed_sources.get(tool_name) if allowed_sources else None
if source_ref is None:
self.ap.logger.warning(f'Tool {tool_name} is not authorized because its Host source is unresolved')
continue
if get_tool_schema is not None:
description, parameters = await get_tool_schema(tool_name)
description, parameters = await get_tool_schema(tool_name, source_ref=source_ref)
else:
description, parameters = None, None
tools.append({
'tool_name': tool_name,
'tool_type': None,
'description': description,
'operations': tool_operations,
'parameters': parameters,
})
tools.append(
{
'tool_name': tool_name,
'tool_type': source_ref['source'],
'description': description,
'operations': tool_operations,
'parameters': parameters,
'source': source_ref['source'],
'source_id': source_ref.get('source_id'),
}
)
return tools
@staticmethod
def _catalog_source_ref(item: dict[str, typing.Any]) -> ToolSourceRef | None:
source = item.get('source')
if not isinstance(source, str) or not source:
return None
source_id = item.get('source_id')
return {
'source': source,
'source_id': source_id if isinstance(source_id, str) and source_id else None,
}
async def _build_knowledge_bases_from_binding(
self,
manifest_perms: typing.Any,
@@ -196,12 +249,16 @@ class AgentResourceBuilder:
try:
kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
if kb:
kb_resources.append({
'kb_id': kb_uuid,
'kb_name': kb.get_name(),
'kb_type': kb.knowledge_base_entity.kb_type if hasattr(kb.knowledge_base_entity, 'kb_type') else None,
'operations': kb_operations,
})
kb_resources.append(
{
'kb_id': kb_uuid,
'kb_name': kb.get_name(),
'kb_type': kb.knowledge_base_entity.kb_type
if hasattr(kb.knowledge_base_entity, 'kb_type')
else None,
'operations': kb_operations,
}
)
except Exception as e:
self.ap.logger.warning(f'Failed to build knowledge base resource {kb_uuid}: {e}')
@@ -233,11 +290,13 @@ class AgentResourceBuilder:
skills: list[SkillResource] = []
for skill_name in names:
skill_data = loaded_skills.get(skill_name) or {}
skills.append({
'skill_name': skill_name,
'display_name': skill_data.get('display_name') or skill_data.get('name') or skill_name,
'description': skill_data.get('description') or None,
})
skills.append(
{
'skill_name': skill_name,
'display_name': skill_data.get('display_name') or skill_data.get('name') or skill_name,
'description': skill_data.get('description') or None,
}
)
return skills
def _build_storage_from_binding(
@@ -285,12 +344,16 @@ class AgentResourceBuilder:
try:
model = await self.ap.model_mgr.get_model_by_uuid(model_uuid)
if model and model.model_entity:
models.append({
'model_id': model_uuid,
'model_type': getattr(model.model_entity, 'model_type', None),
'provider': getattr(model.provider_entity, 'name', None) if hasattr(model, 'provider_entity') else None,
'operations': operations,
})
models.append(
{
'model_id': model_uuid,
'model_type': getattr(model.model_entity, 'model_type', None),
'provider': getattr(model.provider_entity, 'name', None)
if hasattr(model, 'provider_entity')
else None,
'operations': operations,
}
)
seen_model_ids.add(model_uuid)
except Exception as e:
self.ap.logger.warning(f'Failed to build LLM model resource {model_uuid}: {e}')
@@ -308,12 +371,16 @@ class AgentResourceBuilder:
try:
model = await self.ap.model_mgr.get_rerank_model_by_uuid(model_uuid)
if model and model.model_entity:
models.append({
'model_id': model_uuid,
'model_type': getattr(model.model_entity, 'model_type', 'rerank') or 'rerank',
'provider': getattr(model.provider_entity, 'name', None) if hasattr(model, 'provider_entity') else None,
'operations': ['rerank'],
})
models.append(
{
'model_id': model_uuid,
'model_type': getattr(model.model_entity, 'model_type', 'rerank') or 'rerank',
'provider': getattr(model.provider_entity, 'name', None)
if hasattr(model, 'provider_entity')
else None,
'operations': ['rerank'],
}
)
seen_model_ids.add(model_uuid)
except Exception as e:
self.ap.logger.warning(f'Failed to build rerank model resource {model_uuid}: {e}')
@@ -0,0 +1,146 @@
"""Project AgentRunner configuration into Host resource policy."""
from __future__ import annotations
import collections.abc
import typing
from .host_models import ResourcePolicy
class ResourcePolicyProjector:
"""Build one generic resource policy for Pipeline and Agent bindings."""
@classmethod
def from_runner_config(
cls,
runner_config: dict[str, typing.Any] | None,
*,
resolved_model_uuids: collections.abc.Iterable[typing.Any] | None = None,
resolved_tool_names: collections.abc.Iterable[typing.Any] | None = None,
resolved_tool_sources: typing.Mapping[str, typing.Any] | None = None,
resolved_kb_uuids: collections.abc.Iterable[typing.Any] | None = None,
resolved_skill_names: collections.abc.Iterable[typing.Any] | None = None,
) -> ResourcePolicy:
"""Project standard resource fields without depending on a runner ID.
Resolved values are supplied by entry paths that already narrowed the
available resources, such as Pipeline preprocessing. Independent Agent
bindings omit them so the Host can resolve an explicit all-tools grant
against the live tool catalog when the run starts.
"""
config = runner_config if isinstance(runner_config, dict) else {}
selected_tool_names = cls.normalize_names(config.get('tools'))
enable_all_tools = config.get('enable-all-tools', True) is True
if resolved_tool_names is not None:
available_tool_names = cls.normalize_names(resolved_tool_names)
if enable_all_tools:
allowed_tool_names = available_tool_names
else:
selected = set(selected_tool_names)
allowed_tool_names = [name for name in available_tool_names if name in selected]
allow_all_tools = False
elif enable_all_tools:
allowed_tool_names = None
allow_all_tools = True
else:
allowed_tool_names = selected_tool_names
allow_all_tools = False
allowed_tool_sources = cls.normalize_tool_sources(resolved_tool_sources)
if allowed_tool_sources is not None:
allowed = set(allowed_tool_names or [])
allowed_tool_sources = {name: ref for name, ref in allowed_tool_sources.items() if name in allowed}
if resolved_kb_uuids is not None:
allowed_kb_uuids = cls.normalize_names(resolved_kb_uuids)
elif 'knowledge-bases' in config:
allowed_kb_uuids = cls.normalize_names(config.get('knowledge-bases'))
else:
allowed_kb_uuids = None
return ResourcePolicy(
allowed_model_uuids=cls.normalize_optional_names(resolved_model_uuids),
allowed_tool_names=allowed_tool_names,
allowed_tool_sources=allowed_tool_sources,
allow_all_tools=allow_all_tools,
allowed_kb_uuids=allowed_kb_uuids,
allowed_skill_names=cls.normalize_optional_names(resolved_skill_names),
)
@staticmethod
def normalize_names(values: typing.Any) -> list[str]:
"""Return unique, non-empty string identifiers in source order."""
if not isinstance(values, collections.abc.Iterable) or isinstance(values, (str, bytes, dict)):
return []
names: list[str] = []
seen: set[str] = set()
for value in values:
if not isinstance(value, str) or not value or value in seen:
continue
seen.add(value)
names.append(value)
return names
@classmethod
def normalize_optional_names(
cls,
values: collections.abc.Iterable[typing.Any] | None,
) -> list[str] | None:
if values is None:
return None
return cls.normalize_names(values)
@classmethod
def extract_tool_names(cls, tools: collections.abc.Iterable[typing.Any] | None) -> list[str]:
"""Extract tool identifiers from dictionary and SDK object shapes."""
if tools is None:
return []
names: list[str] = []
for tool in tools:
name = tool.get('name') if isinstance(tool, dict) else getattr(tool, 'name', None)
if isinstance(name, str):
names.append(name)
return cls.normalize_names(names)
@staticmethod
def normalize_tool_sources(
values: typing.Mapping[str, typing.Any] | None,
) -> dict[str, dict[str, str | None]] | None:
if values is None:
return None
normalized: dict[str, dict[str, str | None]] = {}
for name, value in values.items():
if not isinstance(name, str) or not name or not isinstance(value, collections.abc.Mapping):
continue
source = value.get('source')
if not isinstance(source, str) or not source:
continue
source_id = value.get('source_id')
normalized[name] = {
'source': source,
'source_id': source_id if isinstance(source_id, str) and source_id else None,
}
return normalized
@classmethod
def filter_tools(
cls,
tools: collections.abc.Iterable[typing.Any],
policy: ResourcePolicy,
) -> list[typing.Any]:
"""Apply a projected policy to an already scoped tool collection."""
tool_list = list(tools)
if policy.allow_all_tools:
return tool_list
allowed_names = set(policy.allowed_tool_names or [])
return [
tool
for tool in tool_list
if (tool.get('name') if isinstance(tool, dict) else getattr(tool, 'name', None)) in allowed_names
]
+1 -9
View File
@@ -304,13 +304,10 @@ class AgentRunJournal:
for item in items:
event = item.get('event') if isinstance(item.get('event'), dict) else {}
input_data = item.get('input') if isinstance(item.get('input'), dict) else {}
conversation = item.get('conversation') if isinstance(item.get('conversation'), dict) else {}
actor = item.get('actor') if isinstance(item.get('actor'), dict) else {}
subject = item.get('subject') if isinstance(item.get('subject'), dict) else {}
text = input_data.get('text')
input_summary = text[:1000] if isinstance(text, str) and text else 'Unconsumed steering input dropped'
event_time = None
raw_event_time = event.get('event_time')
if raw_event_time:
@@ -335,12 +332,7 @@ class AgentRunJournal:
actor_name=actor.get('actor_name'),
subject_type=subject.get('subject_type'),
subject_id=subject.get('subject_id'),
input_summary=input_summary,
input_json={
'text': text,
'contents': self._sanitize_contents(input_data.get('contents') or []),
'attachments': self._sanitize_attachments(input_data.get('attachments') or []),
},
input_summary='Unconsumed steering input dropped',
run_id=run_id,
runner_id=runner_id,
event_time=event_time,
@@ -1,4 +1,5 @@
"""Agent run session registry for proxy action permission validation."""
from __future__ import annotations
import asyncio
@@ -7,7 +8,10 @@ import typing
import time
import threading
from langbot_plugin.api.entities.builtin.pipeline import query as pipeline_query
from .context_builder import AgentResources
from ...provider.tools.toolmgr import ToolSourceRef
MAX_STEERING_QUEUE_ITEMS = 100
@@ -22,6 +26,7 @@ DEFAULT_RESOURCE_OPERATIONS: dict[str, set[str]] = {
class AgentRunSessionStatus(typing.TypedDict):
"""Status tracking for agent run session."""
started_at: int
last_activity_at: int
@@ -58,13 +63,16 @@ class AgentRunSession(typing.TypedDict):
run_id: Unique run identifier (UUID from AgentRunContext)
runner_id: Runner descriptor ID (plugin:author/name/runner)
query_id: Host entry query ID, only present for query-based adapters
execution_query: Host-only Query used by providers and tool loaders
plugin_identity: Plugin identifier (author/name) of the runner
authorization: Run-scoped authorization snapshot; runtime auth truth
status: Session status tracking
"""
run_id: str
runner_id: str
query_id: int | None
execution_query: pipeline_query.Query | None
plugin_identity: str # author/name
authorization: RunAuthorizationSnapshot
status: AgentRunSessionStatus
@@ -104,6 +112,7 @@ class AgentRunSessionRegistry:
available_apis: dict[str, bool] | None = None,
state_policy: dict[str, typing.Any] | None = None,
state_context: dict[str, typing.Any] | None = None,
execution_query: pipeline_query.Query | None = None,
) -> None:
"""Register a new agent run session.
@@ -120,6 +129,7 @@ class AgentRunSessionRegistry:
available_apis: Run-scoped pull APIs exposed in AgentRunContext
state_policy: State policy from binding (enable_state, state_scopes)
state_context: Context for state API (scope_keys, binding_identity, etc.)
execution_query: Host-only Query used for provider and tool execution
"""
if not isinstance(plugin_identity, str) or not plugin_identity.strip():
raise ValueError('plugin_identity is required for agent run sessions')
@@ -153,6 +163,7 @@ class AgentRunSessionRegistry:
'run_id': run_id,
'runner_id': runner_id,
'query_id': query_id,
'execution_query': execution_query,
'plugin_identity': plugin_identity,
'authorization': authorization,
'status': {
@@ -371,6 +382,26 @@ class AgentRunSessionRegistry:
return False
@staticmethod
def get_tool_source_ref(
session: AgentRunSession,
tool_name: str,
) -> ToolSourceRef | None:
"""Return the implementation identity frozen in this run's grant."""
resources = session['authorization']['resources']
for tool in resources.get('tools', []):
if tool.get('tool_name') != tool_name:
continue
source = tool.get('source')
if not isinstance(source, str) or not source:
return None
source_id = tool.get('source_id')
return {
'source': source,
'source_id': source_id if isinstance(source_id, str) and source_id else None,
}
return None
async def list_active_runs(self) -> list[AgentRunSession]:
"""List all active run sessions.
@@ -16,7 +16,10 @@ class AgentsRouterGroup(group.RouterGroup):
return self.success(data={'agents': await self.ap.agent_service.get_agents(sort_by, sort_order)})
json_data = await quart.request.json
created = await self.ap.agent_service.create_agent(json_data)
try:
created = await self.ap.agent_service.create_agent(json_data)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
return self.success(data=created)
@self.route('/_/metadata', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
@@ -33,7 +36,10 @@ class AgentsRouterGroup(group.RouterGroup):
if quart.request.method == 'PUT':
json_data = await quart.request.json
await self.ap.agent_service.update_agent(agent_uuid, json_data)
try:
await self.ap.agent_service.update_agent(agent_uuid, json_data)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
return self.success()
await self.ap.agent_service.delete_agent(agent_uuid)
@@ -3,6 +3,7 @@ from __future__ import annotations
import quart
from ... import group
from ......pipeline.extension_preferences import normalize_extension_preferences
@group.group_class('pipelines', '/api/v1/pipelines')
@@ -19,7 +20,10 @@ class PipelinesRouterGroup(group.RouterGroup):
elif quart.request.method == 'POST':
json_data = await quart.request.json
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(json_data)
try:
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(json_data)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
return self.success(data={'uuid': pipeline_uuid})
@@ -41,7 +45,10 @@ class PipelinesRouterGroup(group.RouterGroup):
elif quart.request.method == 'PUT':
json_data = await quart.request.json
await self.ap.pipeline_service.update_pipeline(pipeline_uuid, json_data)
try:
await self.ap.pipeline_service.update_pipeline(pipeline_uuid, json_data)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
return self.success()
elif quart.request.method == 'DELETE':
@@ -81,21 +88,19 @@ class PipelinesRouterGroup(group.RouterGroup):
self.ap.logger.warning('Unable to list skills for pipeline extensions: %s', exc)
available_skills = []
extensions_prefs = pipeline.get('extensions_preferences', {})
extensions_prefs = normalize_extension_preferences(pipeline.get('extensions_preferences'))
return self.success(
data={
'enable_all_plugins': extensions_prefs.get('enable_all_plugins', True),
'enable_all_mcp_servers': extensions_prefs.get('enable_all_mcp_servers', True),
'enable_all_skills': extensions_prefs.get('enable_all_skills', True),
'bound_plugins': extensions_prefs.get('plugins', []),
'enable_all_plugins': extensions_prefs['enable_all_plugins'],
'enable_all_mcp_servers': extensions_prefs['enable_all_mcp_servers'],
'enable_all_skills': extensions_prefs['enable_all_skills'],
'bound_plugins': extensions_prefs['plugins'],
'available_plugins': plugins,
'bound_mcp_servers': extensions_prefs.get('mcp_servers', []),
'bound_mcp_servers': extensions_prefs['mcp_servers'],
'available_mcp_servers': mcp_servers,
'bound_mcp_resources': extensions_prefs.get('mcp_resources', []),
'mcp_resource_agent_read_enabled': extensions_prefs.get(
'mcp_resource_agent_read_enabled', True
),
'bound_skills': extensions_prefs.get('skills', []),
'bound_mcp_resources': extensions_prefs['mcp_resources'],
'mcp_resource_agent_read_enabled': extensions_prefs['mcp_resource_agent_read_enabled'],
'bound_skills': extensions_prefs['skills'],
'available_skills': available_skills,
}
)
@@ -111,16 +116,41 @@ class PipelinesRouterGroup(group.RouterGroup):
bound_mcp_resources = json_data.get('bound_mcp_resources')
mcp_resource_agent_read_enabled = json_data.get('mcp_resource_agent_read_enabled')
await self.ap.pipeline_service.update_pipeline_extensions(
pipeline_uuid,
bound_plugins,
bound_mcp_servers,
enable_all_plugins,
enable_all_mcp_servers,
bound_skills=bound_skills,
enable_all_skills=enable_all_skills,
bound_mcp_resources=bound_mcp_resources,
mcp_resource_agent_read_enabled=mcp_resource_agent_read_enabled,
)
extension_flags = {
'enable_all_plugins': enable_all_plugins,
'enable_all_mcp_servers': enable_all_mcp_servers,
'enable_all_skills': enable_all_skills,
}
for field, value in extension_flags.items():
if field in json_data and not isinstance(value, bool):
return self.http_status(
400,
-1,
f"Pipeline extension field '{field}' must be a boolean",
)
if 'mcp_resource_agent_read_enabled' in json_data and not isinstance(
mcp_resource_agent_read_enabled, bool
):
return self.http_status(
400,
-1,
"Pipeline extension field 'mcp_resource_agent_read_enabled' must be a boolean",
)
try:
await self.ap.pipeline_service.update_pipeline_extensions(
pipeline_uuid,
bound_plugins,
bound_mcp_servers,
enable_all_plugins,
enable_all_mcp_servers,
bound_skills=bound_skills,
enable_all_skills=enable_all_skills,
bound_mcp_resources=bound_mcp_resources,
mcp_resource_agent_read_enabled=mcp_resource_agent_read_enabled,
)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
return self.success()
@@ -3,59 +3,61 @@ from __future__ import annotations
import quart
from ... import group
from ......pipeline.extension_preferences import normalize_extension_preferences
@group.group_class('tools', '/api/v1/tools')
class ToolsRouterGroup(group.RouterGroup):
async def _get_scoped_tool_catalog(self) -> list[dict] | None:
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
bound_plugins: list[str] | None = None
bound_mcp_servers: list[str] | None = None
if pipeline_uuid:
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
if pipeline is None:
return None
extensions_prefs = normalize_extension_preferences(pipeline.get('extensions_preferences'))
if not extensions_prefs['enable_all_plugins']:
bound_plugins = [f'{plugin["author"]}/{plugin["name"]}' for plugin in extensions_prefs['plugins']]
if not extensions_prefs['enable_all_mcp_servers']:
bound_mcp_servers = extensions_prefs['mcp_servers']
return await self.ap.tool_mgr.get_resolved_tool_catalog(
bound_plugins,
bound_mcp_servers,
include_skill_authoring=True,
)
async def initialize(self) -> None:
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
async def _() -> str:
"""获取所有可用工具列表"""
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
bound_plugins: list[str] | None = None
bound_mcp_servers: list[str] | None = None
catalog = await self._get_scoped_tool_catalog()
if catalog is None:
return self.http_status(404, -1, 'pipeline not found')
return self.success(data={'tools': catalog})
if pipeline_uuid:
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
if pipeline is None:
return self.http_status(404, -1, 'pipeline not found')
extensions_prefs = pipeline.get('extensions_preferences', {}) or {}
if not extensions_prefs.get('enable_all_plugins', True):
bound_plugins = [
f'{plugin.get("author", "")}/{plugin.get("name", "")}'
for plugin in extensions_prefs.get('plugins', [])
if isinstance(plugin, dict) and plugin.get('name')
]
if not extensions_prefs.get('enable_all_mcp_servers', True):
bound_mcp_servers = [
server for server in (extensions_prefs.get('mcp_servers', []) or []) if isinstance(server, str)
]
return self.success(
data={
'tools': await self.ap.tool_mgr.get_tool_catalog(
bound_plugins,
bound_mcp_servers,
include_skill_authoring=True,
)
}
)
@self.route('/<tool_name>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
@self.route('/<path:tool_name>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
async def _(tool_name: str) -> str:
"""获取特定工具详情"""
tools = await self.ap.tool_mgr.get_all_tools()
catalog = await self._get_scoped_tool_catalog()
if catalog is None:
return self.http_status(404, -1, 'pipeline not found')
for tool in tools:
if tool.name == tool_name:
for tool in catalog:
if tool.get('name') == tool_name:
return self.success(
data={
'tool': {
'name': tool.name,
'description': tool.description,
'human_desc': tool.human_desc,
'parameters': tool.parameters,
'name': tool['name'],
'description': tool.get('description') or '',
'human_desc': tool.get('human_desc') or '',
'parameters': tool.get('parameters') or {},
'source': tool.get('source'),
'source_name': tool.get('source_name'),
'source_id': tool.get('source_id'),
}
}
)
+10 -16
View File
@@ -7,6 +7,7 @@ import typing
import sqlalchemy
from ....core import app
from ....agent.runner.config_resolver import RunnerConfigResolver
from ....entity.persistence import agent as persistence_agent
@@ -82,8 +83,8 @@ class AgentService:
if kind != AGENT_KIND_AGENT:
raise ValueError(f'Unsupported agent kind: {kind}')
config = agent_data.get('config') or await self._get_default_agent_config()
runner_id = self._resolve_runner_id(config)
config = agent_data['config'] if 'config' in agent_data else await self._get_default_agent_config()
config, runner_id, _ = RunnerConfigResolver.resolve_agent_runner_config(config)
new_uuid = str(uuid.uuid4())
values = {
'uuid': new_uuid,
@@ -91,7 +92,7 @@ class AgentService:
'description': agent_data.get('description') or '',
'emoji': agent_data.get('emoji') or '🤖',
'kind': AGENT_KIND_AGENT,
'component_ref': agent_data.get('component_ref') or runner_id,
'component_ref': runner_id,
'config': config,
'enabled': agent_data.get('enabled', True),
'supported_event_patterns': agent_data.get('supported_event_patterns') or AGENT_DEFAULT_EVENT_PATTERNS,
@@ -109,12 +110,14 @@ class AgentService:
return
update_data = agent_data.copy()
for protected_field in ('uuid', 'kind', 'created_at', 'updated_at', 'capability'):
for protected_field in ('uuid', 'kind', 'component_ref', 'created_at', 'updated_at', 'capability'):
update_data.pop(protected_field, None)
if 'config' in update_data:
update_data['component_ref'] = update_data.get('component_ref') or self._resolve_runner_id(
update_data['config']
)
config, runner_id, _ = RunnerConfigResolver.resolve_agent_runner_config(update_data['config'])
update_data['config'] = config
else:
_, runner_id, _ = RunnerConfigResolver.resolve_agent_runner_config(existing_agent.config)
update_data['component_ref'] = runner_id
if 'supported_event_patterns' in update_data and not update_data['supported_event_patterns']:
update_data['supported_event_patterns'] = AGENT_DEFAULT_EVENT_PATTERNS
@@ -169,15 +172,6 @@ class AgentService:
},
}
@staticmethod
def _resolve_runner_id(config: dict[str, typing.Any]) -> str | None:
runner = config.get('runner') if isinstance(config, dict) else None
if isinstance(runner, dict):
runner_id = runner.get('id')
if runner_id:
return runner_id
return None
def _agent_to_product_item(
self,
agent: persistence_agent.Agent,
+66 -13
View File
@@ -6,7 +6,12 @@ import sqlalchemy
import typing
from ....core import app
from ....agent.runner.config_resolver import RunnerConfigResolver
from ....entity.persistence import pipeline as persistence_pipeline
from ....pipeline.extension_preferences import (
normalize_extension_preferences,
validate_extension_preferences,
)
default_stage_order = [
@@ -163,6 +168,11 @@ class PipelineService:
return self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
async def create_pipeline(self, pipeline_data: dict, default: bool = False) -> str:
if 'extensions_preferences' in pipeline_data:
self._validate_extension_preferences(pipeline_data['extensions_preferences'])
if 'config' in pipeline_data:
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
# Check limitation
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
max_pipelines = limitation.get('max_pipelines', -1)
@@ -177,6 +187,7 @@ class PipelineService:
pipeline_data['is_default'] = default
pipeline_data['config'] = await self.get_default_pipeline_config()
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
# Ensure extensions_preferences is set with enable_all_plugins and enable_all_mcp_servers=True by default
if 'extensions_preferences' not in pipeline_data:
@@ -203,6 +214,10 @@ class PipelineService:
pipeline_data = pipeline_data.copy()
for protected_field in ('uuid', 'for_version', 'stages', 'is_default'):
pipeline_data.pop(protected_field, None)
if 'config' in pipeline_data:
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
if 'extensions_preferences' in pipeline_data:
self._validate_extension_preferences(pipeline_data['extensions_preferences'])
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
@@ -259,18 +274,7 @@ class PipelineService:
'stages': original_pipeline.stages.copy() if original_pipeline.stages else default_stage_order.copy(),
'config': original_pipeline.config.copy() if original_pipeline.config else {},
'is_default': False,
'extensions_preferences': (
original_pipeline.extensions_preferences.copy()
if original_pipeline.extensions_preferences
else {
'enable_all_plugins': True,
'enable_all_mcp_servers': True,
'plugins': [],
'mcp_servers': [],
'mcp_resources': [],
'mcp_resource_agent_read_enabled': True,
}
),
'extensions_preferences': normalize_extension_preferences(original_pipeline.extensions_preferences),
}
# Insert the new pipeline
@@ -297,6 +301,36 @@ class PipelineService:
mcp_resource_agent_read_enabled: bool | None = None,
) -> None:
"""Update the bound plugins and MCP servers for a pipeline"""
extension_updates: dict[str, typing.Any] = {
'enable_all_plugins': enable_all_plugins,
'enable_all_mcp_servers': enable_all_mcp_servers,
'enable_all_skills': enable_all_skills,
'plugins': bound_plugins,
}
if bound_mcp_servers is not None:
extension_updates['mcp_servers'] = bound_mcp_servers
if bound_skills is not None:
extension_updates['skills'] = bound_skills
if bound_mcp_resources is not None:
extension_updates['mcp_resources'] = bound_mcp_resources
RunnerConfigResolver.validate_mcp_resource_attachments(
bound_mcp_resources,
context='Pipeline extension',
field_name='bound_mcp_resources',
)
if mcp_resource_agent_read_enabled is not None:
extension_updates['mcp_resource_agent_read_enabled'] = mcp_resource_agent_read_enabled
self._validate_extension_preferences(
extension_updates,
context='Pipeline extension',
field_aliases={
'plugins': 'bound_plugins',
'mcp_servers': 'bound_mcp_servers',
'skills': 'bound_skills',
'mcp_resources': 'bound_mcp_resources',
},
)
# Get current pipeline
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
@@ -309,7 +343,7 @@ class PipelineService:
raise ValueError(f'Pipeline {pipeline_uuid} not found')
# Update extensions_preferences
extensions_preferences = pipeline.extensions_preferences or {}
extensions_preferences = normalize_extension_preferences(pipeline.extensions_preferences)
extensions_preferences['enable_all_plugins'] = enable_all_plugins
extensions_preferences['enable_all_mcp_servers'] = enable_all_mcp_servers
extensions_preferences['enable_all_skills'] = enable_all_skills
@@ -333,3 +367,22 @@ class PipelineService:
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
pipeline = await self.get_pipeline(pipeline_uuid)
await self.ap.pipeline_mgr.load_pipeline(pipeline)
@staticmethod
def _validate_extension_preferences(
value: typing.Any,
*,
context: str = 'Pipeline extensions_preferences',
field_aliases: typing.Mapping[str, str] | None = None,
) -> dict[str, typing.Any]:
validated = validate_extension_preferences(
value,
context=context,
field_aliases=field_aliases,
)
RunnerConfigResolver.validate_mcp_resource_attachments(
validated.get('mcp_resources'),
context=context,
field_name='mcp_resources',
)
return validated
+46 -84
View File
@@ -4,6 +4,7 @@ import asyncio
import collections
import datetime as _dt
import enum
import hashlib
import json
import os
from typing import TYPE_CHECKING
@@ -13,6 +14,7 @@ import pydantic
from langbot_plugin.box.client import BoxRuntimeClient
from .connector import BoxRuntimeConnector, _get_box_config
from ..telemetry import features as telemetry_features
from ..utils import constants
from langbot_plugin.box.errors import BoxError, BoxValidationError
from langbot_plugin.box.models import (
BUILTIN_PROFILES,
@@ -27,6 +29,8 @@ _INT_ADAPTER = pydantic.TypeAdapter(int)
_UTC = _dt.timezone.utc
_MAX_RECENT_ERRORS = 50
_MIB = 1024 * 1024
_HOST_BOX_SCOPE_VARIABLE = '_host_box_scope'
_BOX_SESSION_ID_PREFIX = 'lb-box-'
def _is_path_under(path: str, root: str) -> bool:
@@ -224,45 +228,53 @@ class BoxService:
return self._serialize_result(result)
def resolve_box_session_id(self, query: pipeline_query.Query) -> str:
"""Resolve the Box session_id from the pipeline's template and query variables.
"""Resolve a Host-owned Box session ID for the current conversation."""
if query is None:
raise BoxValidationError('Box execution requires a Host session context.')
When ``system.limitation.force_box_session_id_template`` is set to a
non-empty value, that template overrides whatever the pipeline
configured. This is the authoritative SaaS guard: it runs on every
``exec`` call, so a tenant cannot escape a single shared sandbox even
by editing the pipeline config directly through the API (which only
gates the web UI).
"""
forced_template = self._forced_box_session_id_template()
if forced_template:
template = forced_template
else:
template = '{launcher_type}_{launcher_id}'
pipeline_config = query.pipeline_config or {}
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
runner_selector = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
runner_id = runner_selector.get('id') if isinstance(runner_selector, dict) else None
runner_configs = ai_config.get('runner_config', {}) if isinstance(ai_config, dict) else {}
runner_config = runner_configs.get(runner_id, {}) if isinstance(runner_configs, dict) else {}
configured_template = (
runner_config.get('box-session-id-template') if isinstance(runner_config, dict) else None
)
if isinstance(configured_template, str) and configured_template:
template = configured_template
variables = dict(query.variables or {})
variables = getattr(query, 'variables', None)
if isinstance(variables, dict) and _HOST_BOX_SCOPE_VARIABLE in variables:
private_scope = variables[_HOST_BOX_SCOPE_VARIABLE]
if not isinstance(private_scope, str) or not private_scope.strip():
raise BoxValidationError('Box execution requires a Host conversation scope.')
return self._hash_box_session_scope(f'host:{private_scope}')
session = getattr(query, 'session', None)
launcher_type = getattr(query, 'launcher_type', None)
launcher_id = getattr(query, 'launcher_id', None)
if launcher_type is None:
launcher_type = getattr(session, 'launcher_type', None)
if launcher_id is None:
launcher_id = getattr(session, 'launcher_id', None)
if hasattr(launcher_type, 'value'):
launcher_type = launcher_type.value
launcher_id = getattr(query, 'launcher_id', None)
sender_id = getattr(query, 'sender_id', None)
query_id = getattr(query, 'query_id', None)
variables.setdefault('query_id', str(query_id or 'unknown'))
variables.setdefault('launcher_type', str(launcher_type or 'query'))
variables.setdefault('launcher_id', str(launcher_id or query_id or 'unknown'))
variables.setdefault('sender_id', str(sender_id or launcher_id or query_id or 'unknown'))
variables.setdefault('global', 'global')
return template.format_map(collections.defaultdict(lambda: 'unknown', variables))
if launcher_type is None or launcher_id is None or not str(launcher_id).strip():
raise BoxValidationError('Box execution requires a Host session context.')
adapter = getattr(query, 'adapter', None)
adapter_identity = adapter.__class__.__name__ if adapter is not None else None
scope = json.dumps(
{
'instance_id': str(constants.instance_id or '') or None,
'workspace_id': None,
'bot_id': getattr(query, 'bot_uuid', None),
'platform_adapter': adapter_identity,
'target_type': str(launcher_type),
'target_id': str(launcher_id),
'thread_id': None,
},
ensure_ascii=False,
sort_keys=True,
separators=(',', ':'),
)
return self._hash_box_session_scope(f'host:{scope}')
@staticmethod
def _hash_box_session_scope(scope: str) -> str:
digest = hashlib.sha256(scope.encode('utf-8')).hexdigest()
return f'{_BOX_SESSION_ID_PREFIX}{digest}'
def build_skill_extra_mounts(self, query: pipeline_query.Query) -> list[dict]:
"""Build extra_mounts entries for all pipeline-bound skills.
@@ -1128,20 +1140,6 @@ class BoxService:
raw = str(self._local_config().get('image', '') or '').strip()
return raw or None
def _forced_box_session_id_template(self) -> str:
"""Return the SaaS-forced sandbox-scope template, or '' when unset.
Read from ``system.limitation.force_box_session_id_template``. A
non-empty value pins every pipeline to a single sandbox scope
(e.g. ``'{global}'``) and cannot be overridden per-pipeline.
"""
limitation = (
(self.ap.instance_config.data or {}).get('system', {}).get('limitation', {})
if getattr(self.ap, 'instance_config', None) is not None
else {}
)
return str(limitation.get('force_box_session_id_template', '') or '').strip()
def _load_workspace_quota_mb(self) -> int | None:
raw_value = self._local_config().get('workspace_quota_mb')
if raw_value in (None, ''):
@@ -1311,42 +1309,6 @@ class BoxService:
def get_recent_errors(self) -> list[dict]:
return list(self._recent_errors)
def get_system_guidance(self, query_id=None) -> str:
"""Return LLM system-prompt guidance for the exec tool.
All execution-specific prompt text is kept here so that callers
(e.g. LocalAgentRunner) stay free of box domain knowledge.
``query_id`` is the current turn's pipeline query id. When provided,
the guidance ALWAYS advertises the per-query outbox path so the agent
knows how to deliver generated files back to the user — even on turns
where the user sent no inbound attachment (e.g. "generate a QR code"),
which is exactly when the inbound-attachment note never fires. Outbound
collection in the wrapper runs on every turn regardless of inbound
files, so without this the file would be produced and silently dropped.
"""
guidance = (
'When the exec tool is available, use it for exact calculations, statistics, structured data parsing, '
'and code execution instead of estimating mentally. If the user provides numbers, tables, CSV-like text, '
'JSON, or other data and asks for a computed answer, prefer running a short Python script via exec '
'and then answer from the tool result. Unless the user explicitly asks for the script, code, or implementation '
'details, do not include the generated script in the final answer; return the result and a brief explanation only.'
)
if self.default_workspace:
guidance += (
' A default workspace is mounted at /workspace for file tasks. When the user asks to read, create, or '
'modify local files in the working directory, use exec with /workspace paths directly; do not ask the '
'user for directory parameters unless they explicitly need a different directory.'
)
if query_id is not None:
outbox_dir = f'{self.OUTBOX_MOUNT_DIR}/{query_id}'
guidance += (
f' If you produce any file (image, audio, document, etc.) that should be sent back to the user, '
f'write it into {outbox_dir}/ (create the directory if needed). Every file placed there will be '
'delivered to the user automatically; do not paste file contents or base64 into your reply.'
)
return guidance
async def get_status(self) -> dict:
if not self._available:
return {
@@ -110,44 +110,6 @@ class LoadConfigStage(stage.BootingStage):
async def run(self, ap: app.Application):
"""Load config file"""
# # ======= deprecated =======
# if os.path.exists('data/config/command.json'):
# ap.command_cfg = await config.load_json_config(
# 'data/config/command.json',
# 'templates/legacy/command.json',
# completion=False,
# )
# if os.path.exists('data/config/pipeline.json'):
# ap.pipeline_cfg = await config.load_json_config(
# 'data/config/pipeline.json',
# 'templates/legacy/pipeline.json',
# completion=False,
# )
# if os.path.exists('data/config/platform.json'):
# ap.platform_cfg = await config.load_json_config(
# 'data/config/platform.json',
# 'templates/legacy/platform.json',
# completion=False,
# )
# if os.path.exists('data/config/provider.json'):
# ap.provider_cfg = await config.load_json_config(
# 'data/config/provider.json',
# 'templates/legacy/provider.json',
# completion=False,
# )
# if os.path.exists('data/config/system.json'):
# ap.system_cfg = await config.load_json_config(
# 'data/config/system.json',
# 'templates/legacy/system.json',
# completion=False,
# )
# # ======= deprecated =======
ap.instance_config = await config.load_yaml_config('data/config.yaml', 'config.yaml', completion=False)
# Apply environment variable overrides to data/config.yaml
@@ -1,15 +1,6 @@
import sqlalchemy
from .base import Base
from ...utils import constants
initial_metadata = [
{
'key': 'database_version',
'value': str(constants.required_database_version),
},
]
class Metadata(Base):
@@ -1,7 +1,7 @@
"""baseline: stamp existing schema (db version 25)
"""baseline: stamp the supported 4.x schema
This is a no-op migration that marks the starting point for Alembic.
All tables already exist via create_all() + legacy DBMigration system.
Current tables already exist via SQLAlchemy metadata create_all().
Revision ID: 0001_baseline
Revises: None
@@ -15,8 +15,7 @@ depends_on = None
def upgrade() -> None:
# No-op: existing schema is already at database_version=25
# This revision serves as the Alembic baseline.
# No-op: this revision serves as the Alembic baseline.
pass
@@ -10,7 +10,7 @@ ssereadtimeout) live in ``extra_args`` and are left untouched — the
auto-detecting remote transport consumes them regardless.
Revision ID: 0006_normalize_mcp_remote_mode
Revises: 8d3a1f2c4b6e
Revises: 0005_add_llm_context_length
Create Date: 2026-06-21
"""
@@ -18,7 +18,7 @@ import sqlalchemy as sa
from alembic import op
revision = '0006_normalize_mcp_remote_mode'
down_revision = '8d3a1f2c4b6e'
down_revision = '0005_add_llm_context_length'
branch_labels = None
depends_on = None
@@ -5,6 +5,8 @@ Revises: 0006_normalize_mcp_remote_mode
Create Date: 2026-06-26
"""
import json
import sqlalchemy as sa
from alembic import op
@@ -14,19 +16,64 @@ branch_labels = None
depends_on = None
_BOT_ADMINS = sa.table(
'bot_admins',
sa.column('bot_uuid', sa.String(255)),
sa.column('launcher_type', sa.String(64)),
sa.column('launcher_id', sa.String(255)),
)
def _upsert_admin(conn: sa.Connection, *, bot_uuid: str, launcher_type: str, launcher_id: str) -> None:
values = {
'bot_uuid': bot_uuid,
'launcher_type': launcher_type,
'launcher_id': launcher_id,
}
conflict_columns = [
_BOT_ADMINS.c.bot_uuid,
_BOT_ADMINS.c.launcher_type,
_BOT_ADMINS.c.launcher_id,
]
if conn.dialect.name == 'postgresql':
from sqlalchemy.dialects.postgresql import insert
statement = insert(_BOT_ADMINS).values(**values).on_conflict_do_nothing(index_elements=conflict_columns)
elif conn.dialect.name == 'sqlite':
from sqlalchemy.dialects.sqlite import insert
statement = insert(_BOT_ADMINS).values(**values).on_conflict_do_nothing(index_elements=conflict_columns)
else:
existing = conn.execute(
sa.select(sa.literal(1))
.select_from(_BOT_ADMINS)
.where(
_BOT_ADMINS.c.bot_uuid == bot_uuid,
_BOT_ADMINS.c.launcher_type == launcher_type,
_BOT_ADMINS.c.launcher_id == launcher_id,
)
.limit(1)
).first()
if existing is not None:
return
statement = sa.insert(_BOT_ADMINS).values(**values)
conn.execute(statement)
def upgrade() -> None:
conn = op.get_bind()
if 'bot_admins' in sa.inspect(conn).get_table_names():
return
op.create_table(
'bot_admins',
sa.Column('id', sa.Integer, primary_key=True, autoincrement=True),
sa.Column('bot_uuid', sa.String(255), nullable=False),
sa.Column('launcher_type', sa.String(64), nullable=False),
sa.Column('launcher_id', sa.String(255), nullable=False),
sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()),
sa.UniqueConstraint('bot_uuid', 'launcher_type', 'launcher_id', name='uq_bot_admin'),
)
if 'bot_admins' not in sa.inspect(conn).get_table_names():
op.create_table(
'bot_admins',
sa.Column('id', sa.Integer, primary_key=True, autoincrement=True),
sa.Column('bot_uuid', sa.String(255), nullable=False),
sa.Column('launcher_type', sa.String(64), nullable=False),
sa.Column('launcher_id', sa.String(255), nullable=False),
sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()),
sa.UniqueConstraint('bot_uuid', 'launcher_type', 'launcher_id', name='uq_bot_admin'),
)
# Migrate old config-based admins into the first bot (best-effort)
inspector = sa.inspect(conn)
@@ -48,28 +95,27 @@ def upgrade() -> None:
if meta_row is None:
return
import json
try:
cfg = json.loads(meta_row[0])
except Exception:
except (TypeError, json.JSONDecodeError):
return
admins = cfg.get('admins', [])
if not isinstance(admins, list):
return
for entry in admins:
if not isinstance(entry, str):
continue
parts = entry.split('_', 1)
if len(parts) != 2:
continue
launcher_type, launcher_id = parts
try:
conn.execute(
sa.text(
'INSERT OR IGNORE INTO bot_admins (bot_uuid, launcher_type, launcher_id) VALUES (:bu, :lt, :li)'
),
{'bu': first_bot_uuid, 'lt': launcher_type, 'li': launcher_id},
)
except Exception:
pass
_upsert_admin(
conn,
bot_uuid=first_bot_uuid,
launcher_type=launcher_type,
launcher_id=launcher_id,
)
# Remove admins key from stored config
if 'admins' in cfg:
@@ -81,4 +127,5 @@ def upgrade() -> None:
def downgrade() -> None:
op.drop_table('bot_admins')
if 'bot_admins' in sa.inspect(op.get_bind()).get_table_names():
op.drop_table('bot_admins')
@@ -3,7 +3,7 @@
Bindings keep referencing the original Pipeline UUIDs. This migration never
creates Agent rows or copies Pipeline runner configuration.
Revision ID: 0009_migrate_routing_to_event_bindings
Revision ID: 0009_migrate_event_bindings
Revises: 0008_agent_product_surface
Create Date: 2026-06-26
"""
@@ -14,7 +14,7 @@ import uuid
import sqlalchemy as sa
from alembic import op
revision = '0009_migrate_routing_to_event_bindings'
revision = '0009_migrate_event_bindings'
down_revision = '0008_agent_product_surface'
depends_on = None
@@ -30,24 +30,28 @@ def _column_exists(table_name: str, column_name: str) -> bool:
def _rule_to_filters(rule: dict) -> list[dict] | None:
"""Convert a pipeline_routing_rule to event_binding filters (best effort).
Rules that don't map cleanly (message_content, message_has_element) are
skipped callers should handle None as "cannot migrate".
"""
"""Convert one legacy Pipeline routing rule to an EBA event filter."""
rule_type = rule.get('type')
operator = rule.get('operator', 'eq')
value = rule.get('value', '')
if rule_type == 'launcher_type':
if value == 'group':
return [{'field': 'group', 'operator': 'neq', 'value': None}]
if value == 'person':
return [{'field': 'group', 'operator': 'eq', 'value': None}]
elif rule_type == 'launcher_id':
return [{'field': 'chat_type', 'operator': operator, 'value': value}]
if rule_type == 'launcher_id':
return [{'field': 'chat_id', 'operator': operator, 'value': value}]
if rule_type == 'message_content':
return [{'field': 'message_text', 'operator': operator, 'value': value}]
if rule_type == 'message_has_element':
element_operator = {
'eq': 'contains',
'neq': 'not_contains',
}.get(operator)
if element_operator is None:
# The legacy matcher treated every other operator as non-matching.
element_operator = 'unsupported_legacy_operator'
return [{'field': 'message_element_types', 'operator': element_operator, 'value': value}]
return None # message_content / message_has_element: no clean mapping
return None
def upgrade() -> None:
@@ -0,0 +1,19 @@
"""merge mcp resource and agent product migration heads
Revision ID: 0010_merge_mcp_agent_heads
Revises: 0008_mcp_resource_prefs, 0009_migrate_event_bindings
Create Date: 2026-06-30
"""
revision = '0010_merge_mcp_agent_heads'
down_revision = ('0008_mcp_resource_prefs', '0009_migrate_event_bindings')
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -1,19 +0,0 @@
"""merge mcp resource and agent product migration heads
Revision ID: 0010_merge_mcp_resource_agent_heads
Revises: 0008_mcp_resource_prefs, 0009_migrate_routing_to_event_bindings
Create Date: 2026-06-30
"""
revision = '0010_merge_mcp_resource_agent_heads'
down_revision = ('0008_mcp_resource_prefs', '0009_migrate_routing_to_event_bindings')
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
pass
@@ -1,7 +1,7 @@
"""drop legacy bot pipeline routing columns
Revision ID: 0011_drop_legacy_bot_routing
Revises: 0010_merge_mcp_resource_agent_heads
Revises: 0010_merge_mcp_agent_heads
Create Date: 2026-07-01
"""
@@ -10,7 +10,7 @@ import sqlalchemy as sa
revision = '0011_drop_legacy_bot_routing'
down_revision = '0010_merge_mcp_resource_agent_heads'
down_revision = '0010_merge_mcp_agent_heads'
branch_labels = None
depends_on = None
@@ -0,0 +1,67 @@
"""add monitoring tool calls
Revision ID: 0012_monitoring_tool_calls
Revises: 0011_drop_legacy_bot_routing
Create Date: 2026-07-12
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = '0012_monitoring_tool_calls'
down_revision = '0011_drop_legacy_bot_routing'
branch_labels = None
depends_on = None
_INDEXES = {
'ix_monitoring_tool_calls_timestamp': ['timestamp'],
'ix_monitoring_tool_calls_bot_id': ['bot_id'],
'ix_monitoring_tool_calls_pipeline_id': ['pipeline_id'],
'ix_monitoring_tool_calls_session_id': ['session_id'],
'ix_monitoring_tool_calls_message_id': ['message_id'],
}
def _table_exists() -> bool:
return 'monitoring_tool_calls' in sa.inspect(op.get_bind()).get_table_names()
def _index_names() -> set[str]:
if not _table_exists():
return set()
return {index['name'] for index in sa.inspect(op.get_bind()).get_indexes('monitoring_tool_calls')}
def upgrade() -> None:
if not _table_exists():
op.create_table(
'monitoring_tool_calls',
sa.Column('id', sa.String(255), primary_key=True),
sa.Column('timestamp', sa.DateTime(), nullable=False),
sa.Column('tool_name', sa.String(255), nullable=False),
sa.Column('tool_source', sa.String(50), nullable=False),
sa.Column('duration', sa.Integer(), nullable=False),
sa.Column('status', sa.String(50), nullable=False),
sa.Column('bot_id', sa.String(255), nullable=False),
sa.Column('bot_name', sa.String(255), nullable=False),
sa.Column('pipeline_id', sa.String(255), nullable=False),
sa.Column('pipeline_name', sa.String(255), nullable=False),
sa.Column('session_id', sa.String(255), nullable=True),
sa.Column('message_id', sa.String(255), nullable=True),
sa.Column('arguments', sa.Text(), nullable=True),
sa.Column('result', sa.Text(), nullable=True),
sa.Column('error_message', sa.Text(), nullable=True),
)
existing_indexes = _index_names()
for index_name, columns in _INDEXES.items():
if index_name not in existing_indexes:
op.create_index(index_name, 'monitoring_tool_calls', columns, unique=False)
def downgrade() -> None:
if _table_exists():
op.drop_table('monitoring_tool_calls')
+5 -53
View File
@@ -7,15 +7,14 @@ import typing
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
import sqlalchemy
from . import database, migration
from ..entity.persistence import base, metadata, model as persistence_model
from . import database
from ..entity.persistence import base, model as persistence_model
from ..entity import persistence
from ..core import app
from ..utils import constants, importutil
from . import databases, migrations
from ..utils import importutil
from . import databases
importutil.import_modules_in_pkg(databases)
importutil.import_modules_in_pkg(migrations)
importutil.import_modules_in_pkg(persistence)
@@ -43,40 +42,6 @@ class PersistenceManager:
break
await self.create_tables()
# run migrations
database_version = await self.execute_async(
sqlalchemy.select(metadata.Metadata).where(metadata.Metadata.key == 'database_version')
)
database_version = int(database_version.fetchone()[1])
required_database_version = constants.required_database_version
if database_version < required_database_version:
migrations = migration.preregistered_db_migrations
migrations.sort(key=lambda x: x.number)
last_migration_number = database_version
for migration_cls in migrations:
migration_instance = migration_cls(self.ap)
if (
migration_instance.number > database_version
and migration_instance.number <= required_database_version
):
await migration_instance.upgrade()
await self.execute_async(
sqlalchemy.update(metadata.Metadata)
.where(metadata.Metadata.key == 'database_version')
.values({'value': str(migration_instance.number)})
)
last_migration_number = migration_instance.number
self.ap.logger.info(f'Migration {migration_instance.number} completed.')
self.ap.logger.info(f'Successfully upgraded database to version {last_migration_number}.')
# Run Alembic migrations (new migration system)
await self._run_alembic_migrations()
await self.write_space_model_providers()
@@ -88,19 +53,6 @@ class PersistenceManager:
await conn.commit()
# ======= write initial data =======
# write initial metadata
self.ap.logger.info('Creating initial metadata...')
for item in metadata.initial_metadata:
# check if the item exists
result = await self.execute_async(
sqlalchemy.select(metadata.Metadata).where(metadata.Metadata.key == item['key'])
)
row = result.first()
if row is None:
await self.execute_async(sqlalchemy.insert(metadata.Metadata).values(item))
async def write_space_model_providers(self):
space_models_gateway_api_url = self.ap.instance_config.data.get('space', {}).get(
'models_gateway_api_url', 'https://api.langbot.cloud/v1'
@@ -139,7 +91,7 @@ class PersistenceManager:
# =================================
async def _run_alembic_migrations(self):
"""Run Alembic-based migrations after legacy migrations complete."""
"""Run Alembic migrations for supported 4.x databases."""
from . import alembic_runner
engine = self.get_db_engine()
-40
View File
@@ -1,40 +0,0 @@
from __future__ import annotations
import typing
import abc
from ..core import app
preregistered_db_migrations: list[typing.Type[DBMigration]] = []
def migration_class(number: int):
"""Migration class decorator"""
def wrapper(cls: typing.Type[DBMigration]) -> typing.Type[DBMigration]:
cls.number = number
preregistered_db_migrations.append(cls)
return cls
return wrapper
class DBMigration(abc.ABC):
"""Database migration"""
number: int
"""Migration number"""
def __init__(self, ap: app.Application):
self.ap = ap
@abc.abstractmethod
async def upgrade(self):
"""Upgrade"""
pass
@abc.abstractmethod
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,36 +0,0 @@
# Legacy migrations (DEPRECATED — do not add new files here)
This directory holds the **legacy 3.x database migration system**
(`DBMigration` subclasses in `dbmXXX_*.py`, registered via
`@migration.migration_class(N)` and run from `pkg/persistence/mgr.py`).
**This system is frozen. Do not add new `dbmXXX_*.py` migrations.**
The chain is capped at version 25 (`required_database_version = 25` in
`pkg/utils/constants.py`). These files exist only to upgrade pre-existing
3.x databases up to the Alembic baseline (`0001_baseline`). Removing them
would break in-place upgrades from old installations, so they are kept
read-only.
## All new schema changes use Alembic
Migrations now live in `pkg/persistence/alembic/versions/`. To create one:
```bash
uv run python -m langbot.pkg.persistence.alembic_runner autogenerate "description of your change"
```
(requires `data/config.yaml` to exist). Review and edit the generated
script before committing — Alembic migrations run automatically on startup
and must be idempotent and guard against missing tables (the test suite
runs them against empty databases).
### Rules for Alembic revision ids
- Keep the revision id **≤ 32 characters** — PostgreSQL stores
`alembic_version.version_num` as `varchar(32)` and will raise
`StringDataRightTruncationError` on overflow.
- Guard every `op` call against a missing table / missing column
(`inspector.get_table_names()` / `inspector.get_columns()`); fresh
installs create the schema via `create_all()` and stamp the baseline,
so migrations may run against tables that already match or do not exist.
@@ -1,218 +0,0 @@
from .. import migration
from copy import deepcopy
import uuid
import os
import sqlalchemy
import shutil
from ...config import manager as config_manager
from ...entity.persistence import (
model as persistence_model,
pipeline as persistence_pipeline,
bot as persistence_bot,
)
@migration.migration_class(1)
class DBMigrateV3Config(migration.DBMigration):
"""Migrate v3 config to v4 database"""
async def upgrade(self):
"""Upgrade"""
"""
Migrate all config files under data/config.
After migration, all previous config files are saved under data/legacy/config.
After migration, all config files under data/metadata/ are saved under data/legacy/metadata.
"""
if self.ap.provider_cfg is None:
return
# ======= Migrate model =======
# Only migrate the currently selected model
model_name = self.ap.provider_cfg.data.get('model', 'gpt-4o')
model_requester = 'openai-chat-completions'
model_requester_config = {}
model_api_keys = ['sk-proj-1234567890']
model_abilities = []
model_extra_args = {}
if os.path.exists('data/metadata/llm-models.json'):
_llm_model_meta = await config_manager.load_json_config('data/metadata/llm-models.json', completion=False)
for item in _llm_model_meta.data.get('list', []):
if item.get('name') == model_name:
if 'model_name' in item:
model_name = item['model_name']
if 'requester' in item:
model_requester = item['requester']
if 'token_mgr' in item:
_token_mgr = item['token_mgr']
if _token_mgr in self.ap.provider_cfg.data.get('keys', {}):
model_api_keys = self.ap.provider_cfg.data.get('keys', {})[_token_mgr]
if 'tool_call_supported' in item and item['tool_call_supported']:
model_abilities.append('func_call')
if 'vision_supported' in item and item['vision_supported']:
model_abilities.append('vision')
if (
model_requester in self.ap.provider_cfg.data.get('requester', {})
and 'args' in self.ap.provider_cfg.data.get('requester', {})[model_requester]
):
model_extra_args = self.ap.provider_cfg.data.get('requester', {})[model_requester]['args']
if model_requester in self.ap.provider_cfg.data.get('requester', {}):
model_requester_config = self.ap.provider_cfg.data.get('requester', {})[model_requester]
model_requester_config = {
'base_url': model_requester_config['base-url'],
'timeout': model_requester_config['timeout'],
}
break
model_uuid = str(uuid.uuid4())
llm_model_data = {
'uuid': model_uuid,
'name': model_name,
'description': '由 LangBot v3 迁移而来',
'requester': model_requester,
'requester_config': model_requester_config,
'api_keys': model_api_keys,
'abilities': model_abilities,
'extra_args': model_extra_args,
}
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(persistence_model.LLMModel).values(**llm_model_data)
)
# ======= Migrate pipeline config =======
# Modify to default pipeline
default_pipeline = [
self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
for pipeline in (
await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
persistence_pipeline.LegacyPipeline.is_default == True
)
)
).all()
][0]
pipeline_uuid = str(uuid.uuid4())
pipeline_name = 'ChatPipeline'
if default_pipeline:
pipeline_name = default_pipeline['name']
pipeline_uuid = default_pipeline['uuid']
pipeline_config = default_pipeline['config']
# trigger
pipeline_config['trigger']['group-respond-rules'] = self.ap.pipeline_cfg.data['respond-rules']['default']
pipeline_config['trigger']['access-control'] = self.ap.pipeline_cfg.data['access-control']
pipeline_config['trigger']['ignore-rules'] = self.ap.pipeline_cfg.data['ignore-rules']
# safety
pipeline_config['safety']['content-filter'] = {
'scope': 'all',
'check-sensitive-words': self.ap.pipeline_cfg.data['check-sensitive-words'],
}
pipeline_config['safety']['rate-limit'] = {
'window-length': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['window-size'],
'limitation': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['limit'],
'strategy': self.ap.pipeline_cfg.data['rate-limit']['strategy'],
}
# output
pipeline_config['output']['long-text-processing'] = self.ap.platform_cfg.data['long-text-process']
pipeline_config['output']['force-delay'] = self.ap.platform_cfg.data['force-delay']
pipeline_config['output']['misc'] = {
'hide-exception': self.ap.platform_cfg.data['hide-exception-info'],
'quote-origin': self.ap.platform_cfg.data['quote-origin'],
'at-sender': self.ap.platform_cfg.data['at-sender'],
'track-function-calls': self.ap.platform_cfg.data['track-function-calls'],
}
default_pipeline['description'] = default_pipeline['description'] + ' [已迁移 LangBot v3 配置]'
default_pipeline['config'] = pipeline_config
default_pipeline.pop('created_at')
default_pipeline.pop('updated_at')
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
.values(default_pipeline)
.where(persistence_pipeline.LegacyPipeline.uuid == default_pipeline['uuid'])
)
# ======= Migrate bot =======
# Only migrate enabled bots
for adapter in self.ap.platform_cfg.data.get('platform-adapters', []):
if not adapter.get('enable'):
continue
args = deepcopy(adapter)
args.pop('adapter')
args.pop('enable')
bot_data = {
'uuid': str(uuid.uuid4()),
'name': adapter.get('adapter'),
'description': '由 LangBot v3 迁移而来',
'adapter': adapter.get('adapter'),
'adapter_config': args,
'enable': True,
'use_pipeline_uuid': pipeline_uuid,
'use_pipeline_name': pipeline_name,
}
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(**bot_data))
# ======= Migrate system settings =======
self.ap.instance_config.data['admins'] = self.ap.system_cfg.data['admin-sessions']
self.ap.instance_config.data['api']['port'] = self.ap.system_cfg.data['http-api']['port']
self.ap.instance_config.data['command'] = {
'prefix': self.ap.command_cfg.data['command-prefix'],
'enable': self.ap.command_cfg.data['command-enable']
if 'command-enable' in self.ap.command_cfg.data
else True,
'privilege': self.ap.command_cfg.data['privilege'],
}
self.ap.instance_config.data['concurrency']['pipeline'] = self.ap.system_cfg.data['pipeline-concurrency']
self.ap.instance_config.data['concurrency']['session'] = self.ap.system_cfg.data['session-concurrency'][
'default'
]
self.ap.instance_config.data['mcp'] = self.ap.provider_cfg.data['mcp']
self.ap.instance_config.data['proxy'] = self.ap.system_cfg.data['network-proxies']
await self.ap.instance_config.dump_config()
# ======= move files =======
# Migrate all config files under data/config
all_legacy_dir_name = [
'config',
# 'metadata',
'prompts',
'scenario',
]
def move_legacy_files(dir_name: str):
if not os.path.exists(f'data/legacy/{dir_name}'):
os.makedirs(f'data/legacy/{dir_name}')
if os.path.exists(f'data/{dir_name}'):
for file in os.listdir(f'data/{dir_name}'):
if file.endswith('.json'):
shutil.move(f'data/{dir_name}/{file}', f'data/legacy/{dir_name}/{file}')
os.rmdir(f'data/{dir_name}')
for dir_name in all_legacy_dir_name:
move_legacy_files(dir_name)
async def downgrade(self):
"""Downgrade"""
@@ -1,55 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(2)
class DBMigrateCombineQuoteMsgConfig(migration.DBMigration):
"""Combine quote message config"""
async def upgrade(self):
"""Upgrade"""
# Read all pipelines using raw SQL
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
# Ensure 'trigger' exists
if 'trigger' not in config:
config['trigger'] = {}
# Ensure 'misc' exists in 'trigger'
if 'misc' not in config['trigger']:
config['trigger']['misc'] = {}
# Add 'combine-quote-message' if not exists
if 'combine-quote-message' not in config['trigger']['misc']:
config['trigger']['misc']['combine-quote-message'] = False
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,62 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(3)
class DBMigrateN8nConfig(migration.DBMigration):
"""N8n config"""
async def upgrade(self):
"""Upgrade"""
# Read all pipelines using raw SQL
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
# Ensure 'ai' exists
if 'ai' not in config:
config['ai'] = {}
# Add 'n8n-service-api' if not exists
if 'n8n-service-api' not in config['ai']:
config['ai']['n8n-service-api'] = {
'webhook-url': 'http://your-n8n-webhook-url',
'auth-type': 'none',
'basic-username': '',
'basic-password': '',
'jwt-secret': '',
'jwt-algorithm': 'HS256',
'header-name': '',
'header-value': '',
'timeout': 120,
'output-key': 'response',
}
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,53 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(4)
class DBMigrateRAGKBUUID(migration.DBMigration):
"""RAG知识库UUID"""
async def upgrade(self):
"""升级"""
# Read all pipelines using raw SQL
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
# Ensure nested structure exists
if 'ai' not in config:
config['ai'] = {}
if 'local-agent' not in config['ai']:
config['ai']['local-agent'] = {}
# Add 'knowledge-base' if not exists
if 'knowledge-base' not in config['ai']['local-agent']:
config['ai']['local-agent']['knowledge-base'] = ''
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
async def downgrade(self):
"""降级"""
pass
@@ -1,53 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(5)
class DBMigratePipelineRemoveCotConfig(migration.DBMigration):
"""Pipeline remove cot config"""
async def upgrade(self):
"""Upgrade"""
# Read all pipelines using raw SQL
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
# Ensure nested structure exists
if 'output' not in config:
config['output'] = {}
if 'misc' not in config['output']:
config['output']['misc'] = {}
# Add 'remove-think' if not exists
if 'remove-think' not in config['output']['misc']:
config['output']['misc']['remove-think'] = False
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,58 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(6)
class DBMigrateLangflowApiConfig(migration.DBMigration):
"""Langflow API config"""
async def upgrade(self):
"""Upgrade"""
# Read all pipelines using raw SQL
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
# Ensure 'ai' exists
if 'ai' not in config:
config['ai'] = {}
# Add 'langflow-api' if not exists
if 'langflow-api' not in config['ai']:
config['ai']['langflow-api'] = {
'base-url': 'http://localhost:7860',
'api-key': 'your-api-key',
'flow-id': 'your-flow-id',
'input-type': 'chat',
'output-type': 'chat',
'tweaks': '{}',
}
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,44 +0,0 @@
import sqlalchemy
from .. import migration
@migration.migration_class(7)
class DBMigratePluginInstallSource(migration.DBMigration):
"""插件安装来源"""
async def upgrade(self):
"""升级"""
# 查询表结构获取所有列名(异步执行 SQL)
columns = []
if self.ap.persistence_mgr.db.name == 'postgresql':
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
"SELECT column_name FROM information_schema.columns WHERE table_name = 'plugin_settings';"
)
)
all_result = result.fetchall()
columns = [row[0] for row in all_result]
else:
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('PRAGMA table_info(plugin_settings);'))
all_result = result.fetchall()
columns = [row[1] for row in all_result]
# 检查并添加 install_source 列
if 'install_source' not in columns:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
"ALTER TABLE plugin_settings ADD COLUMN install_source VARCHAR(255) NOT NULL DEFAULT 'github'"
)
)
# 检查并添加 install_info 列
if 'install_info' not in columns:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("ALTER TABLE plugin_settings ADD COLUMN install_info JSON NOT NULL DEFAULT '{}'")
)
async def downgrade(self):
"""降级"""
pass
@@ -1,22 +0,0 @@
from .. import migration
@migration.migration_class(8)
class DBMigratePluginConfig(migration.DBMigration):
"""插件配置"""
async def upgrade(self):
"""升级"""
if 'plugin' not in self.ap.instance_config.data:
self.ap.instance_config.data['plugin'] = {
'runtime_ws_url': 'ws://langbot_plugin_runtime:5400/control/ws',
'enable_marketplace': True,
'cloud_service_url': 'https://space.langbot.app',
}
await self.ap.instance_config.dump_config()
async def downgrade(self):
"""降级"""
pass
@@ -1,20 +0,0 @@
import sqlalchemy
from .. import migration
@migration.migration_class(9)
class DBMigratePipelineExtensionPreferences(migration.DBMigration):
"""Pipeline extension preferences"""
async def upgrade(self):
"""Upgrade"""
sql_text = sqlalchemy.text(
"ALTER TABLE legacy_pipelines ADD COLUMN extensions_preferences JSON NOT NULL DEFAULT '{}'"
)
await self.ap.persistence_mgr.execute_async(sql_text)
async def downgrade(self):
"""Downgrade"""
sql_text = sqlalchemy.text('ALTER TABLE legacy_pipelines DROP COLUMN extensions_preferences')
await self.ap.persistence_mgr.execute_async(sql_text)
@@ -1,105 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(10)
class DBMigratePipelineMultiKnowledgeBase(migration.DBMigration):
"""Pipeline support multiple knowledge base binding"""
async def upgrade(self):
"""Upgrade"""
# Read all pipelines using raw SQL
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
# Convert knowledge-base from string to array
if 'ai' in config and 'local-agent' in config['ai']:
current_kb = config['ai']['local-agent'].get('knowledge-base', '')
# If it's already a list, skip
if isinstance(current_kb, list):
continue
# Convert string to list
if current_kb and current_kb != '__none__':
config['ai']['local-agent']['knowledge-bases'] = [current_kb]
else:
config['ai']['local-agent']['knowledge-bases'] = []
# Remove old field
if 'knowledge-base' in config['ai']['local-agent']:
del config['ai']['local-agent']['knowledge-base']
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
async def downgrade(self):
"""Downgrade"""
# Read all pipelines using raw SQL
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
# Convert knowledge-bases from array back to string
if 'ai' in config and 'local-agent' in config['ai']:
current_kbs = config['ai']['local-agent'].get('knowledge-bases', [])
# If it's already a string, skip
if isinstance(current_kbs, str):
continue
# Convert list to string (take first one or empty)
if current_kbs and len(current_kbs) > 0:
config['ai']['local-agent']['knowledge-base'] = current_kbs[0]
else:
config['ai']['local-agent']['knowledge-base'] = ''
# Remove new field
if 'knowledge-bases' in config['ai']['local-agent']:
del config['ai']['local-agent']['knowledge-bases']
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
@@ -1,55 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(11)
class DBMigrateDifyApiConfig(migration.DBMigration):
"""Dify base prompt config"""
async def upgrade(self):
"""Upgrade"""
# Read all pipelines using raw SQL
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
# Ensure nested structure exists
if 'ai' not in config:
config['ai'] = {}
if 'dify-service-api' not in config['ai']:
config['ai']['dify-service-api'] = {}
# Add 'base-prompt' if not exists
if 'base-prompt' not in config['ai']['dify-service-api']:
config['ai']['dify-service-api']['base-prompt'] = (
'When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image.',
)
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,73 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(12)
class DBMigratePipelineExtensionsEnableAll(migration.DBMigration):
"""Pipeline extensions enable all"""
async def upgrade(self):
"""Upgrade"""
# Read all pipelines using raw SQL
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, extensions_preferences FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
extensions_preferences = (
json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
)
# Ensure extensions_preferences is a dict
if extensions_preferences is None:
extensions_preferences = {}
# Add 'enable_all_plugins' if not exists
if 'enable_all_plugins' not in extensions_preferences:
if 'plugins' in extensions_preferences:
extensions_preferences['enable_all_plugins'] = False
else:
extensions_preferences['enable_all_plugins'] = True
extensions_preferences['plugins'] = []
# Add 'enable_all_mcp_servers' if not exists
if 'enable_all_mcp_servers' not in extensions_preferences:
if 'mcp_servers' in extensions_preferences:
extensions_preferences['enable_all_mcp_servers'] = False
else:
extensions_preferences['enable_all_mcp_servers'] = True
extensions_preferences['mcp_servers'] = []
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET extensions_preferences = :extensions_preferences::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{
'extensions_preferences': json.dumps(extensions_preferences),
'for_version': current_version,
'uuid': uuid,
},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET extensions_preferences = :extensions_preferences, for_version = :for_version WHERE uuid = :uuid'
),
{
'extensions_preferences': json.dumps(extensions_preferences),
'for_version': current_version,
'uuid': uuid,
},
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,49 +0,0 @@
import sqlalchemy
from .. import migration
@migration.migration_class(13)
class DBMigrateKnowledgeBaseUpdatedAt(migration.DBMigration):
"""Add updated_at field to knowledge_bases table"""
async def upgrade(self):
"""Upgrade"""
# Get all column names from the table
columns = []
if self.ap.persistence_mgr.db.name == 'postgresql':
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
"SELECT column_name FROM information_schema.columns WHERE table_name = 'knowledge_bases';"
)
)
all_result = result.fetchall()
columns = [row[0] for row in all_result]
else:
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('PRAGMA table_info(knowledge_bases);'))
all_result = result.fetchall()
columns = [row[1] for row in all_result]
# Check and add updated_at column
if 'updated_at' not in columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'ALTER TABLE knowledge_bases ADD COLUMN updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP'
)
)
else:
# SQLite doesn't support DEFAULT CURRENT_TIMESTAMP in ALTER TABLE
# Add column without default first
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE knowledge_bases ADD COLUMN updated_at DATETIME')
)
# Set initial updated_at values to created_at for existing records
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('UPDATE knowledge_bases SET updated_at = created_at WHERE updated_at IS NULL')
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,94 +0,0 @@
import sqlalchemy
from .. import migration
@migration.migration_class(14)
class DBMigrateSpaceAccountSupport(migration.DBMigration):
"""Add Space account support fields to users table"""
async def upgrade(self):
"""Upgrade"""
# Get all column names from the users table
columns = []
if self.ap.persistence_mgr.db.name == 'postgresql':
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("SELECT column_name FROM information_schema.columns WHERE table_name = 'users';")
)
all_result = result.fetchall()
columns = [row[0] for row in all_result]
else:
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('PRAGMA table_info(users);'))
all_result = result.fetchall()
columns = [row[1] for row in all_result]
# Add account_type column
if 'account_type' not in columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("ALTER TABLE users ADD COLUMN account_type VARCHAR(32) DEFAULT 'local' NOT NULL")
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("ALTER TABLE users ADD COLUMN account_type VARCHAR(32) DEFAULT 'local' NOT NULL")
)
# Add space_account_uuid column
if 'space_account_uuid' not in columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_account_uuid VARCHAR(255)')
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_account_uuid VARCHAR(255)')
)
# Add space_access_token column
if 'space_access_token' not in columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_access_token TEXT')
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_access_token TEXT')
)
# Add space_refresh_token column
if 'space_refresh_token' not in columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_refresh_token TEXT')
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_refresh_token TEXT')
)
# Add space_access_token_expires_at column
if 'space_access_token_expires_at' not in columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_access_token_expires_at TIMESTAMP')
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_access_token_expires_at DATETIME')
)
# Add space_api_key column
if 'space_api_key' not in columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_api_key VARCHAR(255)')
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_api_key VARCHAR(255)')
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,15 +0,0 @@
from .. import migration
# this is a deprecated migration
@migration.migration_class(15)
class DBMigrateModelSourceTracking(migration.DBMigration):
"""Add source tracking fields to models tables for Space integration"""
async def upgrade(self):
"""Upgrade"""
pass
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,305 +0,0 @@
import uuid as uuid_lib
import sqlalchemy
from .. import migration
@migration.migration_class(16)
class DBMigrateModelProviderRefactor(migration.DBMigration):
"""Refactor model structure: create providers from existing models and update references"""
async def upgrade(self):
"""Upgrade"""
# Step 1: Create model_providers table if not exists
await self._create_providers_table()
# Step 2: Migrate existing models to use providers
await self._migrate_llm_models()
await self._migrate_embedding_models()
# Step 3: Remove deprecated columns
await self._cleanup_columns()
async def _create_providers_table(self):
"""Create model_providers table"""
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("""
CREATE TABLE IF NOT EXISTS model_providers (
uuid VARCHAR(255) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
requester VARCHAR(255) NOT NULL,
base_url VARCHAR(512) NOT NULL,
api_keys JSONB NOT NULL DEFAULT '[]',
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""")
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("""
CREATE TABLE IF NOT EXISTS model_providers (
uuid VARCHAR(255) PRIMARY KEY,
name VARCHAR(255) NOT NULL,
requester VARCHAR(255) NOT NULL,
base_url VARCHAR(512) NOT NULL,
api_keys JSON NOT NULL DEFAULT '[]',
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
)
""")
)
async def _migrate_llm_models(self):
"""Migrate LLM models to use providers"""
llm_columns = await self._get_columns('llm_models')
# Add provider_uuid column if not exists
if 'provider_uuid' not in llm_columns:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE llm_models ADD COLUMN provider_uuid VARCHAR(255)')
)
# Add prefered_ranking column if not exists
if 'prefered_ranking' not in llm_columns:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE llm_models ADD COLUMN prefered_ranking INTEGER NOT NULL DEFAULT 0')
)
# Only migrate if old columns exist
if 'requester' not in llm_columns:
return
# Get all LLM models with old structure
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, name, requester, requester_config, api_keys FROM llm_models')
)
models = result.fetchall()
# Create providers and update models
provider_cache = {} # (requester, base_url, api_keys_str) -> provider_uuid
for model in models:
model_uuid, model_name, requester, requester_config, api_keys = model
# Extract base_url from requester_config
base_url = ''
if requester_config:
if isinstance(requester_config, str):
import json
requester_config = json.loads(requester_config)
base_url = requester_config.get('base_url', '') or requester_config.get('base-url', '')
# Parse api_keys if it's a string
if isinstance(api_keys, str):
import json
try:
api_keys = json.loads(api_keys)
except Exception:
api_keys = []
if not api_keys:
api_keys = []
# Create cache key
api_keys_str = str(sorted(api_keys)) if api_keys else '[]'
cache_key = (requester, base_url, api_keys_str)
if cache_key in provider_cache:
provider_uuid = provider_cache[cache_key]
else:
# Create new provider
provider_uuid = str(uuid_lib.uuid4())
provider_name = f'{requester}'
if base_url:
# Extract domain for name
try:
from urllib.parse import urlparse
parsed = urlparse(base_url)
provider_name = parsed.netloc or requester
except Exception:
pass
import json
api_keys_json = json.dumps(api_keys) if api_keys else '[]'
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("""
INSERT INTO model_providers (uuid, name, requester, base_url, api_keys)
VALUES (:uuid, :name, :requester, :base_url, :api_keys)
"""),
{
'uuid': provider_uuid,
'name': provider_name,
'requester': requester,
'base_url': base_url,
'api_keys': api_keys_json,
},
)
provider_cache[cache_key] = provider_uuid
# Update model with provider_uuid
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('UPDATE llm_models SET provider_uuid = :provider_uuid WHERE uuid = :uuid'),
{'provider_uuid': provider_uuid, 'uuid': model_uuid},
)
async def _migrate_embedding_models(self):
"""Migrate embedding models to use providers"""
embedding_columns = await self._get_columns('embedding_models')
# Add provider_uuid column if not exists
if 'provider_uuid' not in embedding_columns:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE embedding_models ADD COLUMN provider_uuid VARCHAR(255)')
)
# Add prefered_ranking column if not exists
if 'prefered_ranking' not in embedding_columns:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE embedding_models ADD COLUMN prefered_ranking INTEGER NOT NULL DEFAULT 0')
)
# Only migrate if old columns exist
if 'requester' not in embedding_columns:
return
# Get all embedding models with old structure
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, name, requester, requester_config, api_keys FROM embedding_models')
)
models = result.fetchall()
# Get existing providers
provider_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, requester, base_url, api_keys FROM model_providers')
)
existing_providers = provider_result.fetchall()
provider_cache = {}
for p in existing_providers:
p_uuid, p_requester, p_base_url, p_api_keys = p
api_keys_str = str(sorted(p_api_keys)) if p_api_keys else '[]'
provider_cache[(p_requester, p_base_url, api_keys_str)] = p_uuid
for model in models:
model_uuid, model_name, requester, requester_config, api_keys = model
base_url = ''
if requester_config:
if isinstance(requester_config, str):
import json
requester_config = json.loads(requester_config)
base_url = requester_config.get('base_url', '') or requester_config.get('base-url', '')
# Parse api_keys if it's a string
if isinstance(api_keys, str):
import json
try:
api_keys = json.loads(api_keys)
except Exception:
api_keys = []
if not api_keys:
api_keys = []
api_keys_str = str(sorted(api_keys)) if api_keys else '[]'
cache_key = (requester, base_url, api_keys_str)
if cache_key in provider_cache:
provider_uuid = provider_cache[cache_key]
else:
provider_uuid = str(uuid_lib.uuid4())
provider_name = f'{requester}'
if base_url:
try:
from urllib.parse import urlparse
parsed = urlparse(base_url)
provider_name = parsed.netloc or requester
except Exception:
pass
import json
api_keys_json = json.dumps(api_keys) if api_keys else '[]'
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("""
INSERT INTO model_providers (uuid, name, requester, base_url, api_keys)
VALUES (:uuid, :name, :requester, :base_url, :api_keys)
"""),
{
'uuid': provider_uuid,
'name': provider_name,
'requester': requester,
'base_url': base_url,
'api_keys': api_keys_json,
},
)
provider_cache[cache_key] = provider_uuid
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('UPDATE embedding_models SET provider_uuid = :provider_uuid WHERE uuid = :uuid'),
{'provider_uuid': provider_uuid, 'uuid': model_uuid},
)
async def _cleanup_columns(self):
"""Remove deprecated columns from model tables"""
llm_columns = await self._get_columns('llm_models')
deprecated_llm_cols = ['requester', 'requester_config', 'api_keys', 'description', 'source', 'space_model_id']
for col in deprecated_llm_cols:
if col in llm_columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(f'ALTER TABLE llm_models DROP COLUMN IF EXISTS {col}')
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(f'ALTER TABLE llm_models DROP COLUMN {col}')
)
embedding_columns = await self._get_columns('embedding_models')
deprecated_embedding_cols = [
'requester',
'requester_config',
'api_keys',
'description',
'source',
'space_model_id',
]
for col in deprecated_embedding_cols:
if col in embedding_columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(f'ALTER TABLE embedding_models DROP COLUMN IF EXISTS {col}')
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(f'ALTER TABLE embedding_models DROP COLUMN {col}')
)
async def _get_columns(self, table_name: str) -> list:
"""Get column names for a table"""
if self.ap.persistence_mgr.db.name == 'postgresql':
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
f"SELECT column_name FROM information_schema.columns WHERE table_name = '{table_name}';"
)
)
all_result = result.fetchall()
return [row[0] for row in all_result]
else:
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text(f'PRAGMA table_info({table_name});'))
all_result = result.fetchall()
return [row[1] for row in all_result]
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,25 +0,0 @@
from .. import migration
@migration.migration_class(17)
class MoveCloudServiceUrl(migration.DBMigration):
"""迁移云服务 URL 配置"""
async def upgrade(self):
"""升级"""
if 'space' not in self.ap.instance_config.data:
self.ap.instance_config.data['space'] = {
'url': 'https://space.langbot.app',
'models_gateway_api_url': 'https://api.langbot.cloud/v1',
'oauth_authorize_url': 'https://space.langbot.app/auth/authorize',
'disable_models_service': False,
}
if 'plugin' in self.ap.instance_config.data:
self.ap.instance_config.data['plugin'].pop('cloud_service_url', None)
await self.ap.instance_config.dump_config()
async def downgrade(self):
"""降级"""
pass
@@ -1,58 +0,0 @@
import sqlalchemy
from .. import migration
@migration.migration_class(18)
class DBMigrateAddEmojiSupport(migration.DBMigration):
"""Add emoji field to knowledge_bases, external_knowledge_bases and legacy_pipelines tables"""
async def upgrade(self):
"""Upgrade"""
# Add emoji field to knowledge_bases
await self._add_emoji_to_table('knowledge_bases', '📚')
# Add emoji field to external_knowledge_bases
await self._add_emoji_to_table('external_knowledge_bases', '🔗')
# Add emoji field to legacy_pipelines
await self._add_emoji_to_table('legacy_pipelines', '⚙️')
async def _add_emoji_to_table(self, table_name: str, default_emoji: str):
"""Add emoji column to specified table if it doesn't exist"""
# Get all column names from the table
columns = []
if self.ap.persistence_mgr.db.name == 'postgresql':
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
f"SELECT column_name FROM information_schema.columns WHERE table_name = '{table_name}';"
)
)
all_result = result.fetchall()
columns = [row[0] for row in all_result]
else:
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text(f'PRAGMA table_info({table_name});'))
all_result = result.fetchall()
columns = [row[1] for row in all_result]
# Check and add emoji column
if 'emoji' not in columns:
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(f"ALTER TABLE {table_name} ADD COLUMN emoji VARCHAR(10) DEFAULT '{default_emoji}'")
)
else:
# SQLite doesn't support DEFAULT with emoji directly in ALTER TABLE
# Add column without default first
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(f'ALTER TABLE {table_name} ADD COLUMN emoji VARCHAR(10)')
)
# Set default emoji value for existing records
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(f"UPDATE {table_name} SET emoji = '{default_emoji}' WHERE emoji IS NULL")
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,24 +0,0 @@
import sqlalchemy
from .. import migration
@migration.migration_class(19)
class DBMigrateMonitoringMessageRole(migration.DBMigration):
"""Add role column to monitoring_messages table"""
async def upgrade(self):
"""Upgrade"""
try:
sql_text = sqlalchemy.text("ALTER TABLE monitoring_messages ADD COLUMN role VARCHAR(50) DEFAULT 'user'")
await self.ap.persistence_mgr.execute_async(sql_text)
except Exception:
# Column may already exist
pass
async def downgrade(self):
"""Downgrade"""
try:
sql_text = sqlalchemy.text('ALTER TABLE monitoring_messages DROP COLUMN role')
await self.ap.persistence_mgr.execute_async(sql_text)
except Exception:
pass
@@ -1,161 +0,0 @@
import sqlalchemy
from .. import migration
@migration.migration_class(20)
class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
"""Migrate to unified Knowledge Engine plugin architecture.
Changes:
- Backup existing knowledge_bases data to knowledge_bases_backup
- Clear knowledge_bases table and add new plugin architecture columns
- Drop old columns (PostgreSQL only; SQLite leaves them unmapped)
- Preserve external_knowledge_bases table as-is for future migration
- Set rag_plugin_migration_needed flag in metadata if old data exists
"""
async def upgrade(self):
"""Upgrade"""
has_internal_data = await self._backup_knowledge_bases()
has_external_data = await self._check_external_knowledge_bases()
await self._clear_knowledge_bases()
await self._add_columns_to_knowledge_bases()
await self._drop_old_columns()
if has_internal_data or has_external_data:
await self._set_migration_flag()
async def _get_table_columns(self, table_name: str) -> list[str]:
"""Get column names from a table (works for both SQLite and PostgreSQL)."""
if self.ap.persistence_mgr.db.name == 'postgresql':
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'SELECT column_name FROM information_schema.columns WHERE table_name = :table_name;'
).bindparams(table_name=table_name)
)
return [row[0] for row in result.fetchall()]
else:
# SQLite PRAGMA does not support bind parameters; validate identifier.
if not table_name.isidentifier():
raise ValueError(f'Invalid table name: {table_name}')
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text(f'PRAGMA table_info({table_name});'))
return [row[1] for row in result.fetchall()]
async def _table_exists(self, table_name: str) -> bool:
"""Check if a table exists."""
if self.ap.persistence_mgr.db.name == 'postgresql':
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);'
).bindparams(table_name=table_name)
)
return result.scalar()
else:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;").bindparams(
table_name=table_name
)
)
return result.first() is not None
async def _backup_knowledge_bases(self) -> bool:
"""Backup knowledge_bases data. Returns True if data was backed up."""
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases;'))
count = result.scalar()
if count == 0:
return False
# Drop backup table if it already exists (from a previous failed migration)
if await self._table_exists('knowledge_bases_backup'):
await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DROP TABLE knowledge_bases_backup;'))
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('CREATE TABLE knowledge_bases_backup AS SELECT * FROM knowledge_bases;')
)
self.ap.logger.info(
'Backed up %d knowledge base(s) to knowledge_bases_backup table.',
count,
)
return True
async def _check_external_knowledge_bases(self) -> bool:
"""Check if external_knowledge_bases table exists and has data.
The table is preserved as-is (not dropped) for future migration.
"""
if not await self._table_exists('external_knowledge_bases'):
return False
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
)
count = result.scalar()
if count > 0:
self.ap.logger.info(
'Found %d external knowledge base(s) in external_knowledge_bases table. '
'Table preserved for future migration.',
count,
)
return count > 0
async def _clear_knowledge_bases(self):
"""Clear all rows from knowledge_bases table (preserve table structure)."""
await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DELETE FROM knowledge_bases;'))
async def _add_columns_to_knowledge_bases(self):
"""Add new RAG plugin architecture columns to knowledge_bases table."""
columns = await self._get_table_columns('knowledge_bases')
new_columns = {
'knowledge_engine_plugin_id': 'VARCHAR',
'collection_id': 'VARCHAR',
'creation_settings': 'TEXT', # JSON stored as TEXT for SQLite compatibility
'retrieval_settings': 'TEXT',
}
for col_name, col_type in new_columns.items():
if col_name not in columns:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(f'ALTER TABLE knowledge_bases ADD COLUMN {col_name} {col_type};')
)
async def _drop_old_columns(self):
"""Drop embedding_model_uuid and top_k columns (PostgreSQL only).
SQLite does not support DROP COLUMN in older versions, so we leave the
columns in place — the SQLAlchemy entity simply won't map them.
"""
if self.ap.persistence_mgr.db.name != 'postgresql':
return
columns = await self._get_table_columns('knowledge_bases')
if 'embedding_model_uuid' in columns:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE knowledge_bases DROP COLUMN embedding_model_uuid;')
)
if 'top_k' in columns:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('ALTER TABLE knowledge_bases DROP COLUMN top_k;')
)
async def _set_migration_flag(self):
"""Set rag_plugin_migration_needed flag in metadata table."""
# Check if the key already exists
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("SELECT value FROM metadata WHERE key = 'rag_plugin_migration_needed';")
)
row = result.first()
if row is not None:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("UPDATE metadata SET value = 'true' WHERE key = 'rag_plugin_migration_needed';")
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("INSERT INTO metadata (key, value) VALUES ('rag_plugin_migration_needed', 'true');")
)
self.ap.logger.info('Set rag_plugin_migration_needed=true in metadata.')
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,74 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(21)
class DBMigrateMergeExceptionHandling(migration.DBMigration):
"""Merge hide-exception and block-failed-request-output into a single exception-handling select option,
and add failure-hint field.
Conversion logic:
- block-failed-request-output=true -> exception-handling: hide
- hide-exception=true -> exception-handling: show-hint
- hide-exception=false -> exception-handling: show-error
"""
async def upgrade(self):
"""Upgrade"""
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
if 'output' not in config:
config['output'] = {}
if 'misc' not in config['output']:
config['output']['misc'] = {}
misc = config['output']['misc']
# Determine new exception-handling value from legacy fields
hide_exception = misc.get('hide-exception', True)
block_failed = misc.get('block-failed-request-output', False)
if block_failed:
exception_handling = 'hide'
elif hide_exception:
exception_handling = 'show-hint'
else:
exception_handling = 'show-error'
misc['exception-handling'] = exception_handling
# Add failure-hint with default value
misc['failure-hint'] = 'Request failed.'
# Remove legacy fields
misc.pop('hide-exception', None)
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,73 +0,0 @@
import sqlalchemy
from .. import migration
@migration.migration_class(22)
class DBMigrateMonitoringUserId(migration.DBMigration):
"""Add user_id and user_name columns to monitoring_sessions table
This migration adds the missing user_id column and also ensures user_name
column exists (in case migration 21 failed or was skipped).
"""
async def _table_exists(self, table_name: str) -> bool:
"""Check if a table exists (works for both SQLite and PostgreSQL)."""
if self.ap.persistence_mgr.db.name == 'postgresql':
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);'
).bindparams(table_name=table_name)
)
return bool(result.scalar())
else:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;").bindparams(
table_name=table_name
)
)
return result.first() is not None
async def _get_table_columns(self, table_name: str) -> list[str]:
"""Get column names from a table (works for both SQLite and PostgreSQL)."""
if self.ap.persistence_mgr.db.name == 'postgresql':
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'SELECT column_name FROM information_schema.columns WHERE table_name = :table_name;'
).bindparams(table_name=table_name)
)
return [row[0] for row in result.fetchall()]
else:
if not table_name.isidentifier():
raise ValueError(f'Invalid table name: {table_name}')
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text(f'PRAGMA table_info({table_name});'))
return [row[1] for row in result.fetchall()]
async def _add_column_if_not_exists(self, table_name: str, column_name: str, column_type: str):
"""Add a column to a table if it does not already exist."""
columns = await self._get_table_columns(table_name)
if column_name in columns:
self.ap.logger.debug('%s column already exists in %s.', column_name, table_name)
return
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(f'ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type};')
)
self.ap.logger.info('Added %s column to %s table.', column_name, table_name)
async def upgrade(self):
# Check if monitoring_sessions table exists
if not await self._table_exists('monitoring_sessions'):
self.ap.logger.warning('monitoring_sessions table does not exist, skipping migration.')
return
# Add user_id column to monitoring_sessions table
await self._add_column_if_not_exists('monitoring_sessions', 'user_id', 'VARCHAR(255)')
# Add user_name column to monitoring_sessions table (in case migration 21 failed)
await self._add_column_if_not_exists('monitoring_sessions', 'user_name', 'VARCHAR(255)')
# Add user_name column to monitoring_messages table (in case migration 21 failed)
if await self._table_exists('monitoring_messages'):
await self._add_column_if_not_exists('monitoring_messages', 'user_name', 'VARCHAR(255)')
async def downgrade(self):
pass
@@ -1,102 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(23)
class DBMigrateModelFallbackConfig(migration.DBMigration):
"""Convert model field from plain UUID string to object with primary/fallbacks"""
async def upgrade(self):
"""Upgrade"""
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
if 'ai' not in config or 'local-agent' not in config['ai']:
continue
local_agent = config['ai']['local-agent']
changed = False
# Convert model from string to object
model_value = local_agent.get('model', '')
if isinstance(model_value, str):
local_agent['model'] = {
'primary': model_value,
'fallbacks': [],
}
changed = True
# Remove leftover fallback-models field if present
if 'fallback-models' in local_agent:
del local_agent['fallback-models']
changed = True
if not changed:
continue
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
async def downgrade(self):
"""Downgrade"""
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
)
pipelines = result.fetchall()
current_version = self.ap.ver_mgr.get_current_version()
for pipeline_row in pipelines:
uuid = pipeline_row[0]
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
if 'ai' not in config or 'local-agent' not in config['ai']:
continue
local_agent = config['ai']['local-agent']
# Convert model from object back to string
model_value = local_agent.get('model', '')
if isinstance(model_value, dict):
local_agent['model'] = model_value.get('primary', '')
else:
continue
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text(
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
),
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
)
@@ -1,49 +0,0 @@
from .. import migration
import sqlalchemy
import json
@migration.migration_class(24)
class DBMigrateWecomBotWebSocketMode(migration.DBMigration):
"""Add enable-webhook field to existing wecombot adapter configs.
Existing wecombot bots were all using webhook mode, so we set
enable-webhook=true to preserve their behavior after the new
WebSocket long connection mode is introduced as default.
"""
async def upgrade(self):
"""Upgrade"""
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.text("SELECT uuid, adapter_config FROM bots WHERE adapter = 'wecombot'")
)
bots = result.fetchall()
for bot_row in bots:
bot_uuid = bot_row[0]
adapter_config = json.loads(bot_row[1]) if isinstance(bot_row[1], str) else bot_row[1]
if 'enable-webhook' in adapter_config:
continue
# Determine mode based on existing config: if webhook fields are present, keep webhook mode
has_webhook_config = bool(
adapter_config.get('Token') and adapter_config.get('EncodingAESKey') and adapter_config.get('Corpid')
)
adapter_config['enable-webhook'] = has_webhook_config
if self.ap.persistence_mgr.db.name == 'postgresql':
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('UPDATE bots SET adapter_config = :config::jsonb WHERE uuid = :uuid'),
{'config': json.dumps(adapter_config), 'uuid': bot_uuid},
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.text('UPDATE bots SET adapter_config = :config WHERE uuid = :uuid'),
{'config': json.dumps(adapter_config), 'uuid': bot_uuid},
)
async def downgrade(self):
"""Downgrade"""
pass
@@ -1,15 +0,0 @@
import sqlalchemy
from .. import migration
@migration.migration_class(25)
class DBMigrateBotPipelineRoutingRules(migration.DBMigration):
"""Add pipeline_routing_rules column to bots table"""
async def upgrade(self):
sql_text = sqlalchemy.text("ALTER TABLE bots ADD COLUMN pipeline_routing_rules JSON NOT NULL DEFAULT '[]'")
await self.ap.persistence_mgr.execute_async(sql_text)
async def downgrade(self):
sql_text = sqlalchemy.text('ALTER TABLE bots DROP COLUMN pipeline_routing_rules')
await self.ap.persistence_mgr.execute_async(sql_text)
@@ -1,17 +0,0 @@
from langbot.pkg.entity.persistence import monitoring as persistence_monitoring
from .. import migration
@migration.migration_class(26)
class DBMigrateMonitoringToolCalls(migration.DBMigration):
"""Add monitoring_tool_calls table"""
async def upgrade(self):
"""Upgrade"""
async with self.ap.persistence_mgr.get_db_engine().begin() as conn:
await conn.run_sync(persistence_monitoring.MonitoringToolCall.__table__.create, checkfirst=True)
async def downgrade(self):
"""Downgrade"""
async with self.ap.persistence_mgr.get_db_engine().begin() as conn:
await conn.run_sync(persistence_monitoring.MonitoringToolCall.__table__.drop, checkfirst=True)
@@ -0,0 +1,156 @@
"""Validation and fail-closed reads for Pipeline extension preferences."""
from __future__ import annotations
import typing
_BOOLEAN_FIELDS = (
'enable_all_plugins',
'enable_all_mcp_servers',
'enable_all_skills',
'mcp_resource_agent_read_enabled',
)
def _valid_plugin_binding(value: typing.Any) -> bool:
return (
isinstance(value, dict)
and isinstance(value.get('author'), str)
and bool(value['author'])
and isinstance(value.get('name'), str)
and bool(value['name'])
)
def _valid_name(value: typing.Any) -> bool:
return isinstance(value, str) and bool(value)
def normalize_extension_preferences(value: typing.Any) -> dict[str, typing.Any]:
"""Return a safe runtime view of persisted extension preferences.
Missing fields in a valid object retain the established defaults. A
malformed root is treated as an empty allowlist with every extension
capability disabled.
"""
if not isinstance(value, dict):
return {
'enable_all_plugins': False,
'enable_all_mcp_servers': False,
'enable_all_skills': False,
'mcp_resource_agent_read_enabled': False,
'plugins': [],
'mcp_servers': [],
'skills': [],
'mcp_resources': [],
}
normalized = dict(value)
normalized['enable_all_plugins'] = value.get('enable_all_plugins', True) is True
normalized['enable_all_mcp_servers'] = value.get('enable_all_mcp_servers', True) is True
normalized['enable_all_skills'] = value.get('enable_all_skills', True) is True
normalized['mcp_resource_agent_read_enabled'] = (
value.get('mcp_resource_agent_read_enabled', True) is True
)
plugins = value.get('plugins', [])
plugins_are_valid = isinstance(plugins, list) and all(
_valid_plugin_binding(plugin) for plugin in plugins
)
normalized['plugins'] = list(plugins) if plugins_are_valid else []
if not plugins_are_valid:
normalized['enable_all_plugins'] = False
mcp_servers = value.get('mcp_servers', [])
mcp_servers_are_valid = isinstance(mcp_servers, list) and all(
_valid_name(server) for server in mcp_servers
)
normalized['mcp_servers'] = list(mcp_servers) if mcp_servers_are_valid else []
if not mcp_servers_are_valid:
normalized['enable_all_mcp_servers'] = False
skills = value.get('skills', [])
skills_are_valid = isinstance(skills, list) and all(_valid_name(skill) for skill in skills)
normalized['skills'] = list(skills) if skills_are_valid else []
if not skills_are_valid:
normalized['enable_all_skills'] = False
mcp_resources = value.get('mcp_resources', [])
mcp_resources_are_valid = isinstance(mcp_resources, list) and all(
isinstance(resource, dict) for resource in mcp_resources
)
normalized['mcp_resources'] = list(mcp_resources) if mcp_resources_are_valid else []
if not mcp_resources_are_valid:
normalized['mcp_resource_agent_read_enabled'] = False
return normalized
def validate_extension_preferences(
value: typing.Any,
*,
context: str = 'Pipeline extensions_preferences',
field_aliases: typing.Mapping[str, str] | None = None,
) -> dict[str, typing.Any]:
"""Reject extension preferences that cannot be consumed unambiguously."""
if not isinstance(value, dict):
raise ValueError(f'{context} must be an object')
for field_name in _BOOLEAN_FIELDS:
if field_name in value and not isinstance(value[field_name], bool):
raise ValueError(f"{context} field '{field_name}' must be a boolean")
aliases = field_aliases or {}
_validate_list_field(
value,
'plugins',
aliases.get('plugins', 'plugins'),
_valid_plugin_binding,
'a plugin object with non-empty author and name',
context,
)
_validate_list_field(
value,
'mcp_servers',
aliases.get('mcp_servers', 'mcp_servers'),
_valid_name,
'a non-empty string',
context,
)
_validate_list_field(
value,
'skills',
aliases.get('skills', 'skills'),
_valid_name,
'a non-empty string',
context,
)
_validate_list_field(
value,
'mcp_resources',
aliases.get('mcp_resources', 'mcp_resources'),
lambda item: isinstance(item, dict),
'an object',
context,
)
return value
def _validate_list_field(
value: dict[str, typing.Any],
field_name: str,
field_label: str,
item_validator: typing.Callable[[typing.Any], bool],
item_description: str,
context: str,
) -> None:
if field_name not in value:
return
items = value[field_name]
if not isinstance(items, list):
raise ValueError(f"{context} field '{field_label}' must be a list")
for index, item in enumerate(items):
if not item_validator(item):
raise ValueError(
f"{context} field '{field_label}[{index}]' must be {item_description}"
)
+20 -15
View File
@@ -14,7 +14,8 @@ import langbot_plugin.api.entities.builtin.platform.events as platform_events
import langbot_plugin.api.entities.events as events
from ..utils import importutil
from .config_coercion import coerce_pipeline_config
from ..agent.runner.config_migration import ConfigMigration
from ..agent.runner.config_resolver import RunnerConfigResolver
from .extension_preferences import normalize_extension_preferences
import langbot_plugin.api.entities.builtin.provider.session as provider_session
import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
@@ -92,14 +93,16 @@ class RuntimePipeline:
self.stage_containers = stage_containers
# Extract bound plugins and MCP servers from extensions_preferences
extensions_prefs = pipeline_entity.extensions_preferences or {}
self.enable_all_plugins = extensions_prefs.get('enable_all_plugins', True)
self.enable_all_mcp_servers = extensions_prefs.get('enable_all_mcp_servers', True)
extensions_prefs = normalize_extension_preferences(pipeline_entity.extensions_preferences)
self.enable_all_plugins = extensions_prefs.get('enable_all_plugins', True) is True
self.enable_all_mcp_servers = extensions_prefs.get('enable_all_mcp_servers', True) is True
pipeline_config = pipeline_entity.config or {}
runner_config: dict[str, typing.Any] = {}
runner_id = ConfigMigration.resolve_runner_id(pipeline_config) if isinstance(pipeline_config, dict) else None
runner_id = (
RunnerConfigResolver.resolve_runner_id(pipeline_config) if isinstance(pipeline_config, dict) else None
)
if runner_id:
resolved_runner_config = ConfigMigration.resolve_runner_config(pipeline_config, runner_id)
resolved_runner_config = RunnerConfigResolver.resolve_runner_config(pipeline_config, runner_id)
if isinstance(resolved_runner_config, dict):
runner_config = resolved_runner_config
@@ -107,24 +110,26 @@ class RuntimePipeline:
'mcp-resources',
extensions_prefs.get('mcp_resources', []),
)
self.mcp_resource_agent_read_enabled = runner_config.get(
'mcp-resource-agent-read-enabled',
extensions_prefs.get('mcp_resource_agent_read_enabled', True),
self.mcp_resource_agent_read_enabled = (
runner_config.get(
'mcp-resource-agent-read-enabled',
extensions_prefs.get('mcp_resource_agent_read_enabled', True),
)
is True
)
if self.enable_all_plugins:
# None indicates to use all available plugins
self.bound_plugins = None
else:
plugin_list = extensions_prefs.get('plugins', [])
self.bound_plugins = [f'{p["author"]}/{p["name"]}' for p in plugin_list] if plugin_list else []
plugin_list = extensions_prefs['plugins']
self.bound_plugins = [f'{p["author"]}/{p["name"]}' for p in plugin_list]
if self.enable_all_mcp_servers:
# None indicates to use all available MCP servers
self.bound_mcp_servers = None
else:
mcp_server_list = extensions_prefs.get('mcp_servers', [])
self.bound_mcp_servers = mcp_server_list if mcp_server_list else []
self.bound_mcp_servers = extensions_prefs['mcp_servers']
async def run(self, query: pipeline_query.Query):
query.pipeline_config = self.pipeline_entity.config
@@ -296,9 +301,9 @@ class RuntimePipeline:
# Get runner name from pipeline config
runner_name = None
if query.pipeline_config:
from ..agent.runner.config_migration import ConfigMigration
from ..agent.runner.config_resolver import RunnerConfigResolver
runner_name = ConfigMigration.resolve_runner_id(query.pipeline_config)
runner_name = RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
# Record query start and store message_id
message_id = ''
+48 -17
View File
@@ -11,8 +11,11 @@ import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
import langbot_plugin.api.entities.builtin.platform.events as platform_events
from ...agent.runner.descriptor import AgentRunnerDescriptor
from ...agent.runner.config_migration import ConfigMigration
from ...agent.runner.config_resolver import RunnerConfigResolver
from ...agent.runner import config_schema
from ...agent.runner.resource_policy import ResourcePolicyProjector
from ...provider.tools.toolmgr import ToolManager
from ..extension_preferences import normalize_extension_preferences
DEFAULT_PROMPT_CONFIG = [
@@ -77,6 +80,28 @@ class PreProcessor(stage.PipelineStage):
self.ap.logger.warning(f'Fallback model {fallback_uuid} not found, skipping')
return valid_fallbacks
async def _resolve_host_tools(
self,
query: pipeline_query.Query,
runner_config: dict[str, typing.Any],
bound_plugins: list[str] | None,
bound_mcp_servers: list[str] | None,
include_mcp_resource_tools: bool,
) -> list[typing.Any]:
catalog = await self.ap.tool_mgr.get_resolved_tool_catalog(
bound_plugins,
bound_mcp_servers,
include_skill_authoring=True,
include_mcp_resource_tools=include_mcp_resource_tools,
)
policy = ResourcePolicyProjector.from_runner_config(
runner_config,
resolved_tool_names=ResourcePolicyProjector.extract_tool_names(catalog),
)
catalog = ResourcePolicyProjector.filter_tools(catalog, policy)
ToolManager.bind_query_tool_sources(query, catalog)
return ToolManager.tools_from_catalog(catalog)
def _runner_accepts_multimodal_input(self, descriptor: AgentRunnerDescriptor | None) -> bool:
if descriptor is None:
return True
@@ -136,9 +161,7 @@ class PreProcessor(stage.PipelineStage):
strict_thread=True,
)
except Exception as e:
self.ap.logger.warning(
f'Unable to load Transcript history view for conversation {conversation_uuid}: {e}'
)
self.ap.logger.warning(f'Unable to load Transcript history view for conversation {conversation_uuid}: {e}')
return None
return messages or None
@@ -168,14 +191,16 @@ class PreProcessor(stage.PipelineStage):
) -> entities.StageProcessResult:
"""Process"""
# Resolve runner ID from the current ai.runner.id shape.
runner_id = ConfigMigration.resolve_runner_id(query.pipeline_config)
runner_id = RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
# Get runner config from ai.runner_config[runner_id].
runner_config = ConfigMigration.resolve_runner_config(query.pipeline_config, runner_id) if runner_id else {}
runner_config = (
RunnerConfigResolver.resolve_runner_config(query.pipeline_config, runner_id) if runner_id else {}
)
query.variables = query.variables or {}
bound_plugins = query.variables.get('_pipeline_bound_plugins', None)
bound_mcp_servers = query.variables.get('_pipeline_bound_mcp_servers', None)
include_mcp_resource_tools = query.variables.get('_pipeline_mcp_resource_agent_read_enabled', True)
include_mcp_resource_tools = query.variables.get('_pipeline_mcp_resource_agent_read_enabled', True) is True
descriptor = await self._get_runner_descriptor(runner_id, bound_plugins)
session = await self.ap.sess_mgr.get_session(query)
@@ -204,7 +229,7 @@ class PreProcessor(stage.PipelineStage):
# been idle for longer than the configured conversation expire time.
# The idle window is measured from the last preprocess/update time, not
# from the conversation creation time.
conversation_expire_time = ConfigMigration.get_expire_time(query.pipeline_config)
conversation_expire_time = RunnerConfigResolver.get_expire_time(query.pipeline_config)
now = datetime.datetime.now()
if conversation_expire_time is not None and conversation_expire_time > 0:
last_update_time = getattr(conversation, 'update_time', None) or getattr(conversation, 'create_time', None)
@@ -236,10 +261,12 @@ class PreProcessor(stage.PipelineStage):
query.use_llm_model_uuid = llm_model.model_entity.uuid
if uses_host_tools and 'func_call' in (llm_model.model_entity.abilities or []):
query.use_funcs = await self.ap.tool_mgr.get_all_tools(
query.use_funcs = await self._resolve_host_tools(
query,
runner_config,
bound_plugins,
bound_mcp_servers,
include_mcp_resource_tools=include_mcp_resource_tools,
include_mcp_resource_tools,
)
self.ap.logger.debug(f'Bound plugins: {bound_plugins}')
@@ -249,16 +276,20 @@ class PreProcessor(stage.PipelineStage):
# If primary model doesn't support func_call but fallback models exist,
# load tools anyway since fallback models may support them
if uses_host_tools and not query.use_funcs and query.variables.get('_fallback_model_uuids'):
query.use_funcs = await self.ap.tool_mgr.get_all_tools(
query.use_funcs = await self._resolve_host_tools(
query,
runner_config,
bound_plugins,
bound_mcp_servers,
include_mcp_resource_tools=include_mcp_resource_tools,
include_mcp_resource_tools,
)
elif uses_host_tools:
query.use_funcs = await self.ap.tool_mgr.get_all_tools(
query.use_funcs = await self._resolve_host_tools(
query,
runner_config,
bound_plugins,
bound_mcp_servers,
include_mcp_resource_tools=include_mcp_resource_tools,
include_mcp_resource_tools,
)
self.ap.logger.debug(f'Bound plugins: {bound_plugins}')
@@ -372,13 +403,13 @@ class PreProcessor(stage.PipelineStage):
if getattr(self.ap, 'skill_mgr', None) is not None:
pipeline_data = await self.ap.pipeline_service.get_pipeline(query.pipeline_uuid)
extensions_prefs = (pipeline_data or {}).get('extensions_preferences', {})
enable_all_skills = extensions_prefs.get('enable_all_skills', True)
extensions_prefs = normalize_extension_preferences((pipeline_data or {}).get('extensions_preferences'))
enable_all_skills = extensions_prefs['enable_all_skills']
if enable_all_skills:
bound_skills = None # None = all loaded skills are visible
else:
bound_skills = extensions_prefs.get('skills', [])
bound_skills = extensions_prefs['skills']
query.variables['_pipeline_bound_skills'] = bound_skills
@@ -12,7 +12,7 @@ from ... import entities
from ... import plugin_diagnostics
import langbot_plugin.api.entities.events as events
from ....agent.runner.config_migration import ConfigMigration
from ....agent.runner.config_resolver import RunnerConfigResolver
from ....agent.runner import config_schema
from ....utils import constants, runner as runner_utils
from ....telemetry import features as telemetry_features
@@ -312,8 +312,10 @@ class ChatMessageHandler(handler.MessageHandler):
if prompt_config:
return prompt_config
runner_id = ConfigMigration.resolve_runner_id(query.pipeline_config)
runner_config = ConfigMigration.resolve_runner_config(query.pipeline_config, runner_id) if runner_id else {}
runner_id = RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
runner_config = (
RunnerConfigResolver.resolve_runner_config(query.pipeline_config, runner_id) if runner_id else {}
)
bound_plugins = query.variables.get('_pipeline_bound_plugins', None)
descriptor = await self._get_runner_descriptor(runner_id, bound_plugins)
return config_schema.extract_prompt_config(descriptor, runner_config, DEFAULT_PROMPT_CONFIG)
+40 -16
View File
@@ -15,14 +15,15 @@ from ..discover import engine
from ..entity.persistence import bot as persistence_bot
from ..entity.errors import platform as platform_errors
from ..agent.runner.config_resolver import RunnerConfigResolver
from ..agent.runner.host_models import (
AgentBinding,
AgentEventEnvelope,
BindingScope,
DeliveryPolicy,
ResourcePolicy,
StatePolicy,
)
from ..agent.runner.resource_policy import ResourcePolicyProjector
from .logger import EventLogger
@@ -52,6 +53,8 @@ class SyntheticRouteTestAdapter:
'create_message_card',
'edit_message',
'delete_message',
'add_reaction',
'remove_reaction',
'forward_message',
'set_group_name',
'mute_member',
@@ -985,6 +988,7 @@ class RuntimeBot:
getattr(event, 'message_id', None) or getattr(event, 'feedback_id', None) or f'{event_type}:{uuid.uuid4()}'
)
target_type, target_id, target_metadata = self._infer_reply_target(event)
supported_apis = self._get_adapter_supported_apis(adapter)
conversation_id = None
if target_type and target_id:
@@ -1014,17 +1018,33 @@ class RuntimeBot:
**target_metadata,
},
supports_streaming=False,
supports_edit=False,
supports_reaction=False,
supports_edit='edit_message' in supported_apis,
supports_reaction=bool({'add_reaction', 'remove_reaction'} & set(supported_apis)),
platform_capabilities={
'adapter': adapter.__class__.__name__,
'event_type': event_type,
'supported_apis': supported_apis,
},
),
raw_ref=RawEventRef(ref_id=str(event_id), storage_key=None),
data=self._compact_event_data(event),
)
@staticmethod
def _get_adapter_supported_apis(
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
) -> list[str]:
get_supported_apis = getattr(adapter, 'get_supported_apis', None)
if not callable(get_supported_apis):
return []
try:
declared_apis = get_supported_apis()
except Exception:
return []
if not isinstance(declared_apis, (list, tuple, set)):
return []
return list(dict.fromkeys(api_name for api_name in declared_apis if isinstance(api_name, str) and api_name))
@staticmethod
def _agent_product_to_binding(
agent: dict[str, typing.Any],
@@ -1033,29 +1053,20 @@ class RuntimeBot:
bot_uuid: str,
) -> AgentBinding | None:
config = agent.get('config') if isinstance(agent, dict) else None
if not isinstance(config, dict):
if config is None:
return None
runner = config.get('runner')
runner_id = None
if isinstance(runner, dict):
runner_id = runner.get('id')
runner_id = runner_id or agent.get('component_ref')
_, runner_id, runner_config = RunnerConfigResolver.resolve_agent_runner_config(config)
if not runner_id:
return None
runner_config_map = config.get('runner_config')
runner_config = {}
if isinstance(runner_config_map, dict):
runner_config = runner_config_map.get(runner_id) or {}
return AgentBinding(
binding_id=f'bot:{bot_uuid}:{event_binding.get("id") or uuid.uuid4()}',
scope=BindingScope(scope_type='bot', scope_id=bot_uuid),
event_types=[event_type],
runner_id=runner_id,
runner_config=runner_config,
resource_policy=ResourcePolicy(),
resource_policy=ResourcePolicyProjector.from_runner_config(runner_config),
state_policy=StatePolicy(state_scopes=['conversation', 'actor', 'subject', 'runner']),
delivery_policy=DeliveryPolicy(enable_streaming=False, enable_reply=True),
enabled=True,
@@ -1262,7 +1273,20 @@ class RuntimeBot:
text=f'EBA event {event_type} target agent does not support this event: {target_uuid}',
)
binding = self._agent_product_to_binding(agent, event_binding, event_type, self.bot_entity.uuid)
try:
binding = self._agent_product_to_binding(agent, event_binding, event_type, self.bot_entity.uuid)
except Exception:
return await self._record_event_route_trace(
event_type=event_type,
status='failed',
level='error',
binding=event_binding,
target_type=target_type,
target_uuid=target_uuid,
failure_code='runner_failed',
reason='Agent configuration is invalid',
text=f'Failed to build Agent binding for EBA event {event_type}: {traceback.format_exc()}',
)
if binding is None:
return await self._record_event_route_trace(
event_type=event_type,
+114 -8
View File
@@ -27,7 +27,7 @@ from ..provider.modelmgr import requester as model_requester
from ..core import app
from ..utils import constants
from ..agent.runner.session_registry import get_session_registry
from ..agent.runner.config_migration import ConfigMigration
from ..agent.runner.config_resolver import RunnerConfigResolver
from ..agent.runner import config_schema
@@ -37,6 +37,29 @@ from .agent_run_support import (
)
_HOST_RESERVED_QUERY_VAR_PREFIXES = (
'_host_',
'_pipeline_bound_',
'_pipeline_mcp_',
'_monitoring_',
'_sandbox_',
'_authorized',
'_permission',
)
_HOST_RESERVED_QUERY_VAR_KEYS = frozenset(
{
'_activated_skills',
'_fallback_model_uuids',
'_routed_by_rule',
}
)
def _is_host_reserved_query_var(key: str) -> bool:
"""Return whether a Query variable controls Host authorization or runtime state."""
return key in _HOST_RESERVED_QUERY_VAR_KEYS or key.startswith(_HOST_RESERVED_QUERY_VAR_PREFIXES)
class _RawAction:
def __init__(self, value: str):
self.value = value
@@ -105,11 +128,11 @@ async def _get_pipeline_knowledge_base_uuids(ap: app.Application, query: Any) ->
if not pipeline_config:
return []
runner_id = ConfigMigration.resolve_runner_id(pipeline_config)
runner_id = RunnerConfigResolver.resolve_runner_id(pipeline_config)
if not runner_id:
return []
runner_config = ConfigMigration.resolve_runner_config(pipeline_config, runner_id)
runner_config = RunnerConfigResolver.resolve_runner_config(pipeline_config, runner_id)
registry = getattr(ap, 'agent_runner_registry', None)
if registry is None:
return []
@@ -189,6 +212,60 @@ async def _validate_run_authorization(
return session, None
def _validate_frozen_tool_source_identity(
session: Any,
tool_name: str,
ap: app.Application,
) -> Union[tuple[None, handler.ActionResponse], tuple[dict[str, str | None], None]]:
"""Resolve the exact Host tool implementation frozen for this run."""
authorization = session.get('authorization')
resources = authorization.get('resources') if isinstance(authorization, dict) else None
tools = resources.get('tools') if isinstance(resources, dict) else None
matching_tools = (
[tool for tool in tools if isinstance(tool, dict) and tool.get('tool_name') == tool_name]
if isinstance(tools, list)
else []
)
source_ref: dict[str, str | None] | None = None
if len(matching_tools) == 1:
tool = matching_tools[0]
source = tool.get('source')
source_id = tool.get('source_id')
has_complete_shape = (
'source' in tool
and 'source_id' in tool
and isinstance(source, str)
and bool(source)
and source == source.strip()
and (
source_id is None or (isinstance(source_id, str) and bool(source_id) and source_id == source_id.strip())
)
)
if has_complete_shape:
if source in {'builtin', 'native', 'skill'} and source_id is None:
source_ref = {'source': source, 'source_id': None}
elif source == 'plugin' and isinstance(source_id, str):
source_ref = {'source': source, 'source_id': source_id}
elif source == 'mcp':
from ..provider.tools.loaders.mcp import MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE
if isinstance(source_id, str) or tool_name in {
MCP_TOOL_LIST_RESOURCES,
MCP_TOOL_READ_RESOURCE,
}:
source_ref = {'source': source, 'source_id': source_id}
if source_ref is not None:
return source_ref, None
run_id = session.get('run_id')
ap.logger.warning(f'TOOL: {tool_name} has an invalid frozen source identity for run_id {run_id}')
return None, handler.ActionResponse.error(
message=f'Tool {tool_name} has an invalid frozen source identity for this agent run',
)
def _get_cached_query(ap: app.Application, query_id: int | None) -> Any | None:
"""Return a cached Query for query-based runtime actions when available."""
if query_id is None:
@@ -208,6 +285,8 @@ def _resolve_action_query(data: dict[str, Any], session: Any | None, ap: app.App
if query_id is None:
query_id = data.get('query_id')
query = _get_cached_query(ap, query_id)
if query is None and session is not None:
query = session.get('execution_query')
if query is not None and session is not None:
object.__setattr__(query, '_agent_run_session', session)
return query
@@ -393,6 +472,16 @@ class RuntimeConnectionHandler(handler.Handler):
query = self.ap.query_pool.cached_queries[query_id]
if not isinstance(key, str) or not key:
return handler.ActionResponse.error(
message='Query variable key must be a non-empty string',
)
if _is_host_reserved_query_var(key):
self.ap.logger.warning(f'Plugin attempted to write Host-reserved Query variable {key!r}')
return handler.ActionResponse.error(
message=f'Query variable {key!r} is reserved for LangBot Host',
)
query.variables[key] = value
return handler.ActionResponse.success(
@@ -760,6 +849,7 @@ class RuntimeConnectionHandler(handler.Handler):
run_id = data.get('run_id') # Optional: present for AgentRunner calls
caller_plugin_identity = data.get('caller_plugin_identity') # Optional: for cross-plugin validation
session = None
source_ref = None
is_agent_runner_call = bool(run_id)
if is_agent_runner_call:
@@ -778,16 +868,24 @@ class RuntimeConnectionHandler(handler.Handler):
)
if error:
return error
source_ref, error = _validate_frozen_tool_source_identity(session, tool_name, self.ap)
if error:
return error
# Convert session_data to Session object (simplified)
# In real implementation, you would reconstruct the full session
# For now, we'll call the tool manager's execute method
try:
query = _resolve_action_query(data, session, self.ap)
execute_kwargs: dict[str, Any] = {
'name': tool_name,
'parameters': parameters,
'query': query,
}
if source_ref is not None:
execute_kwargs['source_ref'] = source_ref
result = await self.ap.tool_mgr.execute_func_call(
name=tool_name,
parameters=parameters,
query=query,
**execute_kwargs,
)
if is_agent_runner_call:
return handler.ActionResponse.success(data={'result': result})
@@ -810,6 +908,8 @@ class RuntimeConnectionHandler(handler.Handler):
tool_name = data['tool_name']
run_id = data.get('run_id') # Optional: present for AgentRunner calls
caller_plugin_identity = data.get('caller_plugin_identity') # Optional: for cross-plugin validation
session = None
source_ref = None
# Permission validation for AgentRunner calls
if run_id:
@@ -818,9 +918,15 @@ class RuntimeConnectionHandler(handler.Handler):
)
if error:
return error
source_ref, error = _validate_frozen_tool_source_identity(session, tool_name, self.ap)
if error:
return error
try:
tool_detail = await self.ap.tool_mgr.get_tool_detail(tool_name)
detail_kwargs: dict[str, Any] = {}
if source_ref is not None:
detail_kwargs['source_ref'] = source_ref
tool_detail = await self.ap.tool_mgr.get_tool_detail(tool_name, **detail_kwargs)
if tool_detail is None:
return handler.ActionResponse.error(
message=f'Tool {tool_name} not found',
@@ -1310,7 +1416,7 @@ class RuntimeConnectionHandler(handler.Handler):
Note: This action has dual validation paths:
- AgentRunner: uses session_registry for permission check
- Regular plugin: uses ConfigMigration.resolve_runner_config for pipeline-level check
- Regular plugin: uses RunnerConfigResolver.resolve_runner_config for pipeline-level check
"""
kb_id = data['kb_id']
query_text = data['query_text']
+9
View File
@@ -4,3 +4,12 @@ class ToolNotFoundError(ValueError):
def __init__(self, name: str):
self.name = name
super().__init__(f'Tool not found: {name}')
class ToolExecutionDeniedError(PermissionError):
"""Raised when a known tool is not allowed in the current execution context."""
def __init__(self, name: str, reason: str):
self.name = name
self.reason = reason
super().__init__(f'Tool execution denied for {name}: {reason}')
+65 -17
View File
@@ -22,6 +22,7 @@ from mcp.shared.exceptions import McpError
from pydantic import AnyUrl
from .. import loader
from ..errors import ToolExecutionDeniedError
from ....core import app
import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
import langbot_plugin.api.entities.builtin.provider.message as provider_message
@@ -1417,35 +1418,67 @@ class MCPLoader(loader.ToolLoader):
return items
async def has_tool(self, name: str) -> bool:
def _session_by_source_id(self, source_id: str) -> RuntimeMCPSession | None:
return next(
(session for session in self.sessions.values() if session.server_uuid == source_id),
None,
)
async def has_tool(self, name: str, source_id: str | None = None) -> bool:
"""检查工具是否存在"""
if name in (MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE):
if source_id is None and name in (MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE):
return bool(self._eligible_resource_sessions_for_bound(None))
for session in self.sessions.values():
sessions = [self._session_by_source_id(source_id)] if source_id is not None else list(self.sessions.values())
for session in sessions:
if session is None:
continue
for function in session.get_tools():
if function.name == name:
return True
return False
async def get_tool(self, name: str) -> resource_tool.LLMTool | None:
for session in self.sessions.values():
async def get_tool(
self,
name: str,
source_id: str | None = None,
) -> resource_tool.LLMTool | None:
if source_id is None and name in (MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE):
if not self._eligible_resource_sessions_for_bound(None):
return None
return next(
(tool for tool in self._mcp_synthetic_resource_tools() if tool.name == name),
None,
)
sessions = [self._session_by_source_id(source_id)] if source_id is not None else list(self.sessions.values())
for session in sessions:
if session is None:
continue
for function in session.get_tools():
if function.name == name:
return function
return None
async def invoke_tool(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
async def invoke_tool(
self,
name: str,
parameters: dict,
query: pipeline_query.Query,
source_id: str | None = None,
) -> typing.Any:
"""执行工具调用"""
if name == MCP_TOOL_LIST_RESOURCES:
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is False:
return [provider_message.ContentElement.from_text('Error: MCP resource agent reads are disabled.')]
if source_id is None and name == MCP_TOOL_LIST_RESOURCES:
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is not True:
raise ToolExecutionDeniedError(name, 'MCP resource agent reads are disabled')
return await self._invoke_mcp_list_resources(parameters, query)
if name == MCP_TOOL_READ_RESOURCE:
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is False:
return [provider_message.ContentElement.from_text('Error: MCP resource agent reads are disabled.')]
if source_id is None and name == MCP_TOOL_READ_RESOURCE:
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is not True:
raise ToolExecutionDeniedError(name, 'MCP resource agent reads are disabled')
return await self._invoke_mcp_read_resource(parameters, query)
for session in self.sessions.values():
sessions = [self._session_by_source_id(source_id)] if source_id is not None else list(self.sessions.values())
for session in sessions:
if session is None:
continue
for function in session.get_tools():
if function.name == name:
self.ap.logger.debug(f'Invoking MCP tool: {name} with parameters: {parameters}')
@@ -1525,7 +1558,7 @@ class MCPLoader(loader.ToolLoader):
default_max_bytes: int = MCP_RESOURCE_CONTEXT_MAX_BYTES,
) -> str:
"""Build host-controlled MCP resource context for the current query."""
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is False:
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is not True:
return ''
attachments = (query.variables or {}).get('_pipeline_mcp_resource_attachments', [])
@@ -1543,7 +1576,7 @@ class MCPLoader(loader.ToolLoader):
for raw_attachment in attachments:
if remaining_tokens <= 0:
break
if not isinstance(raw_attachment, dict) or raw_attachment.get('enabled') is False:
if not isinstance(raw_attachment, dict) or raw_attachment.get('enabled', True) is not True:
continue
attachment = raw_attachment.copy()
@@ -1561,8 +1594,23 @@ class MCPLoader(loader.ToolLoader):
if session.server_uuid not in eligible_by_uuid and session.server_name not in eligible_by_name:
continue
max_tokens = min(int(attachment.get('max_tokens') or remaining_tokens), remaining_tokens)
max_bytes = int(attachment.get('max_bytes') or default_max_bytes)
raw_max_tokens = attachment.get('max_tokens')
raw_max_bytes = attachment.get('max_bytes')
if raw_max_tokens is None:
max_tokens = remaining_tokens
elif isinstance(raw_max_tokens, bool) or not isinstance(raw_max_tokens, int) or raw_max_tokens <= 0:
self.ap.logger.warning(f'Ignoring MCP resource {uri!r}: max_tokens must be a positive integer')
continue
else:
max_tokens = min(raw_max_tokens, remaining_tokens)
if raw_max_bytes is None:
max_bytes = default_max_bytes
elif isinstance(raw_max_bytes, bool) or not isinstance(raw_max_bytes, int) or raw_max_bytes <= 0:
self.ap.logger.warning(f'Ignoring MCP resource {uri!r}: max_bytes must be a positive integer')
continue
else:
max_bytes = raw_max_bytes
try:
envelope = await session.read_resource_envelope(
@@ -50,17 +50,35 @@ class PluginToolLoader(loader.ToolLoader):
return catalog
async def has_tool(self, name: str) -> bool:
async def get_tool(
self,
name: str,
source_id: str | None = None,
) -> resource_tool.LLMTool | None:
tools = await self.get_tools([source_id] if source_id else None)
return next((tool for tool in tools if tool.name == name), None)
async def has_tool(self, name: str, source_id: str | None = None) -> bool:
"""检查工具是否存在"""
for tool in await self.ap.plugin_connector.list_tools():
for tool in await self.ap.plugin_connector.list_tools([source_id] if source_id else None):
if tool.metadata.name == name:
return True
return False
async def invoke_tool(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
async def invoke_tool(
self,
name: str,
parameters: dict,
query: pipeline_query.Query,
source_id: str | None = None,
) -> typing.Any:
try:
return await self.ap.plugin_connector.call_tool(
name, parameters, session=query.session, query_id=query.query_id
name,
parameters,
session=query.session,
query_id=query.query_id,
bound_plugins=[source_id] if source_id else None,
)
except Exception as e:
self.ap.logger.error(f'执行函数 {name} 时发生错误: {e}')
@@ -134,7 +134,11 @@ async def persist_activated_skill(
if not isinstance(session, dict):
return
state_context = session.get('state_context')
authorization = session.get('authorization')
if not isinstance(authorization, dict):
return
state_context = authorization.get('state_context')
if not isinstance(state_context, dict):
return
+207 -5
View File
@@ -20,6 +20,16 @@ if TYPE_CHECKING:
)
TOOL_SOURCE_REFS_QUERY_KEY = '_host_tool_source_refs'
class ToolSourceRef(typing.TypedDict):
"""Stable Host-side identity for one tool implementation."""
source: str
source_id: str | None
class ToolManager:
"""LLM工具管理器"""
@@ -115,6 +125,121 @@ class ToolManager:
return catalog
async def get_resolved_tool_catalog(
self,
bound_plugins: list[str] | None = None,
bound_mcp_servers: list[str] | None = None,
include_skill_authoring: bool = True,
include_mcp_resource_tools: bool = False,
) -> list[dict[str, typing.Any]]:
"""Return scoped tools with one unambiguous implementation per name.
LLM tool calls only carry a function name. If two implementations with
the same name remain inside the current Host scope, choosing one by
loader or registration order would authorize one resource and execute
another. Such names are therefore omitted until the scope is narrowed.
"""
catalog = await self.get_tool_catalog(
bound_plugins,
bound_mcp_servers,
include_skill_authoring=include_skill_authoring,
include_mcp_resource_tools=include_mcp_resource_tools,
)
tools_by_name: dict[str, list[dict[str, typing.Any]]] = {}
for item in catalog:
name = item.get('name')
if isinstance(name, str) and name:
tools_by_name.setdefault(name, []).append(item)
resolved: list[dict[str, typing.Any]] = []
for name, candidates in tools_by_name.items():
implementations = {
(str(item.get('source') or ''), self._normalize_source_id(item.get('source_id'))) for item in candidates
}
if len(implementations) != 1:
self.ap.logger.warning(
f'Tool {name} is hidden because multiple implementations are visible: '
f'{sorted(implementations, key=lambda item: (item[0], item[1] or ""))}'
)
continue
resolved.append(candidates[0])
return resolved
@staticmethod
def _normalize_source_id(source_id: typing.Any) -> str | None:
return source_id if isinstance(source_id, str) and source_id else None
@classmethod
def source_ref_from_catalog_item(cls, item: dict[str, typing.Any]) -> ToolSourceRef | None:
source = item.get('source')
if not isinstance(source, str) or not source:
return None
return {
'source': source,
'source_id': cls._normalize_source_id(item.get('source_id')),
}
@classmethod
def source_refs_from_catalog(
cls,
catalog: typing.Iterable[dict[str, typing.Any]],
) -> dict[str, ToolSourceRef]:
refs: dict[str, ToolSourceRef] = {}
for item in catalog:
name = item.get('name')
ref = cls.source_ref_from_catalog_item(item)
if isinstance(name, str) and name and ref is not None:
refs[name] = ref
return refs
@staticmethod
def tools_from_catalog(
catalog: typing.Iterable[dict[str, typing.Any]],
) -> list[resource_tool.LLMTool]:
"""Materialize LLM schemas from an already authorized Host catalog."""
return [
resource_tool.LLMTool(
name=item['name'],
human_desc=item.get('human_desc') or item.get('description') or item['name'],
description=item.get('description') or '',
parameters=item.get('parameters') or {},
func=lambda parameters: {},
)
for item in catalog
]
@classmethod
def bind_query_tool_sources(
cls,
query: pipeline_query.Query,
catalog: typing.Iterable[dict[str, typing.Any]],
) -> None:
query.variables = query.variables or {}
query.variables[TOOL_SOURCE_REFS_QUERY_KEY] = cls.source_refs_from_catalog(catalog)
@staticmethod
def get_query_tool_source(
query: pipeline_query.Query,
name: str,
) -> ToolSourceRef | None:
variables = getattr(query, 'variables', None)
if not isinstance(variables, dict):
return None
refs = variables.get(TOOL_SOURCE_REFS_QUERY_KEY)
if not isinstance(refs, dict):
return None
ref = refs.get(name)
if not isinstance(ref, dict):
return None
source = ref.get('source')
if not isinstance(source, str) or not source:
return None
source_id = ref.get('source_id')
return {
'source': source,
'source_id': source_id if isinstance(source_id, str) and source_id else None,
}
async def get_tool_by_name(self, name: str) -> tool_loader.ToolLookupResult | None:
"""Get tool by name from any active loader."""
for active_loader in (
@@ -129,7 +254,11 @@ class ToolManager:
return None
async def get_tool_schema(self, name: str) -> tuple[str | None, dict | None]:
async def get_tool_schema(
self,
name: str,
source_ref: ToolSourceRef | None = None,
) -> tuple[str | None, dict | None]:
"""Return (description, parameters JSON schema) for a tool by name.
Used by the host to prefill ToolResource so a runner can build LLM tool
@@ -137,19 +266,23 @@ class ToolManager:
return resource_tool.LLMTool, so no per-shape branching is needed.
Returns (None, None) when the tool is not found.
"""
tool = await self.get_tool_by_name(name)
tool = await self.get_tool_by_source(name, source_ref) if source_ref else await self.get_tool_by_name(name)
if tool is None:
return None, None
return tool.description, (tool.parameters or None)
async def get_tool_detail(self, name: str) -> dict | None:
async def get_tool_detail(
self,
name: str,
source_ref: ToolSourceRef | None = None,
) -> dict | None:
"""Return the host-level tool detail shape for a tool by name.
All loaders return resource_tool.LLMTool, so the shape is uniform:
{name, description, human_desc, parameters}. Returns None when the tool
is not found.
"""
tool = await self.get_tool_by_name(name)
tool = await self.get_tool_by_source(name, source_ref) if source_ref else await self.get_tool_by_name(name)
if tool is None:
return None
return {
@@ -159,6 +292,26 @@ class ToolManager:
'parameters': tool.parameters or {},
}
async def get_tool_by_source(
self,
name: str,
source_ref: ToolSourceRef,
) -> tool_loader.ToolLookupResult | None:
"""Resolve a tool only from the implementation frozen at authorization."""
source = source_ref['source']
source_id = source_ref.get('source_id')
if source in {'builtin', 'native'}:
return await self.native_tool_loader.get_tool(name)
if source == 'skill':
return await self.skill_tool_loader.get_tool(name)
if source == 'plugin':
if not source_id:
return None
return await self.plugin_tool_loader.get_tool(name, source_id=source_id)
if source == 'mcp':
return await self.mcp_tool_loader.get_tool(name, source_id=source_id)
return None
async def generate_tools_for_openai(self, use_funcs: list[resource_tool.LLMTool]) -> list:
tools = []
@@ -260,9 +413,58 @@ class ToolManager:
)
return result
async def execute_func_call(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
async def execute_func_call(
self,
name: str,
parameters: dict,
query: pipeline_query.Query,
source_ref: ToolSourceRef | None = None,
) -> typing.Any:
from langbot.pkg.telemetry import features as telemetry_features
source_ref = source_ref or self.get_query_tool_source(query, name)
if source_ref is not None:
source = source_ref['source']
source_id = source_ref.get('source_id')
uses_source_id = False
if source in {'builtin', 'native'}:
loader = self.native_tool_loader
telemetry_source = 'native'
exists = await loader.has_tool(name)
elif source == 'skill':
loader = self.skill_tool_loader
telemetry_source = 'skill'
exists = await loader.has_tool(name)
elif source == 'plugin' and source_id:
loader = self.plugin_tool_loader
telemetry_source = 'plugin'
uses_source_id = True
exists = await loader.has_tool(name, source_id=source_id)
elif source == 'mcp':
loader = self.mcp_tool_loader
telemetry_source = 'mcp'
uses_source_id = True
exists = await loader.has_tool(name, source_id=source_id)
else:
raise ToolNotFoundError(name)
if not exists:
raise ToolNotFoundError(name)
async def invoke_selected_tool() -> typing.Any:
if uses_source_id:
return await loader.invoke_tool(name, parameters, query, source_id=source_id)
return await loader.invoke_tool(name, parameters, query)
telemetry_features.increment(query, 'tool_calls', telemetry_source)
return await self._invoke_tool_with_monitoring(
source=telemetry_source,
name=name,
parameters=parameters,
query=query,
invoke=invoke_selected_tool,
)
if await self.native_tool_loader.has_tool(name):
telemetry_features.increment(query, 'tool_calls', 'native')
return await self._invoke_tool_with_monitoring(
-9
View File
@@ -2,15 +2,6 @@ import langbot
semantic_version = f'v{langbot.__version__}'
required_database_version = 25
"""Tag the version of the legacy (3.x) database schema migration chain.
Frozen at 25: the legacy ``pkg/persistence/migrations`` system (DBMigration /
dbmXXX_*.py) is deprecated and no longer accepts new migrations. All schema
changes from here on are managed by Alembic (see
``pkg/persistence/alembic/versions``). This value only gates the one-time
upgrade of pre-existing 3.x databases up to the Alembic baseline."""
debug_mode = False
edition = 'community'
-6
View File
@@ -37,12 +37,6 @@ system:
max_bots: -1
max_pipelines: -1
max_extensions: -1
# When set to a non-empty string, every pipeline is forced to use this
# Box sandbox-scope template regardless of its own configuration, and
# the per-pipeline "Sandbox Scope" selector is locked in the web UI.
# Used by SaaS deployments to confine a tenant to a single shared
# sandbox (set to '{global}'). Empty string = no restriction.
force_box_session_id_template: ''
task_retention:
# Keep at most this many completed async task records in memory
completed_limit: 200
@@ -1,7 +0,0 @@
{
"privilege": {},
"command-prefix": [
"!",
""
]
}
@@ -1,38 +0,0 @@
{
"access-control":{
"mode": "blacklist",
"blacklist": [],
"whitelist": []
},
"respond-rules": {
"default": {
"at": true,
"prefix": [
"/ai", "!ai", "ai", "ai"
],
"regexp": [],
"random": 0.0
}
},
"income-msg-check": true,
"ignore-rules": {
"prefix": [],
"regexp": []
},
"check-sensitive-words": true,
"baidu-cloud-examine": {
"enable": false,
"api-key": "",
"api-secret": ""
},
"rate-limit": {
"strategy": "drop",
"algo": "fixwin",
"fixwin": {
"default": {
"window-size": 60,
"limit": 60
}
}
}
}
-130
View File
@@ -1,130 +0,0 @@
{
"platform-adapters": [
{
"adapter": "nakuru",
"enable": false,
"host": "127.0.0.1",
"ws_port": 8080,
"http_port": 5700,
"token": ""
},
{
"adapter": "aiocqhttp",
"enable": true,
"host": "0.0.0.0",
"port": 2280,
"access-token": ""
},
{
"adapter": "qq-botpy",
"enable": false,
"appid": "",
"secret": "",
"intents": [
"public_guild_messages",
"direct_message"
]
},
{
"adapter": "qqofficial",
"enable": false,
"appid": "1234567890",
"secret": "xxxxxxx",
"port": 2284,
"token": "abcdefg"
},
{
"adapter": "wecom",
"enable": false,
"host": "0.0.0.0",
"port": 2290,
"corpid": "",
"secret": "",
"token": "",
"EncodingAESKey": "",
"contacts_secret": ""
},
{
"adapter": "lark",
"enable": false,
"app_id": "cli_abcdefgh",
"app_secret": "XXXXXXXXXX",
"bot_name": "LangBot",
"enable-webhook": false,
"port": 2285,
"encrypt-key": "xxxxxxxxx"
},
{
"adapter": "discord",
"enable": false,
"client_id": "1234567890",
"token": "XXXXXXXXXX"
},
{
"adapter": "gewechat",
"enable": false,
"gewechat_url": "http://your-gewechat-server:2531",
"gewechat_file_url": "http://your-gewechat-server:2532",
"port": 2286,
"callback_url": "http://your-callback-url:2286/gewechat/callback",
"app_id": "",
"token": ""
},
{
"adapter": "officialaccount",
"enable": false,
"token": "",
"EncodingAESKey": "",
"AppID": "",
"AppSecret": "",
"Mode": "drop",
"LoadingMessage": "AI正在思考中,请发送任意内容获取回复。",
"host": "0.0.0.0",
"port": 2287
},
{
"adapter": "dingtalk",
"enable": false,
"client_id": "",
"client_secret": "",
"robot_code": "",
"robot_name": "",
"markdown_card": false
},
{
"adapter": "telegram",
"enable": false,
"token": "",
"markdown_card": false
},
{
"adapter": "slack",
"enable": false,
"bot_token": "",
"signing_secret": "",
"port": 2288
},
{
"adapter": "wecomcs",
"enable": false,
"port": 2289,
"corpid": "",
"secret": "",
"token": "",
"EncodingAESKey": ""
}
],
"track-function-calls": true,
"quote-origin": false,
"at-sender": false,
"force-delay": {
"min": 0,
"max": 0
},
"long-text-process": {
"threshold": 2560,
"strategy": "forward",
"font-path": ""
},
"hide-exception-info": true
}
-161
View File
@@ -1,161 +0,0 @@
{
"enable-chat": true,
"enable-vision": true,
"keys": {
"openai": [
"sk-1234567890"
],
"anthropic": [
"sk-1234567890"
],
"moonshot": [
"sk-1234567890"
],
"deepseek": [
"sk-1234567890"
],
"gitee-ai": [
"XXXXX"
],
"xai": [
"xai-1234567890"
],
"zhipuai": [
"xxxxxxx"
],
"siliconflow": [
"xxxxxxx"
],
"bailian": [
"sk-xxxxxxx"
],
"volcark": [
"xxxxxxxx"
],
"modelscope": [
"xxxxxxxx"
],
"ppio": [
"xxxxxxxx"
]
},
"requester": {
"openai-chat-completions": {
"base-url": "https://api.openai.com/v1",
"args": {},
"timeout": 120
},
"anthropic-messages": {
"base-url": "https://api.anthropic.com",
"args": {
"max_tokens": 1024
},
"timeout": 120
},
"moonshot-chat-completions": {
"base-url": "https://api.moonshot.cn/v1",
"args": {},
"timeout": 120
},
"deepseek-chat-completions": {
"base-url": "https://api.deepseek.com",
"args": {},
"timeout": 120
},
"ollama-chat": {
"base-url": "http://127.0.0.1:11434",
"args": {},
"timeout": 600
},
"gitee-ai-chat-completions": {
"base-url": "https://ai.gitee.com/v1",
"args": {},
"timeout": 120
},
"xai-chat-completions": {
"base-url": "https://api.x.ai/v1",
"args": {},
"timeout": 120
},
"zhipuai-chat-completions": {
"base-url": "https://open.bigmodel.cn/api/paas/v4",
"args": {},
"timeout": 120
},
"lmstudio-chat-completions": {
"base-url": "http://127.0.0.1:1234/v1",
"args": {},
"timeout": 120
},
"siliconflow-chat-completions": {
"base-url": "https://api.siliconflow.cn/v1",
"args": {},
"timeout": 120
},
"bailian-chat-completions": {
"args": {},
"base-url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
"timeout": 120
},
"volcark-chat-completions": {
"args": {},
"base-url": "https://ark.cn-beijing.volces.com/api/v3",
"timeout": 120
},
"modelscope-chat-completions": {
"base-url": "https://api-inference.modelscope.cn/v1",
"args": {},
"timeout": 120
},
"ppio-chat-completions": {
"base-url": "https://api.ppinfra.com/v3/openai",
"args": {},
"timeout": 120
}
},
"model": "gpt-4o",
"prompt-mode": "normal",
"prompt": {
"default": "You are a helpful assistant."
},
"runner": "local-agent",
"dify-service-api": {
"base-url": "https://api.dify.ai/v1",
"app-type": "chat",
"options": {
"convert-thinking-tips": "plain"
},
"chat": {
"api-key": "app-1234567890",
"timeout": 120
},
"agent": {
"api-key": "app-1234567890",
"timeout": 120
},
"workflow": {
"api-key": "app-1234567890",
"output-key": "summary",
"timeout": 120
}
},
"dashscope-app-api": {
"app-type": "agent",
"api-key": "sk-1234567890",
"agent": {
"app-id": "Your_app_id",
"references_quote": "参考资料来自:"
},
"workflow": {
"app-id": "Your_app_id",
"references_quote": "参考资料来自:",
"biz_params": {
"city": "北京",
"date": "2023-08-10"
}
}
},
"mcp": {
"servers": []
}
}
-27
View File
@@ -1,27 +0,0 @@
{
"admin-sessions": [],
"network-proxies": {
"http": null,
"https": null
},
"report-usage": true,
"logging-level": "info",
"session-concurrency": {
"default": 1
},
"pipeline-concurrency": 20,
"qcg-center-url": "https://api.qchatgpt.rockchin.top/api/v2",
"help-message": "LangBot - 😎高稳定性、🧩支持插件、🌏实时联网的 ChatGPT QQ 机器人🤖\n链接:https://q.rkcn.top",
"http-api": {
"enable": true,
"host": "0.0.0.0",
"port": 5300,
"jwt-expire": 604800
},
"persistence": {
"sqlite": {
"path": "data/langbot.db"
},
"use": "sqlite"
}
}