mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-07-17 01:46:07 +00:00
feat(agent-runner): enforce 4.x host-owned execution
This commit is contained in:
@@ -1,4 +1,5 @@
|
||||
"""Agent runner subsystem for LangBot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from .runner.descriptor import AgentRunnerDescriptor
|
||||
@@ -15,7 +16,7 @@ from .runner.context_builder import AgentRunContextBuilder
|
||||
from .runner.resource_builder import AgentResourceBuilder
|
||||
from .runner.result_normalizer import AgentResultNormalizer
|
||||
from .runner.orchestrator import AgentRunOrchestrator
|
||||
from .runner.config_migration import ConfigMigration
|
||||
from .runner.config_resolver import RunnerConfigResolver
|
||||
|
||||
__all__ = [
|
||||
'AgentRunnerDescriptor',
|
||||
@@ -33,5 +34,5 @@ __all__ = [
|
||||
'AgentResourceBuilder',
|
||||
'AgentResultNormalizer',
|
||||
'AgentRunOrchestrator',
|
||||
'ConfigMigration',
|
||||
]
|
||||
'RunnerConfigResolver',
|
||||
]
|
||||
|
||||
@@ -16,7 +16,7 @@ from .context_builder import AgentRunContextBuilder
|
||||
from .resource_builder import AgentResourceBuilder
|
||||
from .result_normalizer import AgentResultNormalizer
|
||||
from .orchestrator import AgentRunOrchestrator
|
||||
from .config_migration import ConfigMigration
|
||||
from .config_resolver import RunnerConfigResolver
|
||||
from .default_config import AgentRunnerDefaultConfigService
|
||||
from .binding_resolver import AgentBindingResolver, AgentBindingResolutionError
|
||||
from .session_registry import (
|
||||
@@ -49,7 +49,7 @@ __all__ = [
|
||||
'AgentResourceBuilder',
|
||||
'AgentResultNormalizer',
|
||||
'AgentRunOrchestrator',
|
||||
'ConfigMigration',
|
||||
'RunnerConfigResolver',
|
||||
'AgentRunnerDefaultConfigService',
|
||||
'AgentBindingResolver',
|
||||
'AgentBindingResolutionError',
|
||||
|
||||
@@ -1,64 +0,0 @@
|
||||
"""Helpers for the current AgentRunner config shape."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
|
||||
class ConfigMigration:
|
||||
"""Configuration helpers for the current AgentRunner shape.
|
||||
|
||||
Responsibilities:
|
||||
- Resolve runner ID from ai.runner.id
|
||||
- Extract Agent/runner config from ai.runner_config
|
||||
- Read current conversation expiry settings
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def resolve_runner_id(pipeline_config: dict[str, typing.Any]) -> str | None:
|
||||
"""Resolve runner ID from current configuration.
|
||||
|
||||
Args:
|
||||
pipeline_config: Current configuration container
|
||||
|
||||
Returns:
|
||||
Runner ID string, or None if not configured
|
||||
"""
|
||||
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
|
||||
runner = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
|
||||
runner_id = runner.get('id') if isinstance(runner, dict) else None
|
||||
return runner_id if isinstance(runner_id, str) and runner_id else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_runner_config(
|
||||
pipeline_config: dict[str, typing.Any],
|
||||
runner_id: str,
|
||||
) -> dict[str, typing.Any]:
|
||||
"""Resolve Agent/runner configuration from the current container.
|
||||
|
||||
Args:
|
||||
pipeline_config: Current configuration container
|
||||
runner_id: Resolved runner ID
|
||||
|
||||
Returns:
|
||||
Runner configuration dict (empty if not found)
|
||||
"""
|
||||
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
|
||||
runner_configs = ai_config.get('runner_config', {}) if isinstance(ai_config, dict) else {}
|
||||
runner_config = runner_configs.get(runner_id, {}) if isinstance(runner_configs, dict) else {}
|
||||
return runner_config if isinstance(runner_config, dict) else {}
|
||||
|
||||
@staticmethod
|
||||
def get_expire_time(pipeline_config: dict[str, typing.Any]) -> int:
|
||||
"""Get conversation expire time from configuration.
|
||||
|
||||
Args:
|
||||
pipeline_config: Current configuration container
|
||||
|
||||
Returns:
|
||||
Expire time in seconds (0 means no expiry)
|
||||
"""
|
||||
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
|
||||
runner = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
|
||||
expire_time = runner.get('expire-time', 0) if isinstance(runner, dict) else 0
|
||||
return expire_time if isinstance(expire_time, int) else 0
|
||||
@@ -0,0 +1,199 @@
|
||||
"""Resolve the current AgentRunner configuration shape."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
|
||||
HOST_SECURITY_BOOLEAN_FIELDS = (
|
||||
'enable-all-tools',
|
||||
'mcp-resource-agent-read-enabled',
|
||||
)
|
||||
|
||||
|
||||
class RunnerConfigResolver:
|
||||
"""Configuration helpers for the current AgentRunner shape.
|
||||
|
||||
Responsibilities:
|
||||
- Resolve runner ID from ai.runner.id
|
||||
- Extract Agent/runner config from ai.runner_config
|
||||
- Read current conversation expiry settings
|
||||
- Validate the persisted 4.x Agent binding container
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def validate_agent_config(config: typing.Any) -> dict[str, typing.Any]:
|
||||
"""Validate and return the persisted 4.x Agent config container."""
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError('Agent config must be an object')
|
||||
|
||||
runner = config.get('runner')
|
||||
if not isinstance(runner, dict):
|
||||
raise ValueError("Agent config field 'runner' must be an object")
|
||||
|
||||
runner_id = runner.get('id')
|
||||
if not isinstance(runner_id, str):
|
||||
raise ValueError("Agent config field 'runner.id' must be a string")
|
||||
|
||||
runner_configs = config.get('runner_config')
|
||||
if not isinstance(runner_configs, dict):
|
||||
raise ValueError("Agent config field 'runner_config' must be an object")
|
||||
|
||||
for configured_runner_id, runner_config in runner_configs.items():
|
||||
if not isinstance(configured_runner_id, str) or not configured_runner_id:
|
||||
raise ValueError("Agent config field 'runner_config' must use non-empty string runner IDs")
|
||||
if not isinstance(runner_config, dict):
|
||||
raise ValueError(f'Agent runner_config[{configured_runner_id!r}] must be an object')
|
||||
|
||||
if runner_id and runner_id not in runner_configs:
|
||||
raise ValueError(f'Agent runner_config is missing selected runner {runner_id!r}')
|
||||
|
||||
if runner_id:
|
||||
RunnerConfigResolver.validate_runner_security_fields(
|
||||
runner_configs[runner_id],
|
||||
context=f'Agent runner_config[{runner_id!r}]',
|
||||
)
|
||||
|
||||
return config
|
||||
|
||||
@staticmethod
|
||||
def validate_runner_security_fields(
|
||||
runner_config: dict[str, typing.Any],
|
||||
*,
|
||||
context: str = 'Runner config',
|
||||
) -> dict[str, typing.Any]:
|
||||
"""Reject malformed Host-owned security toggles instead of enabling them."""
|
||||
for field_name in HOST_SECURITY_BOOLEAN_FIELDS:
|
||||
if field_name in runner_config and not isinstance(runner_config[field_name], bool):
|
||||
raise ValueError(f'{context} field {field_name!r} must be a boolean')
|
||||
|
||||
RunnerConfigResolver.validate_mcp_resource_attachments(
|
||||
runner_config.get('mcp-resources'),
|
||||
context=context,
|
||||
field_name='mcp-resources',
|
||||
)
|
||||
return runner_config
|
||||
|
||||
@staticmethod
|
||||
def validate_mcp_resource_attachments(
|
||||
resources: typing.Any,
|
||||
*,
|
||||
context: str,
|
||||
field_name: str,
|
||||
) -> typing.Any:
|
||||
"""Validate explicit attachment enable flags shared by Agent and Pipeline inputs."""
|
||||
if not isinstance(resources, list):
|
||||
return resources
|
||||
for index, resource in enumerate(resources):
|
||||
if not isinstance(resource, dict) or 'enabled' not in resource:
|
||||
continue
|
||||
if not isinstance(resource['enabled'], bool):
|
||||
raise ValueError(f"{context} field '{field_name}[{index}].enabled' must be a boolean")
|
||||
return resources
|
||||
|
||||
@classmethod
|
||||
def validate_pipeline_config(cls, config: typing.Any) -> dict[str, typing.Any]:
|
||||
"""Validate the selected Runner container in a current 4.x Pipeline config."""
|
||||
if not isinstance(config, dict):
|
||||
raise ValueError('Pipeline config must be an object')
|
||||
|
||||
ai_config = config.get('ai')
|
||||
if ai_config is None:
|
||||
return config
|
||||
if not isinstance(ai_config, dict):
|
||||
raise ValueError("Pipeline config field 'ai' must be an object")
|
||||
|
||||
runner = ai_config.get('runner')
|
||||
if runner is None:
|
||||
return config
|
||||
if not isinstance(runner, dict):
|
||||
raise ValueError("Pipeline config field 'ai.runner' must be an object")
|
||||
|
||||
runner_id = runner.get('id')
|
||||
if not isinstance(runner_id, str):
|
||||
raise ValueError("Pipeline config field 'ai.runner.id' must be a string")
|
||||
if not runner_id:
|
||||
return config
|
||||
|
||||
runner_configs = ai_config.get('runner_config')
|
||||
if not isinstance(runner_configs, dict):
|
||||
raise ValueError("Pipeline config field 'ai.runner_config' must be an object")
|
||||
if runner_id not in runner_configs:
|
||||
raise ValueError(f'Pipeline runner_config is missing selected runner {runner_id!r}')
|
||||
|
||||
selected_config = runner_configs[runner_id]
|
||||
if not isinstance(selected_config, dict):
|
||||
raise ValueError(f'Pipeline runner_config[{runner_id!r}] must be an object')
|
||||
cls.validate_runner_security_fields(
|
||||
selected_config,
|
||||
context=f'Pipeline runner_config[{runner_id!r}]',
|
||||
)
|
||||
return config
|
||||
|
||||
@staticmethod
|
||||
def resolve_agent_runner_id(config: dict[str, typing.Any]) -> str | None:
|
||||
"""Resolve a runner ID from a validated persisted Agent config."""
|
||||
runner = config.get('runner', {})
|
||||
runner_id = runner.get('id') if isinstance(runner, dict) else None
|
||||
return runner_id if isinstance(runner_id, str) and runner_id else None
|
||||
|
||||
@classmethod
|
||||
def resolve_agent_runner_config(
|
||||
cls,
|
||||
config: typing.Any,
|
||||
) -> tuple[dict[str, typing.Any], str | None, dict[str, typing.Any]]:
|
||||
"""Validate an Agent config and return its selected runner configuration."""
|
||||
validated = cls.validate_agent_config(config)
|
||||
runner_id = cls.resolve_agent_runner_id(validated)
|
||||
runner_configs = typing.cast(dict[str, typing.Any], validated['runner_config'])
|
||||
runner_config = runner_configs.get(runner_id, {}) if runner_id else {}
|
||||
return validated, runner_id, typing.cast(dict[str, typing.Any], runner_config)
|
||||
|
||||
@staticmethod
|
||||
def resolve_runner_id(pipeline_config: dict[str, typing.Any]) -> str | None:
|
||||
"""Resolve runner ID from current configuration.
|
||||
|
||||
Args:
|
||||
pipeline_config: Current configuration container
|
||||
|
||||
Returns:
|
||||
Runner ID string, or None if not configured
|
||||
"""
|
||||
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
|
||||
runner = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
|
||||
runner_id = runner.get('id') if isinstance(runner, dict) else None
|
||||
return runner_id if isinstance(runner_id, str) and runner_id else None
|
||||
|
||||
@staticmethod
|
||||
def resolve_runner_config(
|
||||
pipeline_config: dict[str, typing.Any],
|
||||
runner_id: str,
|
||||
) -> dict[str, typing.Any]:
|
||||
"""Resolve Agent/runner configuration from the current container.
|
||||
|
||||
Args:
|
||||
pipeline_config: Current configuration container
|
||||
runner_id: Resolved runner ID
|
||||
|
||||
Returns:
|
||||
Runner configuration dict (empty if not found)
|
||||
"""
|
||||
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
|
||||
runner_configs = ai_config.get('runner_config', {}) if isinstance(ai_config, dict) else {}
|
||||
runner_config = runner_configs.get(runner_id, {}) if isinstance(runner_configs, dict) else {}
|
||||
return runner_config if isinstance(runner_config, dict) else {}
|
||||
|
||||
@staticmethod
|
||||
def get_expire_time(pipeline_config: dict[str, typing.Any]) -> int:
|
||||
"""Get conversation expire time from configuration.
|
||||
|
||||
Args:
|
||||
pipeline_config: Current configuration container
|
||||
|
||||
Returns:
|
||||
Expire time in seconds (0 means no expiry)
|
||||
"""
|
||||
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
|
||||
runner = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
|
||||
expire_time = runner.get('expire-time', 0) if isinstance(runner, dict) else 0
|
||||
return expire_time if isinstance(expire_time, int) else 0
|
||||
@@ -75,6 +75,9 @@ class ToolResource(typing.TypedDict):
|
||||
tool_type: str | None
|
||||
description: str | None
|
||||
operations: list[str]
|
||||
parameters: dict[str, typing.Any] | None
|
||||
source: typing.NotRequired[str]
|
||||
source_id: typing.NotRequired[str | None]
|
||||
|
||||
|
||||
class KnowledgeBaseResource(typing.TypedDict):
|
||||
|
||||
@@ -7,7 +7,7 @@ import sqlalchemy
|
||||
from ...core import app
|
||||
from ...entity.persistence import pipeline as persistence_pipeline
|
||||
from . import config_schema
|
||||
from .config_migration import ConfigMigration
|
||||
from .config_resolver import RunnerConfigResolver
|
||||
|
||||
|
||||
class AgentRunnerDefaultConfigService:
|
||||
@@ -53,7 +53,7 @@ class AgentRunnerDefaultConfigService:
|
||||
if not isinstance(pipeline_config, dict):
|
||||
return False
|
||||
|
||||
runner_id = ConfigMigration.resolve_runner_id(pipeline_config)
|
||||
runner_id = RunnerConfigResolver.resolve_runner_id(pipeline_config)
|
||||
if not runner_id:
|
||||
return False
|
||||
|
||||
|
||||
@@ -0,0 +1,319 @@
|
||||
"""Host-only Query compatibility views for AgentRunner tool execution."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import typing
|
||||
|
||||
from langbot_plugin.api.entities.builtin.pipeline import query as pipeline_query
|
||||
from langbot_plugin.api.entities.builtin.platform import events as platform_events
|
||||
from langbot_plugin.api.entities.builtin.platform import message as platform_message
|
||||
from langbot_plugin.api.entities.builtin.provider import message as provider_message
|
||||
from langbot_plugin.api.entities.builtin.provider import session as provider_session
|
||||
|
||||
from ...utils import constants
|
||||
from .host_models import AgentEventEnvelope
|
||||
|
||||
|
||||
HOST_BOX_SCOPE_VARIABLE = '_host_box_scope'
|
||||
AUTHORIZED_SKILLS_VARIABLE = '_pipeline_bound_skills'
|
||||
MCP_RESOURCE_ATTACHMENTS_VARIABLE = '_pipeline_mcp_resource_attachments'
|
||||
MCP_RESOURCE_AGENT_READ_ENABLED_VARIABLE = '_pipeline_mcp_resource_agent_read_enabled'
|
||||
|
||||
|
||||
def project_mcp_resource_config(
|
||||
query: pipeline_query.Query,
|
||||
runner_config: dict[str, typing.Any],
|
||||
) -> dict[str, typing.Any]:
|
||||
"""Attach standard MCP resource settings to a Host execution Query."""
|
||||
variables = getattr(query, 'variables', None)
|
||||
if not isinstance(variables, dict):
|
||||
variables = {}
|
||||
query.variables = variables
|
||||
|
||||
attachments = runner_config.get('mcp-resources', [])
|
||||
variables.setdefault(
|
||||
MCP_RESOURCE_ATTACHMENTS_VARIABLE,
|
||||
list(attachments) if isinstance(attachments, list) else [],
|
||||
)
|
||||
variables.setdefault(
|
||||
MCP_RESOURCE_AGENT_READ_ENABLED_VARIABLE,
|
||||
runner_config.get('mcp-resource-agent-read-enabled', True) is True,
|
||||
)
|
||||
return variables
|
||||
|
||||
|
||||
async def build_mcp_resource_context_addition(
|
||||
ap: typing.Any,
|
||||
query: pipeline_query.Query,
|
||||
) -> str:
|
||||
"""Build model-facing MCP context without mutating the canonical input."""
|
||||
tool_mgr = getattr(ap, 'tool_mgr', None)
|
||||
if tool_mgr is None:
|
||||
return ''
|
||||
mcp_loader = getattr(tool_mgr, '__dict__', {}).get('mcp_tool_loader')
|
||||
if mcp_loader is None:
|
||||
return ''
|
||||
|
||||
resource_context = await mcp_loader.build_resource_context_for_query(query)
|
||||
if not resource_context:
|
||||
return ''
|
||||
|
||||
addition = (
|
||||
'\n\nMCP resource context selected by LangBot host:\n'
|
||||
f'{resource_context}\n\n'
|
||||
'Use this context as read-only reference material. If it conflicts with the user message, '
|
||||
'ask for clarification before taking external actions.'
|
||||
)
|
||||
return addition
|
||||
|
||||
|
||||
def append_mcp_resource_context_to_event(event: AgentEventEnvelope, addition: str) -> None:
|
||||
"""Append pinned context only to the run-scoped execution input."""
|
||||
if not addition:
|
||||
return
|
||||
has_text = event.input.text is not None
|
||||
has_structured_content = bool(event.input.contents)
|
||||
if has_text:
|
||||
event.input.text += addition
|
||||
if has_structured_content or not has_text:
|
||||
if not _append_text_to_content(event.input.contents, addition):
|
||||
event.input.contents.append(provider_message.ContentElement.from_text(addition.strip()))
|
||||
|
||||
|
||||
def _append_text_to_content(content: typing.Any, addition: str) -> bool:
|
||||
if isinstance(content, str):
|
||||
return False
|
||||
if not isinstance(content, list):
|
||||
return False
|
||||
for content_element in content:
|
||||
if getattr(content_element, 'type', None) == 'text':
|
||||
content_element.text = (content_element.text or '') + addition
|
||||
return True
|
||||
content.append(provider_message.ContentElement.from_text(addition.strip()))
|
||||
return True
|
||||
|
||||
|
||||
def prepare_execution_query(
|
||||
query: pipeline_query.Query,
|
||||
event: AgentEventEnvelope,
|
||||
authorized_skill_names: list[str],
|
||||
) -> pipeline_query.Query:
|
||||
"""Attach Host-owned execution metadata without changing Query identity."""
|
||||
variables = prepare_box_scope(query, event, preserve_existing=True)
|
||||
variables[AUTHORIZED_SKILLS_VARIABLE] = list(dict.fromkeys(authorized_skill_names))
|
||||
return query
|
||||
|
||||
|
||||
def prepare_box_scope(
|
||||
query: pipeline_query.Query,
|
||||
event: AgentEventEnvelope,
|
||||
*,
|
||||
preserve_existing: bool = False,
|
||||
) -> dict[str, typing.Any]:
|
||||
"""Attach the Host Box scope before any Query-side file staging."""
|
||||
variables = getattr(query, 'variables', None)
|
||||
if not isinstance(variables, dict):
|
||||
variables = {}
|
||||
query.variables = variables
|
||||
|
||||
existing_scope = variables.get(HOST_BOX_SCOPE_VARIABLE)
|
||||
if preserve_existing and isinstance(existing_scope, str) and existing_scope.strip():
|
||||
return variables
|
||||
|
||||
variables[HOST_BOX_SCOPE_VARIABLE] = build_host_box_scope(event, query=query)
|
||||
return variables
|
||||
|
||||
|
||||
def build_execution_query(
|
||||
event: AgentEventEnvelope,
|
||||
authorized_skill_names: list[str],
|
||||
) -> pipeline_query.Query:
|
||||
"""Build the minimum Query view required by Host-owned tool loaders."""
|
||||
launcher_type, launcher_id, sender_id = _resolve_session_identity(event)
|
||||
message_chain = _build_message_chain(event)
|
||||
message_event = platform_events.MessageEvent(
|
||||
type=event.source_event_type or event.event_type,
|
||||
message_chain=message_chain,
|
||||
time=float(event.event_time) if event.event_time is not None else None,
|
||||
)
|
||||
session = provider_session.Session(
|
||||
launcher_type=launcher_type,
|
||||
launcher_id=launcher_id,
|
||||
sender_id=sender_id,
|
||||
)
|
||||
|
||||
contents = copy.deepcopy(event.input.contents)
|
||||
user_content: str | list[provider_message.ContentElement]
|
||||
if contents:
|
||||
user_content = contents
|
||||
else:
|
||||
user_content = event.input.text or ''
|
||||
|
||||
query = pipeline_query.Query(
|
||||
query_id=_synthetic_query_id(event.event_id),
|
||||
launcher_type=launcher_type,
|
||||
launcher_id=launcher_id,
|
||||
sender_id=sender_id,
|
||||
message_event=message_event,
|
||||
message_chain=message_chain,
|
||||
bot_uuid=event.bot_id,
|
||||
pipeline_uuid=None,
|
||||
pipeline_config=None,
|
||||
session=session,
|
||||
messages=[],
|
||||
user_message=provider_message.Message(role='user', content=user_content),
|
||||
variables={},
|
||||
resp_messages=[],
|
||||
)
|
||||
query.variables[HOST_BOX_SCOPE_VARIABLE] = build_host_box_scope(event)
|
||||
query.variables[AUTHORIZED_SKILLS_VARIABLE] = list(dict.fromkeys(authorized_skill_names))
|
||||
return query
|
||||
|
||||
|
||||
def build_host_box_scope(
|
||||
event: AgentEventEnvelope,
|
||||
*,
|
||||
query: pipeline_query.Query | None = None,
|
||||
) -> str | None:
|
||||
"""Return a stable Host scope for a Query session or event."""
|
||||
target_type, target_id = _resolve_box_target(event, query)
|
||||
if target_type is None or target_id is None:
|
||||
return None
|
||||
|
||||
capabilities = event.delivery.platform_capabilities or {}
|
||||
adapter_identity = None
|
||||
if query is not None:
|
||||
adapter = getattr(query, 'adapter', None)
|
||||
if adapter is not None:
|
||||
adapter_identity = adapter.__class__.__name__
|
||||
adapter_identity = _first_present(
|
||||
adapter_identity,
|
||||
capabilities.get('adapter'),
|
||||
capabilities.get('source'),
|
||||
event.source if query is None else None,
|
||||
)
|
||||
|
||||
return json.dumps(
|
||||
{
|
||||
'instance_id': _nonempty(constants.instance_id),
|
||||
'workspace_id': _nonempty(event.workspace_id),
|
||||
'bot_id': _nonempty(event.bot_id),
|
||||
'platform_adapter': adapter_identity,
|
||||
'target_type': target_type,
|
||||
'target_id': target_id,
|
||||
'thread_id': _nonempty(event.thread_id),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(',', ':'),
|
||||
)
|
||||
|
||||
|
||||
def _resolve_session_identity(
|
||||
event: AgentEventEnvelope,
|
||||
) -> tuple[provider_session.LauncherTypes, str, str]:
|
||||
subject_data = event.subject.data if event.subject is not None else {}
|
||||
reply_target = event.delivery.reply_target or {}
|
||||
|
||||
launcher_type_value = _first_present(
|
||||
reply_target.get('target_type'),
|
||||
subject_data.get('launcher_type'),
|
||||
event.data.get('launcher_type'),
|
||||
reply_target.get('launcher_type'),
|
||||
)
|
||||
if launcher_type_value == provider_session.LauncherTypes.GROUP.value:
|
||||
launcher_type_value = provider_session.LauncherTypes.GROUP.value
|
||||
else:
|
||||
launcher_type_value = provider_session.LauncherTypes.PERSON.value
|
||||
|
||||
launcher_id = _first_nonempty(
|
||||
reply_target.get('target_id'),
|
||||
subject_data.get('launcher_id'),
|
||||
event.data.get('launcher_id'),
|
||||
reply_target.get('launcher_id'),
|
||||
event.conversation_id,
|
||||
event.subject.subject_id if event.subject is not None else None,
|
||||
event.actor.actor_id if event.actor is not None else None,
|
||||
event.event_id,
|
||||
)
|
||||
sender_id = _first_nonempty(
|
||||
subject_data.get('sender_id'),
|
||||
event.data.get('sender_id'),
|
||||
event.actor.actor_id if event.actor is not None else None,
|
||||
launcher_id,
|
||||
)
|
||||
|
||||
return provider_session.LauncherTypes(launcher_type_value), launcher_id, sender_id
|
||||
|
||||
|
||||
def _resolve_box_target(
|
||||
event: AgentEventEnvelope,
|
||||
query: pipeline_query.Query | None,
|
||||
) -> tuple[str | None, str | None]:
|
||||
if query is not None:
|
||||
launcher_type = getattr(query, 'launcher_type', None)
|
||||
if hasattr(launcher_type, 'value'):
|
||||
launcher_type = launcher_type.value
|
||||
launcher_id = getattr(query, 'launcher_id', None)
|
||||
normalized_type = _nonempty(launcher_type)
|
||||
normalized_id = _nonempty(launcher_id)
|
||||
if normalized_type is not None and normalized_id is not None:
|
||||
return normalized_type, normalized_id
|
||||
|
||||
reply_target = event.delivery.reply_target or {}
|
||||
target_type = _first_present(
|
||||
reply_target.get('target_type'),
|
||||
reply_target.get('launcher_type'),
|
||||
)
|
||||
target_id = _first_present(
|
||||
reply_target.get('target_id'),
|
||||
reply_target.get('launcher_id'),
|
||||
)
|
||||
if target_type is not None and target_id is not None:
|
||||
return target_type, target_id
|
||||
|
||||
conversation_id = _nonempty(event.conversation_id)
|
||||
if conversation_id is not None:
|
||||
return 'conversation', conversation_id
|
||||
|
||||
event_id = _nonempty(event.event_id)
|
||||
if event_id is not None:
|
||||
return 'event', event_id
|
||||
return None, None
|
||||
|
||||
|
||||
def _build_message_chain(event: AgentEventEnvelope) -> platform_message.MessageChain:
|
||||
text = event.input.to_text()
|
||||
if not text:
|
||||
return platform_message.MessageChain([])
|
||||
return platform_message.MessageChain([platform_message.Plain(text=text)])
|
||||
|
||||
|
||||
def _synthetic_query_id(event_id: str) -> int:
|
||||
digest = hashlib.sha256(event_id.encode('utf-8')).hexdigest()
|
||||
return int(digest[:15], 16) or 1
|
||||
|
||||
|
||||
def _first_nonempty(*values: typing.Any) -> str:
|
||||
normalized = _first_present(*values)
|
||||
if normalized is not None:
|
||||
return normalized
|
||||
raise ValueError('Agent event does not contain a usable execution identity')
|
||||
|
||||
|
||||
def _first_present(*values: typing.Any) -> str | None:
|
||||
for value in values:
|
||||
normalized = _nonempty(value)
|
||||
if normalized is not None:
|
||||
return normalized
|
||||
return None
|
||||
|
||||
|
||||
def _nonempty(value: typing.Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
normalized = str(value).strip()
|
||||
return normalized or None
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
These are Host-internal models, not exposed to SDK.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
@@ -73,7 +74,7 @@ class AgentEventEnvelope(pydantic.BaseModel):
|
||||
class BindingScope(pydantic.BaseModel):
|
||||
"""Scope for agent binding."""
|
||||
|
||||
scope_type: typing.Literal["agent", "bot", "workspace", "global"] = "agent"
|
||||
scope_type: typing.Literal['agent', 'bot', 'workspace', 'global'] = 'agent'
|
||||
"""Scope type."""
|
||||
|
||||
scope_id: str | None = None
|
||||
@@ -92,6 +93,12 @@ class ResourcePolicy(pydantic.BaseModel):
|
||||
allowed_tool_names: list[str] | None = None
|
||||
"""Additional tool name grants. None means no additional tool grants."""
|
||||
|
||||
allowed_tool_sources: dict[str, dict[str, str | None]] | None = None
|
||||
"""Host-resolved implementation identity for each allowed tool name."""
|
||||
|
||||
allow_all_tools: bool = False
|
||||
"""Whether all tools visible to the current Host scope are granted."""
|
||||
|
||||
allowed_kb_uuids: list[str] | None = None
|
||||
"""Additional knowledge base UUID grants. None means no additional KB grants."""
|
||||
|
||||
@@ -114,8 +121,8 @@ class StatePolicy(pydantic.BaseModel):
|
||||
enable_state: bool = True
|
||||
"""Whether host-owned state is enabled."""
|
||||
|
||||
state_scopes: list[typing.Literal["conversation", "actor", "subject", "runner"]] = (
|
||||
pydantic.Field(default_factory=lambda: ["conversation", "actor"])
|
||||
state_scopes: list[typing.Literal['conversation', 'actor', 'subject', 'runner']] = pydantic.Field(
|
||||
default_factory=lambda: ['conversation', 'actor']
|
||||
)
|
||||
"""Enabled state scopes."""
|
||||
|
||||
@@ -162,7 +169,7 @@ class AgentConfig(pydantic.BaseModel):
|
||||
delivery_policy: DeliveryPolicy = pydantic.Field(default_factory=DeliveryPolicy)
|
||||
"""Delivery policy for this Agent."""
|
||||
|
||||
event_types: list[str] = pydantic.Field(default_factory=lambda: ["message.received"])
|
||||
event_types: list[str] = pydantic.Field(default_factory=lambda: ['message.received'])
|
||||
"""Event types this Agent handles."""
|
||||
|
||||
enabled: bool = True
|
||||
@@ -185,7 +192,7 @@ class AgentBinding(pydantic.BaseModel):
|
||||
scope: BindingScope = pydantic.Field(default_factory=BindingScope)
|
||||
"""Binding scope."""
|
||||
|
||||
event_types: list[str] = pydantic.Field(default_factory=lambda: ["message.received"])
|
||||
event_types: list[str] = pydantic.Field(default_factory=lambda: ['message.received'])
|
||||
"""Event types this binding handles."""
|
||||
|
||||
runner_id: str
|
||||
|
||||
@@ -12,6 +12,14 @@ from ...core import app
|
||||
from .binding_resolver import AgentBindingResolver
|
||||
from .context_builder import AgentRunContextBuilder, AgentRunContextPayload
|
||||
from .descriptor import AgentRunnerDescriptor
|
||||
from .execution_context import (
|
||||
append_mcp_resource_context_to_event,
|
||||
build_mcp_resource_context_addition,
|
||||
build_execution_query,
|
||||
prepare_box_scope,
|
||||
prepare_execution_query,
|
||||
project_mcp_resource_config,
|
||||
)
|
||||
from .host_models import AgentBinding, AgentEventEnvelope
|
||||
from .invoker import AgentRunnerInvoker
|
||||
from .query_bridge import QueryRunBridge
|
||||
@@ -73,6 +81,17 @@ class AgentRunOrchestrator:
|
||||
runner_id = binding.runner_id
|
||||
descriptor = await self.registry.get(runner_id, bound_plugins)
|
||||
|
||||
execution_query = adapter_context.get('_query') if adapter_context else None
|
||||
if execution_query is None:
|
||||
execution_query = build_execution_query(event, [])
|
||||
project_mcp_resource_config(execution_query, binding.runner_config)
|
||||
|
||||
execution_event = event
|
||||
resource_addition = await build_mcp_resource_context_addition(self.ap, execution_query)
|
||||
if resource_addition:
|
||||
execution_event = event.model_copy(deep=True)
|
||||
append_mcp_resource_context_to_event(execution_event, resource_addition)
|
||||
|
||||
resources = await self.resource_builder.build_resources_from_binding(
|
||||
event=event,
|
||||
binding=binding,
|
||||
@@ -80,7 +99,7 @@ class AgentRunOrchestrator:
|
||||
)
|
||||
|
||||
context = await self.context_builder.build_context_from_event(
|
||||
event=event,
|
||||
event=execution_event,
|
||||
binding=binding,
|
||||
descriptor=descriptor,
|
||||
resources=resources,
|
||||
@@ -88,19 +107,23 @@ class AgentRunOrchestrator:
|
||||
|
||||
session_query_id = None
|
||||
if adapter_context:
|
||||
query = adapter_context.get('_query')
|
||||
if query is not None:
|
||||
if execution_query is not None:
|
||||
skill_loader.restore_activated_skills_from_state(
|
||||
self.ap,
|
||||
query,
|
||||
execution_query,
|
||||
context.get('state', {}),
|
||||
)
|
||||
session_query_id = adapter_context.get('query_id')
|
||||
if query is not None or session_query_id is not None:
|
||||
if execution_query is not None or session_query_id is not None:
|
||||
context['context']['available_apis']['prompt_get'] = True
|
||||
if 'params' in adapter_context:
|
||||
context['adapter']['extra']['params'] = adapter_context['params']
|
||||
|
||||
authorized_skill_names = [
|
||||
str(skill['skill_name']) for skill in resources.get('skills', []) if skill.get('skill_name')
|
||||
]
|
||||
prepare_execution_query(execution_query, event, authorized_skill_names)
|
||||
|
||||
state_context = build_state_context(event, binding, descriptor)
|
||||
run_id = context['run_id']
|
||||
available_apis = context.get('context', {}).get('available_apis')
|
||||
@@ -152,6 +175,7 @@ class AgentRunOrchestrator:
|
||||
'state_scopes': list(binding.state_policy.state_scopes),
|
||||
},
|
||||
state_context=state_context,
|
||||
execution_query=execution_query,
|
||||
)
|
||||
|
||||
event_log_id = await self.journal.write_event_log(
|
||||
@@ -309,6 +333,9 @@ class AgentRunOrchestrator:
|
||||
adapter_context = dict(plan.adapter_context)
|
||||
adapter_context['_query'] = query
|
||||
|
||||
# Inbound files and subsequent runner tools must share one Host scope.
|
||||
prepare_box_scope(query, plan.event)
|
||||
|
||||
# Materialize inbound attachments into sandbox before running
|
||||
await self._materialize_inbound_attachments(query, plan.event)
|
||||
|
||||
@@ -392,7 +419,13 @@ class AgentRunOrchestrator:
|
||||
if target_run_id is None:
|
||||
return False
|
||||
|
||||
steering_item = self._build_steering_item(event, target_run_id, descriptor.id)
|
||||
execution_event = event
|
||||
resource_addition = await build_mcp_resource_context_addition(self.ap, query)
|
||||
if resource_addition:
|
||||
execution_event = event.model_copy(deep=True)
|
||||
append_mcp_resource_context_to_event(execution_event, resource_addition)
|
||||
|
||||
steering_item = self._build_steering_item(execution_event, target_run_id, descriptor.id)
|
||||
if not await self._session_registry.enqueue_steering(target_run_id, steering_item):
|
||||
return False
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ import typing
|
||||
from langbot_plugin.api.entities.builtin.pipeline import query as pipeline_query
|
||||
|
||||
from .binding_resolver import AgentBindingResolver
|
||||
from .config_migration import ConfigMigration
|
||||
from .config_resolver import RunnerConfigResolver
|
||||
from .errors import RunnerNotFoundError
|
||||
from .host_models import AgentBinding, AgentEventEnvelope
|
||||
from .query_entry_adapter import QueryEntryAdapter
|
||||
@@ -34,7 +34,7 @@ class QueryRunBridge:
|
||||
|
||||
def build_plan(self, query: pipeline_query.Query) -> QueryRunPlan:
|
||||
"""Build an event-first run plan from a Pipeline Query."""
|
||||
runner_id = ConfigMigration.resolve_runner_id(query.pipeline_config)
|
||||
runner_id = RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
|
||||
if not runner_id:
|
||||
raise RunnerNotFoundError('no runner configured')
|
||||
|
||||
@@ -53,4 +53,4 @@ class QueryRunBridge:
|
||||
|
||||
def resolve_runner_id_for_telemetry(self, query: pipeline_query.Query) -> str | None:
|
||||
"""Resolve runner ID for telemetry/logging without full execution."""
|
||||
return ConfigMigration.resolve_runner_id(query.pipeline_config)
|
||||
return RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
This adapter bridges the current Query entry point with the event-first
|
||||
Protocol v1 architecture without exposing Query internals to runners.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
@@ -23,12 +24,13 @@ from langbot_plugin.api.entities.builtin.agent_runner.delivery import DeliveryCo
|
||||
from .host_models import (
|
||||
AgentConfig,
|
||||
AgentEventEnvelope,
|
||||
ResourcePolicy,
|
||||
StatePolicy,
|
||||
DeliveryPolicy,
|
||||
)
|
||||
from .config_migration import ConfigMigration
|
||||
from .config_resolver import RunnerConfigResolver
|
||||
from .resource_policy import ResourcePolicyProjector
|
||||
from . import events as runner_events
|
||||
from ...provider.tools.toolmgr import TOOL_SOURCE_REFS_QUERY_KEY
|
||||
|
||||
|
||||
class QueryEntryAdapter:
|
||||
@@ -83,7 +85,7 @@ class QueryEntryAdapter:
|
||||
event_id=event.event_id or str(query.query_id),
|
||||
event_type=event.event_type or runner_events.MESSAGE_RECEIVED,
|
||||
event_time=event.event_time,
|
||||
source="host_adapter",
|
||||
source='host_adapter',
|
||||
source_event_type=event.source_event_type,
|
||||
bot_id=query.bot_uuid,
|
||||
workspace_id=None, # Not available in Query
|
||||
@@ -105,21 +107,22 @@ class QueryEntryAdapter:
|
||||
) -> AgentConfig:
|
||||
"""Project the current Pipeline config container into target Agent config."""
|
||||
pipeline_config = query.pipeline_config or {}
|
||||
runner_config = ConfigMigration.resolve_runner_config(pipeline_config, runner_id)
|
||||
runner_config = RunnerConfigResolver.resolve_runner_config(pipeline_config, runner_id)
|
||||
agent_id = getattr(query, 'pipeline_uuid', None)
|
||||
|
||||
# Build resource policy from current config
|
||||
resource_policy = ResourcePolicy(
|
||||
allowed_model_uuids=cls._extract_allowed_models(query),
|
||||
allowed_tool_names=cls._extract_allowed_tools(query),
|
||||
allowed_kb_uuids=cls._extract_allowed_kbs(query),
|
||||
allowed_skill_names=cls._extract_allowed_skills(query),
|
||||
resource_policy = ResourcePolicyProjector.from_runner_config(
|
||||
runner_config,
|
||||
resolved_model_uuids=cls._extract_allowed_models(query),
|
||||
resolved_tool_names=cls._extract_allowed_tools(query),
|
||||
resolved_tool_sources=cls._extract_allowed_tool_sources(query),
|
||||
resolved_kb_uuids=cls._extract_allowed_kbs(query),
|
||||
resolved_skill_names=cls._extract_allowed_skills(query),
|
||||
)
|
||||
|
||||
# Build state policy
|
||||
state_policy = StatePolicy(
|
||||
enable_state=True,
|
||||
state_scopes=["conversation", "actor", "subject", "runner"],
|
||||
state_scopes=['conversation', 'actor', 'subject', 'runner'],
|
||||
)
|
||||
|
||||
# Build delivery policy
|
||||
@@ -181,10 +184,7 @@ class QueryEntryAdapter:
|
||||
if isinstance(value, (list, tuple)):
|
||||
return all(cls.is_json_serializable(item) for item in value)
|
||||
if isinstance(value, dict):
|
||||
return all(
|
||||
isinstance(k, str) and cls.is_json_serializable(v)
|
||||
for k, v in value.items()
|
||||
)
|
||||
return all(isinstance(k, str) and cls.is_json_serializable(v) for k, v in value.items())
|
||||
return False
|
||||
|
||||
# Private helper methods
|
||||
@@ -228,7 +228,7 @@ class QueryEntryAdapter:
|
||||
event_id=cls._build_scoped_event_id(query, source_event_id, event_time),
|
||||
event_type=runner_events.MESSAGE_RECEIVED,
|
||||
event_time=event_time,
|
||||
source="host_adapter",
|
||||
source='host_adapter',
|
||||
source_event_type=source_event_type,
|
||||
data=event_data,
|
||||
)
|
||||
@@ -345,7 +345,7 @@ class QueryEntryAdapter:
|
||||
actor_name = sender.get_name() if sender and hasattr(sender, 'get_name') else None
|
||||
|
||||
return ActorContext(
|
||||
actor_type="user",
|
||||
actor_type='user',
|
||||
actor_id=str(actor_id) if actor_id is not None else None,
|
||||
actor_name=actor_name,
|
||||
metadata={},
|
||||
@@ -371,13 +371,13 @@ class QueryEntryAdapter:
|
||||
launcher_type_value = getattr(launcher_type, 'value', launcher_type)
|
||||
|
||||
return SubjectContext(
|
||||
subject_type="message",
|
||||
subject_type='message',
|
||||
subject_id=str(message_id or query_id or ''),
|
||||
data={
|
||||
"launcher_type": launcher_type_value,
|
||||
"launcher_id": getattr(query, 'launcher_id', None),
|
||||
"sender_id": str(getattr(query, 'sender_id', '')) if getattr(query, 'sender_id', None) else None,
|
||||
"bot_uuid": getattr(query, 'bot_uuid', None),
|
||||
'launcher_type': launcher_type_value,
|
||||
'launcher_id': getattr(query, 'launcher_id', None),
|
||||
'sender_id': str(getattr(query, 'sender_id', '')) if getattr(query, 'sender_id', None) else None,
|
||||
'bot_uuid': getattr(query, 'bot_uuid', None),
|
||||
},
|
||||
)
|
||||
|
||||
@@ -467,31 +467,39 @@ class QueryEntryAdapter:
|
||||
|
||||
if elem_type == 'image_url':
|
||||
image_url = elem.get('image_url') or {}
|
||||
add_attachment({
|
||||
'type': 'image',
|
||||
'source': 'url',
|
||||
'url': image_url.get('url') if isinstance(image_url, dict) else str(image_url),
|
||||
})
|
||||
add_attachment(
|
||||
{
|
||||
'type': 'image',
|
||||
'source': 'url',
|
||||
'url': image_url.get('url') if isinstance(image_url, dict) else str(image_url),
|
||||
}
|
||||
)
|
||||
elif elem_type == 'image_base64':
|
||||
add_attachment({
|
||||
'type': 'image',
|
||||
'source': 'base64',
|
||||
'content': elem.get('image_base64'),
|
||||
})
|
||||
add_attachment(
|
||||
{
|
||||
'type': 'image',
|
||||
'source': 'base64',
|
||||
'content': elem.get('image_base64'),
|
||||
}
|
||||
)
|
||||
elif elem_type == 'file_url':
|
||||
add_attachment({
|
||||
'type': 'file',
|
||||
'source': 'url',
|
||||
'url': elem.get('file_url'),
|
||||
'name': elem.get('file_name'),
|
||||
})
|
||||
add_attachment(
|
||||
{
|
||||
'type': 'file',
|
||||
'source': 'url',
|
||||
'url': elem.get('file_url'),
|
||||
'name': elem.get('file_name'),
|
||||
}
|
||||
)
|
||||
elif elem_type == 'file_base64':
|
||||
add_attachment({
|
||||
'type': 'file',
|
||||
'source': 'base64',
|
||||
'content': elem.get('file_base64'),
|
||||
'name': elem.get('file_name'),
|
||||
})
|
||||
add_attachment(
|
||||
{
|
||||
'type': 'file',
|
||||
'source': 'base64',
|
||||
'content': elem.get('file_base64'),
|
||||
'name': elem.get('file_name'),
|
||||
}
|
||||
)
|
||||
|
||||
message_chain = getattr(query, 'message_chain', None)
|
||||
if message_chain:
|
||||
@@ -505,30 +513,36 @@ class QueryEntryAdapter:
|
||||
image_id = component.image_id or None
|
||||
image_url = component.url or None
|
||||
image_base64 = component.base64 or None
|
||||
add_attachment({
|
||||
'type': 'image',
|
||||
'source': 'message_chain',
|
||||
'id': image_id,
|
||||
'url': image_url,
|
||||
'content': image_base64,
|
||||
})
|
||||
add_attachment(
|
||||
{
|
||||
'type': 'image',
|
||||
'source': 'message_chain',
|
||||
'id': image_id,
|
||||
'url': image_url,
|
||||
'content': image_base64,
|
||||
}
|
||||
)
|
||||
elif isinstance(component, platform_message.File):
|
||||
add_attachment({
|
||||
'type': 'file',
|
||||
'source': 'message_chain',
|
||||
'id': component.id or None,
|
||||
'name': component.name or None,
|
||||
'url': component.url or None,
|
||||
'content': component.base64 or None,
|
||||
})
|
||||
add_attachment(
|
||||
{
|
||||
'type': 'file',
|
||||
'source': 'message_chain',
|
||||
'id': component.id or None,
|
||||
'name': component.name or None,
|
||||
'url': component.url or None,
|
||||
'content': component.base64 or None,
|
||||
}
|
||||
)
|
||||
elif isinstance(component, platform_message.Voice):
|
||||
add_attachment({
|
||||
'type': 'voice',
|
||||
'source': 'message_chain',
|
||||
'id': component.voice_id or None,
|
||||
'url': component.url or None,
|
||||
'content': component.base64 or None,
|
||||
})
|
||||
add_attachment(
|
||||
{
|
||||
'type': 'voice',
|
||||
'source': 'message_chain',
|
||||
'id': component.voice_id or None,
|
||||
'url': component.url or None,
|
||||
'content': component.base64 or None,
|
||||
}
|
||||
)
|
||||
|
||||
return attachments
|
||||
|
||||
@@ -557,9 +571,9 @@ class QueryEntryAdapter:
|
||||
"""Build DeliveryContext from Query."""
|
||||
message_chain = getattr(query, 'message_chain', None)
|
||||
return DeliveryContext(
|
||||
surface="platform",
|
||||
surface='platform',
|
||||
reply_target={
|
||||
"message_id": getattr(message_chain, 'message_id', None),
|
||||
'message_id': getattr(message_chain, 'message_id', None),
|
||||
},
|
||||
supports_streaming=True,
|
||||
supports_edit=False,
|
||||
@@ -601,22 +615,24 @@ class QueryEntryAdapter:
|
||||
) -> list[str] | None:
|
||||
"""Extract allowed tool names from query."""
|
||||
use_funcs = getattr(query, 'use_funcs', None)
|
||||
if not use_funcs:
|
||||
if use_funcs is None:
|
||||
return None
|
||||
try:
|
||||
tool_names = []
|
||||
for func in use_funcs:
|
||||
if isinstance(func, dict):
|
||||
name = func.get('name')
|
||||
elif hasattr(func, 'name'):
|
||||
name = func.name
|
||||
else:
|
||||
continue
|
||||
if name:
|
||||
tool_names.append(name)
|
||||
return tool_names if tool_names else None
|
||||
return ResourcePolicyProjector.extract_tool_names(use_funcs)
|
||||
except (TypeError, AttributeError):
|
||||
return []
|
||||
|
||||
@classmethod
|
||||
def _extract_allowed_tool_sources(
|
||||
cls,
|
||||
query: pipeline_query.Query,
|
||||
) -> dict[str, typing.Any] | None:
|
||||
"""Extract the Host-frozen implementation for each allowed tool."""
|
||||
variables = getattr(query, 'variables', None)
|
||||
if not isinstance(variables, dict):
|
||||
return None
|
||||
refs = variables.get(TOOL_SOURCE_REFS_QUERY_KEY)
|
||||
return refs if isinstance(refs, dict) else None
|
||||
|
||||
@classmethod
|
||||
def _extract_allowed_kbs(
|
||||
@@ -627,10 +643,9 @@ class QueryEntryAdapter:
|
||||
variables = getattr(query, 'variables', None)
|
||||
if not variables:
|
||||
return None
|
||||
kb_uuids = variables.get('_knowledge_base_uuids')
|
||||
if kb_uuids:
|
||||
return kb_uuids
|
||||
return None
|
||||
if '_knowledge_base_uuids' not in variables:
|
||||
return None
|
||||
return ResourcePolicyProjector.normalize_names(variables.get('_knowledge_base_uuids'))
|
||||
|
||||
@classmethod
|
||||
def _extract_allowed_skills(
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Agent resource builder for constructing authorized resources."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
@@ -15,6 +16,9 @@ from .context_builder import (
|
||||
)
|
||||
from . import config_schema
|
||||
from .host_models import AgentEventEnvelope, AgentBinding
|
||||
from .resource_policy import ResourcePolicyProjector
|
||||
from ...provider.tools.loaders.mcp import MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE
|
||||
from ...provider.tools.toolmgr import ToolSourceRef
|
||||
|
||||
|
||||
class AgentResourceBuilder:
|
||||
@@ -66,18 +70,12 @@ class AgentResourceBuilder:
|
||||
manifest_perms = descriptor.permissions
|
||||
|
||||
# Build each resource category
|
||||
models = await self._build_models_from_binding(
|
||||
manifest_perms, resource_policy, descriptor, runner_config
|
||||
)
|
||||
tools = await self._build_tools_from_binding(
|
||||
manifest_perms, resource_policy, descriptor
|
||||
)
|
||||
models = await self._build_models_from_binding(manifest_perms, resource_policy, descriptor, runner_config)
|
||||
tools = await self._build_tools_from_binding(manifest_perms, resource_policy, descriptor, runner_config)
|
||||
knowledge_bases = await self._build_knowledge_bases_from_binding(
|
||||
manifest_perms, resource_policy, descriptor, runner_config
|
||||
)
|
||||
skills = self._build_skills_from_binding(
|
||||
resource_policy, descriptor
|
||||
)
|
||||
skills = self._build_skills_from_binding(resource_policy, descriptor)
|
||||
storage = self._build_storage_from_binding(manifest_perms, binding)
|
||||
|
||||
return {
|
||||
@@ -133,6 +131,7 @@ class AgentResourceBuilder:
|
||||
manifest_perms: typing.Any,
|
||||
resource_policy: typing.Any,
|
||||
descriptor: AgentRunnerDescriptor,
|
||||
runner_config: dict[str, typing.Any],
|
||||
) -> list[ToolResource]:
|
||||
"""Build tools list from binding."""
|
||||
tools: list[ToolResource] = []
|
||||
@@ -143,8 +142,43 @@ class AgentResourceBuilder:
|
||||
if not config_schema.uses_host_tools(descriptor):
|
||||
return tools
|
||||
|
||||
# Get tool names from resource policy
|
||||
allowed_names = resource_policy.allowed_tool_names
|
||||
allowed_sources = resource_policy.allowed_tool_sources
|
||||
if resource_policy.allow_all_tools or allowed_sources is None:
|
||||
get_catalog = getattr(getattr(self.ap, 'tool_mgr', None), 'get_resolved_tool_catalog', None)
|
||||
if get_catalog is None:
|
||||
return tools
|
||||
try:
|
||||
catalog = await get_catalog(
|
||||
include_skill_authoring=True,
|
||||
include_mcp_resource_tools=True,
|
||||
)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to resolve visible Host tools: {e}')
|
||||
return tools
|
||||
|
||||
if not resource_policy.allow_all_tools:
|
||||
selected_names = set(allowed_names or [])
|
||||
catalog = [item for item in catalog if item.get('name') in selected_names]
|
||||
allowed_names = ResourcePolicyProjector.extract_tool_names(catalog)
|
||||
allowed_sources = {
|
||||
item['name']: ref for item in catalog if (ref := self._catalog_source_ref(item)) is not None
|
||||
}
|
||||
|
||||
if runner_config.get('mcp-resource-agent-read-enabled', True) is not True:
|
||||
denied_resource_tools = {MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE}
|
||||
allowed_names = [
|
||||
name
|
||||
for name in (allowed_names or [])
|
||||
if not (
|
||||
name in denied_resource_tools
|
||||
and allowed_sources
|
||||
and (source_ref := allowed_sources.get(name)) is not None
|
||||
and source_ref['source'] == 'mcp'
|
||||
and source_ref.get('source_id') is None
|
||||
)
|
||||
]
|
||||
|
||||
tool_operations = [operation for operation in ('detail', 'call') if operation in tool_perms]
|
||||
|
||||
# Prefill full tool schema (best-effort) so runners can build LLM tool
|
||||
@@ -153,20 +187,39 @@ class AgentResourceBuilder:
|
||||
get_tool_schema = getattr(getattr(self.ap, 'tool_mgr', None), 'get_tool_schema', None)
|
||||
if allowed_names:
|
||||
for tool_name in allowed_names:
|
||||
source_ref = allowed_sources.get(tool_name) if allowed_sources else None
|
||||
if source_ref is None:
|
||||
self.ap.logger.warning(f'Tool {tool_name} is not authorized because its Host source is unresolved')
|
||||
continue
|
||||
if get_tool_schema is not None:
|
||||
description, parameters = await get_tool_schema(tool_name)
|
||||
description, parameters = await get_tool_schema(tool_name, source_ref=source_ref)
|
||||
else:
|
||||
description, parameters = None, None
|
||||
tools.append({
|
||||
'tool_name': tool_name,
|
||||
'tool_type': None,
|
||||
'description': description,
|
||||
'operations': tool_operations,
|
||||
'parameters': parameters,
|
||||
})
|
||||
tools.append(
|
||||
{
|
||||
'tool_name': tool_name,
|
||||
'tool_type': source_ref['source'],
|
||||
'description': description,
|
||||
'operations': tool_operations,
|
||||
'parameters': parameters,
|
||||
'source': source_ref['source'],
|
||||
'source_id': source_ref.get('source_id'),
|
||||
}
|
||||
)
|
||||
|
||||
return tools
|
||||
|
||||
@staticmethod
|
||||
def _catalog_source_ref(item: dict[str, typing.Any]) -> ToolSourceRef | None:
|
||||
source = item.get('source')
|
||||
if not isinstance(source, str) or not source:
|
||||
return None
|
||||
source_id = item.get('source_id')
|
||||
return {
|
||||
'source': source,
|
||||
'source_id': source_id if isinstance(source_id, str) and source_id else None,
|
||||
}
|
||||
|
||||
async def _build_knowledge_bases_from_binding(
|
||||
self,
|
||||
manifest_perms: typing.Any,
|
||||
@@ -196,12 +249,16 @@ class AgentResourceBuilder:
|
||||
try:
|
||||
kb = await self.ap.rag_mgr.get_knowledge_base_by_uuid(kb_uuid)
|
||||
if kb:
|
||||
kb_resources.append({
|
||||
'kb_id': kb_uuid,
|
||||
'kb_name': kb.get_name(),
|
||||
'kb_type': kb.knowledge_base_entity.kb_type if hasattr(kb.knowledge_base_entity, 'kb_type') else None,
|
||||
'operations': kb_operations,
|
||||
})
|
||||
kb_resources.append(
|
||||
{
|
||||
'kb_id': kb_uuid,
|
||||
'kb_name': kb.get_name(),
|
||||
'kb_type': kb.knowledge_base_entity.kb_type
|
||||
if hasattr(kb.knowledge_base_entity, 'kb_type')
|
||||
else None,
|
||||
'operations': kb_operations,
|
||||
}
|
||||
)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to build knowledge base resource {kb_uuid}: {e}')
|
||||
|
||||
@@ -233,11 +290,13 @@ class AgentResourceBuilder:
|
||||
skills: list[SkillResource] = []
|
||||
for skill_name in names:
|
||||
skill_data = loaded_skills.get(skill_name) or {}
|
||||
skills.append({
|
||||
'skill_name': skill_name,
|
||||
'display_name': skill_data.get('display_name') or skill_data.get('name') or skill_name,
|
||||
'description': skill_data.get('description') or None,
|
||||
})
|
||||
skills.append(
|
||||
{
|
||||
'skill_name': skill_name,
|
||||
'display_name': skill_data.get('display_name') or skill_data.get('name') or skill_name,
|
||||
'description': skill_data.get('description') or None,
|
||||
}
|
||||
)
|
||||
return skills
|
||||
|
||||
def _build_storage_from_binding(
|
||||
@@ -285,12 +344,16 @@ class AgentResourceBuilder:
|
||||
try:
|
||||
model = await self.ap.model_mgr.get_model_by_uuid(model_uuid)
|
||||
if model and model.model_entity:
|
||||
models.append({
|
||||
'model_id': model_uuid,
|
||||
'model_type': getattr(model.model_entity, 'model_type', None),
|
||||
'provider': getattr(model.provider_entity, 'name', None) if hasattr(model, 'provider_entity') else None,
|
||||
'operations': operations,
|
||||
})
|
||||
models.append(
|
||||
{
|
||||
'model_id': model_uuid,
|
||||
'model_type': getattr(model.model_entity, 'model_type', None),
|
||||
'provider': getattr(model.provider_entity, 'name', None)
|
||||
if hasattr(model, 'provider_entity')
|
||||
else None,
|
||||
'operations': operations,
|
||||
}
|
||||
)
|
||||
seen_model_ids.add(model_uuid)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to build LLM model resource {model_uuid}: {e}')
|
||||
@@ -308,12 +371,16 @@ class AgentResourceBuilder:
|
||||
try:
|
||||
model = await self.ap.model_mgr.get_rerank_model_by_uuid(model_uuid)
|
||||
if model and model.model_entity:
|
||||
models.append({
|
||||
'model_id': model_uuid,
|
||||
'model_type': getattr(model.model_entity, 'model_type', 'rerank') or 'rerank',
|
||||
'provider': getattr(model.provider_entity, 'name', None) if hasattr(model, 'provider_entity') else None,
|
||||
'operations': ['rerank'],
|
||||
})
|
||||
models.append(
|
||||
{
|
||||
'model_id': model_uuid,
|
||||
'model_type': getattr(model.model_entity, 'model_type', 'rerank') or 'rerank',
|
||||
'provider': getattr(model.provider_entity, 'name', None)
|
||||
if hasattr(model, 'provider_entity')
|
||||
else None,
|
||||
'operations': ['rerank'],
|
||||
}
|
||||
)
|
||||
seen_model_ids.add(model_uuid)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(f'Failed to build rerank model resource {model_uuid}: {e}')
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
"""Project AgentRunner configuration into Host resource policy."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections.abc
|
||||
import typing
|
||||
|
||||
from .host_models import ResourcePolicy
|
||||
|
||||
|
||||
class ResourcePolicyProjector:
|
||||
"""Build one generic resource policy for Pipeline and Agent bindings."""
|
||||
|
||||
@classmethod
|
||||
def from_runner_config(
|
||||
cls,
|
||||
runner_config: dict[str, typing.Any] | None,
|
||||
*,
|
||||
resolved_model_uuids: collections.abc.Iterable[typing.Any] | None = None,
|
||||
resolved_tool_names: collections.abc.Iterable[typing.Any] | None = None,
|
||||
resolved_tool_sources: typing.Mapping[str, typing.Any] | None = None,
|
||||
resolved_kb_uuids: collections.abc.Iterable[typing.Any] | None = None,
|
||||
resolved_skill_names: collections.abc.Iterable[typing.Any] | None = None,
|
||||
) -> ResourcePolicy:
|
||||
"""Project standard resource fields without depending on a runner ID.
|
||||
|
||||
Resolved values are supplied by entry paths that already narrowed the
|
||||
available resources, such as Pipeline preprocessing. Independent Agent
|
||||
bindings omit them so the Host can resolve an explicit all-tools grant
|
||||
against the live tool catalog when the run starts.
|
||||
"""
|
||||
config = runner_config if isinstance(runner_config, dict) else {}
|
||||
selected_tool_names = cls.normalize_names(config.get('tools'))
|
||||
enable_all_tools = config.get('enable-all-tools', True) is True
|
||||
|
||||
if resolved_tool_names is not None:
|
||||
available_tool_names = cls.normalize_names(resolved_tool_names)
|
||||
if enable_all_tools:
|
||||
allowed_tool_names = available_tool_names
|
||||
else:
|
||||
selected = set(selected_tool_names)
|
||||
allowed_tool_names = [name for name in available_tool_names if name in selected]
|
||||
allow_all_tools = False
|
||||
elif enable_all_tools:
|
||||
allowed_tool_names = None
|
||||
allow_all_tools = True
|
||||
else:
|
||||
allowed_tool_names = selected_tool_names
|
||||
allow_all_tools = False
|
||||
|
||||
allowed_tool_sources = cls.normalize_tool_sources(resolved_tool_sources)
|
||||
if allowed_tool_sources is not None:
|
||||
allowed = set(allowed_tool_names or [])
|
||||
allowed_tool_sources = {name: ref for name, ref in allowed_tool_sources.items() if name in allowed}
|
||||
|
||||
if resolved_kb_uuids is not None:
|
||||
allowed_kb_uuids = cls.normalize_names(resolved_kb_uuids)
|
||||
elif 'knowledge-bases' in config:
|
||||
allowed_kb_uuids = cls.normalize_names(config.get('knowledge-bases'))
|
||||
else:
|
||||
allowed_kb_uuids = None
|
||||
|
||||
return ResourcePolicy(
|
||||
allowed_model_uuids=cls.normalize_optional_names(resolved_model_uuids),
|
||||
allowed_tool_names=allowed_tool_names,
|
||||
allowed_tool_sources=allowed_tool_sources,
|
||||
allow_all_tools=allow_all_tools,
|
||||
allowed_kb_uuids=allowed_kb_uuids,
|
||||
allowed_skill_names=cls.normalize_optional_names(resolved_skill_names),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def normalize_names(values: typing.Any) -> list[str]:
|
||||
"""Return unique, non-empty string identifiers in source order."""
|
||||
if not isinstance(values, collections.abc.Iterable) or isinstance(values, (str, bytes, dict)):
|
||||
return []
|
||||
|
||||
names: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in values:
|
||||
if not isinstance(value, str) or not value or value in seen:
|
||||
continue
|
||||
seen.add(value)
|
||||
names.append(value)
|
||||
return names
|
||||
|
||||
@classmethod
|
||||
def normalize_optional_names(
|
||||
cls,
|
||||
values: collections.abc.Iterable[typing.Any] | None,
|
||||
) -> list[str] | None:
|
||||
if values is None:
|
||||
return None
|
||||
return cls.normalize_names(values)
|
||||
|
||||
@classmethod
|
||||
def extract_tool_names(cls, tools: collections.abc.Iterable[typing.Any] | None) -> list[str]:
|
||||
"""Extract tool identifiers from dictionary and SDK object shapes."""
|
||||
if tools is None:
|
||||
return []
|
||||
|
||||
names: list[str] = []
|
||||
for tool in tools:
|
||||
name = tool.get('name') if isinstance(tool, dict) else getattr(tool, 'name', None)
|
||||
if isinstance(name, str):
|
||||
names.append(name)
|
||||
return cls.normalize_names(names)
|
||||
|
||||
@staticmethod
|
||||
def normalize_tool_sources(
|
||||
values: typing.Mapping[str, typing.Any] | None,
|
||||
) -> dict[str, dict[str, str | None]] | None:
|
||||
if values is None:
|
||||
return None
|
||||
|
||||
normalized: dict[str, dict[str, str | None]] = {}
|
||||
for name, value in values.items():
|
||||
if not isinstance(name, str) or not name or not isinstance(value, collections.abc.Mapping):
|
||||
continue
|
||||
source = value.get('source')
|
||||
if not isinstance(source, str) or not source:
|
||||
continue
|
||||
source_id = value.get('source_id')
|
||||
normalized[name] = {
|
||||
'source': source,
|
||||
'source_id': source_id if isinstance(source_id, str) and source_id else None,
|
||||
}
|
||||
return normalized
|
||||
|
||||
@classmethod
|
||||
def filter_tools(
|
||||
cls,
|
||||
tools: collections.abc.Iterable[typing.Any],
|
||||
policy: ResourcePolicy,
|
||||
) -> list[typing.Any]:
|
||||
"""Apply a projected policy to an already scoped tool collection."""
|
||||
tool_list = list(tools)
|
||||
if policy.allow_all_tools:
|
||||
return tool_list
|
||||
|
||||
allowed_names = set(policy.allowed_tool_names or [])
|
||||
return [
|
||||
tool
|
||||
for tool in tool_list
|
||||
if (tool.get('name') if isinstance(tool, dict) else getattr(tool, 'name', None)) in allowed_names
|
||||
]
|
||||
@@ -304,13 +304,10 @@ class AgentRunJournal:
|
||||
|
||||
for item in items:
|
||||
event = item.get('event') if isinstance(item.get('event'), dict) else {}
|
||||
input_data = item.get('input') if isinstance(item.get('input'), dict) else {}
|
||||
conversation = item.get('conversation') if isinstance(item.get('conversation'), dict) else {}
|
||||
actor = item.get('actor') if isinstance(item.get('actor'), dict) else {}
|
||||
subject = item.get('subject') if isinstance(item.get('subject'), dict) else {}
|
||||
|
||||
text = input_data.get('text')
|
||||
input_summary = text[:1000] if isinstance(text, str) and text else 'Unconsumed steering input dropped'
|
||||
event_time = None
|
||||
raw_event_time = event.get('event_time')
|
||||
if raw_event_time:
|
||||
@@ -335,12 +332,7 @@ class AgentRunJournal:
|
||||
actor_name=actor.get('actor_name'),
|
||||
subject_type=subject.get('subject_type'),
|
||||
subject_id=subject.get('subject_id'),
|
||||
input_summary=input_summary,
|
||||
input_json={
|
||||
'text': text,
|
||||
'contents': self._sanitize_contents(input_data.get('contents') or []),
|
||||
'attachments': self._sanitize_attachments(input_data.get('attachments') or []),
|
||||
},
|
||||
input_summary='Unconsumed steering input dropped',
|
||||
run_id=run_id,
|
||||
runner_id=runner_id,
|
||||
event_time=event_time,
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
"""Agent run session registry for proxy action permission validation."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
@@ -7,7 +8,10 @@ import typing
|
||||
import time
|
||||
import threading
|
||||
|
||||
from langbot_plugin.api.entities.builtin.pipeline import query as pipeline_query
|
||||
|
||||
from .context_builder import AgentResources
|
||||
from ...provider.tools.toolmgr import ToolSourceRef
|
||||
|
||||
|
||||
MAX_STEERING_QUEUE_ITEMS = 100
|
||||
@@ -22,6 +26,7 @@ DEFAULT_RESOURCE_OPERATIONS: dict[str, set[str]] = {
|
||||
|
||||
class AgentRunSessionStatus(typing.TypedDict):
|
||||
"""Status tracking for agent run session."""
|
||||
|
||||
started_at: int
|
||||
last_activity_at: int
|
||||
|
||||
@@ -58,13 +63,16 @@ class AgentRunSession(typing.TypedDict):
|
||||
run_id: Unique run identifier (UUID from AgentRunContext)
|
||||
runner_id: Runner descriptor ID (plugin:author/name/runner)
|
||||
query_id: Host entry query ID, only present for query-based adapters
|
||||
execution_query: Host-only Query used by providers and tool loaders
|
||||
plugin_identity: Plugin identifier (author/name) of the runner
|
||||
authorization: Run-scoped authorization snapshot; runtime auth truth
|
||||
status: Session status tracking
|
||||
"""
|
||||
|
||||
run_id: str
|
||||
runner_id: str
|
||||
query_id: int | None
|
||||
execution_query: pipeline_query.Query | None
|
||||
plugin_identity: str # author/name
|
||||
authorization: RunAuthorizationSnapshot
|
||||
status: AgentRunSessionStatus
|
||||
@@ -104,6 +112,7 @@ class AgentRunSessionRegistry:
|
||||
available_apis: dict[str, bool] | None = None,
|
||||
state_policy: dict[str, typing.Any] | None = None,
|
||||
state_context: dict[str, typing.Any] | None = None,
|
||||
execution_query: pipeline_query.Query | None = None,
|
||||
) -> None:
|
||||
"""Register a new agent run session.
|
||||
|
||||
@@ -120,6 +129,7 @@ class AgentRunSessionRegistry:
|
||||
available_apis: Run-scoped pull APIs exposed in AgentRunContext
|
||||
state_policy: State policy from binding (enable_state, state_scopes)
|
||||
state_context: Context for state API (scope_keys, binding_identity, etc.)
|
||||
execution_query: Host-only Query used for provider and tool execution
|
||||
"""
|
||||
if not isinstance(plugin_identity, str) or not plugin_identity.strip():
|
||||
raise ValueError('plugin_identity is required for agent run sessions')
|
||||
@@ -153,6 +163,7 @@ class AgentRunSessionRegistry:
|
||||
'run_id': run_id,
|
||||
'runner_id': runner_id,
|
||||
'query_id': query_id,
|
||||
'execution_query': execution_query,
|
||||
'plugin_identity': plugin_identity,
|
||||
'authorization': authorization,
|
||||
'status': {
|
||||
@@ -371,6 +382,26 @@ class AgentRunSessionRegistry:
|
||||
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def get_tool_source_ref(
|
||||
session: AgentRunSession,
|
||||
tool_name: str,
|
||||
) -> ToolSourceRef | None:
|
||||
"""Return the implementation identity frozen in this run's grant."""
|
||||
resources = session['authorization']['resources']
|
||||
for tool in resources.get('tools', []):
|
||||
if tool.get('tool_name') != tool_name:
|
||||
continue
|
||||
source = tool.get('source')
|
||||
if not isinstance(source, str) or not source:
|
||||
return None
|
||||
source_id = tool.get('source_id')
|
||||
return {
|
||||
'source': source,
|
||||
'source_id': source_id if isinstance(source_id, str) and source_id else None,
|
||||
}
|
||||
return None
|
||||
|
||||
async def list_active_runs(self) -> list[AgentRunSession]:
|
||||
"""List all active run sessions.
|
||||
|
||||
|
||||
@@ -16,7 +16,10 @@ class AgentsRouterGroup(group.RouterGroup):
|
||||
return self.success(data={'agents': await self.ap.agent_service.get_agents(sort_by, sort_order)})
|
||||
|
||||
json_data = await quart.request.json
|
||||
created = await self.ap.agent_service.create_agent(json_data)
|
||||
try:
|
||||
created = await self.ap.agent_service.create_agent(json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success(data=created)
|
||||
|
||||
@self.route('/_/metadata', methods=['GET'], auth_type=group.AuthType.USER_TOKEN_OR_API_KEY)
|
||||
@@ -33,7 +36,10 @@ class AgentsRouterGroup(group.RouterGroup):
|
||||
|
||||
if quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
await self.ap.agent_service.update_agent(agent_uuid, json_data)
|
||||
try:
|
||||
await self.ap.agent_service.update_agent(agent_uuid, json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
return self.success()
|
||||
|
||||
await self.ap.agent_service.delete_agent(agent_uuid)
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import quart
|
||||
|
||||
from ... import group
|
||||
from ......pipeline.extension_preferences import normalize_extension_preferences
|
||||
|
||||
|
||||
@group.group_class('pipelines', '/api/v1/pipelines')
|
||||
@@ -19,7 +20,10 @@ class PipelinesRouterGroup(group.RouterGroup):
|
||||
elif quart.request.method == 'POST':
|
||||
json_data = await quart.request.json
|
||||
|
||||
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(json_data)
|
||||
try:
|
||||
pipeline_uuid = await self.ap.pipeline_service.create_pipeline(json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
return self.success(data={'uuid': pipeline_uuid})
|
||||
|
||||
@@ -41,7 +45,10 @@ class PipelinesRouterGroup(group.RouterGroup):
|
||||
elif quart.request.method == 'PUT':
|
||||
json_data = await quart.request.json
|
||||
|
||||
await self.ap.pipeline_service.update_pipeline(pipeline_uuid, json_data)
|
||||
try:
|
||||
await self.ap.pipeline_service.update_pipeline(pipeline_uuid, json_data)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
return self.success()
|
||||
elif quart.request.method == 'DELETE':
|
||||
@@ -81,21 +88,19 @@ class PipelinesRouterGroup(group.RouterGroup):
|
||||
self.ap.logger.warning('Unable to list skills for pipeline extensions: %s', exc)
|
||||
available_skills = []
|
||||
|
||||
extensions_prefs = pipeline.get('extensions_preferences', {})
|
||||
extensions_prefs = normalize_extension_preferences(pipeline.get('extensions_preferences'))
|
||||
return self.success(
|
||||
data={
|
||||
'enable_all_plugins': extensions_prefs.get('enable_all_plugins', True),
|
||||
'enable_all_mcp_servers': extensions_prefs.get('enable_all_mcp_servers', True),
|
||||
'enable_all_skills': extensions_prefs.get('enable_all_skills', True),
|
||||
'bound_plugins': extensions_prefs.get('plugins', []),
|
||||
'enable_all_plugins': extensions_prefs['enable_all_plugins'],
|
||||
'enable_all_mcp_servers': extensions_prefs['enable_all_mcp_servers'],
|
||||
'enable_all_skills': extensions_prefs['enable_all_skills'],
|
||||
'bound_plugins': extensions_prefs['plugins'],
|
||||
'available_plugins': plugins,
|
||||
'bound_mcp_servers': extensions_prefs.get('mcp_servers', []),
|
||||
'bound_mcp_servers': extensions_prefs['mcp_servers'],
|
||||
'available_mcp_servers': mcp_servers,
|
||||
'bound_mcp_resources': extensions_prefs.get('mcp_resources', []),
|
||||
'mcp_resource_agent_read_enabled': extensions_prefs.get(
|
||||
'mcp_resource_agent_read_enabled', True
|
||||
),
|
||||
'bound_skills': extensions_prefs.get('skills', []),
|
||||
'bound_mcp_resources': extensions_prefs['mcp_resources'],
|
||||
'mcp_resource_agent_read_enabled': extensions_prefs['mcp_resource_agent_read_enabled'],
|
||||
'bound_skills': extensions_prefs['skills'],
|
||||
'available_skills': available_skills,
|
||||
}
|
||||
)
|
||||
@@ -111,16 +116,41 @@ class PipelinesRouterGroup(group.RouterGroup):
|
||||
bound_mcp_resources = json_data.get('bound_mcp_resources')
|
||||
mcp_resource_agent_read_enabled = json_data.get('mcp_resource_agent_read_enabled')
|
||||
|
||||
await self.ap.pipeline_service.update_pipeline_extensions(
|
||||
pipeline_uuid,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
enable_all_plugins,
|
||||
enable_all_mcp_servers,
|
||||
bound_skills=bound_skills,
|
||||
enable_all_skills=enable_all_skills,
|
||||
bound_mcp_resources=bound_mcp_resources,
|
||||
mcp_resource_agent_read_enabled=mcp_resource_agent_read_enabled,
|
||||
)
|
||||
extension_flags = {
|
||||
'enable_all_plugins': enable_all_plugins,
|
||||
'enable_all_mcp_servers': enable_all_mcp_servers,
|
||||
'enable_all_skills': enable_all_skills,
|
||||
}
|
||||
for field, value in extension_flags.items():
|
||||
if field in json_data and not isinstance(value, bool):
|
||||
return self.http_status(
|
||||
400,
|
||||
-1,
|
||||
f"Pipeline extension field '{field}' must be a boolean",
|
||||
)
|
||||
|
||||
if 'mcp_resource_agent_read_enabled' in json_data and not isinstance(
|
||||
mcp_resource_agent_read_enabled, bool
|
||||
):
|
||||
return self.http_status(
|
||||
400,
|
||||
-1,
|
||||
"Pipeline extension field 'mcp_resource_agent_read_enabled' must be a boolean",
|
||||
)
|
||||
|
||||
try:
|
||||
await self.ap.pipeline_service.update_pipeline_extensions(
|
||||
pipeline_uuid,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
enable_all_plugins,
|
||||
enable_all_mcp_servers,
|
||||
bound_skills=bound_skills,
|
||||
enable_all_skills=enable_all_skills,
|
||||
bound_mcp_resources=bound_mcp_resources,
|
||||
mcp_resource_agent_read_enabled=mcp_resource_agent_read_enabled,
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
return self.success()
|
||||
|
||||
@@ -3,59 +3,61 @@ from __future__ import annotations
|
||||
import quart
|
||||
|
||||
from ... import group
|
||||
from ......pipeline.extension_preferences import normalize_extension_preferences
|
||||
|
||||
|
||||
@group.group_class('tools', '/api/v1/tools')
|
||||
class ToolsRouterGroup(group.RouterGroup):
|
||||
async def _get_scoped_tool_catalog(self) -> list[dict] | None:
|
||||
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
|
||||
bound_plugins: list[str] | None = None
|
||||
bound_mcp_servers: list[str] | None = None
|
||||
|
||||
if pipeline_uuid:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
|
||||
if pipeline is None:
|
||||
return None
|
||||
|
||||
extensions_prefs = normalize_extension_preferences(pipeline.get('extensions_preferences'))
|
||||
if not extensions_prefs['enable_all_plugins']:
|
||||
bound_plugins = [f'{plugin["author"]}/{plugin["name"]}' for plugin in extensions_prefs['plugins']]
|
||||
if not extensions_prefs['enable_all_mcp_servers']:
|
||||
bound_mcp_servers = extensions_prefs['mcp_servers']
|
||||
|
||||
return await self.ap.tool_mgr.get_resolved_tool_catalog(
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_skill_authoring=True,
|
||||
)
|
||||
|
||||
async def initialize(self) -> None:
|
||||
@self.route('', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _() -> str:
|
||||
"""获取所有可用工具列表"""
|
||||
pipeline_uuid = quart.request.args.get('pipeline_uuid') or quart.request.args.get('pipeline_id')
|
||||
bound_plugins: list[str] | None = None
|
||||
bound_mcp_servers: list[str] | None = None
|
||||
catalog = await self._get_scoped_tool_catalog()
|
||||
if catalog is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
return self.success(data={'tools': catalog})
|
||||
|
||||
if pipeline_uuid:
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(pipeline_uuid)
|
||||
if pipeline is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
|
||||
extensions_prefs = pipeline.get('extensions_preferences', {}) or {}
|
||||
if not extensions_prefs.get('enable_all_plugins', True):
|
||||
bound_plugins = [
|
||||
f'{plugin.get("author", "")}/{plugin.get("name", "")}'
|
||||
for plugin in extensions_prefs.get('plugins', [])
|
||||
if isinstance(plugin, dict) and plugin.get('name')
|
||||
]
|
||||
if not extensions_prefs.get('enable_all_mcp_servers', True):
|
||||
bound_mcp_servers = [
|
||||
server for server in (extensions_prefs.get('mcp_servers', []) or []) if isinstance(server, str)
|
||||
]
|
||||
|
||||
return self.success(
|
||||
data={
|
||||
'tools': await self.ap.tool_mgr.get_tool_catalog(
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_skill_authoring=True,
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
@self.route('/<tool_name>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
@self.route('/<path:tool_name>', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
|
||||
async def _(tool_name: str) -> str:
|
||||
"""获取特定工具详情"""
|
||||
tools = await self.ap.tool_mgr.get_all_tools()
|
||||
catalog = await self._get_scoped_tool_catalog()
|
||||
if catalog is None:
|
||||
return self.http_status(404, -1, 'pipeline not found')
|
||||
|
||||
for tool in tools:
|
||||
if tool.name == tool_name:
|
||||
for tool in catalog:
|
||||
if tool.get('name') == tool_name:
|
||||
return self.success(
|
||||
data={
|
||||
'tool': {
|
||||
'name': tool.name,
|
||||
'description': tool.description,
|
||||
'human_desc': tool.human_desc,
|
||||
'parameters': tool.parameters,
|
||||
'name': tool['name'],
|
||||
'description': tool.get('description') or '',
|
||||
'human_desc': tool.get('human_desc') or '',
|
||||
'parameters': tool.get('parameters') or {},
|
||||
'source': tool.get('source'),
|
||||
'source_name': tool.get('source_name'),
|
||||
'source_id': tool.get('source_id'),
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
@@ -7,6 +7,7 @@ import typing
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app
|
||||
from ....agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ....entity.persistence import agent as persistence_agent
|
||||
|
||||
|
||||
@@ -82,8 +83,8 @@ class AgentService:
|
||||
if kind != AGENT_KIND_AGENT:
|
||||
raise ValueError(f'Unsupported agent kind: {kind}')
|
||||
|
||||
config = agent_data.get('config') or await self._get_default_agent_config()
|
||||
runner_id = self._resolve_runner_id(config)
|
||||
config = agent_data['config'] if 'config' in agent_data else await self._get_default_agent_config()
|
||||
config, runner_id, _ = RunnerConfigResolver.resolve_agent_runner_config(config)
|
||||
new_uuid = str(uuid.uuid4())
|
||||
values = {
|
||||
'uuid': new_uuid,
|
||||
@@ -91,7 +92,7 @@ class AgentService:
|
||||
'description': agent_data.get('description') or '',
|
||||
'emoji': agent_data.get('emoji') or '🤖',
|
||||
'kind': AGENT_KIND_AGENT,
|
||||
'component_ref': agent_data.get('component_ref') or runner_id,
|
||||
'component_ref': runner_id,
|
||||
'config': config,
|
||||
'enabled': agent_data.get('enabled', True),
|
||||
'supported_event_patterns': agent_data.get('supported_event_patterns') or AGENT_DEFAULT_EVENT_PATTERNS,
|
||||
@@ -109,12 +110,14 @@ class AgentService:
|
||||
return
|
||||
|
||||
update_data = agent_data.copy()
|
||||
for protected_field in ('uuid', 'kind', 'created_at', 'updated_at', 'capability'):
|
||||
for protected_field in ('uuid', 'kind', 'component_ref', 'created_at', 'updated_at', 'capability'):
|
||||
update_data.pop(protected_field, None)
|
||||
if 'config' in update_data:
|
||||
update_data['component_ref'] = update_data.get('component_ref') or self._resolve_runner_id(
|
||||
update_data['config']
|
||||
)
|
||||
config, runner_id, _ = RunnerConfigResolver.resolve_agent_runner_config(update_data['config'])
|
||||
update_data['config'] = config
|
||||
else:
|
||||
_, runner_id, _ = RunnerConfigResolver.resolve_agent_runner_config(existing_agent.config)
|
||||
update_data['component_ref'] = runner_id
|
||||
if 'supported_event_patterns' in update_data and not update_data['supported_event_patterns']:
|
||||
update_data['supported_event_patterns'] = AGENT_DEFAULT_EVENT_PATTERNS
|
||||
|
||||
@@ -169,15 +172,6 @@ class AgentService:
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _resolve_runner_id(config: dict[str, typing.Any]) -> str | None:
|
||||
runner = config.get('runner') if isinstance(config, dict) else None
|
||||
if isinstance(runner, dict):
|
||||
runner_id = runner.get('id')
|
||||
if runner_id:
|
||||
return runner_id
|
||||
return None
|
||||
|
||||
def _agent_to_product_item(
|
||||
self,
|
||||
agent: persistence_agent.Agent,
|
||||
|
||||
@@ -6,7 +6,12 @@ import sqlalchemy
|
||||
import typing
|
||||
|
||||
from ....core import app
|
||||
from ....agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ....entity.persistence import pipeline as persistence_pipeline
|
||||
from ....pipeline.extension_preferences import (
|
||||
normalize_extension_preferences,
|
||||
validate_extension_preferences,
|
||||
)
|
||||
|
||||
|
||||
default_stage_order = [
|
||||
@@ -163,6 +168,11 @@ class PipelineService:
|
||||
return self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
|
||||
async def create_pipeline(self, pipeline_data: dict, default: bool = False) -> str:
|
||||
if 'extensions_preferences' in pipeline_data:
|
||||
self._validate_extension_preferences(pipeline_data['extensions_preferences'])
|
||||
if 'config' in pipeline_data:
|
||||
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
|
||||
|
||||
# Check limitation
|
||||
limitation = self.ap.instance_config.data.get('system', {}).get('limitation', {})
|
||||
max_pipelines = limitation.get('max_pipelines', -1)
|
||||
@@ -177,6 +187,7 @@ class PipelineService:
|
||||
pipeline_data['is_default'] = default
|
||||
|
||||
pipeline_data['config'] = await self.get_default_pipeline_config()
|
||||
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
|
||||
|
||||
# Ensure extensions_preferences is set with enable_all_plugins and enable_all_mcp_servers=True by default
|
||||
if 'extensions_preferences' not in pipeline_data:
|
||||
@@ -203,6 +214,10 @@ class PipelineService:
|
||||
pipeline_data = pipeline_data.copy()
|
||||
for protected_field in ('uuid', 'for_version', 'stages', 'is_default'):
|
||||
pipeline_data.pop(protected_field, None)
|
||||
if 'config' in pipeline_data:
|
||||
RunnerConfigResolver.validate_pipeline_config(pipeline_data['config'])
|
||||
if 'extensions_preferences' in pipeline_data:
|
||||
self._validate_extension_preferences(pipeline_data['extensions_preferences'])
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
@@ -259,18 +274,7 @@ class PipelineService:
|
||||
'stages': original_pipeline.stages.copy() if original_pipeline.stages else default_stage_order.copy(),
|
||||
'config': original_pipeline.config.copy() if original_pipeline.config else {},
|
||||
'is_default': False,
|
||||
'extensions_preferences': (
|
||||
original_pipeline.extensions_preferences.copy()
|
||||
if original_pipeline.extensions_preferences
|
||||
else {
|
||||
'enable_all_plugins': True,
|
||||
'enable_all_mcp_servers': True,
|
||||
'plugins': [],
|
||||
'mcp_servers': [],
|
||||
'mcp_resources': [],
|
||||
'mcp_resource_agent_read_enabled': True,
|
||||
}
|
||||
),
|
||||
'extensions_preferences': normalize_extension_preferences(original_pipeline.extensions_preferences),
|
||||
}
|
||||
|
||||
# Insert the new pipeline
|
||||
@@ -297,6 +301,36 @@ class PipelineService:
|
||||
mcp_resource_agent_read_enabled: bool | None = None,
|
||||
) -> None:
|
||||
"""Update the bound plugins and MCP servers for a pipeline"""
|
||||
extension_updates: dict[str, typing.Any] = {
|
||||
'enable_all_plugins': enable_all_plugins,
|
||||
'enable_all_mcp_servers': enable_all_mcp_servers,
|
||||
'enable_all_skills': enable_all_skills,
|
||||
'plugins': bound_plugins,
|
||||
}
|
||||
if bound_mcp_servers is not None:
|
||||
extension_updates['mcp_servers'] = bound_mcp_servers
|
||||
if bound_skills is not None:
|
||||
extension_updates['skills'] = bound_skills
|
||||
if bound_mcp_resources is not None:
|
||||
extension_updates['mcp_resources'] = bound_mcp_resources
|
||||
RunnerConfigResolver.validate_mcp_resource_attachments(
|
||||
bound_mcp_resources,
|
||||
context='Pipeline extension',
|
||||
field_name='bound_mcp_resources',
|
||||
)
|
||||
if mcp_resource_agent_read_enabled is not None:
|
||||
extension_updates['mcp_resource_agent_read_enabled'] = mcp_resource_agent_read_enabled
|
||||
self._validate_extension_preferences(
|
||||
extension_updates,
|
||||
context='Pipeline extension',
|
||||
field_aliases={
|
||||
'plugins': 'bound_plugins',
|
||||
'mcp_servers': 'bound_mcp_servers',
|
||||
'skills': 'bound_skills',
|
||||
'mcp_resources': 'bound_mcp_resources',
|
||||
},
|
||||
)
|
||||
|
||||
# Get current pipeline
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
@@ -309,7 +343,7 @@ class PipelineService:
|
||||
raise ValueError(f'Pipeline {pipeline_uuid} not found')
|
||||
|
||||
# Update extensions_preferences
|
||||
extensions_preferences = pipeline.extensions_preferences or {}
|
||||
extensions_preferences = normalize_extension_preferences(pipeline.extensions_preferences)
|
||||
extensions_preferences['enable_all_plugins'] = enable_all_plugins
|
||||
extensions_preferences['enable_all_mcp_servers'] = enable_all_mcp_servers
|
||||
extensions_preferences['enable_all_skills'] = enable_all_skills
|
||||
@@ -333,3 +367,22 @@ class PipelineService:
|
||||
await self.ap.pipeline_mgr.remove_pipeline(pipeline_uuid)
|
||||
pipeline = await self.get_pipeline(pipeline_uuid)
|
||||
await self.ap.pipeline_mgr.load_pipeline(pipeline)
|
||||
|
||||
@staticmethod
|
||||
def _validate_extension_preferences(
|
||||
value: typing.Any,
|
||||
*,
|
||||
context: str = 'Pipeline extensions_preferences',
|
||||
field_aliases: typing.Mapping[str, str] | None = None,
|
||||
) -> dict[str, typing.Any]:
|
||||
validated = validate_extension_preferences(
|
||||
value,
|
||||
context=context,
|
||||
field_aliases=field_aliases,
|
||||
)
|
||||
RunnerConfigResolver.validate_mcp_resource_attachments(
|
||||
validated.get('mcp_resources'),
|
||||
context=context,
|
||||
field_name='mcp_resources',
|
||||
)
|
||||
return validated
|
||||
|
||||
@@ -4,6 +4,7 @@ import asyncio
|
||||
import collections
|
||||
import datetime as _dt
|
||||
import enum
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import TYPE_CHECKING
|
||||
@@ -13,6 +14,7 @@ import pydantic
|
||||
from langbot_plugin.box.client import BoxRuntimeClient
|
||||
from .connector import BoxRuntimeConnector, _get_box_config
|
||||
from ..telemetry import features as telemetry_features
|
||||
from ..utils import constants
|
||||
from langbot_plugin.box.errors import BoxError, BoxValidationError
|
||||
from langbot_plugin.box.models import (
|
||||
BUILTIN_PROFILES,
|
||||
@@ -27,6 +29,8 @@ _INT_ADAPTER = pydantic.TypeAdapter(int)
|
||||
_UTC = _dt.timezone.utc
|
||||
_MAX_RECENT_ERRORS = 50
|
||||
_MIB = 1024 * 1024
|
||||
_HOST_BOX_SCOPE_VARIABLE = '_host_box_scope'
|
||||
_BOX_SESSION_ID_PREFIX = 'lb-box-'
|
||||
|
||||
|
||||
def _is_path_under(path: str, root: str) -> bool:
|
||||
@@ -224,45 +228,53 @@ class BoxService:
|
||||
return self._serialize_result(result)
|
||||
|
||||
def resolve_box_session_id(self, query: pipeline_query.Query) -> str:
|
||||
"""Resolve the Box session_id from the pipeline's template and query variables.
|
||||
"""Resolve a Host-owned Box session ID for the current conversation."""
|
||||
if query is None:
|
||||
raise BoxValidationError('Box execution requires a Host session context.')
|
||||
|
||||
When ``system.limitation.force_box_session_id_template`` is set to a
|
||||
non-empty value, that template overrides whatever the pipeline
|
||||
configured. This is the authoritative SaaS guard: it runs on every
|
||||
``exec`` call, so a tenant cannot escape a single shared sandbox even
|
||||
by editing the pipeline config directly through the API (which only
|
||||
gates the web UI).
|
||||
"""
|
||||
forced_template = self._forced_box_session_id_template()
|
||||
if forced_template:
|
||||
template = forced_template
|
||||
else:
|
||||
template = '{launcher_type}_{launcher_id}'
|
||||
pipeline_config = query.pipeline_config or {}
|
||||
ai_config = pipeline_config.get('ai', {}) if isinstance(pipeline_config, dict) else {}
|
||||
runner_selector = ai_config.get('runner', {}) if isinstance(ai_config, dict) else {}
|
||||
runner_id = runner_selector.get('id') if isinstance(runner_selector, dict) else None
|
||||
runner_configs = ai_config.get('runner_config', {}) if isinstance(ai_config, dict) else {}
|
||||
runner_config = runner_configs.get(runner_id, {}) if isinstance(runner_configs, dict) else {}
|
||||
configured_template = (
|
||||
runner_config.get('box-session-id-template') if isinstance(runner_config, dict) else None
|
||||
)
|
||||
if isinstance(configured_template, str) and configured_template:
|
||||
template = configured_template
|
||||
variables = dict(query.variables or {})
|
||||
variables = getattr(query, 'variables', None)
|
||||
if isinstance(variables, dict) and _HOST_BOX_SCOPE_VARIABLE in variables:
|
||||
private_scope = variables[_HOST_BOX_SCOPE_VARIABLE]
|
||||
if not isinstance(private_scope, str) or not private_scope.strip():
|
||||
raise BoxValidationError('Box execution requires a Host conversation scope.')
|
||||
return self._hash_box_session_scope(f'host:{private_scope}')
|
||||
|
||||
session = getattr(query, 'session', None)
|
||||
launcher_type = getattr(query, 'launcher_type', None)
|
||||
launcher_id = getattr(query, 'launcher_id', None)
|
||||
|
||||
if launcher_type is None:
|
||||
launcher_type = getattr(session, 'launcher_type', None)
|
||||
if launcher_id is None:
|
||||
launcher_id = getattr(session, 'launcher_id', None)
|
||||
if hasattr(launcher_type, 'value'):
|
||||
launcher_type = launcher_type.value
|
||||
launcher_id = getattr(query, 'launcher_id', None)
|
||||
sender_id = getattr(query, 'sender_id', None)
|
||||
query_id = getattr(query, 'query_id', None)
|
||||
|
||||
variables.setdefault('query_id', str(query_id or 'unknown'))
|
||||
variables.setdefault('launcher_type', str(launcher_type or 'query'))
|
||||
variables.setdefault('launcher_id', str(launcher_id or query_id or 'unknown'))
|
||||
variables.setdefault('sender_id', str(sender_id or launcher_id or query_id or 'unknown'))
|
||||
variables.setdefault('global', 'global')
|
||||
return template.format_map(collections.defaultdict(lambda: 'unknown', variables))
|
||||
if launcher_type is None or launcher_id is None or not str(launcher_id).strip():
|
||||
raise BoxValidationError('Box execution requires a Host session context.')
|
||||
|
||||
adapter = getattr(query, 'adapter', None)
|
||||
adapter_identity = adapter.__class__.__name__ if adapter is not None else None
|
||||
scope = json.dumps(
|
||||
{
|
||||
'instance_id': str(constants.instance_id or '') or None,
|
||||
'workspace_id': None,
|
||||
'bot_id': getattr(query, 'bot_uuid', None),
|
||||
'platform_adapter': adapter_identity,
|
||||
'target_type': str(launcher_type),
|
||||
'target_id': str(launcher_id),
|
||||
'thread_id': None,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(',', ':'),
|
||||
)
|
||||
return self._hash_box_session_scope(f'host:{scope}')
|
||||
|
||||
@staticmethod
|
||||
def _hash_box_session_scope(scope: str) -> str:
|
||||
digest = hashlib.sha256(scope.encode('utf-8')).hexdigest()
|
||||
return f'{_BOX_SESSION_ID_PREFIX}{digest}'
|
||||
|
||||
def build_skill_extra_mounts(self, query: pipeline_query.Query) -> list[dict]:
|
||||
"""Build extra_mounts entries for all pipeline-bound skills.
|
||||
@@ -1128,20 +1140,6 @@ class BoxService:
|
||||
raw = str(self._local_config().get('image', '') or '').strip()
|
||||
return raw or None
|
||||
|
||||
def _forced_box_session_id_template(self) -> str:
|
||||
"""Return the SaaS-forced sandbox-scope template, or '' when unset.
|
||||
|
||||
Read from ``system.limitation.force_box_session_id_template``. A
|
||||
non-empty value pins every pipeline to a single sandbox scope
|
||||
(e.g. ``'{global}'``) and cannot be overridden per-pipeline.
|
||||
"""
|
||||
limitation = (
|
||||
(self.ap.instance_config.data or {}).get('system', {}).get('limitation', {})
|
||||
if getattr(self.ap, 'instance_config', None) is not None
|
||||
else {}
|
||||
)
|
||||
return str(limitation.get('force_box_session_id_template', '') or '').strip()
|
||||
|
||||
def _load_workspace_quota_mb(self) -> int | None:
|
||||
raw_value = self._local_config().get('workspace_quota_mb')
|
||||
if raw_value in (None, ''):
|
||||
@@ -1311,42 +1309,6 @@ class BoxService:
|
||||
def get_recent_errors(self) -> list[dict]:
|
||||
return list(self._recent_errors)
|
||||
|
||||
def get_system_guidance(self, query_id=None) -> str:
|
||||
"""Return LLM system-prompt guidance for the exec tool.
|
||||
|
||||
All execution-specific prompt text is kept here so that callers
|
||||
(e.g. LocalAgentRunner) stay free of box domain knowledge.
|
||||
|
||||
``query_id`` is the current turn's pipeline query id. When provided,
|
||||
the guidance ALWAYS advertises the per-query outbox path so the agent
|
||||
knows how to deliver generated files back to the user — even on turns
|
||||
where the user sent no inbound attachment (e.g. "generate a QR code"),
|
||||
which is exactly when the inbound-attachment note never fires. Outbound
|
||||
collection in the wrapper runs on every turn regardless of inbound
|
||||
files, so without this the file would be produced and silently dropped.
|
||||
"""
|
||||
guidance = (
|
||||
'When the exec tool is available, use it for exact calculations, statistics, structured data parsing, '
|
||||
'and code execution instead of estimating mentally. If the user provides numbers, tables, CSV-like text, '
|
||||
'JSON, or other data and asks for a computed answer, prefer running a short Python script via exec '
|
||||
'and then answer from the tool result. Unless the user explicitly asks for the script, code, or implementation '
|
||||
'details, do not include the generated script in the final answer; return the result and a brief explanation only.'
|
||||
)
|
||||
if self.default_workspace:
|
||||
guidance += (
|
||||
' A default workspace is mounted at /workspace for file tasks. When the user asks to read, create, or '
|
||||
'modify local files in the working directory, use exec with /workspace paths directly; do not ask the '
|
||||
'user for directory parameters unless they explicitly need a different directory.'
|
||||
)
|
||||
if query_id is not None:
|
||||
outbox_dir = f'{self.OUTBOX_MOUNT_DIR}/{query_id}'
|
||||
guidance += (
|
||||
f' If you produce any file (image, audio, document, etc.) that should be sent back to the user, '
|
||||
f'write it into {outbox_dir}/ (create the directory if needed). Every file placed there will be '
|
||||
'delivered to the user automatically; do not paste file contents or base64 into your reply.'
|
||||
)
|
||||
return guidance
|
||||
|
||||
async def get_status(self) -> dict:
|
||||
if not self._available:
|
||||
return {
|
||||
|
||||
@@ -110,44 +110,6 @@ class LoadConfigStage(stage.BootingStage):
|
||||
async def run(self, ap: app.Application):
|
||||
"""Load config file"""
|
||||
|
||||
# # ======= deprecated =======
|
||||
# if os.path.exists('data/config/command.json'):
|
||||
# ap.command_cfg = await config.load_json_config(
|
||||
# 'data/config/command.json',
|
||||
# 'templates/legacy/command.json',
|
||||
# completion=False,
|
||||
# )
|
||||
|
||||
# if os.path.exists('data/config/pipeline.json'):
|
||||
# ap.pipeline_cfg = await config.load_json_config(
|
||||
# 'data/config/pipeline.json',
|
||||
# 'templates/legacy/pipeline.json',
|
||||
# completion=False,
|
||||
# )
|
||||
|
||||
# if os.path.exists('data/config/platform.json'):
|
||||
# ap.platform_cfg = await config.load_json_config(
|
||||
# 'data/config/platform.json',
|
||||
# 'templates/legacy/platform.json',
|
||||
# completion=False,
|
||||
# )
|
||||
|
||||
# if os.path.exists('data/config/provider.json'):
|
||||
# ap.provider_cfg = await config.load_json_config(
|
||||
# 'data/config/provider.json',
|
||||
# 'templates/legacy/provider.json',
|
||||
# completion=False,
|
||||
# )
|
||||
|
||||
# if os.path.exists('data/config/system.json'):
|
||||
# ap.system_cfg = await config.load_json_config(
|
||||
# 'data/config/system.json',
|
||||
# 'templates/legacy/system.json',
|
||||
# completion=False,
|
||||
# )
|
||||
|
||||
# # ======= deprecated =======
|
||||
|
||||
ap.instance_config = await config.load_yaml_config('data/config.yaml', 'config.yaml', completion=False)
|
||||
|
||||
# Apply environment variable overrides to data/config.yaml
|
||||
|
||||
@@ -1,15 +1,6 @@
|
||||
import sqlalchemy
|
||||
|
||||
from .base import Base
|
||||
from ...utils import constants
|
||||
|
||||
|
||||
initial_metadata = [
|
||||
{
|
||||
'key': 'database_version',
|
||||
'value': str(constants.required_database_version),
|
||||
},
|
||||
]
|
||||
|
||||
|
||||
class Metadata(Base):
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
"""baseline: stamp existing schema (db version 25)
|
||||
"""baseline: stamp the supported 4.x schema
|
||||
|
||||
This is a no-op migration that marks the starting point for Alembic.
|
||||
All tables already exist via create_all() + legacy DBMigration system.
|
||||
Current tables already exist via SQLAlchemy metadata create_all().
|
||||
|
||||
Revision ID: 0001_baseline
|
||||
Revises: None
|
||||
@@ -15,8 +15,7 @@ depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# No-op: existing schema is already at database_version=25
|
||||
# This revision serves as the Alembic baseline.
|
||||
# No-op: this revision serves as the Alembic baseline.
|
||||
pass
|
||||
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ ssereadtimeout) live in ``extra_args`` and are left untouched — the
|
||||
auto-detecting remote transport consumes them regardless.
|
||||
|
||||
Revision ID: 0006_normalize_mcp_remote_mode
|
||||
Revises: 8d3a1f2c4b6e
|
||||
Revises: 0005_add_llm_context_length
|
||||
Create Date: 2026-06-21
|
||||
"""
|
||||
|
||||
@@ -18,7 +18,7 @@ import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = '0006_normalize_mcp_remote_mode'
|
||||
down_revision = '8d3a1f2c4b6e'
|
||||
down_revision = '0005_add_llm_context_length'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
@@ -5,6 +5,8 @@ Revises: 0006_normalize_mcp_remote_mode
|
||||
Create Date: 2026-06-26
|
||||
"""
|
||||
|
||||
import json
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
@@ -14,19 +16,64 @@ branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_BOT_ADMINS = sa.table(
|
||||
'bot_admins',
|
||||
sa.column('bot_uuid', sa.String(255)),
|
||||
sa.column('launcher_type', sa.String(64)),
|
||||
sa.column('launcher_id', sa.String(255)),
|
||||
)
|
||||
|
||||
|
||||
def _upsert_admin(conn: sa.Connection, *, bot_uuid: str, launcher_type: str, launcher_id: str) -> None:
|
||||
values = {
|
||||
'bot_uuid': bot_uuid,
|
||||
'launcher_type': launcher_type,
|
||||
'launcher_id': launcher_id,
|
||||
}
|
||||
conflict_columns = [
|
||||
_BOT_ADMINS.c.bot_uuid,
|
||||
_BOT_ADMINS.c.launcher_type,
|
||||
_BOT_ADMINS.c.launcher_id,
|
||||
]
|
||||
|
||||
if conn.dialect.name == 'postgresql':
|
||||
from sqlalchemy.dialects.postgresql import insert
|
||||
|
||||
statement = insert(_BOT_ADMINS).values(**values).on_conflict_do_nothing(index_elements=conflict_columns)
|
||||
elif conn.dialect.name == 'sqlite':
|
||||
from sqlalchemy.dialects.sqlite import insert
|
||||
|
||||
statement = insert(_BOT_ADMINS).values(**values).on_conflict_do_nothing(index_elements=conflict_columns)
|
||||
else:
|
||||
existing = conn.execute(
|
||||
sa.select(sa.literal(1))
|
||||
.select_from(_BOT_ADMINS)
|
||||
.where(
|
||||
_BOT_ADMINS.c.bot_uuid == bot_uuid,
|
||||
_BOT_ADMINS.c.launcher_type == launcher_type,
|
||||
_BOT_ADMINS.c.launcher_id == launcher_id,
|
||||
)
|
||||
.limit(1)
|
||||
).first()
|
||||
if existing is not None:
|
||||
return
|
||||
statement = sa.insert(_BOT_ADMINS).values(**values)
|
||||
|
||||
conn.execute(statement)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if 'bot_admins' in sa.inspect(conn).get_table_names():
|
||||
return
|
||||
op.create_table(
|
||||
'bot_admins',
|
||||
sa.Column('id', sa.Integer, primary_key=True, autoincrement=True),
|
||||
sa.Column('bot_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('launcher_type', sa.String(64), nullable=False),
|
||||
sa.Column('launcher_id', sa.String(255), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
sa.UniqueConstraint('bot_uuid', 'launcher_type', 'launcher_id', name='uq_bot_admin'),
|
||||
)
|
||||
if 'bot_admins' not in sa.inspect(conn).get_table_names():
|
||||
op.create_table(
|
||||
'bot_admins',
|
||||
sa.Column('id', sa.Integer, primary_key=True, autoincrement=True),
|
||||
sa.Column('bot_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('launcher_type', sa.String(64), nullable=False),
|
||||
sa.Column('launcher_id', sa.String(255), nullable=False),
|
||||
sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()),
|
||||
sa.UniqueConstraint('bot_uuid', 'launcher_type', 'launcher_id', name='uq_bot_admin'),
|
||||
)
|
||||
|
||||
# Migrate old config-based admins into the first bot (best-effort)
|
||||
inspector = sa.inspect(conn)
|
||||
@@ -48,28 +95,27 @@ def upgrade() -> None:
|
||||
if meta_row is None:
|
||||
return
|
||||
|
||||
import json
|
||||
|
||||
try:
|
||||
cfg = json.loads(meta_row[0])
|
||||
except Exception:
|
||||
except (TypeError, json.JSONDecodeError):
|
||||
return
|
||||
|
||||
admins = cfg.get('admins', [])
|
||||
if not isinstance(admins, list):
|
||||
return
|
||||
for entry in admins:
|
||||
if not isinstance(entry, str):
|
||||
continue
|
||||
parts = entry.split('_', 1)
|
||||
if len(parts) != 2:
|
||||
continue
|
||||
launcher_type, launcher_id = parts
|
||||
try:
|
||||
conn.execute(
|
||||
sa.text(
|
||||
'INSERT OR IGNORE INTO bot_admins (bot_uuid, launcher_type, launcher_id) VALUES (:bu, :lt, :li)'
|
||||
),
|
||||
{'bu': first_bot_uuid, 'lt': launcher_type, 'li': launcher_id},
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
_upsert_admin(
|
||||
conn,
|
||||
bot_uuid=first_bot_uuid,
|
||||
launcher_type=launcher_type,
|
||||
launcher_id=launcher_id,
|
||||
)
|
||||
|
||||
# Remove admins key from stored config
|
||||
if 'admins' in cfg:
|
||||
@@ -81,4 +127,5 @@ def upgrade() -> None:
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
op.drop_table('bot_admins')
|
||||
if 'bot_admins' in sa.inspect(op.get_bind()).get_table_names():
|
||||
op.drop_table('bot_admins')
|
||||
|
||||
+17
-13
@@ -3,7 +3,7 @@
|
||||
Bindings keep referencing the original Pipeline UUIDs. This migration never
|
||||
creates Agent rows or copies Pipeline runner configuration.
|
||||
|
||||
Revision ID: 0009_migrate_routing_to_event_bindings
|
||||
Revision ID: 0009_migrate_event_bindings
|
||||
Revises: 0008_agent_product_surface
|
||||
Create Date: 2026-06-26
|
||||
"""
|
||||
@@ -14,7 +14,7 @@ import uuid
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = '0009_migrate_routing_to_event_bindings'
|
||||
revision = '0009_migrate_event_bindings'
|
||||
down_revision = '0008_agent_product_surface'
|
||||
depends_on = None
|
||||
|
||||
@@ -30,24 +30,28 @@ def _column_exists(table_name: str, column_name: str) -> bool:
|
||||
|
||||
|
||||
def _rule_to_filters(rule: dict) -> list[dict] | None:
|
||||
"""Convert a pipeline_routing_rule to event_binding filters (best effort).
|
||||
|
||||
Rules that don't map cleanly (message_content, message_has_element) are
|
||||
skipped — callers should handle None as "cannot migrate".
|
||||
"""
|
||||
"""Convert one legacy Pipeline routing rule to an EBA event filter."""
|
||||
rule_type = rule.get('type')
|
||||
operator = rule.get('operator', 'eq')
|
||||
value = rule.get('value', '')
|
||||
|
||||
if rule_type == 'launcher_type':
|
||||
if value == 'group':
|
||||
return [{'field': 'group', 'operator': 'neq', 'value': None}]
|
||||
if value == 'person':
|
||||
return [{'field': 'group', 'operator': 'eq', 'value': None}]
|
||||
elif rule_type == 'launcher_id':
|
||||
return [{'field': 'chat_type', 'operator': operator, 'value': value}]
|
||||
if rule_type == 'launcher_id':
|
||||
return [{'field': 'chat_id', 'operator': operator, 'value': value}]
|
||||
if rule_type == 'message_content':
|
||||
return [{'field': 'message_text', 'operator': operator, 'value': value}]
|
||||
if rule_type == 'message_has_element':
|
||||
element_operator = {
|
||||
'eq': 'contains',
|
||||
'neq': 'not_contains',
|
||||
}.get(operator)
|
||||
if element_operator is None:
|
||||
# The legacy matcher treated every other operator as non-matching.
|
||||
element_operator = 'unsupported_legacy_operator'
|
||||
return [{'field': 'message_element_types', 'operator': element_operator, 'value': value}]
|
||||
|
||||
return None # message_content / message_has_element: no clean mapping
|
||||
return None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
@@ -0,0 +1,19 @@
|
||||
"""merge mcp resource and agent product migration heads
|
||||
|
||||
Revision ID: 0010_merge_mcp_agent_heads
|
||||
Revises: 0008_mcp_resource_prefs, 0009_migrate_event_bindings
|
||||
Create Date: 2026-06-30
|
||||
"""
|
||||
|
||||
revision = '0010_merge_mcp_agent_heads'
|
||||
down_revision = ('0008_mcp_resource_prefs', '0009_migrate_event_bindings')
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -1,19 +0,0 @@
|
||||
"""merge mcp resource and agent product migration heads
|
||||
|
||||
Revision ID: 0010_merge_mcp_resource_agent_heads
|
||||
Revises: 0008_mcp_resource_prefs, 0009_migrate_routing_to_event_bindings
|
||||
Create Date: 2026-06-30
|
||||
"""
|
||||
|
||||
revision = '0010_merge_mcp_resource_agent_heads'
|
||||
down_revision = ('0008_mcp_resource_prefs', '0009_migrate_routing_to_event_bindings')
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
pass
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
pass
|
||||
@@ -1,7 +1,7 @@
|
||||
"""drop legacy bot pipeline routing columns
|
||||
|
||||
Revision ID: 0011_drop_legacy_bot_routing
|
||||
Revises: 0010_merge_mcp_resource_agent_heads
|
||||
Revises: 0010_merge_mcp_agent_heads
|
||||
Create Date: 2026-07-01
|
||||
"""
|
||||
|
||||
@@ -10,7 +10,7 @@ import sqlalchemy as sa
|
||||
|
||||
|
||||
revision = '0011_drop_legacy_bot_routing'
|
||||
down_revision = '0010_merge_mcp_resource_agent_heads'
|
||||
down_revision = '0010_merge_mcp_agent_heads'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
@@ -0,0 +1,67 @@
|
||||
"""add monitoring tool calls
|
||||
|
||||
Revision ID: 0012_monitoring_tool_calls
|
||||
Revises: 0011_drop_legacy_bot_routing
|
||||
Create Date: 2026-07-12
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
revision = '0012_monitoring_tool_calls'
|
||||
down_revision = '0011_drop_legacy_bot_routing'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_INDEXES = {
|
||||
'ix_monitoring_tool_calls_timestamp': ['timestamp'],
|
||||
'ix_monitoring_tool_calls_bot_id': ['bot_id'],
|
||||
'ix_monitoring_tool_calls_pipeline_id': ['pipeline_id'],
|
||||
'ix_monitoring_tool_calls_session_id': ['session_id'],
|
||||
'ix_monitoring_tool_calls_message_id': ['message_id'],
|
||||
}
|
||||
|
||||
|
||||
def _table_exists() -> bool:
|
||||
return 'monitoring_tool_calls' in sa.inspect(op.get_bind()).get_table_names()
|
||||
|
||||
|
||||
def _index_names() -> set[str]:
|
||||
if not _table_exists():
|
||||
return set()
|
||||
return {index['name'] for index in sa.inspect(op.get_bind()).get_indexes('monitoring_tool_calls')}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
if not _table_exists():
|
||||
op.create_table(
|
||||
'monitoring_tool_calls',
|
||||
sa.Column('id', sa.String(255), primary_key=True),
|
||||
sa.Column('timestamp', sa.DateTime(), nullable=False),
|
||||
sa.Column('tool_name', sa.String(255), nullable=False),
|
||||
sa.Column('tool_source', sa.String(50), nullable=False),
|
||||
sa.Column('duration', sa.Integer(), nullable=False),
|
||||
sa.Column('status', sa.String(50), nullable=False),
|
||||
sa.Column('bot_id', sa.String(255), nullable=False),
|
||||
sa.Column('bot_name', sa.String(255), nullable=False),
|
||||
sa.Column('pipeline_id', sa.String(255), nullable=False),
|
||||
sa.Column('pipeline_name', sa.String(255), nullable=False),
|
||||
sa.Column('session_id', sa.String(255), nullable=True),
|
||||
sa.Column('message_id', sa.String(255), nullable=True),
|
||||
sa.Column('arguments', sa.Text(), nullable=True),
|
||||
sa.Column('result', sa.Text(), nullable=True),
|
||||
sa.Column('error_message', sa.Text(), nullable=True),
|
||||
)
|
||||
|
||||
existing_indexes = _index_names()
|
||||
for index_name, columns in _INDEXES.items():
|
||||
if index_name not in existing_indexes:
|
||||
op.create_index(index_name, 'monitoring_tool_calls', columns, unique=False)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
if _table_exists():
|
||||
op.drop_table('monitoring_tool_calls')
|
||||
@@ -7,15 +7,14 @@ import typing
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
import sqlalchemy
|
||||
|
||||
from . import database, migration
|
||||
from ..entity.persistence import base, metadata, model as persistence_model
|
||||
from . import database
|
||||
from ..entity.persistence import base, model as persistence_model
|
||||
from ..entity import persistence
|
||||
from ..core import app
|
||||
from ..utils import constants, importutil
|
||||
from . import databases, migrations
|
||||
from ..utils import importutil
|
||||
from . import databases
|
||||
|
||||
importutil.import_modules_in_pkg(databases)
|
||||
importutil.import_modules_in_pkg(migrations)
|
||||
importutil.import_modules_in_pkg(persistence)
|
||||
|
||||
|
||||
@@ -43,40 +42,6 @@ class PersistenceManager:
|
||||
break
|
||||
|
||||
await self.create_tables()
|
||||
|
||||
# run migrations
|
||||
database_version = await self.execute_async(
|
||||
sqlalchemy.select(metadata.Metadata).where(metadata.Metadata.key == 'database_version')
|
||||
)
|
||||
|
||||
database_version = int(database_version.fetchone()[1])
|
||||
required_database_version = constants.required_database_version
|
||||
|
||||
if database_version < required_database_version:
|
||||
migrations = migration.preregistered_db_migrations
|
||||
migrations.sort(key=lambda x: x.number)
|
||||
|
||||
last_migration_number = database_version
|
||||
|
||||
for migration_cls in migrations:
|
||||
migration_instance = migration_cls(self.ap)
|
||||
|
||||
if (
|
||||
migration_instance.number > database_version
|
||||
and migration_instance.number <= required_database_version
|
||||
):
|
||||
await migration_instance.upgrade()
|
||||
await self.execute_async(
|
||||
sqlalchemy.update(metadata.Metadata)
|
||||
.where(metadata.Metadata.key == 'database_version')
|
||||
.values({'value': str(migration_instance.number)})
|
||||
)
|
||||
last_migration_number = migration_instance.number
|
||||
self.ap.logger.info(f'Migration {migration_instance.number} completed.')
|
||||
|
||||
self.ap.logger.info(f'Successfully upgraded database to version {last_migration_number}.')
|
||||
|
||||
# Run Alembic migrations (new migration system)
|
||||
await self._run_alembic_migrations()
|
||||
|
||||
await self.write_space_model_providers()
|
||||
@@ -88,19 +53,6 @@ class PersistenceManager:
|
||||
|
||||
await conn.commit()
|
||||
|
||||
# ======= write initial data =======
|
||||
|
||||
# write initial metadata
|
||||
self.ap.logger.info('Creating initial metadata...')
|
||||
for item in metadata.initial_metadata:
|
||||
# check if the item exists
|
||||
result = await self.execute_async(
|
||||
sqlalchemy.select(metadata.Metadata).where(metadata.Metadata.key == item['key'])
|
||||
)
|
||||
row = result.first()
|
||||
if row is None:
|
||||
await self.execute_async(sqlalchemy.insert(metadata.Metadata).values(item))
|
||||
|
||||
async def write_space_model_providers(self):
|
||||
space_models_gateway_api_url = self.ap.instance_config.data.get('space', {}).get(
|
||||
'models_gateway_api_url', 'https://api.langbot.cloud/v1'
|
||||
@@ -139,7 +91,7 @@ class PersistenceManager:
|
||||
# =================================
|
||||
|
||||
async def _run_alembic_migrations(self):
|
||||
"""Run Alembic-based migrations after legacy migrations complete."""
|
||||
"""Run Alembic migrations for supported 4.x databases."""
|
||||
from . import alembic_runner
|
||||
|
||||
engine = self.get_db_engine()
|
||||
|
||||
@@ -1,40 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
import abc
|
||||
|
||||
from ..core import app
|
||||
|
||||
|
||||
preregistered_db_migrations: list[typing.Type[DBMigration]] = []
|
||||
|
||||
|
||||
def migration_class(number: int):
|
||||
"""Migration class decorator"""
|
||||
|
||||
def wrapper(cls: typing.Type[DBMigration]) -> typing.Type[DBMigration]:
|
||||
cls.number = number
|
||||
preregistered_db_migrations.append(cls)
|
||||
return cls
|
||||
|
||||
return wrapper
|
||||
|
||||
|
||||
class DBMigration(abc.ABC):
|
||||
"""Database migration"""
|
||||
|
||||
number: int
|
||||
"""Migration number"""
|
||||
|
||||
def __init__(self, ap: app.Application):
|
||||
self.ap = ap
|
||||
|
||||
@abc.abstractmethod
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
pass
|
||||
|
||||
@abc.abstractmethod
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,36 +0,0 @@
|
||||
# Legacy migrations (DEPRECATED — do not add new files here)
|
||||
|
||||
This directory holds the **legacy 3.x database migration system**
|
||||
(`DBMigration` subclasses in `dbmXXX_*.py`, registered via
|
||||
`@migration.migration_class(N)` and run from `pkg/persistence/mgr.py`).
|
||||
|
||||
**This system is frozen. Do not add new `dbmXXX_*.py` migrations.**
|
||||
|
||||
The chain is capped at version 25 (`required_database_version = 25` in
|
||||
`pkg/utils/constants.py`). These files exist only to upgrade pre-existing
|
||||
3.x databases up to the Alembic baseline (`0001_baseline`). Removing them
|
||||
would break in-place upgrades from old installations, so they are kept
|
||||
read-only.
|
||||
|
||||
## All new schema changes use Alembic
|
||||
|
||||
Migrations now live in `pkg/persistence/alembic/versions/`. To create one:
|
||||
|
||||
```bash
|
||||
uv run python -m langbot.pkg.persistence.alembic_runner autogenerate "description of your change"
|
||||
```
|
||||
|
||||
(requires `data/config.yaml` to exist). Review and edit the generated
|
||||
script before committing — Alembic migrations run automatically on startup
|
||||
and must be idempotent and guard against missing tables (the test suite
|
||||
runs them against empty databases).
|
||||
|
||||
### Rules for Alembic revision ids
|
||||
|
||||
- Keep the revision id **≤ 32 characters** — PostgreSQL stores
|
||||
`alembic_version.version_num` as `varchar(32)` and will raise
|
||||
`StringDataRightTruncationError` on overflow.
|
||||
- Guard every `op` call against a missing table / missing column
|
||||
(`inspector.get_table_names()` / `inspector.get_columns()`); fresh
|
||||
installs create the schema via `create_all()` and stamp the baseline,
|
||||
so migrations may run against tables that already match or do not exist.
|
||||
@@ -1,218 +0,0 @@
|
||||
from .. import migration
|
||||
from copy import deepcopy
|
||||
import uuid
|
||||
import os
|
||||
import sqlalchemy
|
||||
import shutil
|
||||
|
||||
from ...config import manager as config_manager
|
||||
from ...entity.persistence import (
|
||||
model as persistence_model,
|
||||
pipeline as persistence_pipeline,
|
||||
bot as persistence_bot,
|
||||
)
|
||||
|
||||
|
||||
@migration.migration_class(1)
|
||||
class DBMigrateV3Config(migration.DBMigration):
|
||||
"""Migrate v3 config to v4 database"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
"""
|
||||
Migrate all config files under data/config.
|
||||
After migration, all previous config files are saved under data/legacy/config.
|
||||
After migration, all config files under data/metadata/ are saved under data/legacy/metadata.
|
||||
"""
|
||||
|
||||
if self.ap.provider_cfg is None:
|
||||
return
|
||||
|
||||
# ======= Migrate model =======
|
||||
# Only migrate the currently selected model
|
||||
model_name = self.ap.provider_cfg.data.get('model', 'gpt-4o')
|
||||
|
||||
model_requester = 'openai-chat-completions'
|
||||
model_requester_config = {}
|
||||
model_api_keys = ['sk-proj-1234567890']
|
||||
model_abilities = []
|
||||
model_extra_args = {}
|
||||
|
||||
if os.path.exists('data/metadata/llm-models.json'):
|
||||
_llm_model_meta = await config_manager.load_json_config('data/metadata/llm-models.json', completion=False)
|
||||
|
||||
for item in _llm_model_meta.data.get('list', []):
|
||||
if item.get('name') == model_name:
|
||||
if 'model_name' in item:
|
||||
model_name = item['model_name']
|
||||
if 'requester' in item:
|
||||
model_requester = item['requester']
|
||||
if 'token_mgr' in item:
|
||||
_token_mgr = item['token_mgr']
|
||||
|
||||
if _token_mgr in self.ap.provider_cfg.data.get('keys', {}):
|
||||
model_api_keys = self.ap.provider_cfg.data.get('keys', {})[_token_mgr]
|
||||
|
||||
if 'tool_call_supported' in item and item['tool_call_supported']:
|
||||
model_abilities.append('func_call')
|
||||
|
||||
if 'vision_supported' in item and item['vision_supported']:
|
||||
model_abilities.append('vision')
|
||||
|
||||
if (
|
||||
model_requester in self.ap.provider_cfg.data.get('requester', {})
|
||||
and 'args' in self.ap.provider_cfg.data.get('requester', {})[model_requester]
|
||||
):
|
||||
model_extra_args = self.ap.provider_cfg.data.get('requester', {})[model_requester]['args']
|
||||
|
||||
if model_requester in self.ap.provider_cfg.data.get('requester', {}):
|
||||
model_requester_config = self.ap.provider_cfg.data.get('requester', {})[model_requester]
|
||||
model_requester_config = {
|
||||
'base_url': model_requester_config['base-url'],
|
||||
'timeout': model_requester_config['timeout'],
|
||||
}
|
||||
|
||||
break
|
||||
|
||||
model_uuid = str(uuid.uuid4())
|
||||
|
||||
llm_model_data = {
|
||||
'uuid': model_uuid,
|
||||
'name': model_name,
|
||||
'description': '由 LangBot v3 迁移而来',
|
||||
'requester': model_requester,
|
||||
'requester_config': model_requester_config,
|
||||
'api_keys': model_api_keys,
|
||||
'abilities': model_abilities,
|
||||
'extra_args': model_extra_args,
|
||||
}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_model.LLMModel).values(**llm_model_data)
|
||||
)
|
||||
|
||||
# ======= Migrate pipeline config =======
|
||||
# Modify to default pipeline
|
||||
default_pipeline = [
|
||||
self.ap.persistence_mgr.serialize_model(persistence_pipeline.LegacyPipeline, pipeline)
|
||||
for pipeline in (
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline).where(
|
||||
persistence_pipeline.LegacyPipeline.is_default == True
|
||||
)
|
||||
)
|
||||
).all()
|
||||
][0]
|
||||
|
||||
pipeline_uuid = str(uuid.uuid4())
|
||||
pipeline_name = 'ChatPipeline'
|
||||
|
||||
if default_pipeline:
|
||||
pipeline_name = default_pipeline['name']
|
||||
pipeline_uuid = default_pipeline['uuid']
|
||||
|
||||
pipeline_config = default_pipeline['config']
|
||||
|
||||
# trigger
|
||||
pipeline_config['trigger']['group-respond-rules'] = self.ap.pipeline_cfg.data['respond-rules']['default']
|
||||
pipeline_config['trigger']['access-control'] = self.ap.pipeline_cfg.data['access-control']
|
||||
pipeline_config['trigger']['ignore-rules'] = self.ap.pipeline_cfg.data['ignore-rules']
|
||||
|
||||
# safety
|
||||
pipeline_config['safety']['content-filter'] = {
|
||||
'scope': 'all',
|
||||
'check-sensitive-words': self.ap.pipeline_cfg.data['check-sensitive-words'],
|
||||
}
|
||||
pipeline_config['safety']['rate-limit'] = {
|
||||
'window-length': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['window-size'],
|
||||
'limitation': self.ap.pipeline_cfg.data['rate-limit']['fixwin']['default']['limit'],
|
||||
'strategy': self.ap.pipeline_cfg.data['rate-limit']['strategy'],
|
||||
}
|
||||
|
||||
# output
|
||||
pipeline_config['output']['long-text-processing'] = self.ap.platform_cfg.data['long-text-process']
|
||||
pipeline_config['output']['force-delay'] = self.ap.platform_cfg.data['force-delay']
|
||||
pipeline_config['output']['misc'] = {
|
||||
'hide-exception': self.ap.platform_cfg.data['hide-exception-info'],
|
||||
'quote-origin': self.ap.platform_cfg.data['quote-origin'],
|
||||
'at-sender': self.ap.platform_cfg.data['at-sender'],
|
||||
'track-function-calls': self.ap.platform_cfg.data['track-function-calls'],
|
||||
}
|
||||
|
||||
default_pipeline['description'] = default_pipeline['description'] + ' [已迁移 LangBot v3 配置]'
|
||||
default_pipeline['config'] = pipeline_config
|
||||
default_pipeline.pop('created_at')
|
||||
default_pipeline.pop('updated_at')
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_pipeline.LegacyPipeline)
|
||||
.values(default_pipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.uuid == default_pipeline['uuid'])
|
||||
)
|
||||
|
||||
# ======= Migrate bot =======
|
||||
# Only migrate enabled bots
|
||||
for adapter in self.ap.platform_cfg.data.get('platform-adapters', []):
|
||||
if not adapter.get('enable'):
|
||||
continue
|
||||
|
||||
args = deepcopy(adapter)
|
||||
args.pop('adapter')
|
||||
args.pop('enable')
|
||||
|
||||
bot_data = {
|
||||
'uuid': str(uuid.uuid4()),
|
||||
'name': adapter.get('adapter'),
|
||||
'description': '由 LangBot v3 迁移而来',
|
||||
'adapter': adapter.get('adapter'),
|
||||
'adapter_config': args,
|
||||
'enable': True,
|
||||
'use_pipeline_uuid': pipeline_uuid,
|
||||
'use_pipeline_name': pipeline_name,
|
||||
}
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_bot.Bot).values(**bot_data))
|
||||
|
||||
# ======= Migrate system settings =======
|
||||
self.ap.instance_config.data['admins'] = self.ap.system_cfg.data['admin-sessions']
|
||||
self.ap.instance_config.data['api']['port'] = self.ap.system_cfg.data['http-api']['port']
|
||||
self.ap.instance_config.data['command'] = {
|
||||
'prefix': self.ap.command_cfg.data['command-prefix'],
|
||||
'enable': self.ap.command_cfg.data['command-enable']
|
||||
if 'command-enable' in self.ap.command_cfg.data
|
||||
else True,
|
||||
'privilege': self.ap.command_cfg.data['privilege'],
|
||||
}
|
||||
self.ap.instance_config.data['concurrency']['pipeline'] = self.ap.system_cfg.data['pipeline-concurrency']
|
||||
self.ap.instance_config.data['concurrency']['session'] = self.ap.system_cfg.data['session-concurrency'][
|
||||
'default'
|
||||
]
|
||||
self.ap.instance_config.data['mcp'] = self.ap.provider_cfg.data['mcp']
|
||||
self.ap.instance_config.data['proxy'] = self.ap.system_cfg.data['network-proxies']
|
||||
await self.ap.instance_config.dump_config()
|
||||
|
||||
# ======= move files =======
|
||||
# Migrate all config files under data/config
|
||||
all_legacy_dir_name = [
|
||||
'config',
|
||||
# 'metadata',
|
||||
'prompts',
|
||||
'scenario',
|
||||
]
|
||||
|
||||
def move_legacy_files(dir_name: str):
|
||||
if not os.path.exists(f'data/legacy/{dir_name}'):
|
||||
os.makedirs(f'data/legacy/{dir_name}')
|
||||
|
||||
if os.path.exists(f'data/{dir_name}'):
|
||||
for file in os.listdir(f'data/{dir_name}'):
|
||||
if file.endswith('.json'):
|
||||
shutil.move(f'data/{dir_name}/{file}', f'data/legacy/{dir_name}/{file}')
|
||||
|
||||
os.rmdir(f'data/{dir_name}')
|
||||
|
||||
for dir_name in all_legacy_dir_name:
|
||||
move_legacy_files(dir_name)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
@@ -1,55 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(2)
|
||||
class DBMigrateCombineQuoteMsgConfig(migration.DBMigration):
|
||||
"""Combine quote message config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Read all pipelines using raw SQL
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
# Ensure 'trigger' exists
|
||||
if 'trigger' not in config:
|
||||
config['trigger'] = {}
|
||||
|
||||
# Ensure 'misc' exists in 'trigger'
|
||||
if 'misc' not in config['trigger']:
|
||||
config['trigger']['misc'] = {}
|
||||
|
||||
# Add 'combine-quote-message' if not exists
|
||||
if 'combine-quote-message' not in config['trigger']['misc']:
|
||||
config['trigger']['misc']['combine-quote-message'] = False
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,62 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(3)
|
||||
class DBMigrateN8nConfig(migration.DBMigration):
|
||||
"""N8n config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Read all pipelines using raw SQL
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
# Ensure 'ai' exists
|
||||
if 'ai' not in config:
|
||||
config['ai'] = {}
|
||||
|
||||
# Add 'n8n-service-api' if not exists
|
||||
if 'n8n-service-api' not in config['ai']:
|
||||
config['ai']['n8n-service-api'] = {
|
||||
'webhook-url': 'http://your-n8n-webhook-url',
|
||||
'auth-type': 'none',
|
||||
'basic-username': '',
|
||||
'basic-password': '',
|
||||
'jwt-secret': '',
|
||||
'jwt-algorithm': 'HS256',
|
||||
'header-name': '',
|
||||
'header-value': '',
|
||||
'timeout': 120,
|
||||
'output-key': 'response',
|
||||
}
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,53 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(4)
|
||||
class DBMigrateRAGKBUUID(migration.DBMigration):
|
||||
"""RAG知识库UUID"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""升级"""
|
||||
# Read all pipelines using raw SQL
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
# Ensure nested structure exists
|
||||
if 'ai' not in config:
|
||||
config['ai'] = {}
|
||||
if 'local-agent' not in config['ai']:
|
||||
config['ai']['local-agent'] = {}
|
||||
|
||||
# Add 'knowledge-base' if not exists
|
||||
if 'knowledge-base' not in config['ai']['local-agent']:
|
||||
config['ai']['local-agent']['knowledge-base'] = ''
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""降级"""
|
||||
pass
|
||||
@@ -1,53 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(5)
|
||||
class DBMigratePipelineRemoveCotConfig(migration.DBMigration):
|
||||
"""Pipeline remove cot config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Read all pipelines using raw SQL
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
# Ensure nested structure exists
|
||||
if 'output' not in config:
|
||||
config['output'] = {}
|
||||
if 'misc' not in config['output']:
|
||||
config['output']['misc'] = {}
|
||||
|
||||
# Add 'remove-think' if not exists
|
||||
if 'remove-think' not in config['output']['misc']:
|
||||
config['output']['misc']['remove-think'] = False
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,58 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(6)
|
||||
class DBMigrateLangflowApiConfig(migration.DBMigration):
|
||||
"""Langflow API config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Read all pipelines using raw SQL
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
# Ensure 'ai' exists
|
||||
if 'ai' not in config:
|
||||
config['ai'] = {}
|
||||
|
||||
# Add 'langflow-api' if not exists
|
||||
if 'langflow-api' not in config['ai']:
|
||||
config['ai']['langflow-api'] = {
|
||||
'base-url': 'http://localhost:7860',
|
||||
'api-key': 'your-api-key',
|
||||
'flow-id': 'your-flow-id',
|
||||
'input-type': 'chat',
|
||||
'output-type': 'chat',
|
||||
'tweaks': '{}',
|
||||
}
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,44 +0,0 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(7)
|
||||
class DBMigratePluginInstallSource(migration.DBMigration):
|
||||
"""插件安装来源"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""升级"""
|
||||
# 查询表结构获取所有列名(异步执行 SQL)
|
||||
|
||||
columns = []
|
||||
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name = 'plugin_settings';"
|
||||
)
|
||||
)
|
||||
all_result = result.fetchall()
|
||||
columns = [row[0] for row in all_result]
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('PRAGMA table_info(plugin_settings);'))
|
||||
all_result = result.fetchall()
|
||||
columns = [row[1] for row in all_result]
|
||||
|
||||
# 检查并添加 install_source 列
|
||||
if 'install_source' not in columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
"ALTER TABLE plugin_settings ADD COLUMN install_source VARCHAR(255) NOT NULL DEFAULT 'github'"
|
||||
)
|
||||
)
|
||||
|
||||
# 检查并添加 install_info 列
|
||||
if 'install_info' not in columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("ALTER TABLE plugin_settings ADD COLUMN install_info JSON NOT NULL DEFAULT '{}'")
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""降级"""
|
||||
pass
|
||||
@@ -1,22 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(8)
|
||||
class DBMigratePluginConfig(migration.DBMigration):
|
||||
"""插件配置"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""升级"""
|
||||
|
||||
if 'plugin' not in self.ap.instance_config.data:
|
||||
self.ap.instance_config.data['plugin'] = {
|
||||
'runtime_ws_url': 'ws://langbot_plugin_runtime:5400/control/ws',
|
||||
'enable_marketplace': True,
|
||||
'cloud_service_url': 'https://space.langbot.app',
|
||||
}
|
||||
|
||||
await self.ap.instance_config.dump_config()
|
||||
|
||||
async def downgrade(self):
|
||||
"""降级"""
|
||||
pass
|
||||
@@ -1,20 +0,0 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(9)
|
||||
class DBMigratePipelineExtensionPreferences(migration.DBMigration):
|
||||
"""Pipeline extension preferences"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
|
||||
sql_text = sqlalchemy.text(
|
||||
"ALTER TABLE legacy_pipelines ADD COLUMN extensions_preferences JSON NOT NULL DEFAULT '{}'"
|
||||
)
|
||||
await self.ap.persistence_mgr.execute_async(sql_text)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
sql_text = sqlalchemy.text('ALTER TABLE legacy_pipelines DROP COLUMN extensions_preferences')
|
||||
await self.ap.persistence_mgr.execute_async(sql_text)
|
||||
@@ -1,105 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(10)
|
||||
class DBMigratePipelineMultiKnowledgeBase(migration.DBMigration):
|
||||
"""Pipeline support multiple knowledge base binding"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Read all pipelines using raw SQL
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
# Convert knowledge-base from string to array
|
||||
if 'ai' in config and 'local-agent' in config['ai']:
|
||||
current_kb = config['ai']['local-agent'].get('knowledge-base', '')
|
||||
|
||||
# If it's already a list, skip
|
||||
if isinstance(current_kb, list):
|
||||
continue
|
||||
|
||||
# Convert string to list
|
||||
if current_kb and current_kb != '__none__':
|
||||
config['ai']['local-agent']['knowledge-bases'] = [current_kb]
|
||||
else:
|
||||
config['ai']['local-agent']['knowledge-bases'] = []
|
||||
|
||||
# Remove old field
|
||||
if 'knowledge-base' in config['ai']['local-agent']:
|
||||
del config['ai']['local-agent']['knowledge-base']
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
# Read all pipelines using raw SQL
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
# Convert knowledge-bases from array back to string
|
||||
if 'ai' in config and 'local-agent' in config['ai']:
|
||||
current_kbs = config['ai']['local-agent'].get('knowledge-bases', [])
|
||||
|
||||
# If it's already a string, skip
|
||||
if isinstance(current_kbs, str):
|
||||
continue
|
||||
|
||||
# Convert list to string (take first one or empty)
|
||||
if current_kbs and len(current_kbs) > 0:
|
||||
config['ai']['local-agent']['knowledge-base'] = current_kbs[0]
|
||||
else:
|
||||
config['ai']['local-agent']['knowledge-base'] = ''
|
||||
|
||||
# Remove new field
|
||||
if 'knowledge-bases' in config['ai']['local-agent']:
|
||||
del config['ai']['local-agent']['knowledge-bases']
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
@@ -1,55 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(11)
|
||||
class DBMigrateDifyApiConfig(migration.DBMigration):
|
||||
"""Dify base prompt config"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Read all pipelines using raw SQL
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
# Ensure nested structure exists
|
||||
if 'ai' not in config:
|
||||
config['ai'] = {}
|
||||
if 'dify-service-api' not in config['ai']:
|
||||
config['ai']['dify-service-api'] = {}
|
||||
|
||||
# Add 'base-prompt' if not exists
|
||||
if 'base-prompt' not in config['ai']['dify-service-api']:
|
||||
config['ai']['dify-service-api']['base-prompt'] = (
|
||||
'When the file content is readable, please read the content of this file. When the file is an image, describe the content of this image.',
|
||||
)
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,73 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(12)
|
||||
class DBMigratePipelineExtensionsEnableAll(migration.DBMigration):
|
||||
"""Pipeline extensions enable all"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Read all pipelines using raw SQL
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, extensions_preferences FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
extensions_preferences = (
|
||||
json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
)
|
||||
|
||||
# Ensure extensions_preferences is a dict
|
||||
if extensions_preferences is None:
|
||||
extensions_preferences = {}
|
||||
|
||||
# Add 'enable_all_plugins' if not exists
|
||||
if 'enable_all_plugins' not in extensions_preferences:
|
||||
if 'plugins' in extensions_preferences:
|
||||
extensions_preferences['enable_all_plugins'] = False
|
||||
else:
|
||||
extensions_preferences['enable_all_plugins'] = True
|
||||
extensions_preferences['plugins'] = []
|
||||
|
||||
# Add 'enable_all_mcp_servers' if not exists
|
||||
if 'enable_all_mcp_servers' not in extensions_preferences:
|
||||
if 'mcp_servers' in extensions_preferences:
|
||||
extensions_preferences['enable_all_mcp_servers'] = False
|
||||
else:
|
||||
extensions_preferences['enable_all_mcp_servers'] = True
|
||||
extensions_preferences['mcp_servers'] = []
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET extensions_preferences = :extensions_preferences::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{
|
||||
'extensions_preferences': json.dumps(extensions_preferences),
|
||||
'for_version': current_version,
|
||||
'uuid': uuid,
|
||||
},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET extensions_preferences = :extensions_preferences, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{
|
||||
'extensions_preferences': json.dumps(extensions_preferences),
|
||||
'for_version': current_version,
|
||||
'uuid': uuid,
|
||||
},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,49 +0,0 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(13)
|
||||
class DBMigrateKnowledgeBaseUpdatedAt(migration.DBMigration):
|
||||
"""Add updated_at field to knowledge_bases table"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Get all column names from the table
|
||||
columns = []
|
||||
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
"SELECT column_name FROM information_schema.columns WHERE table_name = 'knowledge_bases';"
|
||||
)
|
||||
)
|
||||
all_result = result.fetchall()
|
||||
columns = [row[0] for row in all_result]
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('PRAGMA table_info(knowledge_bases);'))
|
||||
all_result = result.fetchall()
|
||||
columns = [row[1] for row in all_result]
|
||||
|
||||
# Check and add updated_at column
|
||||
if 'updated_at' not in columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'ALTER TABLE knowledge_bases ADD COLUMN updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP'
|
||||
)
|
||||
)
|
||||
else:
|
||||
# SQLite doesn't support DEFAULT CURRENT_TIMESTAMP in ALTER TABLE
|
||||
# Add column without default first
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE knowledge_bases ADD COLUMN updated_at DATETIME')
|
||||
)
|
||||
|
||||
# Set initial updated_at values to created_at for existing records
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('UPDATE knowledge_bases SET updated_at = created_at WHERE updated_at IS NULL')
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,94 +0,0 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(14)
|
||||
class DBMigrateSpaceAccountSupport(migration.DBMigration):
|
||||
"""Add Space account support fields to users table"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Get all column names from the users table
|
||||
columns = []
|
||||
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT column_name FROM information_schema.columns WHERE table_name = 'users';")
|
||||
)
|
||||
all_result = result.fetchall()
|
||||
columns = [row[0] for row in all_result]
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('PRAGMA table_info(users);'))
|
||||
all_result = result.fetchall()
|
||||
columns = [row[1] for row in all_result]
|
||||
|
||||
# Add account_type column
|
||||
if 'account_type' not in columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("ALTER TABLE users ADD COLUMN account_type VARCHAR(32) DEFAULT 'local' NOT NULL")
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("ALTER TABLE users ADD COLUMN account_type VARCHAR(32) DEFAULT 'local' NOT NULL")
|
||||
)
|
||||
|
||||
# Add space_account_uuid column
|
||||
if 'space_account_uuid' not in columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_account_uuid VARCHAR(255)')
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_account_uuid VARCHAR(255)')
|
||||
)
|
||||
|
||||
# Add space_access_token column
|
||||
if 'space_access_token' not in columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_access_token TEXT')
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_access_token TEXT')
|
||||
)
|
||||
|
||||
# Add space_refresh_token column
|
||||
if 'space_refresh_token' not in columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_refresh_token TEXT')
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_refresh_token TEXT')
|
||||
)
|
||||
|
||||
# Add space_access_token_expires_at column
|
||||
if 'space_access_token_expires_at' not in columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_access_token_expires_at TIMESTAMP')
|
||||
)
|
||||
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_access_token_expires_at DATETIME')
|
||||
)
|
||||
|
||||
# Add space_api_key column
|
||||
if 'space_api_key' not in columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_api_key VARCHAR(255)')
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE users ADD COLUMN space_api_key VARCHAR(255)')
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,15 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
|
||||
# this is a deprecated migration
|
||||
@migration.migration_class(15)
|
||||
class DBMigrateModelSourceTracking(migration.DBMigration):
|
||||
"""Add source tracking fields to models tables for Space integration"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
pass
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,305 +0,0 @@
|
||||
import uuid as uuid_lib
|
||||
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(16)
|
||||
class DBMigrateModelProviderRefactor(migration.DBMigration):
|
||||
"""Refactor model structure: create providers from existing models and update references"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Step 1: Create model_providers table if not exists
|
||||
await self._create_providers_table()
|
||||
|
||||
# Step 2: Migrate existing models to use providers
|
||||
await self._migrate_llm_models()
|
||||
await self._migrate_embedding_models()
|
||||
|
||||
# Step 3: Remove deprecated columns
|
||||
await self._cleanup_columns()
|
||||
|
||||
async def _create_providers_table(self):
|
||||
"""Create model_providers table"""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("""
|
||||
CREATE TABLE IF NOT EXISTS model_providers (
|
||||
uuid VARCHAR(255) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
requester VARCHAR(255) NOT NULL,
|
||||
base_url VARCHAR(512) NOT NULL,
|
||||
api_keys JSONB NOT NULL DEFAULT '[]',
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("""
|
||||
CREATE TABLE IF NOT EXISTS model_providers (
|
||||
uuid VARCHAR(255) PRIMARY KEY,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
requester VARCHAR(255) NOT NULL,
|
||||
base_url VARCHAR(512) NOT NULL,
|
||||
api_keys JSON NOT NULL DEFAULT '[]',
|
||||
created_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
)
|
||||
""")
|
||||
)
|
||||
|
||||
async def _migrate_llm_models(self):
|
||||
"""Migrate LLM models to use providers"""
|
||||
llm_columns = await self._get_columns('llm_models')
|
||||
|
||||
# Add provider_uuid column if not exists
|
||||
if 'provider_uuid' not in llm_columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE llm_models ADD COLUMN provider_uuid VARCHAR(255)')
|
||||
)
|
||||
|
||||
# Add prefered_ranking column if not exists
|
||||
if 'prefered_ranking' not in llm_columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE llm_models ADD COLUMN prefered_ranking INTEGER NOT NULL DEFAULT 0')
|
||||
)
|
||||
|
||||
# Only migrate if old columns exist
|
||||
if 'requester' not in llm_columns:
|
||||
return
|
||||
|
||||
# Get all LLM models with old structure
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, name, requester, requester_config, api_keys FROM llm_models')
|
||||
)
|
||||
models = result.fetchall()
|
||||
|
||||
# Create providers and update models
|
||||
provider_cache = {} # (requester, base_url, api_keys_str) -> provider_uuid
|
||||
|
||||
for model in models:
|
||||
model_uuid, model_name, requester, requester_config, api_keys = model
|
||||
|
||||
# Extract base_url from requester_config
|
||||
base_url = ''
|
||||
if requester_config:
|
||||
if isinstance(requester_config, str):
|
||||
import json
|
||||
|
||||
requester_config = json.loads(requester_config)
|
||||
base_url = requester_config.get('base_url', '') or requester_config.get('base-url', '')
|
||||
|
||||
# Parse api_keys if it's a string
|
||||
if isinstance(api_keys, str):
|
||||
import json
|
||||
|
||||
try:
|
||||
api_keys = json.loads(api_keys)
|
||||
except Exception:
|
||||
api_keys = []
|
||||
if not api_keys:
|
||||
api_keys = []
|
||||
|
||||
# Create cache key
|
||||
api_keys_str = str(sorted(api_keys)) if api_keys else '[]'
|
||||
cache_key = (requester, base_url, api_keys_str)
|
||||
|
||||
if cache_key in provider_cache:
|
||||
provider_uuid = provider_cache[cache_key]
|
||||
else:
|
||||
# Create new provider
|
||||
provider_uuid = str(uuid_lib.uuid4())
|
||||
provider_name = f'{requester}'
|
||||
if base_url:
|
||||
# Extract domain for name
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(base_url)
|
||||
provider_name = parsed.netloc or requester
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
import json
|
||||
|
||||
api_keys_json = json.dumps(api_keys) if api_keys else '[]'
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("""
|
||||
INSERT INTO model_providers (uuid, name, requester, base_url, api_keys)
|
||||
VALUES (:uuid, :name, :requester, :base_url, :api_keys)
|
||||
"""),
|
||||
{
|
||||
'uuid': provider_uuid,
|
||||
'name': provider_name,
|
||||
'requester': requester,
|
||||
'base_url': base_url,
|
||||
'api_keys': api_keys_json,
|
||||
},
|
||||
)
|
||||
provider_cache[cache_key] = provider_uuid
|
||||
|
||||
# Update model with provider_uuid
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('UPDATE llm_models SET provider_uuid = :provider_uuid WHERE uuid = :uuid'),
|
||||
{'provider_uuid': provider_uuid, 'uuid': model_uuid},
|
||||
)
|
||||
|
||||
async def _migrate_embedding_models(self):
|
||||
"""Migrate embedding models to use providers"""
|
||||
embedding_columns = await self._get_columns('embedding_models')
|
||||
|
||||
# Add provider_uuid column if not exists
|
||||
if 'provider_uuid' not in embedding_columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE embedding_models ADD COLUMN provider_uuid VARCHAR(255)')
|
||||
)
|
||||
|
||||
# Add prefered_ranking column if not exists
|
||||
if 'prefered_ranking' not in embedding_columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE embedding_models ADD COLUMN prefered_ranking INTEGER NOT NULL DEFAULT 0')
|
||||
)
|
||||
|
||||
# Only migrate if old columns exist
|
||||
if 'requester' not in embedding_columns:
|
||||
return
|
||||
|
||||
# Get all embedding models with old structure
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, name, requester, requester_config, api_keys FROM embedding_models')
|
||||
)
|
||||
models = result.fetchall()
|
||||
|
||||
# Get existing providers
|
||||
provider_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, requester, base_url, api_keys FROM model_providers')
|
||||
)
|
||||
existing_providers = provider_result.fetchall()
|
||||
|
||||
provider_cache = {}
|
||||
for p in existing_providers:
|
||||
p_uuid, p_requester, p_base_url, p_api_keys = p
|
||||
api_keys_str = str(sorted(p_api_keys)) if p_api_keys else '[]'
|
||||
provider_cache[(p_requester, p_base_url, api_keys_str)] = p_uuid
|
||||
|
||||
for model in models:
|
||||
model_uuid, model_name, requester, requester_config, api_keys = model
|
||||
|
||||
base_url = ''
|
||||
if requester_config:
|
||||
if isinstance(requester_config, str):
|
||||
import json
|
||||
|
||||
requester_config = json.loads(requester_config)
|
||||
base_url = requester_config.get('base_url', '') or requester_config.get('base-url', '')
|
||||
|
||||
# Parse api_keys if it's a string
|
||||
if isinstance(api_keys, str):
|
||||
import json
|
||||
|
||||
try:
|
||||
api_keys = json.loads(api_keys)
|
||||
except Exception:
|
||||
api_keys = []
|
||||
if not api_keys:
|
||||
api_keys = []
|
||||
|
||||
api_keys_str = str(sorted(api_keys)) if api_keys else '[]'
|
||||
cache_key = (requester, base_url, api_keys_str)
|
||||
|
||||
if cache_key in provider_cache:
|
||||
provider_uuid = provider_cache[cache_key]
|
||||
else:
|
||||
provider_uuid = str(uuid_lib.uuid4())
|
||||
provider_name = f'{requester}'
|
||||
if base_url:
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
|
||||
parsed = urlparse(base_url)
|
||||
provider_name = parsed.netloc or requester
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
import json
|
||||
|
||||
api_keys_json = json.dumps(api_keys) if api_keys else '[]'
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("""
|
||||
INSERT INTO model_providers (uuid, name, requester, base_url, api_keys)
|
||||
VALUES (:uuid, :name, :requester, :base_url, :api_keys)
|
||||
"""),
|
||||
{
|
||||
'uuid': provider_uuid,
|
||||
'name': provider_name,
|
||||
'requester': requester,
|
||||
'base_url': base_url,
|
||||
'api_keys': api_keys_json,
|
||||
},
|
||||
)
|
||||
provider_cache[cache_key] = provider_uuid
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('UPDATE embedding_models SET provider_uuid = :provider_uuid WHERE uuid = :uuid'),
|
||||
{'provider_uuid': provider_uuid, 'uuid': model_uuid},
|
||||
)
|
||||
|
||||
async def _cleanup_columns(self):
|
||||
"""Remove deprecated columns from model tables"""
|
||||
|
||||
llm_columns = await self._get_columns('llm_models')
|
||||
deprecated_llm_cols = ['requester', 'requester_config', 'api_keys', 'description', 'source', 'space_model_id']
|
||||
for col in deprecated_llm_cols:
|
||||
if col in llm_columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(f'ALTER TABLE llm_models DROP COLUMN IF EXISTS {col}')
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(f'ALTER TABLE llm_models DROP COLUMN {col}')
|
||||
)
|
||||
|
||||
embedding_columns = await self._get_columns('embedding_models')
|
||||
deprecated_embedding_cols = [
|
||||
'requester',
|
||||
'requester_config',
|
||||
'api_keys',
|
||||
'description',
|
||||
'source',
|
||||
'space_model_id',
|
||||
]
|
||||
for col in deprecated_embedding_cols:
|
||||
if col in embedding_columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(f'ALTER TABLE embedding_models DROP COLUMN IF EXISTS {col}')
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(f'ALTER TABLE embedding_models DROP COLUMN {col}')
|
||||
)
|
||||
|
||||
async def _get_columns(self, table_name: str) -> list:
|
||||
"""Get column names for a table"""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
f"SELECT column_name FROM information_schema.columns WHERE table_name = '{table_name}';"
|
||||
)
|
||||
)
|
||||
all_result = result.fetchall()
|
||||
return [row[0] for row in all_result]
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text(f'PRAGMA table_info({table_name});'))
|
||||
all_result = result.fetchall()
|
||||
return [row[1] for row in all_result]
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,25 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(17)
|
||||
class MoveCloudServiceUrl(migration.DBMigration):
|
||||
"""迁移云服务 URL 配置"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""升级"""
|
||||
if 'space' not in self.ap.instance_config.data:
|
||||
self.ap.instance_config.data['space'] = {
|
||||
'url': 'https://space.langbot.app',
|
||||
'models_gateway_api_url': 'https://api.langbot.cloud/v1',
|
||||
'oauth_authorize_url': 'https://space.langbot.app/auth/authorize',
|
||||
'disable_models_service': False,
|
||||
}
|
||||
|
||||
if 'plugin' in self.ap.instance_config.data:
|
||||
self.ap.instance_config.data['plugin'].pop('cloud_service_url', None)
|
||||
|
||||
await self.ap.instance_config.dump_config()
|
||||
|
||||
async def downgrade(self):
|
||||
"""降级"""
|
||||
pass
|
||||
@@ -1,58 +0,0 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(18)
|
||||
class DBMigrateAddEmojiSupport(migration.DBMigration):
|
||||
"""Add emoji field to knowledge_bases, external_knowledge_bases and legacy_pipelines tables"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
# Add emoji field to knowledge_bases
|
||||
await self._add_emoji_to_table('knowledge_bases', '📚')
|
||||
|
||||
# Add emoji field to external_knowledge_bases
|
||||
await self._add_emoji_to_table('external_knowledge_bases', '🔗')
|
||||
|
||||
# Add emoji field to legacy_pipelines
|
||||
await self._add_emoji_to_table('legacy_pipelines', '⚙️')
|
||||
|
||||
async def _add_emoji_to_table(self, table_name: str, default_emoji: str):
|
||||
"""Add emoji column to specified table if it doesn't exist"""
|
||||
# Get all column names from the table
|
||||
columns = []
|
||||
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
f"SELECT column_name FROM information_schema.columns WHERE table_name = '{table_name}';"
|
||||
)
|
||||
)
|
||||
all_result = result.fetchall()
|
||||
columns = [row[0] for row in all_result]
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text(f'PRAGMA table_info({table_name});'))
|
||||
all_result = result.fetchall()
|
||||
columns = [row[1] for row in all_result]
|
||||
|
||||
# Check and add emoji column
|
||||
if 'emoji' not in columns:
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(f"ALTER TABLE {table_name} ADD COLUMN emoji VARCHAR(10) DEFAULT '{default_emoji}'")
|
||||
)
|
||||
else:
|
||||
# SQLite doesn't support DEFAULT with emoji directly in ALTER TABLE
|
||||
# Add column without default first
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(f'ALTER TABLE {table_name} ADD COLUMN emoji VARCHAR(10)')
|
||||
)
|
||||
|
||||
# Set default emoji value for existing records
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(f"UPDATE {table_name} SET emoji = '{default_emoji}' WHERE emoji IS NULL")
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,24 +0,0 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(19)
|
||||
class DBMigrateMonitoringMessageRole(migration.DBMigration):
|
||||
"""Add role column to monitoring_messages table"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
try:
|
||||
sql_text = sqlalchemy.text("ALTER TABLE monitoring_messages ADD COLUMN role VARCHAR(50) DEFAULT 'user'")
|
||||
await self.ap.persistence_mgr.execute_async(sql_text)
|
||||
except Exception:
|
||||
# Column may already exist
|
||||
pass
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
try:
|
||||
sql_text = sqlalchemy.text('ALTER TABLE monitoring_messages DROP COLUMN role')
|
||||
await self.ap.persistence_mgr.execute_async(sql_text)
|
||||
except Exception:
|
||||
pass
|
||||
-161
@@ -1,161 +0,0 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(20)
|
||||
class DBMigrateKnowledgeEnginePluginArchitecture(migration.DBMigration):
|
||||
"""Migrate to unified Knowledge Engine plugin architecture.
|
||||
|
||||
Changes:
|
||||
- Backup existing knowledge_bases data to knowledge_bases_backup
|
||||
- Clear knowledge_bases table and add new plugin architecture columns
|
||||
- Drop old columns (PostgreSQL only; SQLite leaves them unmapped)
|
||||
- Preserve external_knowledge_bases table as-is for future migration
|
||||
- Set rag_plugin_migration_needed flag in metadata if old data exists
|
||||
"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
has_internal_data = await self._backup_knowledge_bases()
|
||||
has_external_data = await self._check_external_knowledge_bases()
|
||||
await self._clear_knowledge_bases()
|
||||
await self._add_columns_to_knowledge_bases()
|
||||
await self._drop_old_columns()
|
||||
if has_internal_data or has_external_data:
|
||||
await self._set_migration_flag()
|
||||
|
||||
async def _get_table_columns(self, table_name: str) -> list[str]:
|
||||
"""Get column names from a table (works for both SQLite and PostgreSQL)."""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT column_name FROM information_schema.columns WHERE table_name = :table_name;'
|
||||
).bindparams(table_name=table_name)
|
||||
)
|
||||
return [row[0] for row in result.fetchall()]
|
||||
else:
|
||||
# SQLite PRAGMA does not support bind parameters; validate identifier.
|
||||
if not table_name.isidentifier():
|
||||
raise ValueError(f'Invalid table name: {table_name}')
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text(f'PRAGMA table_info({table_name});'))
|
||||
return [row[1] for row in result.fetchall()]
|
||||
|
||||
async def _table_exists(self, table_name: str) -> bool:
|
||||
"""Check if a table exists."""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);'
|
||||
).bindparams(table_name=table_name)
|
||||
)
|
||||
return result.scalar()
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;").bindparams(
|
||||
table_name=table_name
|
||||
)
|
||||
)
|
||||
return result.first() is not None
|
||||
|
||||
async def _backup_knowledge_bases(self) -> bool:
|
||||
"""Backup knowledge_bases data. Returns True if data was backed up."""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text('SELECT COUNT(*) FROM knowledge_bases;'))
|
||||
count = result.scalar()
|
||||
if count == 0:
|
||||
return False
|
||||
|
||||
# Drop backup table if it already exists (from a previous failed migration)
|
||||
if await self._table_exists('knowledge_bases_backup'):
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DROP TABLE knowledge_bases_backup;'))
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('CREATE TABLE knowledge_bases_backup AS SELECT * FROM knowledge_bases;')
|
||||
)
|
||||
self.ap.logger.info(
|
||||
'Backed up %d knowledge base(s) to knowledge_bases_backup table.',
|
||||
count,
|
||||
)
|
||||
return True
|
||||
|
||||
async def _check_external_knowledge_bases(self) -> bool:
|
||||
"""Check if external_knowledge_bases table exists and has data.
|
||||
|
||||
The table is preserved as-is (not dropped) for future migration.
|
||||
"""
|
||||
if not await self._table_exists('external_knowledge_bases'):
|
||||
return False
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT COUNT(*) FROM external_knowledge_bases;')
|
||||
)
|
||||
count = result.scalar()
|
||||
if count > 0:
|
||||
self.ap.logger.info(
|
||||
'Found %d external knowledge base(s) in external_knowledge_bases table. '
|
||||
'Table preserved for future migration.',
|
||||
count,
|
||||
)
|
||||
return count > 0
|
||||
|
||||
async def _clear_knowledge_bases(self):
|
||||
"""Clear all rows from knowledge_bases table (preserve table structure)."""
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.text('DELETE FROM knowledge_bases;'))
|
||||
|
||||
async def _add_columns_to_knowledge_bases(self):
|
||||
"""Add new RAG plugin architecture columns to knowledge_bases table."""
|
||||
columns = await self._get_table_columns('knowledge_bases')
|
||||
|
||||
new_columns = {
|
||||
'knowledge_engine_plugin_id': 'VARCHAR',
|
||||
'collection_id': 'VARCHAR',
|
||||
'creation_settings': 'TEXT', # JSON stored as TEXT for SQLite compatibility
|
||||
'retrieval_settings': 'TEXT',
|
||||
}
|
||||
|
||||
for col_name, col_type in new_columns.items():
|
||||
if col_name not in columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(f'ALTER TABLE knowledge_bases ADD COLUMN {col_name} {col_type};')
|
||||
)
|
||||
|
||||
async def _drop_old_columns(self):
|
||||
"""Drop embedding_model_uuid and top_k columns (PostgreSQL only).
|
||||
|
||||
SQLite does not support DROP COLUMN in older versions, so we leave the
|
||||
columns in place — the SQLAlchemy entity simply won't map them.
|
||||
"""
|
||||
if self.ap.persistence_mgr.db.name != 'postgresql':
|
||||
return
|
||||
|
||||
columns = await self._get_table_columns('knowledge_bases')
|
||||
|
||||
if 'embedding_model_uuid' in columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE knowledge_bases DROP COLUMN embedding_model_uuid;')
|
||||
)
|
||||
|
||||
if 'top_k' in columns:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('ALTER TABLE knowledge_bases DROP COLUMN top_k;')
|
||||
)
|
||||
|
||||
async def _set_migration_flag(self):
|
||||
"""Set rag_plugin_migration_needed flag in metadata table."""
|
||||
# Check if the key already exists
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT value FROM metadata WHERE key = 'rag_plugin_migration_needed';")
|
||||
)
|
||||
row = result.first()
|
||||
if row is not None:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("UPDATE metadata SET value = 'true' WHERE key = 'rag_plugin_migration_needed';")
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("INSERT INTO metadata (key, value) VALUES ('rag_plugin_migration_needed', 'true');")
|
||||
)
|
||||
self.ap.logger.info('Set rag_plugin_migration_needed=true in metadata.')
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,74 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(21)
|
||||
class DBMigrateMergeExceptionHandling(migration.DBMigration):
|
||||
"""Merge hide-exception and block-failed-request-output into a single exception-handling select option,
|
||||
and add failure-hint field.
|
||||
|
||||
Conversion logic:
|
||||
- block-failed-request-output=true -> exception-handling: hide
|
||||
- hide-exception=true -> exception-handling: show-hint
|
||||
- hide-exception=false -> exception-handling: show-error
|
||||
"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
if 'output' not in config:
|
||||
config['output'] = {}
|
||||
if 'misc' not in config['output']:
|
||||
config['output']['misc'] = {}
|
||||
|
||||
misc = config['output']['misc']
|
||||
|
||||
# Determine new exception-handling value from legacy fields
|
||||
hide_exception = misc.get('hide-exception', True)
|
||||
block_failed = misc.get('block-failed-request-output', False)
|
||||
|
||||
if block_failed:
|
||||
exception_handling = 'hide'
|
||||
elif hide_exception:
|
||||
exception_handling = 'show-hint'
|
||||
else:
|
||||
exception_handling = 'show-error'
|
||||
|
||||
misc['exception-handling'] = exception_handling
|
||||
|
||||
# Add failure-hint with default value
|
||||
misc['failure-hint'] = 'Request failed.'
|
||||
|
||||
# Remove legacy fields
|
||||
misc.pop('hide-exception', None)
|
||||
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,73 +0,0 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(22)
|
||||
class DBMigrateMonitoringUserId(migration.DBMigration):
|
||||
"""Add user_id and user_name columns to monitoring_sessions table
|
||||
|
||||
This migration adds the missing user_id column and also ensures user_name
|
||||
column exists (in case migration 21 failed or was skipped).
|
||||
"""
|
||||
|
||||
async def _table_exists(self, table_name: str) -> bool:
|
||||
"""Check if a table exists (works for both SQLite and PostgreSQL)."""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = :table_name);'
|
||||
).bindparams(table_name=table_name)
|
||||
)
|
||||
return bool(result.scalar())
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT name FROM sqlite_master WHERE type='table' AND name=:table_name;").bindparams(
|
||||
table_name=table_name
|
||||
)
|
||||
)
|
||||
return result.first() is not None
|
||||
|
||||
async def _get_table_columns(self, table_name: str) -> list[str]:
|
||||
"""Get column names from a table (works for both SQLite and PostgreSQL)."""
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'SELECT column_name FROM information_schema.columns WHERE table_name = :table_name;'
|
||||
).bindparams(table_name=table_name)
|
||||
)
|
||||
return [row[0] for row in result.fetchall()]
|
||||
else:
|
||||
if not table_name.isidentifier():
|
||||
raise ValueError(f'Invalid table name: {table_name}')
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.text(f'PRAGMA table_info({table_name});'))
|
||||
return [row[1] for row in result.fetchall()]
|
||||
|
||||
async def _add_column_if_not_exists(self, table_name: str, column_name: str, column_type: str):
|
||||
"""Add a column to a table if it does not already exist."""
|
||||
columns = await self._get_table_columns(table_name)
|
||||
if column_name in columns:
|
||||
self.ap.logger.debug('%s column already exists in %s.', column_name, table_name)
|
||||
return
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(f'ALTER TABLE {table_name} ADD COLUMN {column_name} {column_type};')
|
||||
)
|
||||
self.ap.logger.info('Added %s column to %s table.', column_name, table_name)
|
||||
|
||||
async def upgrade(self):
|
||||
# Check if monitoring_sessions table exists
|
||||
if not await self._table_exists('monitoring_sessions'):
|
||||
self.ap.logger.warning('monitoring_sessions table does not exist, skipping migration.')
|
||||
return
|
||||
|
||||
# Add user_id column to monitoring_sessions table
|
||||
await self._add_column_if_not_exists('monitoring_sessions', 'user_id', 'VARCHAR(255)')
|
||||
|
||||
# Add user_name column to monitoring_sessions table (in case migration 21 failed)
|
||||
await self._add_column_if_not_exists('monitoring_sessions', 'user_name', 'VARCHAR(255)')
|
||||
|
||||
# Add user_name column to monitoring_messages table (in case migration 21 failed)
|
||||
if await self._table_exists('monitoring_messages'):
|
||||
await self._add_column_if_not_exists('monitoring_messages', 'user_name', 'VARCHAR(255)')
|
||||
|
||||
async def downgrade(self):
|
||||
pass
|
||||
@@ -1,102 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(23)
|
||||
class DBMigrateModelFallbackConfig(migration.DBMigration):
|
||||
"""Convert model field from plain UUID string to object with primary/fallbacks"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
if 'ai' not in config or 'local-agent' not in config['ai']:
|
||||
continue
|
||||
|
||||
local_agent = config['ai']['local-agent']
|
||||
changed = False
|
||||
|
||||
# Convert model from string to object
|
||||
model_value = local_agent.get('model', '')
|
||||
if isinstance(model_value, str):
|
||||
local_agent['model'] = {
|
||||
'primary': model_value,
|
||||
'fallbacks': [],
|
||||
}
|
||||
changed = True
|
||||
|
||||
# Remove leftover fallback-models field if present
|
||||
if 'fallback-models' in local_agent:
|
||||
del local_agent['fallback-models']
|
||||
changed = True
|
||||
|
||||
if not changed:
|
||||
continue
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('SELECT uuid, config FROM legacy_pipelines')
|
||||
)
|
||||
pipelines = result.fetchall()
|
||||
|
||||
current_version = self.ap.ver_mgr.get_current_version()
|
||||
|
||||
for pipeline_row in pipelines:
|
||||
uuid = pipeline_row[0]
|
||||
config = json.loads(pipeline_row[1]) if isinstance(pipeline_row[1], str) else pipeline_row[1]
|
||||
|
||||
if 'ai' not in config or 'local-agent' not in config['ai']:
|
||||
continue
|
||||
|
||||
local_agent = config['ai']['local-agent']
|
||||
|
||||
# Convert model from object back to string
|
||||
model_value = local_agent.get('model', '')
|
||||
if isinstance(model_value, dict):
|
||||
local_agent['model'] = model_value.get('primary', '')
|
||||
else:
|
||||
continue
|
||||
|
||||
# Update using raw SQL with compatibility for both SQLite and PostgreSQL
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config::jsonb, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text(
|
||||
'UPDATE legacy_pipelines SET config = :config, for_version = :for_version WHERE uuid = :uuid'
|
||||
),
|
||||
{'config': json.dumps(config), 'for_version': current_version, 'uuid': uuid},
|
||||
)
|
||||
@@ -1,49 +0,0 @@
|
||||
from .. import migration
|
||||
|
||||
import sqlalchemy
|
||||
import json
|
||||
|
||||
|
||||
@migration.migration_class(24)
|
||||
class DBMigrateWecomBotWebSocketMode(migration.DBMigration):
|
||||
"""Add enable-webhook field to existing wecombot adapter configs.
|
||||
|
||||
Existing wecombot bots were all using webhook mode, so we set
|
||||
enable-webhook=true to preserve their behavior after the new
|
||||
WebSocket long connection mode is introduced as default.
|
||||
"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text("SELECT uuid, adapter_config FROM bots WHERE adapter = 'wecombot'")
|
||||
)
|
||||
bots = result.fetchall()
|
||||
|
||||
for bot_row in bots:
|
||||
bot_uuid = bot_row[0]
|
||||
adapter_config = json.loads(bot_row[1]) if isinstance(bot_row[1], str) else bot_row[1]
|
||||
|
||||
if 'enable-webhook' in adapter_config:
|
||||
continue
|
||||
|
||||
# Determine mode based on existing config: if webhook fields are present, keep webhook mode
|
||||
has_webhook_config = bool(
|
||||
adapter_config.get('Token') and adapter_config.get('EncodingAESKey') and adapter_config.get('Corpid')
|
||||
)
|
||||
adapter_config['enable-webhook'] = has_webhook_config
|
||||
|
||||
if self.ap.persistence_mgr.db.name == 'postgresql':
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('UPDATE bots SET adapter_config = :config::jsonb WHERE uuid = :uuid'),
|
||||
{'config': json.dumps(adapter_config), 'uuid': bot_uuid},
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.text('UPDATE bots SET adapter_config = :config WHERE uuid = :uuid'),
|
||||
{'config': json.dumps(adapter_config), 'uuid': bot_uuid},
|
||||
)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
pass
|
||||
@@ -1,15 +0,0 @@
|
||||
import sqlalchemy
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(25)
|
||||
class DBMigrateBotPipelineRoutingRules(migration.DBMigration):
|
||||
"""Add pipeline_routing_rules column to bots table"""
|
||||
|
||||
async def upgrade(self):
|
||||
sql_text = sqlalchemy.text("ALTER TABLE bots ADD COLUMN pipeline_routing_rules JSON NOT NULL DEFAULT '[]'")
|
||||
await self.ap.persistence_mgr.execute_async(sql_text)
|
||||
|
||||
async def downgrade(self):
|
||||
sql_text = sqlalchemy.text('ALTER TABLE bots DROP COLUMN pipeline_routing_rules')
|
||||
await self.ap.persistence_mgr.execute_async(sql_text)
|
||||
@@ -1,17 +0,0 @@
|
||||
from langbot.pkg.entity.persistence import monitoring as persistence_monitoring
|
||||
from .. import migration
|
||||
|
||||
|
||||
@migration.migration_class(26)
|
||||
class DBMigrateMonitoringToolCalls(migration.DBMigration):
|
||||
"""Add monitoring_tool_calls table"""
|
||||
|
||||
async def upgrade(self):
|
||||
"""Upgrade"""
|
||||
async with self.ap.persistence_mgr.get_db_engine().begin() as conn:
|
||||
await conn.run_sync(persistence_monitoring.MonitoringToolCall.__table__.create, checkfirst=True)
|
||||
|
||||
async def downgrade(self):
|
||||
"""Downgrade"""
|
||||
async with self.ap.persistence_mgr.get_db_engine().begin() as conn:
|
||||
await conn.run_sync(persistence_monitoring.MonitoringToolCall.__table__.drop, checkfirst=True)
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Validation and fail-closed reads for Pipeline extension preferences."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
|
||||
|
||||
_BOOLEAN_FIELDS = (
|
||||
'enable_all_plugins',
|
||||
'enable_all_mcp_servers',
|
||||
'enable_all_skills',
|
||||
'mcp_resource_agent_read_enabled',
|
||||
)
|
||||
|
||||
|
||||
def _valid_plugin_binding(value: typing.Any) -> bool:
|
||||
return (
|
||||
isinstance(value, dict)
|
||||
and isinstance(value.get('author'), str)
|
||||
and bool(value['author'])
|
||||
and isinstance(value.get('name'), str)
|
||||
and bool(value['name'])
|
||||
)
|
||||
|
||||
|
||||
def _valid_name(value: typing.Any) -> bool:
|
||||
return isinstance(value, str) and bool(value)
|
||||
|
||||
|
||||
def normalize_extension_preferences(value: typing.Any) -> dict[str, typing.Any]:
|
||||
"""Return a safe runtime view of persisted extension preferences.
|
||||
|
||||
Missing fields in a valid object retain the established defaults. A
|
||||
malformed root is treated as an empty allowlist with every extension
|
||||
capability disabled.
|
||||
"""
|
||||
if not isinstance(value, dict):
|
||||
return {
|
||||
'enable_all_plugins': False,
|
||||
'enable_all_mcp_servers': False,
|
||||
'enable_all_skills': False,
|
||||
'mcp_resource_agent_read_enabled': False,
|
||||
'plugins': [],
|
||||
'mcp_servers': [],
|
||||
'skills': [],
|
||||
'mcp_resources': [],
|
||||
}
|
||||
|
||||
normalized = dict(value)
|
||||
normalized['enable_all_plugins'] = value.get('enable_all_plugins', True) is True
|
||||
normalized['enable_all_mcp_servers'] = value.get('enable_all_mcp_servers', True) is True
|
||||
normalized['enable_all_skills'] = value.get('enable_all_skills', True) is True
|
||||
normalized['mcp_resource_agent_read_enabled'] = (
|
||||
value.get('mcp_resource_agent_read_enabled', True) is True
|
||||
)
|
||||
|
||||
plugins = value.get('plugins', [])
|
||||
plugins_are_valid = isinstance(plugins, list) and all(
|
||||
_valid_plugin_binding(plugin) for plugin in plugins
|
||||
)
|
||||
normalized['plugins'] = list(plugins) if plugins_are_valid else []
|
||||
if not plugins_are_valid:
|
||||
normalized['enable_all_plugins'] = False
|
||||
|
||||
mcp_servers = value.get('mcp_servers', [])
|
||||
mcp_servers_are_valid = isinstance(mcp_servers, list) and all(
|
||||
_valid_name(server) for server in mcp_servers
|
||||
)
|
||||
normalized['mcp_servers'] = list(mcp_servers) if mcp_servers_are_valid else []
|
||||
if not mcp_servers_are_valid:
|
||||
normalized['enable_all_mcp_servers'] = False
|
||||
|
||||
skills = value.get('skills', [])
|
||||
skills_are_valid = isinstance(skills, list) and all(_valid_name(skill) for skill in skills)
|
||||
normalized['skills'] = list(skills) if skills_are_valid else []
|
||||
if not skills_are_valid:
|
||||
normalized['enable_all_skills'] = False
|
||||
|
||||
mcp_resources = value.get('mcp_resources', [])
|
||||
mcp_resources_are_valid = isinstance(mcp_resources, list) and all(
|
||||
isinstance(resource, dict) for resource in mcp_resources
|
||||
)
|
||||
normalized['mcp_resources'] = list(mcp_resources) if mcp_resources_are_valid else []
|
||||
if not mcp_resources_are_valid:
|
||||
normalized['mcp_resource_agent_read_enabled'] = False
|
||||
return normalized
|
||||
|
||||
|
||||
def validate_extension_preferences(
|
||||
value: typing.Any,
|
||||
*,
|
||||
context: str = 'Pipeline extensions_preferences',
|
||||
field_aliases: typing.Mapping[str, str] | None = None,
|
||||
) -> dict[str, typing.Any]:
|
||||
"""Reject extension preferences that cannot be consumed unambiguously."""
|
||||
if not isinstance(value, dict):
|
||||
raise ValueError(f'{context} must be an object')
|
||||
|
||||
for field_name in _BOOLEAN_FIELDS:
|
||||
if field_name in value and not isinstance(value[field_name], bool):
|
||||
raise ValueError(f"{context} field '{field_name}' must be a boolean")
|
||||
|
||||
aliases = field_aliases or {}
|
||||
_validate_list_field(
|
||||
value,
|
||||
'plugins',
|
||||
aliases.get('plugins', 'plugins'),
|
||||
_valid_plugin_binding,
|
||||
'a plugin object with non-empty author and name',
|
||||
context,
|
||||
)
|
||||
_validate_list_field(
|
||||
value,
|
||||
'mcp_servers',
|
||||
aliases.get('mcp_servers', 'mcp_servers'),
|
||||
_valid_name,
|
||||
'a non-empty string',
|
||||
context,
|
||||
)
|
||||
_validate_list_field(
|
||||
value,
|
||||
'skills',
|
||||
aliases.get('skills', 'skills'),
|
||||
_valid_name,
|
||||
'a non-empty string',
|
||||
context,
|
||||
)
|
||||
_validate_list_field(
|
||||
value,
|
||||
'mcp_resources',
|
||||
aliases.get('mcp_resources', 'mcp_resources'),
|
||||
lambda item: isinstance(item, dict),
|
||||
'an object',
|
||||
context,
|
||||
)
|
||||
return value
|
||||
|
||||
|
||||
def _validate_list_field(
|
||||
value: dict[str, typing.Any],
|
||||
field_name: str,
|
||||
field_label: str,
|
||||
item_validator: typing.Callable[[typing.Any], bool],
|
||||
item_description: str,
|
||||
context: str,
|
||||
) -> None:
|
||||
if field_name not in value:
|
||||
return
|
||||
items = value[field_name]
|
||||
if not isinstance(items, list):
|
||||
raise ValueError(f"{context} field '{field_label}' must be a list")
|
||||
for index, item in enumerate(items):
|
||||
if not item_validator(item):
|
||||
raise ValueError(
|
||||
f"{context} field '{field_label}[{index}]' must be {item_description}"
|
||||
)
|
||||
@@ -14,7 +14,8 @@ import langbot_plugin.api.entities.builtin.platform.events as platform_events
|
||||
import langbot_plugin.api.entities.events as events
|
||||
from ..utils import importutil
|
||||
from .config_coercion import coerce_pipeline_config
|
||||
from ..agent.runner.config_migration import ConfigMigration
|
||||
from ..agent.runner.config_resolver import RunnerConfigResolver
|
||||
from .extension_preferences import normalize_extension_preferences
|
||||
|
||||
import langbot_plugin.api.entities.builtin.provider.session as provider_session
|
||||
import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
|
||||
@@ -92,14 +93,16 @@ class RuntimePipeline:
|
||||
self.stage_containers = stage_containers
|
||||
|
||||
# Extract bound plugins and MCP servers from extensions_preferences
|
||||
extensions_prefs = pipeline_entity.extensions_preferences or {}
|
||||
self.enable_all_plugins = extensions_prefs.get('enable_all_plugins', True)
|
||||
self.enable_all_mcp_servers = extensions_prefs.get('enable_all_mcp_servers', True)
|
||||
extensions_prefs = normalize_extension_preferences(pipeline_entity.extensions_preferences)
|
||||
self.enable_all_plugins = extensions_prefs.get('enable_all_plugins', True) is True
|
||||
self.enable_all_mcp_servers = extensions_prefs.get('enable_all_mcp_servers', True) is True
|
||||
pipeline_config = pipeline_entity.config or {}
|
||||
runner_config: dict[str, typing.Any] = {}
|
||||
runner_id = ConfigMigration.resolve_runner_id(pipeline_config) if isinstance(pipeline_config, dict) else None
|
||||
runner_id = (
|
||||
RunnerConfigResolver.resolve_runner_id(pipeline_config) if isinstance(pipeline_config, dict) else None
|
||||
)
|
||||
if runner_id:
|
||||
resolved_runner_config = ConfigMigration.resolve_runner_config(pipeline_config, runner_id)
|
||||
resolved_runner_config = RunnerConfigResolver.resolve_runner_config(pipeline_config, runner_id)
|
||||
if isinstance(resolved_runner_config, dict):
|
||||
runner_config = resolved_runner_config
|
||||
|
||||
@@ -107,24 +110,26 @@ class RuntimePipeline:
|
||||
'mcp-resources',
|
||||
extensions_prefs.get('mcp_resources', []),
|
||||
)
|
||||
self.mcp_resource_agent_read_enabled = runner_config.get(
|
||||
'mcp-resource-agent-read-enabled',
|
||||
extensions_prefs.get('mcp_resource_agent_read_enabled', True),
|
||||
self.mcp_resource_agent_read_enabled = (
|
||||
runner_config.get(
|
||||
'mcp-resource-agent-read-enabled',
|
||||
extensions_prefs.get('mcp_resource_agent_read_enabled', True),
|
||||
)
|
||||
is True
|
||||
)
|
||||
|
||||
if self.enable_all_plugins:
|
||||
# None indicates to use all available plugins
|
||||
self.bound_plugins = None
|
||||
else:
|
||||
plugin_list = extensions_prefs.get('plugins', [])
|
||||
self.bound_plugins = [f'{p["author"]}/{p["name"]}' for p in plugin_list] if plugin_list else []
|
||||
plugin_list = extensions_prefs['plugins']
|
||||
self.bound_plugins = [f'{p["author"]}/{p["name"]}' for p in plugin_list]
|
||||
|
||||
if self.enable_all_mcp_servers:
|
||||
# None indicates to use all available MCP servers
|
||||
self.bound_mcp_servers = None
|
||||
else:
|
||||
mcp_server_list = extensions_prefs.get('mcp_servers', [])
|
||||
self.bound_mcp_servers = mcp_server_list if mcp_server_list else []
|
||||
self.bound_mcp_servers = extensions_prefs['mcp_servers']
|
||||
|
||||
async def run(self, query: pipeline_query.Query):
|
||||
query.pipeline_config = self.pipeline_entity.config
|
||||
@@ -296,9 +301,9 @@ class RuntimePipeline:
|
||||
# Get runner name from pipeline config
|
||||
runner_name = None
|
||||
if query.pipeline_config:
|
||||
from ..agent.runner.config_migration import ConfigMigration
|
||||
from ..agent.runner.config_resolver import RunnerConfigResolver
|
||||
|
||||
runner_name = ConfigMigration.resolve_runner_id(query.pipeline_config)
|
||||
runner_name = RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
|
||||
|
||||
# Record query start and store message_id
|
||||
message_id = ''
|
||||
|
||||
@@ -11,8 +11,11 @@ import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
|
||||
import langbot_plugin.api.entities.builtin.platform.events as platform_events
|
||||
|
||||
from ...agent.runner.descriptor import AgentRunnerDescriptor
|
||||
from ...agent.runner.config_migration import ConfigMigration
|
||||
from ...agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ...agent.runner import config_schema
|
||||
from ...agent.runner.resource_policy import ResourcePolicyProjector
|
||||
from ...provider.tools.toolmgr import ToolManager
|
||||
from ..extension_preferences import normalize_extension_preferences
|
||||
|
||||
|
||||
DEFAULT_PROMPT_CONFIG = [
|
||||
@@ -77,6 +80,28 @@ class PreProcessor(stage.PipelineStage):
|
||||
self.ap.logger.warning(f'Fallback model {fallback_uuid} not found, skipping')
|
||||
return valid_fallbacks
|
||||
|
||||
async def _resolve_host_tools(
|
||||
self,
|
||||
query: pipeline_query.Query,
|
||||
runner_config: dict[str, typing.Any],
|
||||
bound_plugins: list[str] | None,
|
||||
bound_mcp_servers: list[str] | None,
|
||||
include_mcp_resource_tools: bool,
|
||||
) -> list[typing.Any]:
|
||||
catalog = await self.ap.tool_mgr.get_resolved_tool_catalog(
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_skill_authoring=True,
|
||||
include_mcp_resource_tools=include_mcp_resource_tools,
|
||||
)
|
||||
policy = ResourcePolicyProjector.from_runner_config(
|
||||
runner_config,
|
||||
resolved_tool_names=ResourcePolicyProjector.extract_tool_names(catalog),
|
||||
)
|
||||
catalog = ResourcePolicyProjector.filter_tools(catalog, policy)
|
||||
ToolManager.bind_query_tool_sources(query, catalog)
|
||||
return ToolManager.tools_from_catalog(catalog)
|
||||
|
||||
def _runner_accepts_multimodal_input(self, descriptor: AgentRunnerDescriptor | None) -> bool:
|
||||
if descriptor is None:
|
||||
return True
|
||||
@@ -136,9 +161,7 @@ class PreProcessor(stage.PipelineStage):
|
||||
strict_thread=True,
|
||||
)
|
||||
except Exception as e:
|
||||
self.ap.logger.warning(
|
||||
f'Unable to load Transcript history view for conversation {conversation_uuid}: {e}'
|
||||
)
|
||||
self.ap.logger.warning(f'Unable to load Transcript history view for conversation {conversation_uuid}: {e}')
|
||||
return None
|
||||
|
||||
return messages or None
|
||||
@@ -168,14 +191,16 @@ class PreProcessor(stage.PipelineStage):
|
||||
) -> entities.StageProcessResult:
|
||||
"""Process"""
|
||||
# Resolve runner ID from the current ai.runner.id shape.
|
||||
runner_id = ConfigMigration.resolve_runner_id(query.pipeline_config)
|
||||
runner_id = RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
|
||||
|
||||
# Get runner config from ai.runner_config[runner_id].
|
||||
runner_config = ConfigMigration.resolve_runner_config(query.pipeline_config, runner_id) if runner_id else {}
|
||||
runner_config = (
|
||||
RunnerConfigResolver.resolve_runner_config(query.pipeline_config, runner_id) if runner_id else {}
|
||||
)
|
||||
query.variables = query.variables or {}
|
||||
bound_plugins = query.variables.get('_pipeline_bound_plugins', None)
|
||||
bound_mcp_servers = query.variables.get('_pipeline_bound_mcp_servers', None)
|
||||
include_mcp_resource_tools = query.variables.get('_pipeline_mcp_resource_agent_read_enabled', True)
|
||||
include_mcp_resource_tools = query.variables.get('_pipeline_mcp_resource_agent_read_enabled', True) is True
|
||||
descriptor = await self._get_runner_descriptor(runner_id, bound_plugins)
|
||||
|
||||
session = await self.ap.sess_mgr.get_session(query)
|
||||
@@ -204,7 +229,7 @@ class PreProcessor(stage.PipelineStage):
|
||||
# been idle for longer than the configured conversation expire time.
|
||||
# The idle window is measured from the last preprocess/update time, not
|
||||
# from the conversation creation time.
|
||||
conversation_expire_time = ConfigMigration.get_expire_time(query.pipeline_config)
|
||||
conversation_expire_time = RunnerConfigResolver.get_expire_time(query.pipeline_config)
|
||||
now = datetime.datetime.now()
|
||||
if conversation_expire_time is not None and conversation_expire_time > 0:
|
||||
last_update_time = getattr(conversation, 'update_time', None) or getattr(conversation, 'create_time', None)
|
||||
@@ -236,10 +261,12 @@ class PreProcessor(stage.PipelineStage):
|
||||
query.use_llm_model_uuid = llm_model.model_entity.uuid
|
||||
|
||||
if uses_host_tools and 'func_call' in (llm_model.model_entity.abilities or []):
|
||||
query.use_funcs = await self.ap.tool_mgr.get_all_tools(
|
||||
query.use_funcs = await self._resolve_host_tools(
|
||||
query,
|
||||
runner_config,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_mcp_resource_tools=include_mcp_resource_tools,
|
||||
include_mcp_resource_tools,
|
||||
)
|
||||
|
||||
self.ap.logger.debug(f'Bound plugins: {bound_plugins}')
|
||||
@@ -249,16 +276,20 @@ class PreProcessor(stage.PipelineStage):
|
||||
# If primary model doesn't support func_call but fallback models exist,
|
||||
# load tools anyway since fallback models may support them
|
||||
if uses_host_tools and not query.use_funcs and query.variables.get('_fallback_model_uuids'):
|
||||
query.use_funcs = await self.ap.tool_mgr.get_all_tools(
|
||||
query.use_funcs = await self._resolve_host_tools(
|
||||
query,
|
||||
runner_config,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_mcp_resource_tools=include_mcp_resource_tools,
|
||||
include_mcp_resource_tools,
|
||||
)
|
||||
elif uses_host_tools:
|
||||
query.use_funcs = await self.ap.tool_mgr.get_all_tools(
|
||||
query.use_funcs = await self._resolve_host_tools(
|
||||
query,
|
||||
runner_config,
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_mcp_resource_tools=include_mcp_resource_tools,
|
||||
include_mcp_resource_tools,
|
||||
)
|
||||
|
||||
self.ap.logger.debug(f'Bound plugins: {bound_plugins}')
|
||||
@@ -372,13 +403,13 @@ class PreProcessor(stage.PipelineStage):
|
||||
|
||||
if getattr(self.ap, 'skill_mgr', None) is not None:
|
||||
pipeline_data = await self.ap.pipeline_service.get_pipeline(query.pipeline_uuid)
|
||||
extensions_prefs = (pipeline_data or {}).get('extensions_preferences', {})
|
||||
enable_all_skills = extensions_prefs.get('enable_all_skills', True)
|
||||
extensions_prefs = normalize_extension_preferences((pipeline_data or {}).get('extensions_preferences'))
|
||||
enable_all_skills = extensions_prefs['enable_all_skills']
|
||||
|
||||
if enable_all_skills:
|
||||
bound_skills = None # None = all loaded skills are visible
|
||||
else:
|
||||
bound_skills = extensions_prefs.get('skills', [])
|
||||
bound_skills = extensions_prefs['skills']
|
||||
|
||||
query.variables['_pipeline_bound_skills'] = bound_skills
|
||||
|
||||
|
||||
@@ -12,7 +12,7 @@ from ... import entities
|
||||
from ... import plugin_diagnostics
|
||||
|
||||
import langbot_plugin.api.entities.events as events
|
||||
from ....agent.runner.config_migration import ConfigMigration
|
||||
from ....agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ....agent.runner import config_schema
|
||||
from ....utils import constants, runner as runner_utils
|
||||
from ....telemetry import features as telemetry_features
|
||||
@@ -312,8 +312,10 @@ class ChatMessageHandler(handler.MessageHandler):
|
||||
if prompt_config:
|
||||
return prompt_config
|
||||
|
||||
runner_id = ConfigMigration.resolve_runner_id(query.pipeline_config)
|
||||
runner_config = ConfigMigration.resolve_runner_config(query.pipeline_config, runner_id) if runner_id else {}
|
||||
runner_id = RunnerConfigResolver.resolve_runner_id(query.pipeline_config)
|
||||
runner_config = (
|
||||
RunnerConfigResolver.resolve_runner_config(query.pipeline_config, runner_id) if runner_id else {}
|
||||
)
|
||||
bound_plugins = query.variables.get('_pipeline_bound_plugins', None)
|
||||
descriptor = await self._get_runner_descriptor(runner_id, bound_plugins)
|
||||
return config_schema.extract_prompt_config(descriptor, runner_config, DEFAULT_PROMPT_CONFIG)
|
||||
|
||||
@@ -15,14 +15,15 @@ from ..discover import engine
|
||||
|
||||
from ..entity.persistence import bot as persistence_bot
|
||||
from ..entity.errors import platform as platform_errors
|
||||
from ..agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ..agent.runner.host_models import (
|
||||
AgentBinding,
|
||||
AgentEventEnvelope,
|
||||
BindingScope,
|
||||
DeliveryPolicy,
|
||||
ResourcePolicy,
|
||||
StatePolicy,
|
||||
)
|
||||
from ..agent.runner.resource_policy import ResourcePolicyProjector
|
||||
|
||||
from .logger import EventLogger
|
||||
|
||||
@@ -52,6 +53,8 @@ class SyntheticRouteTestAdapter:
|
||||
'create_message_card',
|
||||
'edit_message',
|
||||
'delete_message',
|
||||
'add_reaction',
|
||||
'remove_reaction',
|
||||
'forward_message',
|
||||
'set_group_name',
|
||||
'mute_member',
|
||||
@@ -985,6 +988,7 @@ class RuntimeBot:
|
||||
getattr(event, 'message_id', None) or getattr(event, 'feedback_id', None) or f'{event_type}:{uuid.uuid4()}'
|
||||
)
|
||||
target_type, target_id, target_metadata = self._infer_reply_target(event)
|
||||
supported_apis = self._get_adapter_supported_apis(adapter)
|
||||
|
||||
conversation_id = None
|
||||
if target_type and target_id:
|
||||
@@ -1014,17 +1018,33 @@ class RuntimeBot:
|
||||
**target_metadata,
|
||||
},
|
||||
supports_streaming=False,
|
||||
supports_edit=False,
|
||||
supports_reaction=False,
|
||||
supports_edit='edit_message' in supported_apis,
|
||||
supports_reaction=bool({'add_reaction', 'remove_reaction'} & set(supported_apis)),
|
||||
platform_capabilities={
|
||||
'adapter': adapter.__class__.__name__,
|
||||
'event_type': event_type,
|
||||
'supported_apis': supported_apis,
|
||||
},
|
||||
),
|
||||
raw_ref=RawEventRef(ref_id=str(event_id), storage_key=None),
|
||||
data=self._compact_event_data(event),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _get_adapter_supported_apis(
|
||||
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
|
||||
) -> list[str]:
|
||||
get_supported_apis = getattr(adapter, 'get_supported_apis', None)
|
||||
if not callable(get_supported_apis):
|
||||
return []
|
||||
try:
|
||||
declared_apis = get_supported_apis()
|
||||
except Exception:
|
||||
return []
|
||||
if not isinstance(declared_apis, (list, tuple, set)):
|
||||
return []
|
||||
return list(dict.fromkeys(api_name for api_name in declared_apis if isinstance(api_name, str) and api_name))
|
||||
|
||||
@staticmethod
|
||||
def _agent_product_to_binding(
|
||||
agent: dict[str, typing.Any],
|
||||
@@ -1033,29 +1053,20 @@ class RuntimeBot:
|
||||
bot_uuid: str,
|
||||
) -> AgentBinding | None:
|
||||
config = agent.get('config') if isinstance(agent, dict) else None
|
||||
if not isinstance(config, dict):
|
||||
if config is None:
|
||||
return None
|
||||
|
||||
runner = config.get('runner')
|
||||
runner_id = None
|
||||
if isinstance(runner, dict):
|
||||
runner_id = runner.get('id')
|
||||
runner_id = runner_id or agent.get('component_ref')
|
||||
_, runner_id, runner_config = RunnerConfigResolver.resolve_agent_runner_config(config)
|
||||
if not runner_id:
|
||||
return None
|
||||
|
||||
runner_config_map = config.get('runner_config')
|
||||
runner_config = {}
|
||||
if isinstance(runner_config_map, dict):
|
||||
runner_config = runner_config_map.get(runner_id) or {}
|
||||
|
||||
return AgentBinding(
|
||||
binding_id=f'bot:{bot_uuid}:{event_binding.get("id") or uuid.uuid4()}',
|
||||
scope=BindingScope(scope_type='bot', scope_id=bot_uuid),
|
||||
event_types=[event_type],
|
||||
runner_id=runner_id,
|
||||
runner_config=runner_config,
|
||||
resource_policy=ResourcePolicy(),
|
||||
resource_policy=ResourcePolicyProjector.from_runner_config(runner_config),
|
||||
state_policy=StatePolicy(state_scopes=['conversation', 'actor', 'subject', 'runner']),
|
||||
delivery_policy=DeliveryPolicy(enable_streaming=False, enable_reply=True),
|
||||
enabled=True,
|
||||
@@ -1262,7 +1273,20 @@ class RuntimeBot:
|
||||
text=f'EBA event {event_type} target agent does not support this event: {target_uuid}',
|
||||
)
|
||||
|
||||
binding = self._agent_product_to_binding(agent, event_binding, event_type, self.bot_entity.uuid)
|
||||
try:
|
||||
binding = self._agent_product_to_binding(agent, event_binding, event_type, self.bot_entity.uuid)
|
||||
except Exception:
|
||||
return await self._record_event_route_trace(
|
||||
event_type=event_type,
|
||||
status='failed',
|
||||
level='error',
|
||||
binding=event_binding,
|
||||
target_type=target_type,
|
||||
target_uuid=target_uuid,
|
||||
failure_code='runner_failed',
|
||||
reason='Agent configuration is invalid',
|
||||
text=f'Failed to build Agent binding for EBA event {event_type}: {traceback.format_exc()}',
|
||||
)
|
||||
if binding is None:
|
||||
return await self._record_event_route_trace(
|
||||
event_type=event_type,
|
||||
|
||||
@@ -27,7 +27,7 @@ from ..provider.modelmgr import requester as model_requester
|
||||
from ..core import app
|
||||
from ..utils import constants
|
||||
from ..agent.runner.session_registry import get_session_registry
|
||||
from ..agent.runner.config_migration import ConfigMigration
|
||||
from ..agent.runner.config_resolver import RunnerConfigResolver
|
||||
from ..agent.runner import config_schema
|
||||
|
||||
|
||||
@@ -37,6 +37,29 @@ from .agent_run_support import (
|
||||
)
|
||||
|
||||
|
||||
_HOST_RESERVED_QUERY_VAR_PREFIXES = (
|
||||
'_host_',
|
||||
'_pipeline_bound_',
|
||||
'_pipeline_mcp_',
|
||||
'_monitoring_',
|
||||
'_sandbox_',
|
||||
'_authorized',
|
||||
'_permission',
|
||||
)
|
||||
_HOST_RESERVED_QUERY_VAR_KEYS = frozenset(
|
||||
{
|
||||
'_activated_skills',
|
||||
'_fallback_model_uuids',
|
||||
'_routed_by_rule',
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _is_host_reserved_query_var(key: str) -> bool:
|
||||
"""Return whether a Query variable controls Host authorization or runtime state."""
|
||||
return key in _HOST_RESERVED_QUERY_VAR_KEYS or key.startswith(_HOST_RESERVED_QUERY_VAR_PREFIXES)
|
||||
|
||||
|
||||
class _RawAction:
|
||||
def __init__(self, value: str):
|
||||
self.value = value
|
||||
@@ -105,11 +128,11 @@ async def _get_pipeline_knowledge_base_uuids(ap: app.Application, query: Any) ->
|
||||
if not pipeline_config:
|
||||
return []
|
||||
|
||||
runner_id = ConfigMigration.resolve_runner_id(pipeline_config)
|
||||
runner_id = RunnerConfigResolver.resolve_runner_id(pipeline_config)
|
||||
if not runner_id:
|
||||
return []
|
||||
|
||||
runner_config = ConfigMigration.resolve_runner_config(pipeline_config, runner_id)
|
||||
runner_config = RunnerConfigResolver.resolve_runner_config(pipeline_config, runner_id)
|
||||
registry = getattr(ap, 'agent_runner_registry', None)
|
||||
if registry is None:
|
||||
return []
|
||||
@@ -189,6 +212,60 @@ async def _validate_run_authorization(
|
||||
return session, None
|
||||
|
||||
|
||||
def _validate_frozen_tool_source_identity(
|
||||
session: Any,
|
||||
tool_name: str,
|
||||
ap: app.Application,
|
||||
) -> Union[tuple[None, handler.ActionResponse], tuple[dict[str, str | None], None]]:
|
||||
"""Resolve the exact Host tool implementation frozen for this run."""
|
||||
authorization = session.get('authorization')
|
||||
resources = authorization.get('resources') if isinstance(authorization, dict) else None
|
||||
tools = resources.get('tools') if isinstance(resources, dict) else None
|
||||
matching_tools = (
|
||||
[tool for tool in tools if isinstance(tool, dict) and tool.get('tool_name') == tool_name]
|
||||
if isinstance(tools, list)
|
||||
else []
|
||||
)
|
||||
|
||||
source_ref: dict[str, str | None] | None = None
|
||||
if len(matching_tools) == 1:
|
||||
tool = matching_tools[0]
|
||||
source = tool.get('source')
|
||||
source_id = tool.get('source_id')
|
||||
has_complete_shape = (
|
||||
'source' in tool
|
||||
and 'source_id' in tool
|
||||
and isinstance(source, str)
|
||||
and bool(source)
|
||||
and source == source.strip()
|
||||
and (
|
||||
source_id is None or (isinstance(source_id, str) and bool(source_id) and source_id == source_id.strip())
|
||||
)
|
||||
)
|
||||
if has_complete_shape:
|
||||
if source in {'builtin', 'native', 'skill'} and source_id is None:
|
||||
source_ref = {'source': source, 'source_id': None}
|
||||
elif source == 'plugin' and isinstance(source_id, str):
|
||||
source_ref = {'source': source, 'source_id': source_id}
|
||||
elif source == 'mcp':
|
||||
from ..provider.tools.loaders.mcp import MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE
|
||||
|
||||
if isinstance(source_id, str) or tool_name in {
|
||||
MCP_TOOL_LIST_RESOURCES,
|
||||
MCP_TOOL_READ_RESOURCE,
|
||||
}:
|
||||
source_ref = {'source': source, 'source_id': source_id}
|
||||
|
||||
if source_ref is not None:
|
||||
return source_ref, None
|
||||
|
||||
run_id = session.get('run_id')
|
||||
ap.logger.warning(f'TOOL: {tool_name} has an invalid frozen source identity for run_id {run_id}')
|
||||
return None, handler.ActionResponse.error(
|
||||
message=f'Tool {tool_name} has an invalid frozen source identity for this agent run',
|
||||
)
|
||||
|
||||
|
||||
def _get_cached_query(ap: app.Application, query_id: int | None) -> Any | None:
|
||||
"""Return a cached Query for query-based runtime actions when available."""
|
||||
if query_id is None:
|
||||
@@ -208,6 +285,8 @@ def _resolve_action_query(data: dict[str, Any], session: Any | None, ap: app.App
|
||||
if query_id is None:
|
||||
query_id = data.get('query_id')
|
||||
query = _get_cached_query(ap, query_id)
|
||||
if query is None and session is not None:
|
||||
query = session.get('execution_query')
|
||||
if query is not None and session is not None:
|
||||
object.__setattr__(query, '_agent_run_session', session)
|
||||
return query
|
||||
@@ -393,6 +472,16 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
|
||||
query = self.ap.query_pool.cached_queries[query_id]
|
||||
|
||||
if not isinstance(key, str) or not key:
|
||||
return handler.ActionResponse.error(
|
||||
message='Query variable key must be a non-empty string',
|
||||
)
|
||||
if _is_host_reserved_query_var(key):
|
||||
self.ap.logger.warning(f'Plugin attempted to write Host-reserved Query variable {key!r}')
|
||||
return handler.ActionResponse.error(
|
||||
message=f'Query variable {key!r} is reserved for LangBot Host',
|
||||
)
|
||||
|
||||
query.variables[key] = value
|
||||
|
||||
return handler.ActionResponse.success(
|
||||
@@ -760,6 +849,7 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
run_id = data.get('run_id') # Optional: present for AgentRunner calls
|
||||
caller_plugin_identity = data.get('caller_plugin_identity') # Optional: for cross-plugin validation
|
||||
session = None
|
||||
source_ref = None
|
||||
is_agent_runner_call = bool(run_id)
|
||||
|
||||
if is_agent_runner_call:
|
||||
@@ -778,16 +868,24 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
)
|
||||
if error:
|
||||
return error
|
||||
source_ref, error = _validate_frozen_tool_source_identity(session, tool_name, self.ap)
|
||||
if error:
|
||||
return error
|
||||
|
||||
# Convert session_data to Session object (simplified)
|
||||
# In real implementation, you would reconstruct the full session
|
||||
# For now, we'll call the tool manager's execute method
|
||||
try:
|
||||
query = _resolve_action_query(data, session, self.ap)
|
||||
execute_kwargs: dict[str, Any] = {
|
||||
'name': tool_name,
|
||||
'parameters': parameters,
|
||||
'query': query,
|
||||
}
|
||||
if source_ref is not None:
|
||||
execute_kwargs['source_ref'] = source_ref
|
||||
result = await self.ap.tool_mgr.execute_func_call(
|
||||
name=tool_name,
|
||||
parameters=parameters,
|
||||
query=query,
|
||||
**execute_kwargs,
|
||||
)
|
||||
if is_agent_runner_call:
|
||||
return handler.ActionResponse.success(data={'result': result})
|
||||
@@ -810,6 +908,8 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
tool_name = data['tool_name']
|
||||
run_id = data.get('run_id') # Optional: present for AgentRunner calls
|
||||
caller_plugin_identity = data.get('caller_plugin_identity') # Optional: for cross-plugin validation
|
||||
session = None
|
||||
source_ref = None
|
||||
|
||||
# Permission validation for AgentRunner calls
|
||||
if run_id:
|
||||
@@ -818,9 +918,15 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
)
|
||||
if error:
|
||||
return error
|
||||
source_ref, error = _validate_frozen_tool_source_identity(session, tool_name, self.ap)
|
||||
if error:
|
||||
return error
|
||||
|
||||
try:
|
||||
tool_detail = await self.ap.tool_mgr.get_tool_detail(tool_name)
|
||||
detail_kwargs: dict[str, Any] = {}
|
||||
if source_ref is not None:
|
||||
detail_kwargs['source_ref'] = source_ref
|
||||
tool_detail = await self.ap.tool_mgr.get_tool_detail(tool_name, **detail_kwargs)
|
||||
if tool_detail is None:
|
||||
return handler.ActionResponse.error(
|
||||
message=f'Tool {tool_name} not found',
|
||||
@@ -1310,7 +1416,7 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
|
||||
Note: This action has dual validation paths:
|
||||
- AgentRunner: uses session_registry for permission check
|
||||
- Regular plugin: uses ConfigMigration.resolve_runner_config for pipeline-level check
|
||||
- Regular plugin: uses RunnerConfigResolver.resolve_runner_config for pipeline-level check
|
||||
"""
|
||||
kb_id = data['kb_id']
|
||||
query_text = data['query_text']
|
||||
|
||||
@@ -4,3 +4,12 @@ class ToolNotFoundError(ValueError):
|
||||
def __init__(self, name: str):
|
||||
self.name = name
|
||||
super().__init__(f'Tool not found: {name}')
|
||||
|
||||
|
||||
class ToolExecutionDeniedError(PermissionError):
|
||||
"""Raised when a known tool is not allowed in the current execution context."""
|
||||
|
||||
def __init__(self, name: str, reason: str):
|
||||
self.name = name
|
||||
self.reason = reason
|
||||
super().__init__(f'Tool execution denied for {name}: {reason}')
|
||||
|
||||
@@ -22,6 +22,7 @@ from mcp.shared.exceptions import McpError
|
||||
from pydantic import AnyUrl
|
||||
|
||||
from .. import loader
|
||||
from ..errors import ToolExecutionDeniedError
|
||||
from ....core import app
|
||||
import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
|
||||
import langbot_plugin.api.entities.builtin.provider.message as provider_message
|
||||
@@ -1417,35 +1418,67 @@ class MCPLoader(loader.ToolLoader):
|
||||
|
||||
return items
|
||||
|
||||
async def has_tool(self, name: str) -> bool:
|
||||
def _session_by_source_id(self, source_id: str) -> RuntimeMCPSession | None:
|
||||
return next(
|
||||
(session for session in self.sessions.values() if session.server_uuid == source_id),
|
||||
None,
|
||||
)
|
||||
|
||||
async def has_tool(self, name: str, source_id: str | None = None) -> bool:
|
||||
"""检查工具是否存在"""
|
||||
if name in (MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE):
|
||||
if source_id is None and name in (MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE):
|
||||
return bool(self._eligible_resource_sessions_for_bound(None))
|
||||
for session in self.sessions.values():
|
||||
sessions = [self._session_by_source_id(source_id)] if source_id is not None else list(self.sessions.values())
|
||||
for session in sessions:
|
||||
if session is None:
|
||||
continue
|
||||
for function in session.get_tools():
|
||||
if function.name == name:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def get_tool(self, name: str) -> resource_tool.LLMTool | None:
|
||||
for session in self.sessions.values():
|
||||
async def get_tool(
|
||||
self,
|
||||
name: str,
|
||||
source_id: str | None = None,
|
||||
) -> resource_tool.LLMTool | None:
|
||||
if source_id is None and name in (MCP_TOOL_LIST_RESOURCES, MCP_TOOL_READ_RESOURCE):
|
||||
if not self._eligible_resource_sessions_for_bound(None):
|
||||
return None
|
||||
return next(
|
||||
(tool for tool in self._mcp_synthetic_resource_tools() if tool.name == name),
|
||||
None,
|
||||
)
|
||||
sessions = [self._session_by_source_id(source_id)] if source_id is not None else list(self.sessions.values())
|
||||
for session in sessions:
|
||||
if session is None:
|
||||
continue
|
||||
for function in session.get_tools():
|
||||
if function.name == name:
|
||||
return function
|
||||
return None
|
||||
|
||||
async def invoke_tool(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
|
||||
async def invoke_tool(
|
||||
self,
|
||||
name: str,
|
||||
parameters: dict,
|
||||
query: pipeline_query.Query,
|
||||
source_id: str | None = None,
|
||||
) -> typing.Any:
|
||||
"""执行工具调用"""
|
||||
if name == MCP_TOOL_LIST_RESOURCES:
|
||||
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is False:
|
||||
return [provider_message.ContentElement.from_text('Error: MCP resource agent reads are disabled.')]
|
||||
if source_id is None and name == MCP_TOOL_LIST_RESOURCES:
|
||||
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is not True:
|
||||
raise ToolExecutionDeniedError(name, 'MCP resource agent reads are disabled')
|
||||
return await self._invoke_mcp_list_resources(parameters, query)
|
||||
if name == MCP_TOOL_READ_RESOURCE:
|
||||
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is False:
|
||||
return [provider_message.ContentElement.from_text('Error: MCP resource agent reads are disabled.')]
|
||||
if source_id is None and name == MCP_TOOL_READ_RESOURCE:
|
||||
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is not True:
|
||||
raise ToolExecutionDeniedError(name, 'MCP resource agent reads are disabled')
|
||||
return await self._invoke_mcp_read_resource(parameters, query)
|
||||
|
||||
for session in self.sessions.values():
|
||||
sessions = [self._session_by_source_id(source_id)] if source_id is not None else list(self.sessions.values())
|
||||
for session in sessions:
|
||||
if session is None:
|
||||
continue
|
||||
for function in session.get_tools():
|
||||
if function.name == name:
|
||||
self.ap.logger.debug(f'Invoking MCP tool: {name} with parameters: {parameters}')
|
||||
@@ -1525,7 +1558,7 @@ class MCPLoader(loader.ToolLoader):
|
||||
default_max_bytes: int = MCP_RESOURCE_CONTEXT_MAX_BYTES,
|
||||
) -> str:
|
||||
"""Build host-controlled MCP resource context for the current query."""
|
||||
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is False:
|
||||
if getattr(query, 'variables', {}).get('_pipeline_mcp_resource_agent_read_enabled', True) is not True:
|
||||
return ''
|
||||
|
||||
attachments = (query.variables or {}).get('_pipeline_mcp_resource_attachments', [])
|
||||
@@ -1543,7 +1576,7 @@ class MCPLoader(loader.ToolLoader):
|
||||
for raw_attachment in attachments:
|
||||
if remaining_tokens <= 0:
|
||||
break
|
||||
if not isinstance(raw_attachment, dict) or raw_attachment.get('enabled') is False:
|
||||
if not isinstance(raw_attachment, dict) or raw_attachment.get('enabled', True) is not True:
|
||||
continue
|
||||
|
||||
attachment = raw_attachment.copy()
|
||||
@@ -1561,8 +1594,23 @@ class MCPLoader(loader.ToolLoader):
|
||||
if session.server_uuid not in eligible_by_uuid and session.server_name not in eligible_by_name:
|
||||
continue
|
||||
|
||||
max_tokens = min(int(attachment.get('max_tokens') or remaining_tokens), remaining_tokens)
|
||||
max_bytes = int(attachment.get('max_bytes') or default_max_bytes)
|
||||
raw_max_tokens = attachment.get('max_tokens')
|
||||
raw_max_bytes = attachment.get('max_bytes')
|
||||
if raw_max_tokens is None:
|
||||
max_tokens = remaining_tokens
|
||||
elif isinstance(raw_max_tokens, bool) or not isinstance(raw_max_tokens, int) or raw_max_tokens <= 0:
|
||||
self.ap.logger.warning(f'Ignoring MCP resource {uri!r}: max_tokens must be a positive integer')
|
||||
continue
|
||||
else:
|
||||
max_tokens = min(raw_max_tokens, remaining_tokens)
|
||||
|
||||
if raw_max_bytes is None:
|
||||
max_bytes = default_max_bytes
|
||||
elif isinstance(raw_max_bytes, bool) or not isinstance(raw_max_bytes, int) or raw_max_bytes <= 0:
|
||||
self.ap.logger.warning(f'Ignoring MCP resource {uri!r}: max_bytes must be a positive integer')
|
||||
continue
|
||||
else:
|
||||
max_bytes = raw_max_bytes
|
||||
|
||||
try:
|
||||
envelope = await session.read_resource_envelope(
|
||||
|
||||
@@ -50,17 +50,35 @@ class PluginToolLoader(loader.ToolLoader):
|
||||
|
||||
return catalog
|
||||
|
||||
async def has_tool(self, name: str) -> bool:
|
||||
async def get_tool(
|
||||
self,
|
||||
name: str,
|
||||
source_id: str | None = None,
|
||||
) -> resource_tool.LLMTool | None:
|
||||
tools = await self.get_tools([source_id] if source_id else None)
|
||||
return next((tool for tool in tools if tool.name == name), None)
|
||||
|
||||
async def has_tool(self, name: str, source_id: str | None = None) -> bool:
|
||||
"""检查工具是否存在"""
|
||||
for tool in await self.ap.plugin_connector.list_tools():
|
||||
for tool in await self.ap.plugin_connector.list_tools([source_id] if source_id else None):
|
||||
if tool.metadata.name == name:
|
||||
return True
|
||||
return False
|
||||
|
||||
async def invoke_tool(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
|
||||
async def invoke_tool(
|
||||
self,
|
||||
name: str,
|
||||
parameters: dict,
|
||||
query: pipeline_query.Query,
|
||||
source_id: str | None = None,
|
||||
) -> typing.Any:
|
||||
try:
|
||||
return await self.ap.plugin_connector.call_tool(
|
||||
name, parameters, session=query.session, query_id=query.query_id
|
||||
name,
|
||||
parameters,
|
||||
session=query.session,
|
||||
query_id=query.query_id,
|
||||
bound_plugins=[source_id] if source_id else None,
|
||||
)
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'执行函数 {name} 时发生错误: {e}')
|
||||
|
||||
@@ -134,7 +134,11 @@ async def persist_activated_skill(
|
||||
if not isinstance(session, dict):
|
||||
return
|
||||
|
||||
state_context = session.get('state_context')
|
||||
authorization = session.get('authorization')
|
||||
if not isinstance(authorization, dict):
|
||||
return
|
||||
|
||||
state_context = authorization.get('state_context')
|
||||
if not isinstance(state_context, dict):
|
||||
return
|
||||
|
||||
|
||||
@@ -20,6 +20,16 @@ if TYPE_CHECKING:
|
||||
)
|
||||
|
||||
|
||||
TOOL_SOURCE_REFS_QUERY_KEY = '_host_tool_source_refs'
|
||||
|
||||
|
||||
class ToolSourceRef(typing.TypedDict):
|
||||
"""Stable Host-side identity for one tool implementation."""
|
||||
|
||||
source: str
|
||||
source_id: str | None
|
||||
|
||||
|
||||
class ToolManager:
|
||||
"""LLM工具管理器"""
|
||||
|
||||
@@ -115,6 +125,121 @@ class ToolManager:
|
||||
|
||||
return catalog
|
||||
|
||||
async def get_resolved_tool_catalog(
|
||||
self,
|
||||
bound_plugins: list[str] | None = None,
|
||||
bound_mcp_servers: list[str] | None = None,
|
||||
include_skill_authoring: bool = True,
|
||||
include_mcp_resource_tools: bool = False,
|
||||
) -> list[dict[str, typing.Any]]:
|
||||
"""Return scoped tools with one unambiguous implementation per name.
|
||||
|
||||
LLM tool calls only carry a function name. If two implementations with
|
||||
the same name remain inside the current Host scope, choosing one by
|
||||
loader or registration order would authorize one resource and execute
|
||||
another. Such names are therefore omitted until the scope is narrowed.
|
||||
"""
|
||||
catalog = await self.get_tool_catalog(
|
||||
bound_plugins,
|
||||
bound_mcp_servers,
|
||||
include_skill_authoring=include_skill_authoring,
|
||||
include_mcp_resource_tools=include_mcp_resource_tools,
|
||||
)
|
||||
tools_by_name: dict[str, list[dict[str, typing.Any]]] = {}
|
||||
for item in catalog:
|
||||
name = item.get('name')
|
||||
if isinstance(name, str) and name:
|
||||
tools_by_name.setdefault(name, []).append(item)
|
||||
|
||||
resolved: list[dict[str, typing.Any]] = []
|
||||
for name, candidates in tools_by_name.items():
|
||||
implementations = {
|
||||
(str(item.get('source') or ''), self._normalize_source_id(item.get('source_id'))) for item in candidates
|
||||
}
|
||||
if len(implementations) != 1:
|
||||
self.ap.logger.warning(
|
||||
f'Tool {name} is hidden because multiple implementations are visible: '
|
||||
f'{sorted(implementations, key=lambda item: (item[0], item[1] or ""))}'
|
||||
)
|
||||
continue
|
||||
resolved.append(candidates[0])
|
||||
return resolved
|
||||
|
||||
@staticmethod
|
||||
def _normalize_source_id(source_id: typing.Any) -> str | None:
|
||||
return source_id if isinstance(source_id, str) and source_id else None
|
||||
|
||||
@classmethod
|
||||
def source_ref_from_catalog_item(cls, item: dict[str, typing.Any]) -> ToolSourceRef | None:
|
||||
source = item.get('source')
|
||||
if not isinstance(source, str) or not source:
|
||||
return None
|
||||
return {
|
||||
'source': source,
|
||||
'source_id': cls._normalize_source_id(item.get('source_id')),
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def source_refs_from_catalog(
|
||||
cls,
|
||||
catalog: typing.Iterable[dict[str, typing.Any]],
|
||||
) -> dict[str, ToolSourceRef]:
|
||||
refs: dict[str, ToolSourceRef] = {}
|
||||
for item in catalog:
|
||||
name = item.get('name')
|
||||
ref = cls.source_ref_from_catalog_item(item)
|
||||
if isinstance(name, str) and name and ref is not None:
|
||||
refs[name] = ref
|
||||
return refs
|
||||
|
||||
@staticmethod
|
||||
def tools_from_catalog(
|
||||
catalog: typing.Iterable[dict[str, typing.Any]],
|
||||
) -> list[resource_tool.LLMTool]:
|
||||
"""Materialize LLM schemas from an already authorized Host catalog."""
|
||||
return [
|
||||
resource_tool.LLMTool(
|
||||
name=item['name'],
|
||||
human_desc=item.get('human_desc') or item.get('description') or item['name'],
|
||||
description=item.get('description') or '',
|
||||
parameters=item.get('parameters') or {},
|
||||
func=lambda parameters: {},
|
||||
)
|
||||
for item in catalog
|
||||
]
|
||||
|
||||
@classmethod
|
||||
def bind_query_tool_sources(
|
||||
cls,
|
||||
query: pipeline_query.Query,
|
||||
catalog: typing.Iterable[dict[str, typing.Any]],
|
||||
) -> None:
|
||||
query.variables = query.variables or {}
|
||||
query.variables[TOOL_SOURCE_REFS_QUERY_KEY] = cls.source_refs_from_catalog(catalog)
|
||||
|
||||
@staticmethod
|
||||
def get_query_tool_source(
|
||||
query: pipeline_query.Query,
|
||||
name: str,
|
||||
) -> ToolSourceRef | None:
|
||||
variables = getattr(query, 'variables', None)
|
||||
if not isinstance(variables, dict):
|
||||
return None
|
||||
refs = variables.get(TOOL_SOURCE_REFS_QUERY_KEY)
|
||||
if not isinstance(refs, dict):
|
||||
return None
|
||||
ref = refs.get(name)
|
||||
if not isinstance(ref, dict):
|
||||
return None
|
||||
source = ref.get('source')
|
||||
if not isinstance(source, str) or not source:
|
||||
return None
|
||||
source_id = ref.get('source_id')
|
||||
return {
|
||||
'source': source,
|
||||
'source_id': source_id if isinstance(source_id, str) and source_id else None,
|
||||
}
|
||||
|
||||
async def get_tool_by_name(self, name: str) -> tool_loader.ToolLookupResult | None:
|
||||
"""Get tool by name from any active loader."""
|
||||
for active_loader in (
|
||||
@@ -129,7 +254,11 @@ class ToolManager:
|
||||
|
||||
return None
|
||||
|
||||
async def get_tool_schema(self, name: str) -> tuple[str | None, dict | None]:
|
||||
async def get_tool_schema(
|
||||
self,
|
||||
name: str,
|
||||
source_ref: ToolSourceRef | None = None,
|
||||
) -> tuple[str | None, dict | None]:
|
||||
"""Return (description, parameters JSON schema) for a tool by name.
|
||||
|
||||
Used by the host to prefill ToolResource so a runner can build LLM tool
|
||||
@@ -137,19 +266,23 @@ class ToolManager:
|
||||
return resource_tool.LLMTool, so no per-shape branching is needed.
|
||||
Returns (None, None) when the tool is not found.
|
||||
"""
|
||||
tool = await self.get_tool_by_name(name)
|
||||
tool = await self.get_tool_by_source(name, source_ref) if source_ref else await self.get_tool_by_name(name)
|
||||
if tool is None:
|
||||
return None, None
|
||||
return tool.description, (tool.parameters or None)
|
||||
|
||||
async def get_tool_detail(self, name: str) -> dict | None:
|
||||
async def get_tool_detail(
|
||||
self,
|
||||
name: str,
|
||||
source_ref: ToolSourceRef | None = None,
|
||||
) -> dict | None:
|
||||
"""Return the host-level tool detail shape for a tool by name.
|
||||
|
||||
All loaders return resource_tool.LLMTool, so the shape is uniform:
|
||||
{name, description, human_desc, parameters}. Returns None when the tool
|
||||
is not found.
|
||||
"""
|
||||
tool = await self.get_tool_by_name(name)
|
||||
tool = await self.get_tool_by_source(name, source_ref) if source_ref else await self.get_tool_by_name(name)
|
||||
if tool is None:
|
||||
return None
|
||||
return {
|
||||
@@ -159,6 +292,26 @@ class ToolManager:
|
||||
'parameters': tool.parameters or {},
|
||||
}
|
||||
|
||||
async def get_tool_by_source(
|
||||
self,
|
||||
name: str,
|
||||
source_ref: ToolSourceRef,
|
||||
) -> tool_loader.ToolLookupResult | None:
|
||||
"""Resolve a tool only from the implementation frozen at authorization."""
|
||||
source = source_ref['source']
|
||||
source_id = source_ref.get('source_id')
|
||||
if source in {'builtin', 'native'}:
|
||||
return await self.native_tool_loader.get_tool(name)
|
||||
if source == 'skill':
|
||||
return await self.skill_tool_loader.get_tool(name)
|
||||
if source == 'plugin':
|
||||
if not source_id:
|
||||
return None
|
||||
return await self.plugin_tool_loader.get_tool(name, source_id=source_id)
|
||||
if source == 'mcp':
|
||||
return await self.mcp_tool_loader.get_tool(name, source_id=source_id)
|
||||
return None
|
||||
|
||||
async def generate_tools_for_openai(self, use_funcs: list[resource_tool.LLMTool]) -> list:
|
||||
tools = []
|
||||
|
||||
@@ -260,9 +413,58 @@ class ToolManager:
|
||||
)
|
||||
return result
|
||||
|
||||
async def execute_func_call(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
|
||||
async def execute_func_call(
|
||||
self,
|
||||
name: str,
|
||||
parameters: dict,
|
||||
query: pipeline_query.Query,
|
||||
source_ref: ToolSourceRef | None = None,
|
||||
) -> typing.Any:
|
||||
from langbot.pkg.telemetry import features as telemetry_features
|
||||
|
||||
source_ref = source_ref or self.get_query_tool_source(query, name)
|
||||
if source_ref is not None:
|
||||
source = source_ref['source']
|
||||
source_id = source_ref.get('source_id')
|
||||
uses_source_id = False
|
||||
if source in {'builtin', 'native'}:
|
||||
loader = self.native_tool_loader
|
||||
telemetry_source = 'native'
|
||||
exists = await loader.has_tool(name)
|
||||
elif source == 'skill':
|
||||
loader = self.skill_tool_loader
|
||||
telemetry_source = 'skill'
|
||||
exists = await loader.has_tool(name)
|
||||
elif source == 'plugin' and source_id:
|
||||
loader = self.plugin_tool_loader
|
||||
telemetry_source = 'plugin'
|
||||
uses_source_id = True
|
||||
exists = await loader.has_tool(name, source_id=source_id)
|
||||
elif source == 'mcp':
|
||||
loader = self.mcp_tool_loader
|
||||
telemetry_source = 'mcp'
|
||||
uses_source_id = True
|
||||
exists = await loader.has_tool(name, source_id=source_id)
|
||||
else:
|
||||
raise ToolNotFoundError(name)
|
||||
|
||||
if not exists:
|
||||
raise ToolNotFoundError(name)
|
||||
|
||||
async def invoke_selected_tool() -> typing.Any:
|
||||
if uses_source_id:
|
||||
return await loader.invoke_tool(name, parameters, query, source_id=source_id)
|
||||
return await loader.invoke_tool(name, parameters, query)
|
||||
|
||||
telemetry_features.increment(query, 'tool_calls', telemetry_source)
|
||||
return await self._invoke_tool_with_monitoring(
|
||||
source=telemetry_source,
|
||||
name=name,
|
||||
parameters=parameters,
|
||||
query=query,
|
||||
invoke=invoke_selected_tool,
|
||||
)
|
||||
|
||||
if await self.native_tool_loader.has_tool(name):
|
||||
telemetry_features.increment(query, 'tool_calls', 'native')
|
||||
return await self._invoke_tool_with_monitoring(
|
||||
|
||||
@@ -2,15 +2,6 @@ import langbot
|
||||
|
||||
semantic_version = f'v{langbot.__version__}'
|
||||
|
||||
required_database_version = 25
|
||||
"""Tag the version of the legacy (3.x) database schema migration chain.
|
||||
|
||||
Frozen at 25: the legacy ``pkg/persistence/migrations`` system (DBMigration /
|
||||
dbmXXX_*.py) is deprecated and no longer accepts new migrations. All schema
|
||||
changes from here on are managed by Alembic (see
|
||||
``pkg/persistence/alembic/versions``). This value only gates the one-time
|
||||
upgrade of pre-existing 3.x databases up to the Alembic baseline."""
|
||||
|
||||
debug_mode = False
|
||||
|
||||
edition = 'community'
|
||||
|
||||
@@ -37,12 +37,6 @@ system:
|
||||
max_bots: -1
|
||||
max_pipelines: -1
|
||||
max_extensions: -1
|
||||
# When set to a non-empty string, every pipeline is forced to use this
|
||||
# Box sandbox-scope template regardless of its own configuration, and
|
||||
# the per-pipeline "Sandbox Scope" selector is locked in the web UI.
|
||||
# Used by SaaS deployments to confine a tenant to a single shared
|
||||
# sandbox (set to '{global}'). Empty string = no restriction.
|
||||
force_box_session_id_template: ''
|
||||
task_retention:
|
||||
# Keep at most this many completed async task records in memory
|
||||
completed_limit: 200
|
||||
|
||||
@@ -1,7 +0,0 @@
|
||||
{
|
||||
"privilege": {},
|
||||
"command-prefix": [
|
||||
"!",
|
||||
"!"
|
||||
]
|
||||
}
|
||||
@@ -1,38 +0,0 @@
|
||||
{
|
||||
"access-control":{
|
||||
"mode": "blacklist",
|
||||
"blacklist": [],
|
||||
"whitelist": []
|
||||
},
|
||||
"respond-rules": {
|
||||
"default": {
|
||||
"at": true,
|
||||
"prefix": [
|
||||
"/ai", "!ai", "!ai", "ai"
|
||||
],
|
||||
"regexp": [],
|
||||
"random": 0.0
|
||||
}
|
||||
},
|
||||
"income-msg-check": true,
|
||||
"ignore-rules": {
|
||||
"prefix": [],
|
||||
"regexp": []
|
||||
},
|
||||
"check-sensitive-words": true,
|
||||
"baidu-cloud-examine": {
|
||||
"enable": false,
|
||||
"api-key": "",
|
||||
"api-secret": ""
|
||||
},
|
||||
"rate-limit": {
|
||||
"strategy": "drop",
|
||||
"algo": "fixwin",
|
||||
"fixwin": {
|
||||
"default": {
|
||||
"window-size": 60,
|
||||
"limit": 60
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
{
|
||||
"platform-adapters": [
|
||||
{
|
||||
"adapter": "nakuru",
|
||||
"enable": false,
|
||||
"host": "127.0.0.1",
|
||||
"ws_port": 8080,
|
||||
"http_port": 5700,
|
||||
"token": ""
|
||||
},
|
||||
{
|
||||
"adapter": "aiocqhttp",
|
||||
"enable": true,
|
||||
"host": "0.0.0.0",
|
||||
"port": 2280,
|
||||
"access-token": ""
|
||||
},
|
||||
{
|
||||
"adapter": "qq-botpy",
|
||||
"enable": false,
|
||||
"appid": "",
|
||||
"secret": "",
|
||||
"intents": [
|
||||
"public_guild_messages",
|
||||
"direct_message"
|
||||
]
|
||||
},
|
||||
{
|
||||
"adapter": "qqofficial",
|
||||
"enable": false,
|
||||
"appid": "1234567890",
|
||||
"secret": "xxxxxxx",
|
||||
"port": 2284,
|
||||
"token": "abcdefg"
|
||||
},
|
||||
{
|
||||
"adapter": "wecom",
|
||||
"enable": false,
|
||||
"host": "0.0.0.0",
|
||||
"port": 2290,
|
||||
"corpid": "",
|
||||
"secret": "",
|
||||
"token": "",
|
||||
"EncodingAESKey": "",
|
||||
"contacts_secret": ""
|
||||
},
|
||||
{
|
||||
"adapter": "lark",
|
||||
"enable": false,
|
||||
"app_id": "cli_abcdefgh",
|
||||
"app_secret": "XXXXXXXXXX",
|
||||
"bot_name": "LangBot",
|
||||
"enable-webhook": false,
|
||||
"port": 2285,
|
||||
"encrypt-key": "xxxxxxxxx"
|
||||
},
|
||||
{
|
||||
"adapter": "discord",
|
||||
"enable": false,
|
||||
"client_id": "1234567890",
|
||||
"token": "XXXXXXXXXX"
|
||||
},
|
||||
{
|
||||
"adapter": "gewechat",
|
||||
"enable": false,
|
||||
"gewechat_url": "http://your-gewechat-server:2531",
|
||||
"gewechat_file_url": "http://your-gewechat-server:2532",
|
||||
"port": 2286,
|
||||
"callback_url": "http://your-callback-url:2286/gewechat/callback",
|
||||
"app_id": "",
|
||||
"token": ""
|
||||
},
|
||||
{
|
||||
"adapter": "officialaccount",
|
||||
"enable": false,
|
||||
"token": "",
|
||||
"EncodingAESKey": "",
|
||||
"AppID": "",
|
||||
"AppSecret": "",
|
||||
"Mode": "drop",
|
||||
"LoadingMessage": "AI正在思考中,请发送任意内容获取回复。",
|
||||
"host": "0.0.0.0",
|
||||
"port": 2287
|
||||
},
|
||||
{
|
||||
"adapter": "dingtalk",
|
||||
"enable": false,
|
||||
"client_id": "",
|
||||
"client_secret": "",
|
||||
"robot_code": "",
|
||||
"robot_name": "",
|
||||
"markdown_card": false
|
||||
},
|
||||
{
|
||||
"adapter": "telegram",
|
||||
"enable": false,
|
||||
"token": "",
|
||||
"markdown_card": false
|
||||
},
|
||||
{
|
||||
"adapter": "slack",
|
||||
"enable": false,
|
||||
"bot_token": "",
|
||||
"signing_secret": "",
|
||||
"port": 2288
|
||||
},
|
||||
{
|
||||
"adapter": "wecomcs",
|
||||
"enable": false,
|
||||
"port": 2289,
|
||||
"corpid": "",
|
||||
"secret": "",
|
||||
"token": "",
|
||||
"EncodingAESKey": ""
|
||||
}
|
||||
],
|
||||
"track-function-calls": true,
|
||||
"quote-origin": false,
|
||||
"at-sender": false,
|
||||
"force-delay": {
|
||||
"min": 0,
|
||||
"max": 0
|
||||
},
|
||||
"long-text-process": {
|
||||
"threshold": 2560,
|
||||
"strategy": "forward",
|
||||
"font-path": ""
|
||||
},
|
||||
"hide-exception-info": true
|
||||
}
|
||||
@@ -1,161 +0,0 @@
|
||||
{
|
||||
"enable-chat": true,
|
||||
"enable-vision": true,
|
||||
"keys": {
|
||||
"openai": [
|
||||
"sk-1234567890"
|
||||
],
|
||||
"anthropic": [
|
||||
"sk-1234567890"
|
||||
],
|
||||
"moonshot": [
|
||||
"sk-1234567890"
|
||||
],
|
||||
"deepseek": [
|
||||
"sk-1234567890"
|
||||
],
|
||||
"gitee-ai": [
|
||||
"XXXXX"
|
||||
],
|
||||
"xai": [
|
||||
"xai-1234567890"
|
||||
],
|
||||
"zhipuai": [
|
||||
"xxxxxxx"
|
||||
],
|
||||
"siliconflow": [
|
||||
"xxxxxxx"
|
||||
],
|
||||
"bailian": [
|
||||
"sk-xxxxxxx"
|
||||
],
|
||||
"volcark": [
|
||||
"xxxxxxxx"
|
||||
],
|
||||
"modelscope": [
|
||||
"xxxxxxxx"
|
||||
],
|
||||
"ppio": [
|
||||
"xxxxxxxx"
|
||||
]
|
||||
},
|
||||
"requester": {
|
||||
"openai-chat-completions": {
|
||||
"base-url": "https://api.openai.com/v1",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
},
|
||||
"anthropic-messages": {
|
||||
"base-url": "https://api.anthropic.com",
|
||||
"args": {
|
||||
"max_tokens": 1024
|
||||
},
|
||||
"timeout": 120
|
||||
},
|
||||
"moonshot-chat-completions": {
|
||||
"base-url": "https://api.moonshot.cn/v1",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
},
|
||||
"deepseek-chat-completions": {
|
||||
"base-url": "https://api.deepseek.com",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
},
|
||||
"ollama-chat": {
|
||||
"base-url": "http://127.0.0.1:11434",
|
||||
"args": {},
|
||||
"timeout": 600
|
||||
},
|
||||
"gitee-ai-chat-completions": {
|
||||
"base-url": "https://ai.gitee.com/v1",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
},
|
||||
"xai-chat-completions": {
|
||||
"base-url": "https://api.x.ai/v1",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
},
|
||||
"zhipuai-chat-completions": {
|
||||
"base-url": "https://open.bigmodel.cn/api/paas/v4",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
},
|
||||
"lmstudio-chat-completions": {
|
||||
"base-url": "http://127.0.0.1:1234/v1",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
},
|
||||
"siliconflow-chat-completions": {
|
||||
"base-url": "https://api.siliconflow.cn/v1",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
},
|
||||
"bailian-chat-completions": {
|
||||
"args": {},
|
||||
"base-url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
|
||||
"timeout": 120
|
||||
},
|
||||
"volcark-chat-completions": {
|
||||
"args": {},
|
||||
"base-url": "https://ark.cn-beijing.volces.com/api/v3",
|
||||
"timeout": 120
|
||||
},
|
||||
"modelscope-chat-completions": {
|
||||
"base-url": "https://api-inference.modelscope.cn/v1",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
},
|
||||
"ppio-chat-completions": {
|
||||
"base-url": "https://api.ppinfra.com/v3/openai",
|
||||
"args": {},
|
||||
"timeout": 120
|
||||
}
|
||||
},
|
||||
"model": "gpt-4o",
|
||||
"prompt-mode": "normal",
|
||||
"prompt": {
|
||||
"default": "You are a helpful assistant."
|
||||
},
|
||||
"runner": "local-agent",
|
||||
"dify-service-api": {
|
||||
"base-url": "https://api.dify.ai/v1",
|
||||
"app-type": "chat",
|
||||
"options": {
|
||||
"convert-thinking-tips": "plain"
|
||||
},
|
||||
"chat": {
|
||||
"api-key": "app-1234567890",
|
||||
"timeout": 120
|
||||
},
|
||||
"agent": {
|
||||
"api-key": "app-1234567890",
|
||||
"timeout": 120
|
||||
},
|
||||
"workflow": {
|
||||
"api-key": "app-1234567890",
|
||||
"output-key": "summary",
|
||||
"timeout": 120
|
||||
}
|
||||
},
|
||||
"dashscope-app-api": {
|
||||
"app-type": "agent",
|
||||
"api-key": "sk-1234567890",
|
||||
"agent": {
|
||||
"app-id": "Your_app_id",
|
||||
"references_quote": "参考资料来自:"
|
||||
},
|
||||
"workflow": {
|
||||
"app-id": "Your_app_id",
|
||||
"references_quote": "参考资料来自:",
|
||||
"biz_params": {
|
||||
"city": "北京",
|
||||
"date": "2023-08-10"
|
||||
}
|
||||
}
|
||||
},
|
||||
"mcp": {
|
||||
"servers": []
|
||||
}
|
||||
}
|
||||
@@ -1,27 +0,0 @@
|
||||
{
|
||||
"admin-sessions": [],
|
||||
"network-proxies": {
|
||||
"http": null,
|
||||
"https": null
|
||||
},
|
||||
"report-usage": true,
|
||||
"logging-level": "info",
|
||||
"session-concurrency": {
|
||||
"default": 1
|
||||
},
|
||||
"pipeline-concurrency": 20,
|
||||
"qcg-center-url": "https://api.qchatgpt.rockchin.top/api/v2",
|
||||
"help-message": "LangBot - 😎高稳定性、🧩支持插件、🌏实时联网的 ChatGPT QQ 机器人🤖\n链接:https://q.rkcn.top",
|
||||
"http-api": {
|
||||
"enable": true,
|
||||
"host": "0.0.0.0",
|
||||
"port": 5300,
|
||||
"jwt-expire": 604800
|
||||
},
|
||||
"persistence": {
|
||||
"sqlite": {
|
||||
"path": "data/langbot.db"
|
||||
},
|
||||
"use": "sqlite"
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user