mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-07-22 12:26:08 +00:00
feat(tenancy): harden shared cloud runtime boundaries
This commit is contained in:
@@ -83,6 +83,7 @@ class RouterGroup(abc.ABC):
|
||||
rule = self.path + rule
|
||||
|
||||
async def handler_error(*args, **kwargs):
|
||||
request_context: RequestContext | None = None
|
||||
if auth_type == AuthType.ACCOUNT_TOKEN:
|
||||
authorization = quart.request.headers.get('Authorization', '')
|
||||
if not authorization.startswith('Bearer '):
|
||||
@@ -183,6 +184,17 @@ class RouterGroup(abc.ABC):
|
||||
return self._auth_error_response(e)
|
||||
|
||||
try:
|
||||
if request_context is not None:
|
||||
persistence_mgr = getattr(self.ap, 'persistence_mgr', None)
|
||||
tenant_scope_descriptor = getattr(type(persistence_mgr), 'tenant_scope', None)
|
||||
if callable(tenant_scope_descriptor):
|
||||
# Authorization discovery is complete. Carry the
|
||||
# trusted Workspace identity across the handler, but
|
||||
# do not reserve a database connection while it waits
|
||||
# on providers, runtimes, uploads, or streamed clients.
|
||||
# Services that need atomic writes open a short UoW.
|
||||
async with persistence_mgr.tenant_scope(request_context.workspace_uuid):
|
||||
return await f(*args, **kwargs)
|
||||
return await f(*args, **kwargs)
|
||||
|
||||
except Exception as e: # 自动 500
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from langbot.pkg.utils import constants
|
||||
from langbot_plugin.box.errors import BoxAdmissionError
|
||||
|
||||
from langbot.pkg.cloud.entitlements import EntitlementUnavailableError
|
||||
from ...authz import Permission
|
||||
from ...context import RequestContext
|
||||
from .. import group
|
||||
@@ -18,7 +20,10 @@ class BoxRouterGroup(group.RouterGroup):
|
||||
permission=Permission.RESOURCE_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
status = await self.ap.box_service.get_status(request_context)
|
||||
try:
|
||||
status = await self.ap.box_service.get_status(request_context)
|
||||
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
|
||||
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
|
||||
status['hidden'] = should_hide_box_runtime_status(constants.edition, status.get('enabled'))
|
||||
return self.success(data=status)
|
||||
|
||||
@@ -29,7 +34,10 @@ class BoxRouterGroup(group.RouterGroup):
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
sessions = await self.ap.box_service.get_sessions(request_context)
|
||||
try:
|
||||
sessions = await self.ap.box_service.get_sessions(request_context)
|
||||
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
|
||||
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
|
||||
return self.success(data=sessions)
|
||||
|
||||
@self.route(
|
||||
@@ -39,5 +47,10 @@ class BoxRouterGroup(group.RouterGroup):
|
||||
permission=Permission.AUDIT_VIEW,
|
||||
)
|
||||
async def _(request_context: RequestContext) -> str:
|
||||
try:
|
||||
if getattr(self.ap.box_service, 'managed_admission_required', False):
|
||||
await self.ap.box_service.require_workspace_sandbox(request_context)
|
||||
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
|
||||
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
|
||||
errors = self.ap.box_service.get_recent_errors(request_context)
|
||||
return self.success(data=errors)
|
||||
|
||||
@@ -13,6 +13,7 @@ import quart
|
||||
from ....authz import Permission, permissions_for_role, require_permission
|
||||
from ....context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
|
||||
from ... import group
|
||||
from ......core.task_boundary import run_in_workspace_uow
|
||||
from ......platform.sources.websocket_manager import WebSocketScope, ws_connection_manager
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
@@ -113,7 +114,11 @@ class WebSocketChatRouterGroup(group.RouterGroup):
|
||||
require_permission(current_context, Permission.RUNTIME_OPERATE)
|
||||
|
||||
async def _get_scoped_adapter(self, request_context: RequestContext, pipeline_uuid: str):
|
||||
pipeline = await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid)
|
||||
pipeline = await run_in_workspace_uow(
|
||||
self.ap,
|
||||
request_context.workspace_uuid,
|
||||
lambda: self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid),
|
||||
)
|
||||
if pipeline is None:
|
||||
return None
|
||||
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(request_context)
|
||||
|
||||
@@ -16,11 +16,13 @@ import posixpath
|
||||
import sqlalchemy
|
||||
|
||||
from .....core import taskmgr
|
||||
from .....core.task_boundary import run_in_workspace_uow
|
||||
from .....entity.persistence import plugin as persistence_plugin
|
||||
from ...authz import Permission
|
||||
from ...context import ExecutionContext, RequestContext
|
||||
from .. import group
|
||||
from .....workspace.errors import WorkspaceNotFoundError
|
||||
from .....plugin.github import validate_github_plugin_install_info
|
||||
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
|
||||
|
||||
|
||||
@@ -308,7 +310,11 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
):
|
||||
"""Revalidate a captured task context immediately before Runtime I/O."""
|
||||
|
||||
await self.ap.plugin_connector.require_workspace_context(execution_context)
|
||||
await run_in_workspace_uow(
|
||||
self.ap,
|
||||
execution_context.workspace_uuid,
|
||||
lambda: self.ap.plugin_connector.require_workspace_context(execution_context),
|
||||
)
|
||||
return await operation()
|
||||
|
||||
async def _require_public_plugin_runtime_context(self) -> ExecutionContext:
|
||||
@@ -757,27 +763,31 @@ class PluginsRouterGroup(group.RouterGroup):
|
||||
if limit_error is not None:
|
||||
return limit_error
|
||||
|
||||
data = await quart.request.json
|
||||
asset_url = data.get('asset_url', '')
|
||||
owner = data.get('owner', '')
|
||||
repo = data.get('repo', '')
|
||||
release_tag = data.get('release_tag', '')
|
||||
data = await quart.request.json or {}
|
||||
try:
|
||||
install_info = validate_github_plugin_install_info(
|
||||
{
|
||||
'asset_url': data.get('asset_url'),
|
||||
'asset_id': data.get('asset_id'),
|
||||
'release_id': data.get('release_id'),
|
||||
'owner': data.get('owner'),
|
||||
'repo': data.get('repo'),
|
||||
'release_tag': data.get('release_tag'),
|
||||
'github_url': f'https://github.com/{data.get("owner", "")}/{data.get("repo", "")}',
|
||||
}
|
||||
)
|
||||
except ValueError as exc:
|
||||
return self.http_status(400, -1, str(exc))
|
||||
|
||||
if not asset_url:
|
||||
return self.http_status(400, -1, 'Missing asset_url parameter')
|
||||
owner = install_info['owner']
|
||||
repo = install_info['repo']
|
||||
release_tag = install_info['release_tag']
|
||||
|
||||
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
|
||||
|
||||
ctx = taskmgr.TaskContext.new()
|
||||
ctx.metadata['plugin_name'] = f'{owner}/{repo}'
|
||||
ctx.metadata['install_source'] = 'github'
|
||||
install_info = {
|
||||
'asset_url': asset_url,
|
||||
'owner': owner,
|
||||
'repo': repo,
|
||||
'release_tag': release_tag,
|
||||
'github_url': f'https://github.com/{owner}/{repo}',
|
||||
}
|
||||
|
||||
wrapper = self.ap.task_mgr.create_user_task(
|
||||
self._run_fenced_plugin_operation(
|
||||
|
||||
@@ -25,12 +25,33 @@ class SystemRouterGroup(group.RouterGroup):
|
||||
account, _ = await self._authenticate_account(authorization.removeprefix('Bearer '))
|
||||
request_context = await self._resolve_account_context(account, group.AuthType.USER_TOKEN)
|
||||
if request_context is not None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(WorkspaceMetadata).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key.in_(['wizard_status', 'wizard_progress']),
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
|
||||
async def load_workspace_metadata():
|
||||
return await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
WorkspaceMetadata.key,
|
||||
WorkspaceMetadata.value,
|
||||
).where(
|
||||
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
|
||||
WorkspaceMetadata.key.in_(['wizard_status', 'wizard_progress']),
|
||||
)
|
||||
)
|
||||
|
||||
cloud_runtime = (
|
||||
getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
)
|
||||
if cloud_runtime:
|
||||
if not callable(tenant_uow):
|
||||
raise RuntimeError('Cloud system metadata requires an explicit tenant UoW')
|
||||
async with tenant_uow(request_context.workspace_uuid):
|
||||
result = await load_workspace_metadata()
|
||||
else:
|
||||
result = await load_workspace_metadata()
|
||||
# ``execute_async`` deliberately preserves its historical
|
||||
# AsyncConnection result shape. Selecting the two fields
|
||||
# explicitly keeps this reader independent of ORM Session
|
||||
# scalar semantics inside a tenant UoW.
|
||||
for row in result:
|
||||
if row.key == 'wizard_status':
|
||||
wizard_status = row.value
|
||||
|
||||
@@ -44,11 +44,27 @@ class WebhookRouterGroup(group.RouterGroup):
|
||||
if not hasattr(runtime_bot.adapter, 'handle_unified_webhook'):
|
||||
return quart.jsonify({'error': 'Adapter does not support unified webhook'}), 501
|
||||
|
||||
response = await runtime_bot.adapter.handle_unified_webhook(
|
||||
bot_uuid=bot_uuid,
|
||||
path=path,
|
||||
request=quart.request,
|
||||
)
|
||||
async def dispatch():
|
||||
await self.ap.workspace_service.get_execution_binding(
|
||||
runtime_bot.workspace_uuid,
|
||||
expected_generation=runtime_bot.placement_generation,
|
||||
)
|
||||
return await runtime_bot.adapter.handle_unified_webhook(
|
||||
bot_uuid=bot_uuid,
|
||||
path=path,
|
||||
request=quart.request,
|
||||
)
|
||||
|
||||
persistence_mgr = self.ap.persistence_mgr
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
tenant_scope = getattr(persistence_mgr, 'tenant_scope', None)
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('Cloud webhook dispatch requires an explicit tenant scope')
|
||||
async with tenant_scope(runtime_bot.workspace_uuid):
|
||||
response = await dispatch()
|
||||
else:
|
||||
response = await dispatch()
|
||||
|
||||
return response
|
||||
|
||||
|
||||
@@ -170,37 +170,89 @@ class ApiKeyService:
|
||||
if not secret.startswith('lbk_'):
|
||||
return None
|
||||
secret_hash = self._hash_secret(secret)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
|
||||
)
|
||||
key = result.first()
|
||||
if key is None or key.status != apikey.ApiKeyStatus.ACTIVE.value:
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
discovery_uow = getattr(self.ap.persistence_mgr, 'api_key_discovery_uow', None)
|
||||
if current_session() is None and callable(discovery_uow):
|
||||
async with discovery_uow(secret_hash) as discovery:
|
||||
key = await discovery.session.scalar(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
|
||||
)
|
||||
else:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
|
||||
)
|
||||
key = result.first()
|
||||
if key is None:
|
||||
return None
|
||||
discovered_workspace_uuid = key.workspace_uuid
|
||||
discovered_key_id = key.id
|
||||
now = self._utcnow()
|
||||
if key.expires_at is not None and key.expires_at <= now:
|
||||
return None
|
||||
|
||||
raw_scopes = list(key.scopes or [])
|
||||
async def bind_and_record_use() -> tuple[typing.Any, typing.Any] | None:
|
||||
# Re-read inside the tenant transaction. A revoke/expiry racing
|
||||
# discovery must not result in an authenticated identity.
|
||||
active_session = current_session()
|
||||
if active_session is not None:
|
||||
scoped_key = await active_session.scalar(
|
||||
sqlalchemy.select(apikey.ApiKey).where(
|
||||
apikey.ApiKey.id == discovered_key_id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
)
|
||||
)
|
||||
else: # compatibility for isolated service tests
|
||||
scoped_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(apikey.ApiKey).where(
|
||||
apikey.ApiKey.id == discovered_key_id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
)
|
||||
)
|
||||
scoped_key = scoped_result.first()
|
||||
if scoped_key is None or scoped_key.status != apikey.ApiKeyStatus.ACTIVE.value:
|
||||
return None
|
||||
if scoped_key.expires_at is not None and scoped_key.expires_at <= now:
|
||||
return None
|
||||
|
||||
binding = await self.ap.workspace_service.get_execution_binding(discovered_workspace_uuid)
|
||||
updated = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(apikey.ApiKey)
|
||||
.where(
|
||||
apikey.ApiKey.id == scoped_key.id,
|
||||
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
apikey.ApiKey.status == apikey.ApiKeyStatus.ACTIVE.value,
|
||||
)
|
||||
.values(last_used_at=now)
|
||||
.returning(apikey.ApiKey.id)
|
||||
)
|
||||
# Authentication and revocation race on this atomic predicate. If
|
||||
# revoke won, no active row is returned and the stale object read
|
||||
# above must never become an authenticated identity.
|
||||
if updated.scalar_one_or_none() is None:
|
||||
return None
|
||||
return binding, scoped_key
|
||||
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if current_session() is None and callable(tenant_uow):
|
||||
async with tenant_uow(discovered_workspace_uuid):
|
||||
bound = await bind_and_record_use()
|
||||
else:
|
||||
bound = await bind_and_record_use()
|
||||
if bound is None:
|
||||
return None
|
||||
binding, scoped_key = bound
|
||||
raw_scopes = list(scoped_key.scopes or [])
|
||||
permissions = (
|
||||
frozenset(permission.value for permission in Permission)
|
||||
if '*' in raw_scopes
|
||||
else frozenset(self._normalize_scopes(raw_scopes))
|
||||
)
|
||||
binding = await self.ap.workspace_service.get_execution_binding(key.workspace_uuid)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(apikey.ApiKey)
|
||||
.where(
|
||||
apikey.ApiKey.id == key.id,
|
||||
apikey.ApiKey.workspace_uuid == key.workspace_uuid,
|
||||
apikey.ApiKey.key_hash == secret_hash,
|
||||
)
|
||||
.values(last_used_at=now)
|
||||
)
|
||||
return ApiKeyIdentity(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
api_key_uuid=key.uuid,
|
||||
api_key_uuid=scoped_key.uuid,
|
||||
permissions=permissions,
|
||||
)
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import datetime
|
||||
import functools
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
@@ -22,6 +23,25 @@ DEFAULT_LOG_RETENTION_DAYS = 3
|
||||
UPLOAD_OWNER_TYPES = ('upload_image', 'upload_document', 'upload')
|
||||
|
||||
|
||||
def _workspace_scope(method):
|
||||
"""Bind maintenance work to a Workspace without spanning external I/O."""
|
||||
|
||||
@functools.wraps(method)
|
||||
async def wrapped(self, context, *args, **kwargs):
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
persistence_mgr = getattr(self.ap, 'persistence_mgr', None)
|
||||
tenant_scope = getattr(persistence_mgr, 'tenant_scope', None)
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('Cloud maintenance requires an explicit tenant scope')
|
||||
async with tenant_scope(workspace_uuid):
|
||||
return await method(self, context, *args, **kwargs)
|
||||
return await method(self, context, *args, **kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
class MaintenanceService:
|
||||
"""Storage maintenance and diagnostics."""
|
||||
|
||||
@@ -30,6 +50,7 @@ class MaintenanceService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
@_workspace_scope
|
||||
async def cleanup_expired_files(self, context: ExecutionContext) -> dict[str, int]:
|
||||
if not isinstance(context, ExecutionContext):
|
||||
raise WorkspaceRequiredError('Storage cleanup requires an ExecutionContext')
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import copy
|
||||
import re
|
||||
import uuid
|
||||
@@ -8,6 +7,7 @@ import uuid
|
||||
import sqlalchemy
|
||||
|
||||
from ....core import app, taskmgr
|
||||
from ....core.task_boundary import create_detached_task
|
||||
from ....entity.persistence import mcp as persistence_mcp
|
||||
from ....entity.persistence import plugin as persistence_plugin
|
||||
from ....provider.tools.loaders.mcp import MCPSessionStatus, RuntimeMCPSession
|
||||
@@ -249,7 +249,10 @@ class MCPService:
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_mcp.MCPServer).values(payload))
|
||||
created = await self._get_mcp_server_by_uuid_raw(execution_context, payload['uuid'])
|
||||
if created and self.ap.tool_mgr.mcp_tool_loader:
|
||||
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(execution_context, created))
|
||||
task = create_detached_task(
|
||||
self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(execution_context, created),
|
||||
after_commit_manager=self.ap.persistence_mgr,
|
||||
)
|
||||
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
|
||||
return payload['uuid']
|
||||
|
||||
@@ -351,7 +354,10 @@ class MCPService:
|
||||
if old_enable and loader.has_session(execution_context, old_name):
|
||||
await loader.remove_mcp_server(execution_context, old_name)
|
||||
if new_enable:
|
||||
task = asyncio.create_task(loader.host_mcp_server(execution_context, updated))
|
||||
task = create_detached_task(
|
||||
loader.host_mcp_server(execution_context, updated),
|
||||
after_commit_manager=self.ap.persistence_mgr,
|
||||
)
|
||||
loader._hosted_mcp_tasks.append(task)
|
||||
|
||||
async def delete_mcp_server(self, context: TenantContext, server_uuid: str) -> None:
|
||||
|
||||
@@ -2,8 +2,11 @@ from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
import datetime
|
||||
import functools
|
||||
import json
|
||||
import sqlalchemy
|
||||
from sqlalchemy.dialects import postgresql as postgresql_dialect
|
||||
from sqlalchemy.dialects import sqlite as sqlite_dialect
|
||||
|
||||
from ....core import app
|
||||
from ....entity.persistence import monitoring as persistence_monitoring
|
||||
@@ -12,6 +15,21 @@ from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
def _workspace_transaction(method):
|
||||
"""Run an explicit service entrypoint in one Workspace transaction."""
|
||||
|
||||
@functools.wraps(method)
|
||||
async def wrapped(self, context, *args, **kwargs):
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if callable(tenant_uow):
|
||||
async with tenant_uow(workspace_uuid):
|
||||
return await method(self, context, *args, **kwargs)
|
||||
return await method(self, context, *args, **kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
|
||||
class MonitoringService:
|
||||
"""Monitoring service"""
|
||||
|
||||
@@ -49,7 +67,7 @@ class MonitoringService:
|
||||
Returns:
|
||||
A dict mapping table name to the number of deleted rows.
|
||||
"""
|
||||
self._require_write_context(context)
|
||||
workspace_uuid = self._require_write_context(context)
|
||||
if retention_days < 1:
|
||||
raise ValueError('retention_days must be >= 1')
|
||||
if batch_size < 1:
|
||||
@@ -104,17 +122,27 @@ class MonitoringService:
|
||||
),
|
||||
]
|
||||
|
||||
deleted_counts: dict[str, int] = {}
|
||||
async def delete_records() -> dict[str, int]:
|
||||
deleted_counts: dict[str, int] = {}
|
||||
for table_name, model_cls, ts_column, pk_column in tables_and_columns:
|
||||
deleted_counts[table_name] = await self._delete_expired_in_batches(
|
||||
context=context,
|
||||
model_cls=model_cls,
|
||||
ts_column=ts_column,
|
||||
pk_column=pk_column,
|
||||
cutoff=cutoff,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
return deleted_counts
|
||||
|
||||
for table_name, model_cls, ts_column, pk_column in tables_and_columns:
|
||||
deleted_counts[table_name] = await self._delete_expired_in_batches(
|
||||
context=context,
|
||||
model_cls=model_cls,
|
||||
ts_column=ts_column,
|
||||
pk_column=pk_column,
|
||||
cutoff=cutoff,
|
||||
batch_size=batch_size,
|
||||
)
|
||||
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
|
||||
if callable(tenant_scope):
|
||||
# Carry the Workspace across the complete cleanup without holding a
|
||||
# connection. Each select+delete batch opens and commits its own UoW.
|
||||
async with tenant_scope(workspace_uuid):
|
||||
deleted_counts = await delete_records()
|
||||
else:
|
||||
deleted_counts = await delete_records()
|
||||
|
||||
if sum(deleted_counts.values()) > 0:
|
||||
await self._release_sqlite_space()
|
||||
@@ -134,25 +162,36 @@ class MonitoringService:
|
||||
deleted_total = 0
|
||||
|
||||
while True:
|
||||
select_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(pk_column)
|
||||
.where(model_cls.workspace_uuid == workspace_uuid, ts_column < cutoff)
|
||||
.limit(batch_size)
|
||||
)
|
||||
pk_values = list(select_result.scalars().all())
|
||||
if not pk_values:
|
||||
break
|
||||
|
||||
delete_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(model_cls).where(
|
||||
model_cls.workspace_uuid == workspace_uuid,
|
||||
pk_column.in_(pk_values),
|
||||
async def delete_batch() -> tuple[int, int]:
|
||||
select_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(pk_column)
|
||||
.where(model_cls.workspace_uuid == workspace_uuid, ts_column < cutoff)
|
||||
.limit(batch_size)
|
||||
)
|
||||
)
|
||||
deleted = delete_result.rowcount or 0
|
||||
deleted_total += deleted
|
||||
pk_values = list(select_result.scalars().all())
|
||||
if not pk_values:
|
||||
return 0, 0
|
||||
|
||||
if len(pk_values) < batch_size:
|
||||
delete_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(model_cls).where(
|
||||
model_cls.workspace_uuid == workspace_uuid,
|
||||
pk_column.in_(pk_values),
|
||||
)
|
||||
)
|
||||
return len(pk_values), int(delete_result.rowcount or 0)
|
||||
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if callable(tenant_uow):
|
||||
async with tenant_uow(workspace_uuid):
|
||||
selected, deleted = await delete_batch()
|
||||
else:
|
||||
selected, deleted = await delete_batch()
|
||||
|
||||
deleted_total += deleted
|
||||
if selected == 0:
|
||||
break
|
||||
if selected < batch_size:
|
||||
break
|
||||
|
||||
return deleted_total
|
||||
@@ -192,22 +231,30 @@ class MonitoringService:
|
||||
session_id: str | None = None,
|
||||
):
|
||||
workspace_uuid = self._require_write_context(context)
|
||||
context_columns = (
|
||||
persistence_monitoring.MonitoringMessage.id,
|
||||
persistence_monitoring.MonitoringMessage.bot_id,
|
||||
persistence_monitoring.MonitoringMessage.bot_name,
|
||||
persistence_monitoring.MonitoringMessage.pipeline_id,
|
||||
persistence_monitoring.MonitoringMessage.pipeline_name,
|
||||
persistence_monitoring.MonitoringMessage.session_id,
|
||||
)
|
||||
if message_id:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_monitoring.MonitoringMessage).where(
|
||||
sqlalchemy.select(*context_columns).where(
|
||||
persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid,
|
||||
persistence_monitoring.MonitoringMessage.id == message_id,
|
||||
)
|
||||
)
|
||||
row = result.first()
|
||||
if row:
|
||||
return row[0]
|
||||
return row
|
||||
|
||||
if not session_id:
|
||||
return None
|
||||
|
||||
user_query = (
|
||||
sqlalchemy.select(persistence_monitoring.MonitoringMessage)
|
||||
sqlalchemy.select(*context_columns)
|
||||
.where(
|
||||
sqlalchemy.and_(
|
||||
persistence_monitoring.MonitoringMessage.session_id == session_id,
|
||||
@@ -221,10 +268,10 @@ class MonitoringService:
|
||||
result = await self.ap.persistence_mgr.execute_async(user_query)
|
||||
row = result.first()
|
||||
if row:
|
||||
return row[0]
|
||||
return row
|
||||
|
||||
any_query = (
|
||||
sqlalchemy.select(persistence_monitoring.MonitoringMessage)
|
||||
sqlalchemy.select(*context_columns)
|
||||
.where(
|
||||
persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid,
|
||||
persistence_monitoring.MonitoringMessage.session_id == session_id,
|
||||
@@ -234,10 +281,11 @@ class MonitoringService:
|
||||
)
|
||||
result = await self.ap.persistence_mgr.execute_async(any_query)
|
||||
row = result.first()
|
||||
return row[0] if row else None
|
||||
return row
|
||||
|
||||
# ========== Recording Methods ==========
|
||||
|
||||
@_workspace_transaction
|
||||
async def record_message(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
@@ -285,6 +333,7 @@ class MonitoringService:
|
||||
|
||||
return message_id
|
||||
|
||||
@_workspace_transaction
|
||||
async def record_llm_call(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
@@ -331,6 +380,7 @@ class MonitoringService:
|
||||
|
||||
return call_id
|
||||
|
||||
@_workspace_transaction
|
||||
async def record_tool_call(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
@@ -389,6 +439,7 @@ class MonitoringService:
|
||||
|
||||
return call_id
|
||||
|
||||
@_workspace_transaction
|
||||
async def record_embedding_call(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
@@ -432,6 +483,7 @@ class MonitoringService:
|
||||
|
||||
return call_id
|
||||
|
||||
@_workspace_transaction
|
||||
async def record_session_start(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
@@ -466,6 +518,7 @@ class MonitoringService:
|
||||
sqlalchemy.insert(persistence_monitoring.MonitoringSession).values(session_data)
|
||||
)
|
||||
|
||||
@_workspace_transaction
|
||||
async def update_session_activity(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
@@ -503,6 +556,7 @@ class MonitoringService:
|
||||
# Check if any rows were updated
|
||||
return result.rowcount > 0
|
||||
|
||||
@_workspace_transaction
|
||||
async def record_error(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
@@ -540,6 +594,7 @@ class MonitoringService:
|
||||
|
||||
return error_id
|
||||
|
||||
@_workspace_transaction
|
||||
async def update_message_status(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
@@ -1802,81 +1857,57 @@ class MonitoringService:
|
||||
)
|
||||
return None
|
||||
|
||||
# Check if record with this feedback_id already exists
|
||||
existing_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(MonitoringFeedback).where(
|
||||
MonitoringFeedback.workspace_uuid == workspace_uuid,
|
||||
MonitoringFeedback.feedback_id == feedback_id,
|
||||
)
|
||||
)
|
||||
existing_row = existing_result.first()
|
||||
|
||||
if existing_row:
|
||||
# UPDATE existing record
|
||||
existing = existing_row[0] if isinstance(existing_row, tuple) else existing_row
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(MonitoringFeedback)
|
||||
.where(
|
||||
MonitoringFeedback.workspace_uuid == workspace_uuid,
|
||||
MonitoringFeedback.feedback_id == feedback_id,
|
||||
)
|
||||
.values(
|
||||
timestamp=now,
|
||||
feedback_type=feedback_type,
|
||||
feedback_content=feedback_content,
|
||||
inaccurate_reasons=reasons_json,
|
||||
bot_id=bot_id or existing.bot_id,
|
||||
bot_name=bot_name or existing.bot_name,
|
||||
pipeline_id=pipeline_id or existing.pipeline_id,
|
||||
pipeline_name=pipeline_name or existing.pipeline_name,
|
||||
session_id=session_id or existing.session_id,
|
||||
message_id=message_id or existing.message_id,
|
||||
stream_id=stream_id or existing.stream_id,
|
||||
user_id=user_id or existing.user_id,
|
||||
platform=platform or existing.platform,
|
||||
)
|
||||
)
|
||||
return existing.id
|
||||
record_data = {
|
||||
'id': str(uuid.uuid4()),
|
||||
'workspace_uuid': workspace_uuid,
|
||||
'timestamp': now,
|
||||
'feedback_id': feedback_id,
|
||||
'feedback_type': feedback_type,
|
||||
'feedback_content': feedback_content,
|
||||
'inaccurate_reasons': reasons_json,
|
||||
'bot_id': bot_id,
|
||||
'bot_name': bot_name,
|
||||
'pipeline_id': pipeline_id,
|
||||
'pipeline_name': pipeline_name,
|
||||
'session_id': session_id,
|
||||
'message_id': message_id,
|
||||
'stream_id': stream_id,
|
||||
'user_id': user_id,
|
||||
'platform': platform,
|
||||
}
|
||||
dialect_name = self.ap.persistence_mgr.get_db_engine().dialect.name
|
||||
if dialect_name == 'postgresql':
|
||||
statement = postgresql_dialect.insert(MonitoringFeedback).values(record_data)
|
||||
elif dialect_name == 'sqlite':
|
||||
statement = sqlite_dialect.insert(MonitoringFeedback).values(record_data)
|
||||
else:
|
||||
# INSERT new record with IntegrityError defense
|
||||
record_id = str(uuid.uuid4())
|
||||
record_data = {
|
||||
'id': record_id,
|
||||
'workspace_uuid': workspace_uuid,
|
||||
'timestamp': now,
|
||||
'feedback_id': feedback_id,
|
||||
'feedback_type': feedback_type,
|
||||
'feedback_content': feedback_content,
|
||||
'inaccurate_reasons': reasons_json,
|
||||
'bot_id': bot_id,
|
||||
'bot_name': bot_name,
|
||||
'pipeline_id': pipeline_id,
|
||||
'pipeline_name': pipeline_name,
|
||||
'session_id': session_id,
|
||||
'message_id': message_id,
|
||||
'stream_id': stream_id,
|
||||
'user_id': user_id,
|
||||
'platform': platform,
|
||||
}
|
||||
try:
|
||||
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(MonitoringFeedback).values(record_data))
|
||||
return record_id
|
||||
except Exception:
|
||||
# UNIQUE constraint conflict (concurrent feedback for same feedback_id)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(MonitoringFeedback)
|
||||
.where(
|
||||
MonitoringFeedback.workspace_uuid == workspace_uuid,
|
||||
MonitoringFeedback.feedback_id == feedback_id,
|
||||
)
|
||||
.values(
|
||||
timestamp=now,
|
||||
feedback_type=feedback_type,
|
||||
feedback_content=feedback_content,
|
||||
inaccurate_reasons=reasons_json,
|
||||
)
|
||||
)
|
||||
return feedback_id
|
||||
raise RuntimeError(f'Monitoring feedback upsert does not support {dialect_name!r}')
|
||||
|
||||
excluded = statement.excluded
|
||||
|
||||
def preserve_existing(column):
|
||||
return sqlalchemy.func.coalesce(sqlalchemy.func.nullif(getattr(excluded, column.key), ''), column)
|
||||
|
||||
statement = statement.on_conflict_do_update(
|
||||
index_elements=[MonitoringFeedback.workspace_uuid, MonitoringFeedback.feedback_id],
|
||||
set_={
|
||||
'timestamp': excluded.timestamp,
|
||||
'feedback_type': excluded.feedback_type,
|
||||
'feedback_content': excluded.feedback_content,
|
||||
'inaccurate_reasons': excluded.inaccurate_reasons,
|
||||
'bot_id': preserve_existing(MonitoringFeedback.bot_id),
|
||||
'bot_name': preserve_existing(MonitoringFeedback.bot_name),
|
||||
'pipeline_id': preserve_existing(MonitoringFeedback.pipeline_id),
|
||||
'pipeline_name': preserve_existing(MonitoringFeedback.pipeline_name),
|
||||
'session_id': preserve_existing(MonitoringFeedback.session_id),
|
||||
'message_id': preserve_existing(MonitoringFeedback.message_id),
|
||||
'stream_id': preserve_existing(MonitoringFeedback.stream_id),
|
||||
'user_id': preserve_existing(MonitoringFeedback.user_id),
|
||||
'platform': preserve_existing(MonitoringFeedback.platform),
|
||||
},
|
||||
).returning(MonitoringFeedback.id)
|
||||
result = await self.ap.persistence_mgr.execute_async(statement)
|
||||
return str(result.scalar_one())
|
||||
|
||||
async def get_feedback_stats(
|
||||
self,
|
||||
|
||||
@@ -4,6 +4,7 @@ import io
|
||||
import inspect
|
||||
import os
|
||||
import posixpath
|
||||
import stat
|
||||
import zipfile
|
||||
from typing import Optional
|
||||
from urllib.parse import quote, unquote, urlparse
|
||||
@@ -34,6 +35,12 @@ _GITHUB_ASSET_HOSTS = {
|
||||
'raw.githubusercontent.com',
|
||||
'codeload.github.com',
|
||||
}
|
||||
_MAX_GITHUB_ARCHIVE_BYTES = 10 * 1024 * 1024
|
||||
_MAX_GITHUB_ARCHIVE_ENTRIES = 4096
|
||||
_MAX_SKILL_ARCHIVE_FILES = 1024
|
||||
_MAX_SKILL_FILE_BYTES = 10 * 1024 * 1024
|
||||
_MAX_SKILL_UNCOMPRESSED_BYTES = 50 * 1024 * 1024
|
||||
_MAX_SKILL_COMPRESSION_RATIO = 200
|
||||
|
||||
|
||||
class SkillService:
|
||||
@@ -322,9 +329,22 @@ class SkillService:
|
||||
|
||||
async def _download_github_asset(self, asset_url: str) -> bytes:
|
||||
async with httpx.AsyncClient(follow_redirects=True, timeout=120) as client:
|
||||
resp = await client.get(asset_url)
|
||||
resp.raise_for_status()
|
||||
return resp.content
|
||||
async with client.stream('GET', asset_url) as resp:
|
||||
resp.raise_for_status()
|
||||
content_length = resp.headers.get('content-length')
|
||||
if content_length is not None:
|
||||
try:
|
||||
if int(content_length) > _MAX_GITHUB_ARCHIVE_BYTES:
|
||||
raise ValueError('GitHub skill archive exceeds the compressed size limit')
|
||||
except ValueError as exc:
|
||||
if 'exceeds' in str(exc):
|
||||
raise
|
||||
content = bytearray()
|
||||
async for chunk in resp.aiter_bytes():
|
||||
content.extend(chunk)
|
||||
if len(content) > _MAX_GITHUB_ARCHIVE_BYTES:
|
||||
raise ValueError('GitHub skill archive exceeds the compressed size limit')
|
||||
return bytes(content)
|
||||
|
||||
async def _download_github_skill_directory_as_zip(
|
||||
self, asset_url: str, *, owner: str, repo: str
|
||||
@@ -339,7 +359,11 @@ class SkillService:
|
||||
raise ValueError('GitHub repository archive must be a valid .zip archive') from exc
|
||||
|
||||
with source_archive as source_zip:
|
||||
if len(source_zip.infolist()) > _MAX_GITHUB_ARCHIVE_ENTRIES:
|
||||
raise ValueError('GitHub repository archive contains too many entries')
|
||||
skill_entry = self._find_github_skill_archive_entry(source_zip, info['file_path'])
|
||||
if skill_entry.file_size > _MAX_SKILL_FILE_BYTES:
|
||||
raise ValueError('GitHub SKILL.md exceeds the file size limit')
|
||||
try:
|
||||
skill_md_content = source_zip.read(skill_entry).decode('utf-8')
|
||||
except UnicodeDecodeError as exc:
|
||||
@@ -377,6 +401,7 @@ class SkillService:
|
||||
normalized_source_dir = posixpath.normpath(source_skill_dir)
|
||||
source_prefix = f'{normalized_source_dir}/'
|
||||
copied_files = 0
|
||||
copied_bytes = 0
|
||||
|
||||
for member in source_zip.infolist():
|
||||
normalized_member = posixpath.normpath(member.filename)
|
||||
@@ -399,10 +424,33 @@ class SkillService:
|
||||
if member.is_dir():
|
||||
target_zip.writestr(target_info, b'')
|
||||
continue
|
||||
|
||||
target_zip.writestr(target_info, source_zip.read(member))
|
||||
if member.flag_bits & 0x1:
|
||||
raise ValueError('Encrypted GitHub skill archive entries are not supported')
|
||||
unix_mode = member.external_attr >> 16
|
||||
if stat.S_IFMT(unix_mode) == stat.S_IFLNK:
|
||||
raise ValueError(f'GitHub archive contains a symbolic link: {member.filename}')
|
||||
if member.file_size > _MAX_SKILL_FILE_BYTES:
|
||||
raise ValueError(f'GitHub skill file exceeds the size limit: {member.filename}')
|
||||
if member.file_size and member.file_size > max(member.compress_size, 1) * _MAX_SKILL_COMPRESSION_RATIO:
|
||||
raise ValueError(f'GitHub skill file exceeds the compression-ratio limit: {member.filename}')
|
||||
copied_files += 1
|
||||
copied_bytes += member.file_size
|
||||
if copied_files > _MAX_SKILL_ARCHIVE_FILES:
|
||||
raise ValueError('GitHub skill directory contains too many files')
|
||||
if copied_bytes > _MAX_SKILL_UNCOMPRESSED_BYTES:
|
||||
raise ValueError('GitHub skill directory exceeds the uncompressed size limit')
|
||||
|
||||
# Copy in bounded chunks instead of materialising a potentially
|
||||
# large member in Core memory. The Box Runtime independently
|
||||
# revalidates the resulting archive before installation.
|
||||
with source_zip.open(member, 'r') as source_file, target_zip.open(target_info, 'w') as target_file:
|
||||
remaining = member.file_size
|
||||
while remaining:
|
||||
chunk = source_file.read(min(64 * 1024, remaining))
|
||||
if not chunk:
|
||||
raise ValueError(f'GitHub skill file is truncated: {member.filename}')
|
||||
target_file.write(chunk)
|
||||
remaining -= len(chunk)
|
||||
if copied_files == 0:
|
||||
raise ValueError('GitHub skill directory is empty')
|
||||
|
||||
|
||||
@@ -151,10 +151,11 @@ class UserService:
|
||||
)
|
||||
|
||||
async def is_initialized(self) -> bool:
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
|
||||
|
||||
result_list = result.all()
|
||||
return result_list is not None and len(result_list) > 0
|
||||
account = await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).limit(1),
|
||||
f'instance:{self._jwt_identity()[1]}',
|
||||
)
|
||||
return account is not None
|
||||
|
||||
def _session_factory(self) -> async_sessionmaker[AsyncSession]:
|
||||
return async_sessionmaker(self.ap.persistence_mgr.get_db_engine(), expire_on_commit=False)
|
||||
@@ -247,28 +248,24 @@ class UserService:
|
||||
|
||||
async def get_user_by_email(self, user_email: str) -> user.User | None:
|
||||
normalized_email = user_email.strip().casefold()
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email)
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list is not None and len(result_list) > 0 else None
|
||||
|
||||
async def get_user_by_uuid(self, account_uuid: str) -> user.User | None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.uuid == account_uuid)
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.uuid == account_uuid),
|
||||
f'uuid:{account_uuid}',
|
||||
)
|
||||
return result.first()
|
||||
|
||||
async def get_user_by_space_account_uuid(self, space_account_uuid: str) -> user.User | None:
|
||||
"""Get user by Space account UUID"""
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid)
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list is not None and len(result_list) > 0 else None
|
||||
|
||||
async def authenticate(self, user_email: str, password: str) -> str | None:
|
||||
user_obj = await self.get_user_by_email(user_email)
|
||||
if user_obj is None:
|
||||
@@ -389,11 +386,13 @@ class UserService:
|
||||
|
||||
async def reset_password(self, user_email: str, new_password: str) -> None:
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalize_email(user_email))
|
||||
.values(password=hashed_password)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
async def change_password(self, user_email: str, current_password: str, new_password: str) -> None:
|
||||
@@ -407,11 +406,13 @@ class UserService:
|
||||
await self._verify_password(user_obj.password, current_password)
|
||||
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
normalized_email = normalize_email(user_email)
|
||||
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalize_email(user_email))
|
||||
.values(password=hashed_password)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
# Space user management
|
||||
@@ -435,7 +436,7 @@ class UserService:
|
||||
|
||||
if existing_user:
|
||||
# Update existing user's tokens
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.space_account_uuid == space_account_uuid)
|
||||
.values(
|
||||
@@ -443,7 +444,8 @@ class UserService:
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'space:{space_account_uuid}',
|
||||
)
|
||||
await self._update_space_provider_for_account(existing_user, api_key)
|
||||
return await self.get_user_by_space_account_uuid(space_account_uuid)
|
||||
@@ -538,9 +540,38 @@ class UserService:
|
||||
|
||||
async def get_first_user(self) -> user.User | None:
|
||||
"""Get the first user (for single-user mode)"""
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
|
||||
result_list = result.all()
|
||||
return result_list[0] if result_list else None
|
||||
return await self._identity_scalar(
|
||||
sqlalchemy.select(user.User).limit(1),
|
||||
f'instance:{self._jwt_identity()[1]}',
|
||||
)
|
||||
|
||||
async def _identity_scalar(
|
||||
self,
|
||||
statement: typing.Any,
|
||||
identity: str,
|
||||
) -> user.User | None:
|
||||
"""Execute one exact Account lookup in an explicit discovery transaction."""
|
||||
|
||||
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
return await discovery.session.scalar(statement)
|
||||
result = await self.ap.persistence_mgr.execute_async(statement)
|
||||
rows = result.all()
|
||||
return rows[0] if rows else None
|
||||
|
||||
async def _identity_execute(self, statement: typing.Any, identity: str) -> typing.Any:
|
||||
"""Execute one exact Account mutation in an explicit transaction."""
|
||||
|
||||
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
|
||||
if current_session() is None and callable(identity_uow):
|
||||
async with identity_uow(digest) as discovery:
|
||||
return await discovery.session.execute(statement)
|
||||
return await self.ap.persistence_mgr.execute_async(statement)
|
||||
|
||||
async def set_password(self, user_email: str, new_password: str, current_password: str | None = None) -> None:
|
||||
"""Set or change password for a user"""
|
||||
@@ -557,10 +588,12 @@ class UserService:
|
||||
await self._verify_password(user_obj.password, current_password)
|
||||
|
||||
hashed_password = await self._hash_password(new_password)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
normalized_email = normalize_email(user_email)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalize_email(user_email))
|
||||
.values(password=hashed_password)
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(password=hashed_password),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
async def bind_space_account(self, user_email: str, code: str) -> user.User:
|
||||
@@ -596,9 +629,10 @@ class UserService:
|
||||
raise ValueError('This Space account is already bound to another user')
|
||||
|
||||
# Update local account to Space account
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
normalized_email = normalize_email(user_email)
|
||||
await self._identity_execute(
|
||||
sqlalchemy.update(user.User)
|
||||
.where(user.User.normalized_email == normalize_email(user_email))
|
||||
.where(user.User.normalized_email == normalized_email)
|
||||
.values(
|
||||
user=normalize_email(space_email), # Update email to Space email
|
||||
normalized_email=normalize_email(space_email),
|
||||
@@ -608,7 +642,8 @@ class UserService:
|
||||
space_refresh_token=refresh_token,
|
||||
space_api_key=api_key,
|
||||
space_access_token_expires_at=expires_at,
|
||||
)
|
||||
),
|
||||
f'email:{normalized_email}',
|
||||
)
|
||||
|
||||
# Update Space model provider API keys
|
||||
|
||||
@@ -31,6 +31,9 @@ if typing.TYPE_CHECKING:
|
||||
|
||||
# JSON-RPC-ish 401 body returned before the MCP app is reached.
|
||||
_UNAUTHORIZED_BODY = b'{"error":"unauthorized","message":"A valid LangBot API key is required for MCP access."}'
|
||||
_ENTITLEMENT_UNAVAILABLE_BODY = (
|
||||
b'{"error":"entitlement_unavailable","message":"Workspace entitlement is unavailable for MCP access."}'
|
||||
)
|
||||
|
||||
|
||||
def _extract_api_key(headers: list[tuple[bytes, bytes]]) -> str:
|
||||
@@ -110,6 +113,29 @@ class MCPMount:
|
||||
await send({'type': 'http.response.body', 'body': _UNAUTHORIZED_BODY})
|
||||
return
|
||||
|
||||
deployment_admission = getattr(self.ap, 'deployment_admission', None)
|
||||
try:
|
||||
if deployment_admission is not None:
|
||||
deployment_admission.require_active()
|
||||
entitlement_revision = 0
|
||||
deployment = getattr(self.ap, 'deployment', None)
|
||||
if deployment is not None and getattr(deployment, 'multi_workspace_enabled', False):
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
if resolver is None or identity.instance_uuid != resolver.instance_uuid:
|
||||
raise RuntimeError('Workspace entitlement resolver is unavailable')
|
||||
entitlement = await resolver.resolve(identity.workspace_uuid)
|
||||
entitlement_revision = entitlement.entitlement_revision
|
||||
except Exception:
|
||||
await send(
|
||||
{
|
||||
'type': 'http.response.start',
|
||||
'status': 403,
|
||||
'headers': [(b'content-type', b'application/json')],
|
||||
}
|
||||
)
|
||||
await send({'type': 'http.response.body', 'body': _ENTITLEMENT_UNAVAILABLE_BODY})
|
||||
return
|
||||
|
||||
request_context = RequestContext(
|
||||
instance_uuid=identity.instance_uuid,
|
||||
placement_generation=identity.placement_generation,
|
||||
@@ -125,11 +151,18 @@ class MCPMount:
|
||||
role=None,
|
||||
permissions=identity.permissions,
|
||||
),
|
||||
entitlement_revision=entitlement_revision,
|
||||
)
|
||||
token = bind_request_context(request_context)
|
||||
try:
|
||||
await mcp_asgi(scope, receive, send)
|
||||
finally:
|
||||
reset_request_context(token)
|
||||
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('MCP request persistence scope is unavailable')
|
||||
async with tenant_scope(identity.workspace_uuid):
|
||||
token = bind_request_context(request_context)
|
||||
try:
|
||||
await mcp_asgi(scope, receive, send)
|
||||
if deployment_admission is not None:
|
||||
deployment_admission.require_active()
|
||||
finally:
|
||||
reset_request_context(token)
|
||||
|
||||
return dispatcher
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import datetime as dt
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from langbot_plugin.box.errors import BoxAdmissionError, BoxRuntimeUnavailableError
|
||||
from langbot_plugin.box.models import (
|
||||
SandboxAdmissionGrant,
|
||||
SandboxAdmissionPolicy,
|
||||
SandboxAdmissionRevocation,
|
||||
)
|
||||
|
||||
from ..api.http.context import ExecutionContext
|
||||
from ..cloud.entitlements import EntitlementSnapshot, EntitlementUnavailableError
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from langbot_plugin.box.client import BoxRuntimeClient
|
||||
|
||||
from ..core.app import Application
|
||||
|
||||
|
||||
_UTC = dt.timezone.utc
|
||||
_MANAGED_SANDBOX_FEATURE = 'managed_sandbox'
|
||||
_MANAGED_SANDBOX_SESSION_LIMIT = 'managed_sandbox_sessions'
|
||||
_MAX_GRANT_TTL_SEC = 300
|
||||
|
||||
|
||||
class SandboxAdmissionController:
|
||||
"""Project Cloud entitlements into short-lived Box Runtime grants.
|
||||
|
||||
Product and plan names intentionally never cross this boundary. The
|
||||
closed Control Plane supplies a versioned generic entitlement, while Core
|
||||
installs only the numeric authority understood by the shared Box Runtime.
|
||||
|
||||
No state is allocated for a Workspace until it attempts to use the
|
||||
managed sandbox. Per-Workspace locks serialize renewal/revocation so a
|
||||
concurrent first use cannot install conflicting grants.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ap: Application,
|
||||
client: BoxRuntimeClient,
|
||||
*,
|
||||
policy: SandboxAdmissionPolicy,
|
||||
wall_time: Callable[[], float] = time.time,
|
||||
) -> None:
|
||||
self.ap = ap
|
||||
self.client = client
|
||||
self.policy = policy
|
||||
self._wall_time = wall_time
|
||||
self._locks: dict[str, asyncio.Lock] = {}
|
||||
self._highest_revisions: dict[str, int] = {}
|
||||
|
||||
def _workspace_lock(self, workspace_uuid: str) -> asyncio.Lock:
|
||||
lock = self._locks.get(workspace_uuid)
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
self._locks[workspace_uuid] = lock
|
||||
return lock
|
||||
|
||||
@staticmethod
|
||||
def _context_revision(context: ExecutionContext) -> int:
|
||||
revision = getattr(context, 'entitlement_revision', 0)
|
||||
if isinstance(revision, bool) or not isinstance(revision, int):
|
||||
return 0
|
||||
return max(revision, 0)
|
||||
|
||||
def _revocation_revision(self, context: ExecutionContext, candidate_revision: int = 0) -> int:
|
||||
return max(
|
||||
1,
|
||||
self._highest_revisions.get(context.workspace_uuid, 0),
|
||||
self._context_revision(context),
|
||||
candidate_revision,
|
||||
)
|
||||
|
||||
async def _revoke_locked(
|
||||
self,
|
||||
context: ExecutionContext,
|
||||
*,
|
||||
candidate_revision: int = 0,
|
||||
) -> None:
|
||||
revision = self._revocation_revision(context, candidate_revision)
|
||||
revocation = SandboxAdmissionRevocation(
|
||||
instance_uuid=context.instance_uuid,
|
||||
workspace_uuid=context.workspace_uuid,
|
||||
entitlement_revision=revision,
|
||||
)
|
||||
try:
|
||||
result = await self.client.revoke_sandbox_admission_grant(revocation)
|
||||
if (
|
||||
not isinstance(result, dict)
|
||||
or result.get('revoked') is not True
|
||||
or result.get('workspace_uuid') != context.workspace_uuid
|
||||
or result.get('entitlement_revision') != revision
|
||||
):
|
||||
raise BoxRuntimeUnavailableError('Box Runtime returned an invalid sandbox revocation receipt')
|
||||
except Exception as exc:
|
||||
# The caller still fails closed even if the control connection is
|
||||
# unavailable. A previously installed grant expires independently
|
||||
# in at most five minutes inside the Runtime.
|
||||
self.ap.logger.warning(
|
||||
'Failed to install Box sandbox admission revocation: '
|
||||
f'workspace_uuid={context.workspace_uuid} revision={revision} error={exc}'
|
||||
)
|
||||
self._highest_revisions[context.workspace_uuid] = revision
|
||||
|
||||
@staticmethod
|
||||
def _require_managed_sandbox(snapshot: EntitlementSnapshot) -> None:
|
||||
snapshot.require_feature(_MANAGED_SANDBOX_FEATURE)
|
||||
sessions = snapshot.limit(_MANAGED_SANDBOX_SESSION_LIMIT)
|
||||
if sessions != 1:
|
||||
raise EntitlementUnavailableError('Workspace entitlement must grant exactly one managed sandbox session')
|
||||
|
||||
def _grant_expiry(self, snapshot: EntitlementSnapshot) -> dt.datetime:
|
||||
now_epoch = int(self._wall_time())
|
||||
ttl_sec = min(self.policy.max_grant_ttl_sec, _MAX_GRANT_TTL_SEC)
|
||||
expires_epoch = min(snapshot.expires_at, now_epoch + ttl_sec)
|
||||
if expires_epoch <= now_epoch:
|
||||
raise EntitlementUnavailableError('Workspace entitlement expired before sandbox admission')
|
||||
return dt.datetime.fromtimestamp(expires_epoch, tz=_UTC)
|
||||
|
||||
async def require(self, context: ExecutionContext) -> SandboxAdmissionGrant:
|
||||
"""Validate entitlement freshness and install/renew one Runtime grant."""
|
||||
|
||||
resolver = getattr(self.ap, 'entitlement_resolver', None)
|
||||
if resolver is None:
|
||||
raise EntitlementUnavailableError('Workspace entitlement resolver is unavailable')
|
||||
if context.instance_uuid != resolver.instance_uuid:
|
||||
raise EntitlementUnavailableError('Workspace entitlement targets another LangBot instance')
|
||||
|
||||
lock = self._workspace_lock(context.workspace_uuid)
|
||||
async with lock:
|
||||
try:
|
||||
snapshot = await resolver.resolve(
|
||||
context.workspace_uuid,
|
||||
minimum_revision=self._context_revision(context),
|
||||
now=int(self._wall_time()),
|
||||
)
|
||||
except EntitlementUnavailableError as exc:
|
||||
# Only a verified, scoped snapshot can authoritatively revoke
|
||||
# a revision. Provider timeouts, malformed responses, and
|
||||
# rollback/equivocation errors fail this request closed but do
|
||||
# not tombstone a still-valid revision forever.
|
||||
authoritative_revision = exc.entitlement_revision
|
||||
if authoritative_revision is not None:
|
||||
await self._revoke_locked(
|
||||
context,
|
||||
candidate_revision=authoritative_revision,
|
||||
)
|
||||
raise
|
||||
|
||||
try:
|
||||
self._require_managed_sandbox(snapshot)
|
||||
except EntitlementUnavailableError:
|
||||
await self._revoke_locked(
|
||||
context,
|
||||
candidate_revision=snapshot.entitlement_revision,
|
||||
)
|
||||
raise
|
||||
|
||||
grant = SandboxAdmissionGrant(
|
||||
instance_uuid=context.instance_uuid,
|
||||
workspace_uuid=context.workspace_uuid,
|
||||
execution_generation=context.placement_generation,
|
||||
entitlement_revision=snapshot.entitlement_revision,
|
||||
expires_at=self._grant_expiry(snapshot),
|
||||
max_sessions=1,
|
||||
max_managed_processes=0,
|
||||
)
|
||||
result = await self.client.upsert_sandbox_admission_grant(grant)
|
||||
if (
|
||||
not isinstance(result, dict)
|
||||
or result.get('installed') is not True
|
||||
or result.get('workspace_uuid') != context.workspace_uuid
|
||||
or result.get('execution_generation') != context.placement_generation
|
||||
or result.get('entitlement_revision') != snapshot.entitlement_revision
|
||||
or result.get('max_sessions') != 1
|
||||
or result.get('max_managed_processes') != 0
|
||||
):
|
||||
raise BoxRuntimeUnavailableError('Box Runtime returned an invalid sandbox admission receipt')
|
||||
self._highest_revisions[context.workspace_uuid] = max(
|
||||
self._highest_revisions.get(context.workspace_uuid, 0),
|
||||
snapshot.entitlement_revision,
|
||||
)
|
||||
return grant
|
||||
|
||||
async def revoke(self, context: ExecutionContext, *, entitlement_revision: int = 0) -> None:
|
||||
"""Explicitly revoke a Workspace grant using a monotonic tombstone."""
|
||||
|
||||
async with self._workspace_lock(context.workspace_uuid):
|
||||
await self._revoke_locked(context, candidate_revision=entitlement_revision)
|
||||
|
||||
|
||||
def require_cloud_admission_policy(raw_policy: object) -> SandboxAdmissionPolicy:
|
||||
"""Parse the Cloud Box policy without permitting an OSS downgrade."""
|
||||
|
||||
try:
|
||||
policy = SandboxAdmissionPolicy.model_validate(raw_policy)
|
||||
except Exception as exc:
|
||||
raise BoxAdmissionError('Cloud Box sandbox admission policy is invalid') from exc
|
||||
if not policy.required:
|
||||
raise BoxAdmissionError('Cloud Box sandbox admission must be required')
|
||||
if policy.logical_session_id != 'global':
|
||||
raise BoxAdmissionError('Cloud Box sandbox session ID must be global')
|
||||
if policy.required_backend != 'nsjail':
|
||||
raise BoxAdmissionError('Cloud Box sandbox backend must be nsjail')
|
||||
if policy.max_sessions != 1 or policy.max_managed_processes != 0:
|
||||
raise BoxAdmissionError('Cloud Box sandbox policy must allow one session and zero managed processes')
|
||||
if policy.max_grant_ttl_sec > _MAX_GRANT_TTL_SEC:
|
||||
raise BoxAdmissionError('Cloud Box sandbox admission grant TTL must not exceed 300 seconds')
|
||||
if policy.workspace_quota_mb <= 0:
|
||||
raise BoxAdmissionError('Cloud Box sandbox workspace quota must be a positive integer')
|
||||
return policy
|
||||
@@ -0,0 +1,285 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import errno
|
||||
import os
|
||||
import stat
|
||||
from collections.abc import Iterable
|
||||
|
||||
|
||||
class UnsafeWorkspacePathError(OSError):
|
||||
"""A tenant-controlled path could not be opened without following links."""
|
||||
|
||||
|
||||
_DIRECTORY_FLAGS = (
|
||||
os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
|
||||
)
|
||||
_FILE_READ_FLAGS = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
|
||||
_FILE_WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
|
||||
_MAX_REMOVAL_ENTRIES = 4096
|
||||
_MAX_REMOVAL_DEPTH = 16
|
||||
|
||||
|
||||
def _component(value: str) -> str:
|
||||
normalized = str(value or '').strip()
|
||||
if (
|
||||
not normalized
|
||||
or normalized in {'.', '..'}
|
||||
or '/' in normalized
|
||||
or '\\' in normalized
|
||||
or '\x00' in normalized
|
||||
or len(os.fsencode(normalized)) > 240
|
||||
):
|
||||
raise UnsafeWorkspacePathError('Unsafe Workspace path component')
|
||||
return normalized
|
||||
|
||||
|
||||
def _unsafe(path: str, exc: BaseException | None = None) -> UnsafeWorkspacePathError:
|
||||
error = UnsafeWorkspacePathError(f'Workspace path is not a link-free directory: {path}')
|
||||
if exc is not None:
|
||||
error.__cause__ = exc
|
||||
return error
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _root_fd(root: str):
|
||||
try:
|
||||
fd = os.open(root, _DIRECTORY_FLAGS)
|
||||
except OSError as exc:
|
||||
raise _unsafe(root, exc)
|
||||
try:
|
||||
if not stat.S_ISDIR(os.fstat(fd).st_mode):
|
||||
raise _unsafe(root)
|
||||
yield fd
|
||||
finally:
|
||||
os.close(fd)
|
||||
|
||||
|
||||
def _open_dir_at(parent_fd: int, name: str, *, create: bool) -> int:
|
||||
name = _component(name)
|
||||
if create:
|
||||
try:
|
||||
os.mkdir(name, mode=0o700, dir_fd=parent_fd)
|
||||
except FileExistsError:
|
||||
pass
|
||||
try:
|
||||
fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
|
||||
except OSError as exc:
|
||||
raise _unsafe(name, exc)
|
||||
if not stat.S_ISDIR(os.fstat(fd).st_mode):
|
||||
os.close(fd)
|
||||
raise _unsafe(name)
|
||||
return fd
|
||||
|
||||
|
||||
def _remove_entry(
|
||||
parent_fd: int,
|
||||
name: str,
|
||||
*,
|
||||
budget: list[int] | None = None,
|
||||
depth: int = 0,
|
||||
) -> None:
|
||||
"""Remove an entry recursively without following a symlink at any depth."""
|
||||
|
||||
name = _component(name)
|
||||
budget = budget if budget is not None else [_MAX_REMOVAL_ENTRIES]
|
||||
if depth > _MAX_REMOVAL_DEPTH or budget[0] <= 0:
|
||||
raise UnsafeWorkspacePathError('Workspace cleanup exceeded its inode budget')
|
||||
budget[0] -= 1
|
||||
for _ in range(4):
|
||||
try:
|
||||
child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except OSError as exc:
|
||||
if exc.errno not in {errno.ELOOP, errno.ENOTDIR, errno.EACCES}:
|
||||
raise
|
||||
try:
|
||||
os.unlink(name, dir_fd=parent_fd)
|
||||
return
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except IsADirectoryError:
|
||||
continue
|
||||
else:
|
||||
try:
|
||||
_clear_dir(child_fd, budget=budget, depth=depth + 1)
|
||||
finally:
|
||||
os.close(child_fd)
|
||||
try:
|
||||
os.rmdir(name, dir_fd=parent_fd)
|
||||
return
|
||||
except FileNotFoundError:
|
||||
return
|
||||
except NotADirectoryError:
|
||||
continue
|
||||
raise UnsafeWorkspacePathError('Workspace entry changed while it was being removed')
|
||||
|
||||
|
||||
def _clear_dir(directory_fd: int, *, budget: list[int], depth: int) -> None:
|
||||
# ``scandir(fd)`` enumerates the already-open directory. Names are then
|
||||
# resolved relative to the same fd, so a tenant cannot redirect the walk by
|
||||
# swapping an ancestor symlink between validation and use.
|
||||
# Do not materialize the whole directory: an attacker-controlled outbox
|
||||
# may contain an inode bomb even when its byte size is tiny. Removal is
|
||||
# deliberately budgeted and fails closed once the per-operation cap is
|
||||
# reached; hard filesystem/inode quota remains a Cloud readiness gate.
|
||||
with os.scandir(directory_fd) as iterator:
|
||||
for entry in iterator:
|
||||
_remove_entry(directory_fd, entry.name, budget=budget, depth=depth)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _query_fd(root: str, subdir: str, query_key: str, *, create: bool, reset: bool = False):
|
||||
subdir = _component(subdir)
|
||||
query_key = _component(query_key)
|
||||
with _root_fd(root) as root_fd:
|
||||
subdir_fd = _open_dir_at(root_fd, subdir, create=create)
|
||||
try:
|
||||
if reset:
|
||||
_remove_entry(subdir_fd, query_key)
|
||||
query_fd = _open_dir_at(subdir_fd, query_key, create=create)
|
||||
try:
|
||||
yield query_fd
|
||||
finally:
|
||||
os.close(query_fd)
|
||||
finally:
|
||||
os.close(subdir_fd)
|
||||
|
||||
|
||||
def write_files(
|
||||
root: str,
|
||||
subdir: str,
|
||||
query_key: str,
|
||||
files: Iterable[tuple[str, bytes]],
|
||||
) -> None:
|
||||
"""Atomically recreate one query directory and write regular files only."""
|
||||
|
||||
with _query_fd(root, subdir, query_key, create=True, reset=True) as query_fd:
|
||||
for raw_name, data in files:
|
||||
name = _component(raw_name)
|
||||
try:
|
||||
file_fd = os.open(name, _FILE_WRITE_FLAGS, 0o600, dir_fd=query_fd)
|
||||
except OSError as exc:
|
||||
raise UnsafeWorkspacePathError(f'Could not create a link-free Workspace file: {name}') from exc
|
||||
with os.fdopen(file_fd, 'wb') as file_obj:
|
||||
file_obj.write(data)
|
||||
|
||||
|
||||
def _read_directory(
|
||||
directory_fd: int,
|
||||
*,
|
||||
prefix: str,
|
||||
max_file_bytes: int,
|
||||
max_files: int,
|
||||
max_total_bytes: int,
|
||||
output: list[tuple[str, bytes]],
|
||||
total: list[int],
|
||||
remaining_entries: list[int],
|
||||
remaining_directories: list[int],
|
||||
depth: int,
|
||||
) -> None:
|
||||
if depth > 8:
|
||||
return
|
||||
with os.scandir(directory_fd) as iterator:
|
||||
for entry in iterator:
|
||||
if len(output) >= max_files or total[0] >= max_total_bytes:
|
||||
return
|
||||
if remaining_entries[0] <= 0:
|
||||
raise UnsafeWorkspacePathError('Sandbox outbox exceeds the directory-entry limit')
|
||||
remaining_entries[0] -= 1
|
||||
name = _component(entry.name)
|
||||
relative = f'{prefix}/{name}' if prefix else name
|
||||
if entry.is_symlink():
|
||||
continue
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
if remaining_directories[0] <= 0:
|
||||
raise UnsafeWorkspacePathError('Sandbox outbox exceeds the directory limit')
|
||||
remaining_directories[0] -= 1
|
||||
try:
|
||||
child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=directory_fd)
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
_read_directory(
|
||||
child_fd,
|
||||
prefix=relative,
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_files=max_files,
|
||||
max_total_bytes=max_total_bytes,
|
||||
output=output,
|
||||
total=total,
|
||||
remaining_entries=remaining_entries,
|
||||
remaining_directories=remaining_directories,
|
||||
depth=depth + 1,
|
||||
)
|
||||
finally:
|
||||
os.close(child_fd)
|
||||
continue
|
||||
try:
|
||||
file_fd = os.open(name, _FILE_READ_FLAGS, dir_fd=directory_fd)
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
metadata = os.fstat(file_fd)
|
||||
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > max_file_bytes:
|
||||
continue
|
||||
remaining = max_total_bytes - total[0]
|
||||
if metadata.st_size > remaining:
|
||||
continue
|
||||
with os.fdopen(file_fd, 'rb', closefd=False) as file_obj:
|
||||
data = file_obj.read(max_file_bytes + 1)
|
||||
if len(data) > max_file_bytes or len(data) > remaining:
|
||||
continue
|
||||
output.append((relative, data))
|
||||
total[0] += len(data)
|
||||
finally:
|
||||
os.close(file_fd)
|
||||
|
||||
|
||||
def read_regular_files(
|
||||
root: str,
|
||||
subdir: str,
|
||||
query_key: str,
|
||||
*,
|
||||
max_file_bytes: int,
|
||||
max_files: int,
|
||||
max_total_bytes: int,
|
||||
max_entries: int = 512,
|
||||
max_directories: int = 64,
|
||||
) -> list[tuple[str, bytes]]:
|
||||
"""Read bounded regular files without following tenant-created links."""
|
||||
|
||||
output: list[tuple[str, bytes]] = []
|
||||
try:
|
||||
with _query_fd(root, subdir, query_key, create=False) as query_fd:
|
||||
_read_directory(
|
||||
query_fd,
|
||||
prefix='',
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_files=max_files,
|
||||
max_total_bytes=max_total_bytes,
|
||||
output=output,
|
||||
total=[0],
|
||||
remaining_entries=[max_entries],
|
||||
remaining_directories=[max_directories],
|
||||
depth=0,
|
||||
)
|
||||
except (FileNotFoundError, UnsafeWorkspacePathError):
|
||||
# A missing directory is an empty outbox. An unsafe existing path is
|
||||
# deliberately surfaced to the caller rather than followed.
|
||||
if os.path.lexists(os.path.join(root, subdir, query_key)):
|
||||
raise
|
||||
return output
|
||||
|
||||
|
||||
def reset_directory(root: str, subdir: str, query_key: str) -> None:
|
||||
with _query_fd(root, subdir, query_key, create=True, reset=True):
|
||||
return
|
||||
|
||||
|
||||
def purge_subdirectory(root: str, subdir: str) -> None:
|
||||
"""Remove one known subtree without following a hostile replacement link."""
|
||||
|
||||
with _root_fd(root) as root_fd:
|
||||
_remove_entry(root_fd, _component(subdir))
|
||||
+377
-69
@@ -4,8 +4,10 @@ import asyncio
|
||||
import collections
|
||||
import datetime as _dt
|
||||
import enum
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import secrets
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import pydantic
|
||||
@@ -13,11 +15,14 @@ import pydantic
|
||||
from langbot_plugin.box.client import BoxRuntimeClient
|
||||
from langbot_plugin.entities.io.context import ActionContext
|
||||
from langbot_plugin.box.tenancy import box_namespace
|
||||
from langbot_plugin.box.security import BOX_SHARED_WORKSPACE_PROBE_PREFIX
|
||||
from .admission import SandboxAdmissionController, require_cloud_admission_policy
|
||||
from .connector import BoxRuntimeConnector, _get_box_config
|
||||
from . import secure_fs
|
||||
from ..telemetry import features as telemetry_features
|
||||
from ..api.http.context import ExecutionContext
|
||||
from ..api.http.service.tenant import TenantContext, require_workspace_uuid
|
||||
from langbot_plugin.box.errors import BoxError, BoxValidationError
|
||||
from langbot_plugin.box.errors import BoxAdmissionError, BoxError, BoxValidationError
|
||||
from langbot_plugin.box.models import (
|
||||
BUILTIN_PROFILES,
|
||||
BoxExecutionResult,
|
||||
@@ -51,6 +56,7 @@ class BoxService:
|
||||
output_limit_chars: int = 4000,
|
||||
):
|
||||
self.ap = ap
|
||||
self._cloud_managed = bool(getattr(getattr(ap, 'deployment', None), 'multi_workspace_enabled', False))
|
||||
self._enabled = self._load_enabled()
|
||||
self._runtime_connector: BoxRuntimeConnector | None = None
|
||||
if client is None:
|
||||
@@ -67,6 +73,18 @@ class BoxService:
|
||||
self.profile = self._load_profile()
|
||||
self.custom_image = self._load_custom_image()
|
||||
self.workspace_quota_mb = self._load_workspace_quota_mb()
|
||||
self._admission_policy = (
|
||||
require_cloud_admission_policy(_get_box_config(ap).get('admission')) if self._cloud_managed else None
|
||||
)
|
||||
self._admission = (
|
||||
SandboxAdmissionController(
|
||||
ap,
|
||||
self.client,
|
||||
policy=self._admission_policy,
|
||||
)
|
||||
if self._cloud_managed and self._admission_policy is not None
|
||||
else None
|
||||
)
|
||||
self._recent_errors: collections.deque[dict] = collections.deque(maxlen=_MAX_RECENT_ERRORS)
|
||||
self._shutdown_task = None
|
||||
self._available = False
|
||||
@@ -85,6 +103,10 @@ class BoxService:
|
||||
``available = False`` to consumers, but distinguished in get_status."""
|
||||
return self._enabled
|
||||
|
||||
@property
|
||||
def managed_admission_required(self) -> bool:
|
||||
return self._cloud_managed
|
||||
|
||||
async def initialize(self):
|
||||
if not self._enabled:
|
||||
# Disabled by config: do NOT connect to a remote runtime, do NOT
|
||||
@@ -103,17 +125,24 @@ class BoxService:
|
||||
else:
|
||||
await self.client.initialize()
|
||||
self._ensure_default_workspace()
|
||||
await self._verify_cloud_runtime()
|
||||
self._available = True
|
||||
self._connector_error = ''
|
||||
self.ap.logger.info(
|
||||
f'LangBot Box runtime initialized: profile={self.profile.name} '
|
||||
f'default_workspace={self.default_workspace or "(none)"}'
|
||||
)
|
||||
await self._purge_attachment_dirs()
|
||||
# Cloud query directories use globally opaque query UUIDs. Never
|
||||
# sweep all tenants when a future replica joins the same logical
|
||||
# instance; that could delete another replica's in-flight files.
|
||||
if not self._cloud_managed:
|
||||
await self._purge_attachment_dirs()
|
||||
except Exception as exc:
|
||||
self.ap.logger.warning(f'LangBot Box runtime unavailable, sandbox features disabled: {exc}')
|
||||
self._available = False
|
||||
self._connector_error = str(exc)
|
||||
if self._cloud_managed:
|
||||
raise
|
||||
|
||||
async def _on_runtime_disconnect(self, connector: BoxRuntimeConnector) -> None:
|
||||
"""Called by the connector when the Box runtime connection drops.
|
||||
@@ -143,6 +172,7 @@ class BoxService:
|
||||
try:
|
||||
connector.dispose()
|
||||
await connector.initialize()
|
||||
await self._verify_cloud_runtime()
|
||||
self._available = True
|
||||
self._connector_error = ''
|
||||
self.ap.logger.info('Box runtime reconnected, sandbox features restored.')
|
||||
@@ -154,6 +184,83 @@ class BoxService:
|
||||
finally:
|
||||
self._reconnecting = False
|
||||
|
||||
async def _verify_cloud_runtime(self) -> None:
|
||||
if not self._cloud_managed:
|
||||
return
|
||||
self._ensure_cloud_shared_workspace()
|
||||
await self._challenge_cloud_shared_workspace()
|
||||
backend_info = await self.client.get_backend_info()
|
||||
if (
|
||||
not isinstance(backend_info, dict)
|
||||
or backend_info.get('name') != 'nsjail'
|
||||
or backend_info.get('available') is not True
|
||||
):
|
||||
raise BoxValidationError('Cloud Box nsjail isolation readiness failed')
|
||||
|
||||
async def _challenge_cloud_shared_workspace(self) -> None:
|
||||
"""Prove Core and Box Runtime see the same durable filesystem.
|
||||
|
||||
Equal configured path strings are not evidence of a shared container
|
||||
volume. Core creates one high-entropy, no-follow marker under its
|
||||
canonical root and the authenticated Runtime host-control action reads
|
||||
that basename only. Any mismatch fails Cloud startup/reconnect closed.
|
||||
"""
|
||||
|
||||
if self.default_workspace is None:
|
||||
raise BoxValidationError('Cloud Box shared default_workspace is unavailable')
|
||||
|
||||
marker_name = f'{BOX_SHARED_WORKSPACE_PROBE_PREFIX}{secrets.token_hex(16)}'
|
||||
marker_payload = secrets.token_bytes(64)
|
||||
expected_digest = hashlib.sha256(marker_payload).hexdigest()
|
||||
directory_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)
|
||||
nofollow = getattr(os, 'O_NOFOLLOW', 0)
|
||||
root_fd: int | None = None
|
||||
marker_fd: int | None = None
|
||||
marker_created = False
|
||||
try:
|
||||
root_fd = os.open(self.default_workspace, directory_flags | nofollow)
|
||||
marker_fd = os.open(
|
||||
marker_name,
|
||||
os.O_WRONLY | os.O_CREAT | os.O_EXCL | nofollow,
|
||||
0o600,
|
||||
dir_fd=root_fd,
|
||||
)
|
||||
marker_created = True
|
||||
remaining = memoryview(marker_payload)
|
||||
while remaining:
|
||||
written = os.write(marker_fd, remaining)
|
||||
if written <= 0:
|
||||
raise BoxValidationError('Failed to write Cloud Box shared-volume probe')
|
||||
remaining = remaining[written:]
|
||||
os.fsync(marker_fd)
|
||||
os.close(marker_fd)
|
||||
marker_fd = None
|
||||
|
||||
result = await self.client.verify_shared_workspace(marker_name)
|
||||
if (
|
||||
not isinstance(result, dict)
|
||||
or result.get('marker_name') != marker_name
|
||||
or result.get('size') != len(marker_payload)
|
||||
or not secrets.compare_digest(str(result.get('sha256') or ''), expected_digest)
|
||||
):
|
||||
raise BoxValidationError(
|
||||
'Cloud Box Core and Runtime do not share the configured durable Workspace volume'
|
||||
)
|
||||
except BoxValidationError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise BoxValidationError('Cloud Box shared durable Workspace volume verification failed') from exc
|
||||
finally:
|
||||
if marker_fd is not None:
|
||||
os.close(marker_fd)
|
||||
if root_fd is not None:
|
||||
if marker_created:
|
||||
try:
|
||||
os.unlink(marker_name, dir_fd=root_fd)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
os.close(root_fd)
|
||||
|
||||
@property
|
||||
def available(self) -> bool:
|
||||
return self._available
|
||||
@@ -200,10 +307,14 @@ class BoxService:
|
||||
bot_uuid=getattr(context, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(context, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(context, 'query_uuid', None),
|
||||
entitlement_revision=getattr(context, 'entitlement_revision', 0),
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _query_execution_context(cls, query: pipeline_query.Query) -> ExecutionContext:
|
||||
attached_context = getattr(query, '_execution_context', None)
|
||||
if isinstance(attached_context, ExecutionContext):
|
||||
return cls._execution_context(attached_context)
|
||||
return cls._execution_context(
|
||||
ExecutionContext(
|
||||
instance_uuid=str(getattr(query, 'instance_uuid', '') or ''),
|
||||
@@ -212,6 +323,7 @@ class BoxService:
|
||||
bot_uuid=getattr(query, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(query, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(query, 'query_uuid', None),
|
||||
entitlement_revision=getattr(query, 'entitlement_revision', 0),
|
||||
)
|
||||
)
|
||||
|
||||
@@ -241,6 +353,116 @@ class BoxService:
|
||||
raise BoxValidationError('Box execution context belongs to a stale Workspace placement')
|
||||
return execution_context
|
||||
|
||||
async def require_workspace_sandbox(self, context: TenantContext) -> ExecutionContext:
|
||||
"""Fence a Workspace and install its short-lived Cloud admission grant.
|
||||
|
||||
OSS keeps the existing singleton behavior and does not require a
|
||||
Control Plane entitlement. Cloud always resolves a fresh generic
|
||||
entitlement before a sandbox-visible operation.
|
||||
"""
|
||||
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
await self._require_validated_workspace_sandbox(execution_context)
|
||||
return execution_context
|
||||
|
||||
async def _require_validated_workspace_sandbox(self, execution_context: ExecutionContext) -> None:
|
||||
if not self._available:
|
||||
raise BoxError('Box runtime is not available. Install and start Docker to use sandbox features.')
|
||||
if self._cloud_managed:
|
||||
if self._admission is None:
|
||||
raise BoxAdmissionError('Cloud Box sandbox admission is unavailable')
|
||||
await self._admission.require(execution_context)
|
||||
|
||||
async def is_workspace_sandbox_available(self, context: TenantContext) -> bool:
|
||||
"""Return tenant-specific availability for UI and tool discovery.
|
||||
|
||||
This method deliberately catches entitlement failures so callers can
|
||||
hide tools without leaking plan details. Direct execution APIs use
|
||||
:meth:`require_workspace_sandbox` and retain an explicit failure.
|
||||
"""
|
||||
|
||||
if not self._available:
|
||||
return False
|
||||
try:
|
||||
await self.require_workspace_sandbox(context)
|
||||
return True
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def _managed_policy_payload(
|
||||
self,
|
||||
context: TenantContext,
|
||||
spec_payload: dict,
|
||||
) -> dict:
|
||||
"""Reject tenant-owned policy fields and apply the Cloud hard policy."""
|
||||
|
||||
payload = dict(spec_payload)
|
||||
if not self._cloud_managed:
|
||||
return payload
|
||||
policy = self._admission_policy
|
||||
if policy is None:
|
||||
raise BoxAdmissionError('Cloud Box sandbox admission policy is unavailable')
|
||||
|
||||
forged_fields = {
|
||||
'plan',
|
||||
'subscription',
|
||||
'managed_sandbox',
|
||||
'entitlement',
|
||||
'entitlement_revision',
|
||||
'max_sessions',
|
||||
'max_managed_processes',
|
||||
'backend',
|
||||
}
|
||||
submitted_forged_fields = sorted(forged_fields.intersection(payload))
|
||||
if submitted_forged_fields:
|
||||
raise BoxAdmissionError(
|
||||
'Managed sandbox policy fields are host-controlled: ' + ', '.join(submitted_forged_fields)
|
||||
)
|
||||
|
||||
submitted_session_id = str(payload.get('session_id', '') or '').strip()
|
||||
if submitted_session_id and submitted_session_id != policy.logical_session_id:
|
||||
raise BoxAdmissionError('Managed sandbox session_id is runtime-owned')
|
||||
submitted_network = str(getattr(payload.get('network'), 'value', payload.get('network', 'off')) or 'off')
|
||||
if submitted_network != 'off':
|
||||
raise BoxAdmissionError('Managed sandbox network access is disabled')
|
||||
if payload.get('extra_mounts'):
|
||||
raise BoxAdmissionError('Managed sandbox additional host mounts are disabled')
|
||||
submitted_mount_path = str(payload.get('mount_path', '/workspace') or '/workspace')
|
||||
if submitted_mount_path != '/workspace':
|
||||
raise BoxAdmissionError('Managed sandbox mount_path is runtime-owned')
|
||||
|
||||
canonical_host_path = self._tenant_workspace(context)
|
||||
if canonical_host_path is None:
|
||||
raise BoxAdmissionError('Managed sandbox Workspace path is unavailable')
|
||||
submitted_host_path = str(payload.get('host_path', '') or '').strip()
|
||||
if submitted_host_path and os.path.realpath(submitted_host_path) != os.path.realpath(canonical_host_path):
|
||||
raise BoxAdmissionError('Managed sandbox host_path is runtime-owned')
|
||||
|
||||
timeout = payload.get('timeout_sec', policy.max_timeout_sec)
|
||||
if isinstance(timeout, bool) or not isinstance(timeout, int):
|
||||
raise BoxValidationError('timeout_sec must be an integer')
|
||||
payload.update(
|
||||
{
|
||||
'session_id': policy.logical_session_id,
|
||||
'network': 'off',
|
||||
'host_path': canonical_host_path,
|
||||
'mount_path': '/workspace',
|
||||
'extra_mounts': [],
|
||||
'persistent': True,
|
||||
'timeout_sec': min(timeout, policy.max_timeout_sec),
|
||||
'cpus': policy.cpus,
|
||||
'memory_mb': policy.memory_mb,
|
||||
'pids_limit': policy.pids_limit,
|
||||
'read_only_rootfs': policy.read_only_rootfs,
|
||||
'workspace_quota_mb': policy.workspace_quota_mb,
|
||||
}
|
||||
)
|
||||
return payload
|
||||
|
||||
def _reject_cloud_managed_process(self) -> None:
|
||||
if self._cloud_managed:
|
||||
raise BoxAdmissionError('Managed processes are disabled for Cloud sandboxes')
|
||||
|
||||
def _tenant_workspace(self, context: TenantContext) -> str | None:
|
||||
if self.default_workspace is None:
|
||||
return None
|
||||
@@ -257,7 +479,8 @@ class BoxService:
|
||||
if not self._available:
|
||||
raise BoxError('Box runtime is not available. Install and start Docker to use sandbox features.')
|
||||
execution_context = await self._validated_execution_context(self._query_execution_context(query))
|
||||
spec_payload = dict(spec_payload)
|
||||
spec_payload = self._managed_policy_payload(execution_context, spec_payload)
|
||||
await self._require_validated_workspace_sandbox(execution_context)
|
||||
if spec_payload.get('host_path') in (None, ''):
|
||||
tenant_workspace = self._tenant_workspace(execution_context)
|
||||
if tenant_workspace is not None:
|
||||
@@ -284,6 +507,11 @@ class BoxService:
|
||||
spec,
|
||||
action_context=self._action_context(execution_context),
|
||||
)
|
||||
# A placement may be cut over while a long-running sandbox call is
|
||||
# in flight. Never accept a result produced by the superseded
|
||||
# generation. Runtime-side generation fencing prevents new work;
|
||||
# this second Core check closes the response race.
|
||||
await self._validated_execution_context(execution_context)
|
||||
except BoxError as exc:
|
||||
self._record_error(exc, query)
|
||||
raise
|
||||
@@ -311,6 +539,8 @@ class BoxService:
|
||||
by editing the pipeline config directly through the API (which only
|
||||
gates the web UI).
|
||||
"""
|
||||
if self._cloud_managed:
|
||||
return 'global'
|
||||
forced_template = self._forced_box_session_id_template()
|
||||
if forced_template:
|
||||
template = forced_template
|
||||
@@ -362,6 +592,8 @@ class BoxService:
|
||||
skills it discovered on its own filesystem, so the path is valid there
|
||||
by construction.
|
||||
"""
|
||||
if self._cloud_managed:
|
||||
return []
|
||||
skill_mgr = getattr(self.ap, 'skill_mgr', None)
|
||||
if skill_mgr is None:
|
||||
return []
|
||||
@@ -392,13 +624,21 @@ class BoxService:
|
||||
)
|
||||
return mounts
|
||||
|
||||
async def execute_tool(self, parameters: dict, query: pipeline_query.Query) -> dict:
|
||||
async def execute_tool(
|
||||
self,
|
||||
parameters: dict,
|
||||
query: pipeline_query.Query,
|
||||
*,
|
||||
skill_name: str | None = None,
|
||||
) -> dict:
|
||||
"""Execute an agent-facing ``exec`` tool call.
|
||||
|
||||
Translates the agent-facing ``command`` field to the internal
|
||||
``BoxSpec.cmd`` field and injects the session id from the query.
|
||||
"""
|
||||
spec_payload: dict = {'cmd': parameters['command']}
|
||||
if skill_name is not None:
|
||||
spec_payload['skill_name'] = skill_name
|
||||
|
||||
# Pass through allowed agent-facing fields
|
||||
for key in ('workdir', 'timeout_sec', 'env'):
|
||||
@@ -424,7 +664,8 @@ class BoxService:
|
||||
"""Execute trusted internal Box work inside one Workspace namespace."""
|
||||
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
payload = dict(spec_payload)
|
||||
payload = self._managed_policy_payload(execution_context, spec_payload)
|
||||
await self._require_validated_workspace_sandbox(execution_context)
|
||||
if payload.get('host_path') in (None, ''):
|
||||
tenant_workspace = self._tenant_workspace(execution_context)
|
||||
if tenant_workspace is not None:
|
||||
@@ -432,7 +673,9 @@ class BoxService:
|
||||
if self.shares_filesystem_with_box:
|
||||
os.makedirs(tenant_workspace, exist_ok=True)
|
||||
spec = self.build_spec(payload, skip_host_mount_validation=skip_host_mount_validation)
|
||||
return await self.client.execute(spec, action_context=self._action_context(execution_context))
|
||||
result = await self.client.execute(spec, action_context=self._action_context(execution_context))
|
||||
await self._validated_execution_context(execution_context)
|
||||
return result
|
||||
|
||||
# ── Attachment passthrough (inbound / outbound) ──────────────────
|
||||
#
|
||||
@@ -462,10 +705,22 @@ class BoxService:
|
||||
# Hard cap on a single attachment. The HTTP upload endpoints already cap
|
||||
# uploads at 10MiB; keep parity.
|
||||
_ATTACHMENT_MAX_BYTES = 10 * _MIB
|
||||
_ATTACHMENT_MAX_FILES = 20
|
||||
_ATTACHMENT_MAX_TOTAL_BYTES = 50 * _MIB
|
||||
# Conservative cap for the exec FALLBACK path only (ARG_MAX / stdout
|
||||
# truncation). The host-filesystem path has no such limit.
|
||||
_EXEC_FALLBACK_MAX_BYTES = 256 * 1024
|
||||
|
||||
def _attachment_query_key(self, query: pipeline_query.Query) -> str:
|
||||
query_uuid = str(getattr(query, 'query_uuid', '') or '').strip()
|
||||
if query_uuid:
|
||||
if query_uuid in {'.', '..'} or '/' in query_uuid or '\\' in query_uuid or '\x00' in query_uuid:
|
||||
raise BoxValidationError('Query attachment identity is invalid')
|
||||
return query_uuid
|
||||
if self._cloud_managed:
|
||||
raise BoxValidationError('Cloud attachment transfer requires query_uuid')
|
||||
return str(query.query_id)
|
||||
|
||||
def _host_query_dir(self, subdir: str, query: pipeline_query.Query) -> str | None:
|
||||
"""Host path for ``/workspace/<subdir>/<query_id>`` when LangBot can
|
||||
access the bind-mounted workspace directly, else ``None``.
|
||||
@@ -477,9 +732,9 @@ class BoxService:
|
||||
E2B and remote runtimes, where we must fall back to the exec channel.
|
||||
"""
|
||||
root = self._tenant_workspace(self._query_execution_context(query))
|
||||
if not root or not os.path.isdir(root):
|
||||
if not root or not os.path.isdir(root) or os.path.islink(root):
|
||||
return None
|
||||
return os.path.join(root, subdir, str(query.query_id))
|
||||
return os.path.join(root, subdir, self._attachment_query_key(query))
|
||||
|
||||
async def _purge_attachment_dirs(self) -> None:
|
||||
"""Remove leftover inbox/outbox directories on startup.
|
||||
@@ -498,14 +753,12 @@ class BoxService:
|
||||
if not root or not os.path.isdir(root):
|
||||
return
|
||||
|
||||
import shutil
|
||||
|
||||
host_survivors: list[str] = []
|
||||
|
||||
def _host_purge() -> list[str]:
|
||||
candidates = [
|
||||
os.path.join(root, self.INBOX_SUBDIR),
|
||||
os.path.join(root, self.OUTBOX_SUBDIR),
|
||||
candidates: list[tuple[str, str]] = [
|
||||
(root, self.INBOX_SUBDIR),
|
||||
(root, self.OUTBOX_SUBDIR),
|
||||
]
|
||||
tenants_root = os.path.join(root, 'tenants')
|
||||
if os.path.isdir(tenants_root):
|
||||
@@ -515,17 +768,18 @@ class BoxService:
|
||||
continue
|
||||
candidates.extend(
|
||||
[
|
||||
os.path.join(tenant_entry.path, self.INBOX_SUBDIR),
|
||||
os.path.join(tenant_entry.path, self.OUTBOX_SUBDIR),
|
||||
(tenant_entry.path, self.INBOX_SUBDIR),
|
||||
(tenant_entry.path, self.OUTBOX_SUBDIR),
|
||||
]
|
||||
)
|
||||
survivors: list[str] = []
|
||||
for path in candidates:
|
||||
if not os.path.isdir(path):
|
||||
continue
|
||||
shutil.rmtree(path, ignore_errors=True)
|
||||
if os.path.exists(path):
|
||||
survivors.append(path)
|
||||
for candidate_root, subdir in candidates:
|
||||
path = os.path.join(candidate_root, subdir)
|
||||
try:
|
||||
secure_fs.purge_subdirectory(candidate_root, subdir)
|
||||
except OSError:
|
||||
if os.path.lexists(path):
|
||||
survivors.append(path)
|
||||
return survivors
|
||||
|
||||
try:
|
||||
@@ -640,14 +894,15 @@ class BoxService:
|
||||
(the webchat session uses small sequential ids) never inherits stale
|
||||
files from an earlier turn.
|
||||
"""
|
||||
import shutil
|
||||
|
||||
shutil.rmtree(host_dir, ignore_errors=True)
|
||||
os.makedirs(host_dir, exist_ok=True)
|
||||
query_key = os.path.basename(host_dir)
|
||||
subdir = os.path.basename(os.path.dirname(host_dir))
|
||||
tenant_root = os.path.dirname(os.path.dirname(host_dir))
|
||||
try:
|
||||
secure_fs.write_files(tenant_root, subdir, query_key, files)
|
||||
except secure_fs.UnsafeWorkspacePathError as exc:
|
||||
raise BoxValidationError('Sandbox attachment path contains an unsafe symbolic link') from exc
|
||||
written: list[str] = []
|
||||
for name, data in files:
|
||||
with open(os.path.join(host_dir, name), 'wb') as fh:
|
||||
fh.write(data)
|
||||
for name, _data in files:
|
||||
written.append(f'{target_mount_dir}/{name}')
|
||||
return written
|
||||
|
||||
@@ -718,6 +973,8 @@ class BoxService:
|
||||
"""
|
||||
if not self._available:
|
||||
return []
|
||||
if self._cloud_managed:
|
||||
await self.require_workspace_sandbox(self._query_execution_context(query))
|
||||
|
||||
import langbot_plugin.api.entities.builtin.platform.message as platform_message
|
||||
|
||||
@@ -766,7 +1023,8 @@ class BoxService:
|
||||
if not pending:
|
||||
return []
|
||||
|
||||
target_dir = f'{self.INBOX_MOUNT_DIR}/{query.query_id}'
|
||||
query_key = self._attachment_query_key(query)
|
||||
target_dir = f'{self.INBOX_MOUNT_DIR}/{query_key}'
|
||||
written = await self._write_files_into_sandbox(query, self.INBOX_SUBDIR, target_dir, pending)
|
||||
written_basenames = {os.path.basename(p) for p in written}
|
||||
|
||||
@@ -794,6 +1052,8 @@ class BoxService:
|
||||
"""
|
||||
if not self._available:
|
||||
return []
|
||||
if self._cloud_managed:
|
||||
await self.require_workspace_sandbox(self._query_execution_context(query))
|
||||
|
||||
host_dir = self._host_query_dir(self.OUTBOX_SUBDIR, query)
|
||||
if host_dir is not None:
|
||||
@@ -817,22 +1077,21 @@ class BoxService:
|
||||
"""Read outbox files straight off the bind-mounted host directory."""
|
||||
import base64 as _b64
|
||||
|
||||
entries: list[dict] = []
|
||||
if not os.path.isdir(host_dir):
|
||||
return entries
|
||||
for root, _dirs, names in os.walk(host_dir):
|
||||
for name in sorted(names):
|
||||
path = os.path.join(root, name)
|
||||
try:
|
||||
if os.path.getsize(path) > self._ATTACHMENT_MAX_BYTES:
|
||||
continue
|
||||
with open(path, 'rb') as fh:
|
||||
data = fh.read()
|
||||
except OSError:
|
||||
continue
|
||||
rel = os.path.relpath(path, host_dir)
|
||||
entries.append({'name': rel, 'b64': _b64.b64encode(data).decode('ascii')})
|
||||
return entries
|
||||
query_key = os.path.basename(host_dir)
|
||||
subdir = os.path.basename(os.path.dirname(host_dir))
|
||||
tenant_root = os.path.dirname(os.path.dirname(host_dir))
|
||||
try:
|
||||
files = secure_fs.read_regular_files(
|
||||
tenant_root,
|
||||
subdir,
|
||||
query_key,
|
||||
max_file_bytes=self._ATTACHMENT_MAX_BYTES,
|
||||
max_files=self._ATTACHMENT_MAX_FILES,
|
||||
max_total_bytes=self._ATTACHMENT_MAX_TOTAL_BYTES,
|
||||
)
|
||||
except secure_fs.UnsafeWorkspacePathError as exc:
|
||||
raise BoxValidationError('Sandbox outbox contains an unsafe symbolic link') from exc
|
||||
return [{'name': name, 'b64': _b64.b64encode(data).decode('ascii')} for name, data in files]
|
||||
|
||||
async def _read_outbox_via_exec(self, query: pipeline_query.Query) -> list[dict]:
|
||||
"""Fallback: read the outbox over the exec channel (E2B / remote).
|
||||
@@ -842,7 +1101,7 @@ class BoxService:
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
target_dir = f'{self.OUTBOX_MOUNT_DIR}/{query.query_id}'
|
||||
target_dir = f'{self.OUTBOX_MOUNT_DIR}/{self._attachment_query_key(query)}'
|
||||
max_bytes = self._EXEC_FALLBACK_MAX_BYTES
|
||||
script = (
|
||||
'import base64, json, os\n'
|
||||
@@ -887,16 +1146,19 @@ class BoxService:
|
||||
container's root can remove its own files. Best-effort: never raise
|
||||
into the pipeline.
|
||||
"""
|
||||
target_dir = f'{self.OUTBOX_MOUNT_DIR}/{query.query_id}'
|
||||
target_dir = f'{self.OUTBOX_MOUNT_DIR}/{self._attachment_query_key(query)}'
|
||||
|
||||
if host_dir is not None:
|
||||
import shutil
|
||||
|
||||
def _clear() -> bool:
|
||||
shutil.rmtree(host_dir, ignore_errors=True)
|
||||
survived = os.path.exists(host_dir) and bool(os.listdir(host_dir))
|
||||
os.makedirs(host_dir, exist_ok=True)
|
||||
return survived
|
||||
query_key = os.path.basename(host_dir)
|
||||
subdir = os.path.basename(os.path.dirname(host_dir))
|
||||
tenant_root = os.path.dirname(os.path.dirname(host_dir))
|
||||
try:
|
||||
secure_fs.reset_directory(tenant_root, subdir, query_key)
|
||||
return False
|
||||
except OSError:
|
||||
return True
|
||||
|
||||
survived = await asyncio.to_thread(_clear)
|
||||
if not survived:
|
||||
@@ -952,9 +1214,9 @@ class BoxService:
|
||||
self._shutdown_task = loop.create_task(self.shutdown())
|
||||
|
||||
async def get_sessions(self, context: TenantContext) -> list[dict]:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
if not self._available:
|
||||
return []
|
||||
execution_context = await self.require_workspace_sandbox(context)
|
||||
try:
|
||||
return await self.client.get_sessions(action_context=self._action_context(execution_context))
|
||||
except Exception:
|
||||
@@ -992,7 +1254,8 @@ class BoxService:
|
||||
skip_host_mount_validation: bool = False,
|
||||
) -> dict:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
spec_payload = dict(spec_payload)
|
||||
spec_payload = self._managed_policy_payload(execution_context, spec_payload)
|
||||
await self._require_validated_workspace_sandbox(execution_context)
|
||||
if spec_payload.get('host_path') in (None, ''):
|
||||
tenant_workspace = self._tenant_workspace(execution_context)
|
||||
if tenant_workspace is not None:
|
||||
@@ -1008,6 +1271,7 @@ class BoxService:
|
||||
session_id: str,
|
||||
process_payload: dict,
|
||||
) -> BoxManagedProcessInfo:
|
||||
self._reject_cloud_managed_process()
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
process_spec = BoxManagedProcessSpec.model_validate(process_payload)
|
||||
return await self.client.start_managed_process(
|
||||
@@ -1022,6 +1286,7 @@ class BoxService:
|
||||
session_id: str,
|
||||
process_id: str = 'default',
|
||||
) -> BoxManagedProcessInfo:
|
||||
self._reject_cloud_managed_process()
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
return await self.client.get_managed_process(
|
||||
session_id,
|
||||
@@ -1035,6 +1300,7 @@ class BoxService:
|
||||
session_id: str,
|
||||
process_id: str = 'default',
|
||||
) -> None:
|
||||
self._reject_cloud_managed_process()
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
return await self.client.stop_managed_process(
|
||||
session_id,
|
||||
@@ -1076,6 +1342,7 @@ class BoxService:
|
||||
execution context used by the action RPC that created the process.
|
||||
"""
|
||||
|
||||
self._reject_cloud_managed_process()
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
if self._runtime_connector is None:
|
||||
raise BoxValidationError(
|
||||
@@ -1092,23 +1359,32 @@ class BoxService:
|
||||
)
|
||||
|
||||
async def list_skills(self, context: TenantContext) -> list[dict]:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
return await self.client.list_skills(action_context=self._action_context(execution_context))
|
||||
|
||||
async def get_skill(self, context: TenantContext, name: str) -> dict | None:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
return await self.client.get_skill(name, action_context=self._action_context(execution_context))
|
||||
|
||||
async def create_skill(self, context: TenantContext, skill: dict) -> dict:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
payload = dict(skill)
|
||||
payload.pop('workspace_uuid', None)
|
||||
if self._cloud_managed and str(payload.get('package_root', '') or '').strip():
|
||||
raise BoxAdmissionError('Cloud skill package_root is runtime-owned')
|
||||
if self._cloud_managed:
|
||||
payload.pop('package_root', None)
|
||||
return await self.client.create_skill(payload, action_context=self._action_context(execution_context))
|
||||
|
||||
async def update_skill(self, context: TenantContext, name: str, skill: dict) -> dict:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
payload = dict(skill)
|
||||
payload.pop('workspace_uuid', None)
|
||||
if self._cloud_managed:
|
||||
# The runtime already owns the package path for an existing skill.
|
||||
# A serialized read response may contain it, but it is never an
|
||||
# authority-bearing update field in shared Cloud mode.
|
||||
payload.pop('package_root', None)
|
||||
return await self.client.update_skill(
|
||||
name,
|
||||
payload,
|
||||
@@ -1116,13 +1392,20 @@ class BoxService:
|
||||
)
|
||||
|
||||
async def delete_skill(self, context: TenantContext, name: str) -> None:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
await self.client.delete_skill(name, action_context=self._action_context(execution_context))
|
||||
|
||||
async def scan_skill_directory(self, context: TenantContext, path: str) -> dict:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
if self._cloud_managed:
|
||||
raise BoxAdmissionError('Scanning arbitrary host skill directories is disabled in Cloud')
|
||||
return await self.client.scan_skill_directory(path, action_context=self._action_context(execution_context))
|
||||
|
||||
async def _validated_skill_execution_context(self, context: TenantContext) -> ExecutionContext:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
await self._require_validated_workspace_sandbox(execution_context)
|
||||
return execution_context
|
||||
|
||||
async def list_skill_files(
|
||||
self,
|
||||
context: TenantContext,
|
||||
@@ -1131,7 +1414,7 @@ class BoxService:
|
||||
include_hidden: bool = False,
|
||||
max_entries: int = 200,
|
||||
) -> dict:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
return await self.client.list_skill_files(
|
||||
name,
|
||||
path,
|
||||
@@ -1141,7 +1424,7 @@ class BoxService:
|
||||
)
|
||||
|
||||
async def read_skill_file(self, context: TenantContext, name: str, path: str) -> dict:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
return await self.client.read_skill_file(
|
||||
name,
|
||||
path,
|
||||
@@ -1149,7 +1432,7 @@ class BoxService:
|
||||
)
|
||||
|
||||
async def write_skill_file(self, context: TenantContext, name: str, path: str, content: str) -> dict:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
return await self.client.write_skill_file(
|
||||
name,
|
||||
path,
|
||||
@@ -1165,7 +1448,7 @@ class BoxService:
|
||||
source_subdir: str = '',
|
||||
target_suffix: str = 'upload',
|
||||
) -> list[dict]:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
return await self.client.preview_skill_zip(
|
||||
file_bytes,
|
||||
filename,
|
||||
@@ -1184,7 +1467,7 @@ class BoxService:
|
||||
source_subdir: str = '',
|
||||
target_suffix: str = 'upload',
|
||||
) -> list[dict]:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
execution_context = await self._validated_skill_execution_context(context)
|
||||
return await self.client.install_skill_zip(
|
||||
file_bytes,
|
||||
filename,
|
||||
@@ -1405,6 +1688,20 @@ class BoxService:
|
||||
allowed_roots = ', '.join(self.allowed_mount_roots)
|
||||
raise BoxValidationError(f'box.local.default_workspace is outside allowed_mount_roots: {allowed_roots}')
|
||||
|
||||
def _ensure_cloud_shared_workspace(self) -> None:
|
||||
"""Require the Core-side view of the Cloud Box durable volume."""
|
||||
|
||||
if self.default_workspace is None:
|
||||
raise BoxValidationError('Cloud Box requires box.local.default_workspace')
|
||||
if not os.path.isabs(self.default_workspace) or not os.path.isdir(self.default_workspace):
|
||||
raise BoxValidationError('Cloud Box shared default_workspace must be an existing absolute directory')
|
||||
if not os.access(self.default_workspace, os.R_OK | os.W_OK | os.X_OK):
|
||||
raise BoxValidationError('Cloud Box shared default_workspace must be writable by LangBot Core')
|
||||
if not self.allowed_mount_roots or not any(
|
||||
_is_path_under(self.default_workspace, allowed_root) for allowed_root in self.allowed_mount_roots
|
||||
):
|
||||
raise BoxValidationError('Cloud Box shared default_workspace is outside allowed_mount_roots')
|
||||
|
||||
def _validate_host_mount(self, spec: BoxSpec):
|
||||
if spec.host_path is None:
|
||||
return
|
||||
@@ -1547,13 +1844,13 @@ class BoxService:
|
||||
and error.get('workspace_uuid') == execution_context.workspace_uuid
|
||||
]
|
||||
|
||||
def get_system_guidance(self, query_id=None) -> str:
|
||||
def get_system_guidance(self, query: pipeline_query.Query | int | str | None = None) -> str:
|
||||
"""Return LLM system-prompt guidance for the exec tool.
|
||||
|
||||
All execution-specific prompt text is kept here so that callers
|
||||
(e.g. LocalAgentRunner) stay free of box domain knowledge.
|
||||
|
||||
``query_id`` is the current turn's pipeline query id. When provided,
|
||||
``query`` is the current turn's pipeline query. When provided,
|
||||
the guidance ALWAYS advertises the per-query outbox path so the agent
|
||||
knows how to deliver generated files back to the user — even on turns
|
||||
where the user sent no inbound attachment (e.g. "generate a QR code"),
|
||||
@@ -1574,8 +1871,17 @@ class BoxService:
|
||||
'modify local files in the working directory, use exec with /workspace paths directly; do not ask the '
|
||||
'user for directory parameters unless they explicitly need a different directory.'
|
||||
)
|
||||
if query_id is not None:
|
||||
outbox_dir = f'{self.OUTBOX_MOUNT_DIR}/{query_id}'
|
||||
if query is not None:
|
||||
if not isinstance(query, (int, str)):
|
||||
query_key = self._attachment_query_key(query)
|
||||
else:
|
||||
# Backwards compatibility for OSS callers/tests that passed
|
||||
# the old process-local integer identity. Cloud callers must
|
||||
# pass the full Query so an opaque UUID is always advertised.
|
||||
if self._cloud_managed:
|
||||
raise BoxValidationError('Cloud outbox guidance requires a pipeline Query')
|
||||
query_key = str(query)
|
||||
outbox_dir = f'{self.OUTBOX_MOUNT_DIR}/{query_key}'
|
||||
guidance += (
|
||||
f' If you produce any file (image, audio, document, etc.) that should be sent back to the user, '
|
||||
f'write it into {outbox_dir}/ (create the directory if needed). Every file placed there will be '
|
||||
@@ -1593,6 +1899,8 @@ class BoxService:
|
||||
|
||||
async def get_status(self, context: TenantContext) -> dict:
|
||||
execution_context = await self._validated_execution_context(context)
|
||||
if self._cloud_managed and self._available:
|
||||
await self._require_validated_workspace_sandbox(execution_context)
|
||||
action_context = self._action_context(execution_context)
|
||||
recent_error_count = len(self.get_recent_errors(execution_context))
|
||||
if not self._available:
|
||||
|
||||
@@ -126,24 +126,30 @@ def should_prepare_python_env(host_path: str | None) -> bool:
|
||||
return bool(list_python_manifest_files(normalized_root))
|
||||
|
||||
|
||||
def wrap_python_command_with_env(command: str, *, mount_path: str = '/workspace') -> str:
|
||||
def wrap_python_command_with_env(
|
||||
command: str,
|
||||
*,
|
||||
mount_path: str = '/workspace',
|
||||
state_path: str | None = None,
|
||||
) -> str:
|
||||
"""Wrap a command with a reusable sandbox-local Python env bootstrap.
|
||||
|
||||
This is the generic "workspace is a Python project" path used by mutable
|
||||
workspaces such as skills. Read-only installation strategies stay in the
|
||||
higher-level caller because they are application policy, not workspace
|
||||
semantics.
|
||||
``mount_path`` is always the source tree used for manifest hashing and
|
||||
installation. ``state_path`` may point at a separate writable directory
|
||||
for read-only source mounts; when omitted, legacy mutable-workspace behavior
|
||||
stores the environment beside the source.
|
||||
"""
|
||||
writable_state_path = state_path or mount_path
|
||||
bootstrap = textwrap.dedent(
|
||||
f"""
|
||||
set -e
|
||||
|
||||
_LB_VENV_DIR="{mount_path}/.venv"
|
||||
_LB_META_DIR="{mount_path}/.langbot"
|
||||
_LB_VENV_DIR="{writable_state_path}/.venv"
|
||||
_LB_META_DIR="{writable_state_path}/.langbot"
|
||||
_LB_META_FILE="$_LB_META_DIR/python-env.json"
|
||||
_LB_LOCK_DIR="$_LB_META_DIR/python-env.lock"
|
||||
_LB_TMP_DIR="{mount_path}/.tmp"
|
||||
_LB_PIP_CACHE_DIR="{mount_path}/.cache/pip"
|
||||
_LB_TMP_DIR="{writable_state_path}/.tmp"
|
||||
_LB_PIP_CACHE_DIR="{writable_state_path}/.cache/pip"
|
||||
|
||||
mkdir -p "$_LB_META_DIR" "$_LB_TMP_DIR" "$_LB_PIP_CACHE_DIR"
|
||||
_LB_SYSTEM_PYTHON="$(command -v python3 || command -v python || true)"
|
||||
|
||||
@@ -3,6 +3,8 @@ from __future__ import annotations
|
||||
import dataclasses
|
||||
import importlib.metadata
|
||||
import inspect
|
||||
import os
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import Any, Protocol
|
||||
@@ -13,12 +15,17 @@ from .entitlements import EntitlementProvider, OpenSourceEntitlementProvider
|
||||
|
||||
CLOUD_BOOTSTRAP_ENTRY_POINT = 'langbot.cloud_bootstrap'
|
||||
REQUIRED_TENANT_ISOLATION_VERSION = 2
|
||||
SUPPORTED_PGVECTOR_DIMENSIONS = frozenset({384, 512, 768, 1024, 1536})
|
||||
|
||||
|
||||
class CloudBootstrapError(RuntimeError):
|
||||
"""Fail-closed Cloud bootstrap validation error."""
|
||||
|
||||
|
||||
class CloudRuntimeUnavailableError(CloudBootstrapError):
|
||||
"""The verified Cloud receipt no longer admits runtime work."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class OpenSourceDeployment:
|
||||
"""Default deployment selected when no closed bootstrap is installed."""
|
||||
@@ -88,8 +95,70 @@ class VerifiedCloudDeployment:
|
||||
raise CloudBootstrapError('Cloud runtime requires database.use=postgresql')
|
||||
if config.get('vdb', {}).get('use') != self.required_vector_backend:
|
||||
raise CloudBootstrapError('Cloud runtime requires vdb.use=pgvector')
|
||||
pgvector_config = config.get('vdb', {}).get('pgvector', {})
|
||||
if pgvector_config.get('use_business_database') is not True:
|
||||
raise CloudBootstrapError('Cloud runtime requires vdb.pgvector.use_business_database=true')
|
||||
dimensions = pgvector_config.get('allowed_dimensions')
|
||||
if (
|
||||
not isinstance(dimensions, list)
|
||||
or not dimensions
|
||||
or any(isinstance(item, bool) or not isinstance(item, int) for item in dimensions)
|
||||
or not set(dimensions).issubset(SUPPORTED_PGVECTOR_DIMENSIONS)
|
||||
):
|
||||
supported = ', '.join(str(item) for item in sorted(SUPPORTED_PGVECTOR_DIMENSIONS))
|
||||
raise CloudBootstrapError(f'Cloud pgvector allowed_dimensions must be a non-empty subset of: {supported}')
|
||||
if config.get('mcp', {}).get('stdio', {}).get('enabled', True) is not False:
|
||||
raise CloudBootstrapError('Cloud runtime requires mcp.stdio.enabled=false')
|
||||
plugin_worker = config.get('plugin', {}).get('worker', {})
|
||||
if plugin_worker.get('require_hard_limits') is not True:
|
||||
raise CloudBootstrapError('Cloud Runtime requires plugin.worker.require_hard_limits=true')
|
||||
box_config = config.get('box', {})
|
||||
if box_config.get('enabled') is not True:
|
||||
raise CloudBootstrapError('Cloud runtime requires box.enabled=true')
|
||||
if box_config.get('backend') != 'nsjail':
|
||||
raise CloudBootstrapError('Cloud runtime requires box.backend=nsjail')
|
||||
runtime_endpoint = str(box_config.get('runtime', {}).get('endpoint', '') or '').strip()
|
||||
if not runtime_endpoint:
|
||||
raise CloudBootstrapError('Cloud runtime requires a shared external box.runtime.endpoint')
|
||||
admission = box_config.get('admission', {})
|
||||
required_admission = {
|
||||
'required': True,
|
||||
'logical_session_id': 'global',
|
||||
'required_backend': 'nsjail',
|
||||
'max_sessions': 1,
|
||||
'max_managed_processes': 0,
|
||||
}
|
||||
if any(admission.get(name) != value for name, value in required_admission.items()):
|
||||
raise CloudBootstrapError(
|
||||
'Cloud runtime requires grant-enforced Box admission with one global session and zero managed processes'
|
||||
)
|
||||
grant_ttl = admission.get('max_grant_ttl_sec')
|
||||
if isinstance(grant_ttl, bool) or not isinstance(grant_ttl, int) or not 1 <= grant_ttl <= 300:
|
||||
raise CloudBootstrapError('Cloud Box admission max_grant_ttl_sec must be between 1 and 300')
|
||||
workspace_quota_mb = admission.get('workspace_quota_mb')
|
||||
if isinstance(workspace_quota_mb, bool) or not isinstance(workspace_quota_mb, int) or workspace_quota_mb <= 0:
|
||||
raise CloudBootstrapError('Cloud Box admission workspace_quota_mb must be a positive integer')
|
||||
local_config = box_config.get('local', {})
|
||||
host_root = str(local_config.get('host_root', '') or '').strip()
|
||||
default_workspace = str(local_config.get('default_workspace', '') or '').strip()
|
||||
allowed_mount_roots = local_config.get('allowed_mount_roots')
|
||||
if not host_root or not os.path.isabs(host_root):
|
||||
raise CloudBootstrapError('Cloud Box local.host_root must be an absolute shared-volume path')
|
||||
if not default_workspace or not os.path.isabs(default_workspace):
|
||||
raise CloudBootstrapError('Cloud Box local.default_workspace must be an absolute shared-volume path')
|
||||
if (
|
||||
not isinstance(allowed_mount_roots, list)
|
||||
or not allowed_mount_roots
|
||||
or any(not isinstance(root, str) or not os.path.isabs(root) for root in allowed_mount_roots)
|
||||
):
|
||||
raise CloudBootstrapError('Cloud Box local.allowed_mount_roots must contain absolute shared-volume paths')
|
||||
resolved_workspace = os.path.realpath(default_workspace)
|
||||
if not any(
|
||||
resolved_workspace == os.path.realpath(root)
|
||||
or resolved_workspace.startswith(f'{os.path.realpath(root)}{os.sep}')
|
||||
for root in allowed_mount_roots
|
||||
):
|
||||
raise CloudBootstrapError('Cloud Box local.default_workspace must be under allowed_mount_roots')
|
||||
|
||||
|
||||
class CloudBootstrapProvider(Protocol):
|
||||
@@ -101,6 +170,98 @@ class CloudBootstrapProvider(Protocol):
|
||||
) -> VerifiedCloudDeployment | Awaitable[VerifiedCloudDeployment]: ...
|
||||
|
||||
|
||||
class DeploymentAdmissionGuard:
|
||||
"""Continuously enforce one verified deployment receipt.
|
||||
|
||||
Startup verification alone is insufficient because a long-running process
|
||||
could otherwise keep serving after the signed Manifest expires. The guard
|
||||
tracks both wall-clock expiry and a monotonic deadline so moving the system
|
||||
clock backwards cannot extend an already admitted receipt.
|
||||
|
||||
A closed bootstrap may atomically replace the receipt with a strictly newer
|
||||
Manifest generation after performing its own signature verification. The
|
||||
logical instance and deployment mode cannot change during the process.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instance_uuid: str,
|
||||
deployment: OpenSourceDeployment | VerifiedCloudDeployment,
|
||||
*,
|
||||
wall_time: Callable[[], float] = time.time,
|
||||
monotonic_time: Callable[[], float] = time.monotonic,
|
||||
) -> None:
|
||||
self.instance_uuid = instance_uuid
|
||||
self._wall_time = wall_time
|
||||
self._monotonic_time = monotonic_time
|
||||
self._lock = threading.Lock()
|
||||
self._deployment = deployment
|
||||
self._deadline: float | None = None
|
||||
self._install_initial(deployment)
|
||||
|
||||
@property
|
||||
def deployment(self) -> OpenSourceDeployment | VerifiedCloudDeployment:
|
||||
with self._lock:
|
||||
return self._deployment
|
||||
|
||||
def _install_initial(self, deployment: OpenSourceDeployment | VerifiedCloudDeployment) -> None:
|
||||
now = int(self._wall_time())
|
||||
if isinstance(deployment, VerifiedCloudDeployment):
|
||||
deployment.validate(self.instance_uuid, now=now)
|
||||
self._deadline = self._monotonic_time() + (deployment.expires_at - now)
|
||||
elif not isinstance(deployment, OpenSourceDeployment):
|
||||
raise TypeError('Deployment admission requires a verified deployment object')
|
||||
|
||||
@staticmethod
|
||||
def _receipt_identity(deployment: VerifiedCloudDeployment) -> tuple[Any, ...]:
|
||||
return (
|
||||
deployment.instance_uuid,
|
||||
deployment.manifest_jti,
|
||||
deployment.manifest_generation,
|
||||
deployment.expires_at,
|
||||
deployment.release,
|
||||
tuple(sorted(deployment.capabilities)),
|
||||
deployment.tenant_isolation_version,
|
||||
deployment.verification_key_id,
|
||||
)
|
||||
|
||||
def replace(self, deployment: VerifiedCloudDeployment) -> None:
|
||||
"""Atomically install a verified, non-rollback Cloud receipt."""
|
||||
|
||||
now = int(self._wall_time())
|
||||
deployment.validate(self.instance_uuid, now=now)
|
||||
with self._lock:
|
||||
current = self._deployment
|
||||
if not isinstance(current, VerifiedCloudDeployment):
|
||||
raise CloudRuntimeUnavailableError('Deployment mode cannot change while LangBot is running')
|
||||
if deployment.manifest_generation < current.manifest_generation:
|
||||
raise CloudRuntimeUnavailableError('Cloud Manifest generation rolled back')
|
||||
if deployment.manifest_generation == current.manifest_generation and self._receipt_identity(
|
||||
deployment
|
||||
) != self._receipt_identity(current):
|
||||
raise CloudRuntimeUnavailableError('Cloud Manifest generation has conflicting contents')
|
||||
self._deployment = deployment
|
||||
self._deadline = self._monotonic_time() + (deployment.expires_at - now)
|
||||
|
||||
def require_active(self) -> OpenSourceDeployment | VerifiedCloudDeployment:
|
||||
"""Return the active deployment or fail closed after Manifest expiry."""
|
||||
|
||||
now = int(self._wall_time())
|
||||
monotonic_now = self._monotonic_time()
|
||||
with self._lock:
|
||||
deployment = self._deployment
|
||||
deadline = self._deadline
|
||||
if isinstance(deployment, OpenSourceDeployment):
|
||||
return deployment
|
||||
try:
|
||||
deployment.validate(self.instance_uuid, now=now)
|
||||
except CloudBootstrapError as exc:
|
||||
raise CloudRuntimeUnavailableError(str(exc)) from exc
|
||||
if deadline is None or monotonic_now >= deadline:
|
||||
raise CloudRuntimeUnavailableError('Verified Cloud Manifest is expired')
|
||||
return deployment
|
||||
|
||||
|
||||
async def _invoke_provider(
|
||||
loaded: Any,
|
||||
*,
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from typing import Protocol, runtime_checkable
|
||||
|
||||
import pydantic
|
||||
@@ -11,6 +12,10 @@ import pydantic
|
||||
class EntitlementUnavailableError(RuntimeError):
|
||||
"""Raised when a trusted, currently-active entitlement is unavailable."""
|
||||
|
||||
def __init__(self, message: str, *, entitlement_revision: int | None = None) -> None:
|
||||
super().__init__(message)
|
||||
self.entitlement_revision = entitlement_revision
|
||||
|
||||
|
||||
class EntitlementSnapshot(pydantic.BaseModel):
|
||||
"""Plan-agnostic capability projection consumed by open-source Core.
|
||||
@@ -60,9 +65,15 @@ class EntitlementSnapshot(pydantic.BaseModel):
|
||||
if self.instance_uuid != instance_uuid or self.workspace_uuid != workspace_uuid:
|
||||
raise EntitlementUnavailableError('Entitlement scope does not match the Workspace execution context')
|
||||
if self.status != 'active':
|
||||
raise EntitlementUnavailableError('Workspace entitlement is not active')
|
||||
raise EntitlementUnavailableError(
|
||||
'Workspace entitlement is not active',
|
||||
entitlement_revision=self.entitlement_revision,
|
||||
)
|
||||
if current_time < self.not_before or current_time >= self.expires_at:
|
||||
raise EntitlementUnavailableError('Workspace entitlement is not currently valid')
|
||||
raise EntitlementUnavailableError(
|
||||
'Workspace entitlement is not currently valid',
|
||||
entitlement_revision=self.entitlement_revision,
|
||||
)
|
||||
return self
|
||||
|
||||
def require_feature(self, feature: str) -> None:
|
||||
@@ -95,9 +106,16 @@ class OpenSourceEntitlementProvider:
|
||||
class EntitlementResolver:
|
||||
"""Validate scope/freshness and reject revision rollback or equivocation."""
|
||||
|
||||
def __init__(self, instance_uuid: str, provider: EntitlementProvider) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
instance_uuid: str,
|
||||
provider: EntitlementProvider,
|
||||
*,
|
||||
deployment_admission: Callable[[], object] | None = None,
|
||||
) -> None:
|
||||
self.instance_uuid = instance_uuid
|
||||
self.provider = provider
|
||||
self._deployment_admission = deployment_admission
|
||||
self._lock = asyncio.Lock()
|
||||
self._snapshots: dict[str, tuple[int, str, EntitlementSnapshot]] = {}
|
||||
|
||||
@@ -112,7 +130,12 @@ class EntitlementResolver:
|
||||
minimum_revision: int = 0,
|
||||
now: int | None = None,
|
||||
) -> EntitlementSnapshot:
|
||||
if self._deployment_admission is not None:
|
||||
self._deployment_admission()
|
||||
candidate = await self.provider.get_workspace_entitlement(workspace_uuid)
|
||||
if self._deployment_admission is not None:
|
||||
# A provider call may cross the Manifest expiry boundary.
|
||||
self._deployment_admission()
|
||||
if not isinstance(candidate, EntitlementSnapshot):
|
||||
raise EntitlementUnavailableError('Entitlement provider returned an invalid snapshot')
|
||||
# Deep-copy untrusted provider-owned containers before caching them.
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import typing
|
||||
import inspect
|
||||
|
||||
from ..core import app
|
||||
from . import operator
|
||||
@@ -63,6 +64,12 @@ class CommandManager:
|
||||
) -> typing.AsyncGenerator[command_context.CommandReturn, None]:
|
||||
"""执行命令"""
|
||||
|
||||
require_context = getattr(self.ap.plugin_connector, 'require_workspace_context', None)
|
||||
if require_context is not None:
|
||||
result = require_context(context)
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
command_list = await self.ap.plugin_connector.list_commands(bound_plugins)
|
||||
|
||||
for command in command_list:
|
||||
|
||||
+12
-24
@@ -4,7 +4,6 @@ import logging
|
||||
import asyncio
|
||||
import traceback
|
||||
import os
|
||||
import sqlalchemy
|
||||
|
||||
from ..platform import botmgr as im_mgr
|
||||
from ..platform.webhook_pusher import WebhookPusher
|
||||
@@ -51,7 +50,6 @@ from ..workspace import collaboration as workspace_collaboration_module
|
||||
from ..cloud import bootstrap as cloud_bootstrap_module
|
||||
from ..cloud import entitlements as cloud_entitlements_module
|
||||
from ..api.http.context import ExecutionContext, PrincipalContext, PrincipalType
|
||||
from ..entity.persistence.workspace import WorkspaceExecutionState, WorkspaceExecutionStatus
|
||||
|
||||
|
||||
class Application:
|
||||
@@ -132,6 +130,8 @@ class Application:
|
||||
|
||||
deployment: cloud_bootstrap_module.OpenSourceDeployment | cloud_bootstrap_module.VerifiedCloudDeployment = None
|
||||
|
||||
deployment_admission: cloud_bootstrap_module.DeploymentAdmissionGuard = None
|
||||
|
||||
entitlement_resolver: cloud_entitlements_module.EntitlementResolver | None = None
|
||||
|
||||
vector_db_mgr: vectordb_mgr.VectorDBManager = None
|
||||
@@ -250,18 +250,12 @@ class Application:
|
||||
check_interval_seconds = check_interval_hours * 3600
|
||||
while True:
|
||||
try:
|
||||
execution_states = await self.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(WorkspaceExecutionState).where(
|
||||
WorkspaceExecutionState.instance_uuid == self.workspace_service.instance_uuid,
|
||||
WorkspaceExecutionState.state == WorkspaceExecutionStatus.ACTIVE.value,
|
||||
WorkspaceExecutionState.write_fenced == sqlalchemy.false(),
|
||||
)
|
||||
)
|
||||
for execution_state in execution_states.all():
|
||||
bindings = await self.workspace_service.list_active_execution_bindings()
|
||||
for binding in bindings:
|
||||
context = ExecutionContext(
|
||||
instance_uuid=execution_state.instance_uuid,
|
||||
workspace_uuid=execution_state.workspace_uuid,
|
||||
placement_generation=execution_state.active_generation,
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
trigger_principal=PrincipalContext(PrincipalType.SYSTEM),
|
||||
)
|
||||
deleted = await self.monitoring_service.cleanup_expired_records(
|
||||
@@ -298,18 +292,12 @@ class Application:
|
||||
check_interval_seconds = check_interval_hours * 3600
|
||||
while True:
|
||||
try:
|
||||
execution_states = await self.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(WorkspaceExecutionState).where(
|
||||
WorkspaceExecutionState.instance_uuid == self.workspace_service.instance_uuid,
|
||||
WorkspaceExecutionState.state == WorkspaceExecutionStatus.ACTIVE.value,
|
||||
WorkspaceExecutionState.write_fenced == sqlalchemy.false(),
|
||||
)
|
||||
)
|
||||
for execution_state in execution_states.all():
|
||||
bindings = await self.workspace_service.list_active_execution_bindings()
|
||||
for binding in bindings:
|
||||
context = ExecutionContext(
|
||||
instance_uuid=execution_state.instance_uuid,
|
||||
workspace_uuid=execution_state.workspace_uuid,
|
||||
placement_generation=execution_state.active_generation,
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
trigger_principal=PrincipalContext(PrincipalType.SYSTEM),
|
||||
)
|
||||
deleted = await self.maintenance_service.cleanup_expired_files(context)
|
||||
|
||||
@@ -61,8 +61,16 @@ class BuildAppStage(stage.BootingStage):
|
||||
instance_config=ap.instance_config.data,
|
||||
)
|
||||
ap.deployment = deployment
|
||||
ap.deployment_admission = cloud_bootstrap.DeploymentAdmissionGuard(
|
||||
constants.instance_id,
|
||||
deployment,
|
||||
)
|
||||
ap.entitlement_resolver = (
|
||||
EntitlementResolver(constants.instance_id, deployment.entitlement_provider)
|
||||
EntitlementResolver(
|
||||
constants.instance_id,
|
||||
deployment.entitlement_provider,
|
||||
deployment_admission=ap.deployment_admission.require_active,
|
||||
)
|
||||
if deployment.multi_workspace_enabled
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -21,6 +21,7 @@ _RUNTIME_POLICY_DEFAULTS = {
|
||||
'max_pids': 128,
|
||||
'max_open_files': 256,
|
||||
'max_file_size_mb': 512,
|
||||
'require_hard_limits': False,
|
||||
}
|
||||
},
|
||||
'mcp': {'stdio': {'enabled': True}},
|
||||
@@ -103,11 +104,19 @@ def _apply_env_overrides_to_config(cfg: dict) -> dict:
|
||||
if '__' not in env_key:
|
||||
continue
|
||||
|
||||
print(f'apply env overrides to config: env_key: {env_key}, env_value: {env_value}')
|
||||
|
||||
# Convert environment variable name to config path
|
||||
# e.g., CONCURRENCY__PIPELINE -> ['concurrency', 'pipeline']
|
||||
keys = [key.lower() for key in env_key.split('__')]
|
||||
# macOS and some launchers expose variables such as
|
||||
# ``__CF_USER_TEXT_ENCODING``. They are not LangBot config paths and
|
||||
# must not create an empty top-level YAML key when config is dumped.
|
||||
if any(not key for key in keys):
|
||||
continue
|
||||
|
||||
# Values may contain database passwords, runtime control tokens, or
|
||||
# provider credentials. Keep the useful audit breadcrumb without ever
|
||||
# copying the secret into startup logs.
|
||||
print(f'apply env override to config: env_key: {env_key}')
|
||||
|
||||
# Navigate to the target value and validate the path
|
||||
current = cfg
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import typing
|
||||
|
||||
|
||||
T = typing.TypeVar('T')
|
||||
|
||||
|
||||
def create_detached_task(
|
||||
coro: typing.Coroutine[typing.Any, typing.Any, T],
|
||||
*,
|
||||
loop: asyncio.AbstractEventLoop | None = None,
|
||||
name: str | None = None,
|
||||
after_commit_manager: typing.Any | None = None,
|
||||
) -> asyncio.Task[T]:
|
||||
"""Create a task that inherits no request-local ContextVars.
|
||||
|
||||
A normal ``asyncio.create_task`` copies the caller's context. That is
|
||||
unsafe for work which outlives an HTTP request because it can copy the
|
||||
request's active database transaction or trusted tenant scope into a
|
||||
different asyncio task. Detached work must receive durable identity such
|
||||
as ``ExecutionContext`` through explicit arguments and establish its own
|
||||
tenant scope or unit of work whenever it accesses persistence.
|
||||
"""
|
||||
|
||||
task_loop = loop or asyncio.get_running_loop()
|
||||
gate: asyncio.Future[None] | None = None
|
||||
# Inspect the type so dynamic Mock/AsyncMock attributes do not turn into a
|
||||
# fake gate in lightweight tests or embedders.
|
||||
gate_factory = getattr(type(after_commit_manager), 'create_after_commit_gate', None)
|
||||
if callable(gate_factory):
|
||||
gate = gate_factory(after_commit_manager)
|
||||
task_coro = _wait_for_commit(coro, gate) if gate is not None else coro
|
||||
return task_loop.create_task(task_coro, name=name, context=contextvars.Context())
|
||||
|
||||
|
||||
async def _wait_for_commit(
|
||||
coro: typing.Coroutine[typing.Any, typing.Any, T],
|
||||
gate: asyncio.Future[None],
|
||||
) -> T:
|
||||
try:
|
||||
await gate
|
||||
except BaseException:
|
||||
coro.close()
|
||||
raise
|
||||
return await coro
|
||||
|
||||
|
||||
async def run_in_workspace_uow(
|
||||
ap: typing.Any,
|
||||
workspace_uuid: str,
|
||||
operation: typing.Callable[[], typing.Awaitable[T]],
|
||||
) -> T:
|
||||
"""Run one short persistence section in a detached Cloud task scope.
|
||||
|
||||
This helper deliberately scopes only the supplied operation. Callers
|
||||
should not wrap long-running network or runtime work in a database
|
||||
transaction.
|
||||
"""
|
||||
|
||||
persistence_mgr = getattr(ap, 'persistence_mgr', None)
|
||||
if persistence_mgr is None:
|
||||
return await operation()
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if not cloud_runtime:
|
||||
return await operation()
|
||||
|
||||
tenant_uow = getattr(persistence_mgr, 'tenant_uow', None)
|
||||
if not callable(tenant_uow):
|
||||
raise RuntimeError('Detached Cloud tasks require an explicit tenant unit of work')
|
||||
async with tenant_uow(workspace_uuid):
|
||||
return await operation()
|
||||
@@ -7,6 +7,7 @@ import time
|
||||
|
||||
from . import app
|
||||
from . import entities as core_entities
|
||||
from .task_boundary import create_detached_task
|
||||
|
||||
|
||||
class TaskContext:
|
||||
@@ -125,7 +126,12 @@ class TaskWrapper:
|
||||
TaskWrapper._id_index += 1
|
||||
self.ap = ap
|
||||
self.task_context = context or TaskContext()
|
||||
self.task = self.ap.event_loop.create_task(coro)
|
||||
self.task = create_detached_task(
|
||||
coro,
|
||||
loop=self.ap.event_loop,
|
||||
name=name or None,
|
||||
after_commit_manager=getattr(self.ap, 'persistence_mgr', None),
|
||||
)
|
||||
self.task_type = task_type
|
||||
self.kind = kind
|
||||
self.name = name
|
||||
|
||||
@@ -1,3 +1,6 @@
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from .base import Base
|
||||
@@ -15,6 +18,17 @@ class PluginSetting(Base):
|
||||
)
|
||||
plugin_author = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
plugin_name = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
installation_uuid = sqlalchemy.Column(
|
||||
sqlalchemy.String(36),
|
||||
nullable=False,
|
||||
default=lambda: str(uuid.uuid4()),
|
||||
)
|
||||
artifact_digest = sqlalchemy.Column(
|
||||
sqlalchemy.String(64),
|
||||
nullable=False,
|
||||
default=lambda: hashlib.sha256(f'pending:{uuid.uuid4()}'.encode()).hexdigest(),
|
||||
)
|
||||
runtime_revision = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=1)
|
||||
enabled = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=True)
|
||||
priority = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0)
|
||||
config = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default=dict)
|
||||
@@ -28,4 +42,15 @@ class PluginSetting(Base):
|
||||
onupdate=sqlalchemy.func.now(),
|
||||
)
|
||||
|
||||
__table_args__ = (sqlalchemy.Index('ix_plugin_settings_workspace_enabled', 'workspace_uuid', 'enabled'),)
|
||||
__table_args__ = (
|
||||
sqlalchemy.UniqueConstraint('installation_uuid', name='uq_plugin_settings_installation_uuid'),
|
||||
sqlalchemy.CheckConstraint('runtime_revision >= 1', name='ck_plugin_settings_runtime_revision_positive'),
|
||||
sqlalchemy.CheckConstraint('length(artifact_digest) = 64', name='ck_plugin_settings_artifact_digest_length'),
|
||||
sqlalchemy.Index('ix_plugin_settings_workspace_enabled', 'workspace_uuid', 'enabled'),
|
||||
sqlalchemy.Index(
|
||||
'ix_plugin_settings_workspace_installation',
|
||||
'workspace_uuid',
|
||||
'installation_uuid',
|
||||
unique=True,
|
||||
),
|
||||
)
|
||||
|
||||
@@ -29,6 +29,9 @@ class KnowledgeBase(Base):
|
||||
)
|
||||
creation_settings = sqlalchemy.Column(sqlalchemy.JSON, nullable=True, default=None)
|
||||
retrieval_settings = sqlalchemy.Column(sqlalchemy.JSON, nullable=True, default=None)
|
||||
# Server-selected pgvector dimension. ``None`` means no embedding has been
|
||||
# written yet; the first pgvector upsert binds it atomically.
|
||||
embedding_dimension = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
|
||||
|
||||
# Field sets for different operations
|
||||
MUTABLE_FIELDS = {'name', 'description', 'retrieval_settings'}
|
||||
@@ -40,6 +43,7 @@ class KnowledgeBase(Base):
|
||||
ALL_DB_FIELDS = CREATE_FIELDS | {
|
||||
'workspace_uuid',
|
||||
'legacy_vector_collection',
|
||||
'embedding_dimension',
|
||||
'emoji',
|
||||
'created_at',
|
||||
'updated_at',
|
||||
@@ -57,6 +61,10 @@ class KnowledgeBase(Base):
|
||||
sqlite_where=sqlalchemy.text('collection_id IS NOT NULL'),
|
||||
postgresql_where=sqlalchemy.text('collection_id IS NOT NULL'),
|
||||
),
|
||||
sqlalchemy.CheckConstraint(
|
||||
'embedding_dimension IS NULL OR embedding_dimension > 0',
|
||||
name='ck_knowledge_bases_embedding_dimension_positive',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,11 +1,13 @@
|
||||
"""enforce PostgreSQL tenant isolation with row-level security
|
||||
"""enforce PostgreSQL tenant isolation with exact discovery contracts
|
||||
|
||||
Revision ID: 0011_postgres_tenant_rls
|
||||
Revises: 0010_scope_resources
|
||||
Create Date: 2026-07-19
|
||||
|
||||
The table list is deliberately duplicated from the runtime contract. Alembic
|
||||
revisions must remain self-contained after application code evolves.
|
||||
The table and policy lists are deliberately duplicated from the runtime
|
||||
contract. Alembic revisions must remain self-contained after application code
|
||||
evolves. Discovery policies are SELECT-only and reveal the minimum rows needed
|
||||
to turn an authenticated credential into one Workspace transaction.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -20,8 +22,18 @@ depends_on = None
|
||||
|
||||
|
||||
_POLICY_NAME = 'langbot_workspace_isolation'
|
||||
_ACCOUNT_POLICY_NAME = 'langbot_account_discovery'
|
||||
_API_KEY_POLICY_NAME = 'langbot_api_key_discovery'
|
||||
_INVITATION_POLICY_NAME = 'langbot_invitation_discovery'
|
||||
_INSTANCE_POLICY_NAME = 'langbot_instance_discovery'
|
||||
|
||||
_TENANT_SETTING = 'langbot.workspace_uuid'
|
||||
_ACCOUNT_SETTING = 'langbot.account_uuid'
|
||||
_API_KEY_HASH_SETTING = 'langbot.api_key_hash'
|
||||
_INVITATION_HASH_SETTING = 'langbot.invitation_hash'
|
||||
_INSTANCE_SETTING = 'langbot.instance_uuid'
|
||||
_OSS_WORKSPACE_METADATA_KEY = 'oss_workspace_uuid'
|
||||
|
||||
_TENANT_TABLE_COLUMNS: dict[str, str] = {
|
||||
'workspaces': 'uuid',
|
||||
'workspace_memberships': 'workspace_uuid',
|
||||
@@ -54,17 +66,42 @@ _TENANT_TABLE_COLUMNS: dict[str, str] = {
|
||||
}
|
||||
|
||||
|
||||
def _setting(name: str) -> str:
|
||||
return f"NULLIF(current_setting('{name}', true), '')"
|
||||
|
||||
|
||||
def _tenant_expression(column: str) -> str:
|
||||
return f'{column}::text = {_setting(_TENANT_SETTING)}'
|
||||
|
||||
|
||||
_DISCOVERY_POLICIES: dict[str, dict[str, str]] = {
|
||||
'workspace_memberships': {
|
||||
_ACCOUNT_POLICY_NAME: (f"account_uuid::text = {_setting(_ACCOUNT_SETTING)} AND status = 'active'"),
|
||||
},
|
||||
'workspace_execution_states': {
|
||||
_INSTANCE_POLICY_NAME: (
|
||||
f"instance_uuid = {_setting(_INSTANCE_SETTING)} AND state = 'active' AND write_fenced = false"
|
||||
),
|
||||
},
|
||||
'api_keys': {
|
||||
_API_KEY_POLICY_NAME: (
|
||||
f"key_hash = {_setting(_API_KEY_HASH_SETTING)} AND status = 'active' "
|
||||
'AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP)'
|
||||
),
|
||||
},
|
||||
'workspace_invitations': {
|
||||
_INVITATION_POLICY_NAME: f'token_hash = {_setting(_INVITATION_HASH_SETTING)}',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _quote_identifier(conn: sa.Connection, identifier: str) -> str:
|
||||
return conn.dialect.identifier_preparer.quote(identifier)
|
||||
|
||||
|
||||
def _record_oss_workspace_scope(conn: sa.Connection) -> None:
|
||||
"""Keep PostgreSQL OSS usable after FORCE RLS is enabled.
|
||||
"""Keep PostgreSQL OSS usable after FORCE RLS is enabled."""
|
||||
|
||||
The runtime reads this instance-level metadata and installs the singleton
|
||||
Workspace as a transaction-local default. Cloud databases have no local
|
||||
Workspace and therefore do not receive this compatibility value.
|
||||
"""
|
||||
local_workspaces = (
|
||||
conn.execute(sa.text("SELECT uuid FROM workspaces WHERE source = 'local' ORDER BY uuid")).scalars().all()
|
||||
)
|
||||
@@ -84,6 +121,43 @@ def _record_oss_workspace_scope(conn: sa.Connection) -> None:
|
||||
raise RuntimeError('Stored OSS Workspace scope does not match the local Workspace')
|
||||
|
||||
|
||||
def _drop_all_policies(conn: sa.Connection, table_name: str) -> None:
|
||||
policy_names = conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT p.polname
|
||||
FROM pg_policy p
|
||||
JOIN pg_class c ON c.oid = p.polrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema() AND c.relname = :table_name
|
||||
"""
|
||||
),
|
||||
{'table_name': table_name},
|
||||
).scalars()
|
||||
table = _quote_identifier(conn, table_name)
|
||||
for policy in policy_names:
|
||||
op.execute(sa.text(f'DROP POLICY {_quote_identifier(conn, policy)} ON {table}'))
|
||||
|
||||
|
||||
def _create_policy(
|
||||
conn: sa.Connection,
|
||||
table_name: str,
|
||||
policy_name: str,
|
||||
expression: str,
|
||||
*,
|
||||
command: str,
|
||||
) -> None:
|
||||
table = _quote_identifier(conn, table_name)
|
||||
policy = _quote_identifier(conn, policy_name)
|
||||
if command == 'ALL':
|
||||
sql = f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR ALL TO PUBLIC USING ({expression}) WITH CHECK ({expression})'
|
||||
elif command == 'SELECT':
|
||||
sql = f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR SELECT TO PUBLIC USING ({expression})'
|
||||
else: # pragma: no cover - migration-local invariant
|
||||
raise AssertionError(f'Unsupported RLS command: {command}')
|
||||
op.execute(sa.text(sql))
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if conn.dialect.name != 'postgresql':
|
||||
@@ -96,22 +170,20 @@ def upgrade() -> None:
|
||||
|
||||
_record_oss_workspace_scope(conn)
|
||||
|
||||
policy_name = _quote_identifier(conn, _POLICY_NAME)
|
||||
for table_name, tenant_column in _TENANT_TABLE_COLUMNS.items():
|
||||
table = _quote_identifier(conn, table_name)
|
||||
column = _quote_identifier(conn, tenant_column)
|
||||
tenant_value = f"CAST(NULLIF(current_setting('{_TENANT_SETTING}', true), '') AS VARCHAR(36))"
|
||||
|
||||
_drop_all_policies(conn, table_name)
|
||||
op.execute(sa.text(f'ALTER TABLE {table} ENABLE ROW LEVEL SECURITY'))
|
||||
op.execute(sa.text(f'ALTER TABLE {table} FORCE ROW LEVEL SECURITY'))
|
||||
op.execute(sa.text(f'DROP POLICY IF EXISTS {policy_name} ON {table}'))
|
||||
op.execute(
|
||||
sa.text(
|
||||
f'CREATE POLICY {policy_name} ON {table} '
|
||||
f'USING ({column} = {tenant_value}) '
|
||||
f'WITH CHECK ({column} = {tenant_value})'
|
||||
)
|
||||
_create_policy(
|
||||
conn,
|
||||
table_name,
|
||||
_POLICY_NAME,
|
||||
_tenant_expression(_quote_identifier(conn, tenant_column)),
|
||||
command='ALL',
|
||||
)
|
||||
for policy_name, expression in _DISCOVERY_POLICIES.get(table_name, {}).items():
|
||||
_create_policy(conn, table_name, policy_name, expression, command='SELECT')
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
@@ -120,11 +192,10 @@ def downgrade() -> None:
|
||||
return
|
||||
|
||||
existing_tables = set(sa.inspect(conn).get_table_names())
|
||||
policy_name = _quote_identifier(conn, _POLICY_NAME)
|
||||
for table_name in _TENANT_TABLE_COLUMNS:
|
||||
if table_name not in existing_tables:
|
||||
continue
|
||||
table = _quote_identifier(conn, table_name)
|
||||
op.execute(sa.text(f'DROP POLICY IF EXISTS {policy_name} ON {table}'))
|
||||
_drop_all_policies(conn, table_name)
|
||||
op.execute(sa.text(f'ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY'))
|
||||
op.execute(sa.text(f'ALTER TABLE {table} DISABLE ROW LEVEL SECURITY'))
|
||||
|
||||
@@ -0,0 +1,179 @@
|
||||
"""add immutable plugin installation identity
|
||||
|
||||
Revision ID: 0012_plugin_identity
|
||||
Revises: 0011_postgres_tenant_rls
|
||||
Create Date: 2026-07-19
|
||||
|
||||
The migration gives every legacy row a random, stable installation UUID. A
|
||||
legacy artifact digest is only a valid SHA-256-shaped recovery marker; Core
|
||||
replaces it with the package digest and increments ``runtime_revision`` before
|
||||
the next package apply.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import uuid
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
|
||||
|
||||
revision = '0012_plugin_identity'
|
||||
down_revision = '0011_postgres_tenant_rls'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_TABLE = 'plugin_settings'
|
||||
_INSTALLATION_INDEX = 'ix_plugin_settings_workspace_installation'
|
||||
_INSTALLATION_UNIQUE = 'uq_plugin_settings_installation_uuid'
|
||||
_REVISION_CHECK = 'ck_plugin_settings_runtime_revision_positive'
|
||||
_DIGEST_CHECK = 'ck_plugin_settings_artifact_digest_length'
|
||||
|
||||
|
||||
def _column_names(conn: sa.Connection) -> set[str]:
|
||||
inspector = sa.inspect(conn)
|
||||
if _TABLE not in inspector.get_table_names():
|
||||
return set()
|
||||
return {column['name'] for column in inspector.get_columns(_TABLE)}
|
||||
|
||||
|
||||
def _legacy_digest(installation_uuid: str) -> str:
|
||||
return hashlib.sha256(f'legacy-installation:{installation_uuid}'.encode()).hexdigest()
|
||||
|
||||
|
||||
def _suspend_postgres_rls(conn: sa.Connection) -> tuple[bool, bool]:
|
||||
"""Let the release migration backfill every tenant row after revision 0011."""
|
||||
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return False, False
|
||||
row = conn.execute(
|
||||
sa.text('SELECT relrowsecurity, relforcerowsecurity FROM pg_class WHERE oid = to_regclass(:table_name)'),
|
||||
{'table_name': _TABLE},
|
||||
).one()
|
||||
rls_enabled, rls_forced = bool(row.relrowsecurity), bool(row.relforcerowsecurity)
|
||||
if rls_forced:
|
||||
op.execute(sa.text(f'ALTER TABLE {_TABLE} NO FORCE ROW LEVEL SECURITY'))
|
||||
if rls_enabled:
|
||||
op.execute(sa.text(f'ALTER TABLE {_TABLE} DISABLE ROW LEVEL SECURITY'))
|
||||
return rls_enabled, rls_forced
|
||||
|
||||
|
||||
def _restore_postgres_rls(conn: sa.Connection, state: tuple[bool, bool]) -> None:
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return
|
||||
rls_enabled, rls_forced = state
|
||||
if rls_enabled:
|
||||
op.execute(sa.text(f'ALTER TABLE {_TABLE} ENABLE ROW LEVEL SECURITY'))
|
||||
if rls_forced:
|
||||
op.execute(sa.text(f'ALTER TABLE {_TABLE} FORCE ROW LEVEL SECURITY'))
|
||||
|
||||
|
||||
def _backfill(conn: sa.Connection) -> None:
|
||||
table = sa.table(
|
||||
_TABLE,
|
||||
sa.column('workspace_uuid', sa.String(36)),
|
||||
sa.column('plugin_author', sa.String(255)),
|
||||
sa.column('plugin_name', sa.String(255)),
|
||||
sa.column('installation_uuid', sa.String(36)),
|
||||
sa.column('artifact_digest', sa.String(64)),
|
||||
sa.column('runtime_revision', sa.Integer()),
|
||||
)
|
||||
rows = conn.execute(
|
||||
sa.select(
|
||||
table.c.workspace_uuid,
|
||||
table.c.plugin_author,
|
||||
table.c.plugin_name,
|
||||
table.c.installation_uuid,
|
||||
table.c.artifact_digest,
|
||||
table.c.runtime_revision,
|
||||
)
|
||||
).all()
|
||||
for row in rows:
|
||||
installation_uuid = str(row.installation_uuid or uuid.uuid4())
|
||||
values: dict[str, object] = {}
|
||||
if not row.installation_uuid:
|
||||
values['installation_uuid'] = installation_uuid
|
||||
if not row.artifact_digest or len(str(row.artifact_digest)) != 64:
|
||||
values['artifact_digest'] = _legacy_digest(installation_uuid)
|
||||
if row.runtime_revision is None or row.runtime_revision < 1:
|
||||
values['runtime_revision'] = 1
|
||||
if values:
|
||||
conn.execute(
|
||||
table.update()
|
||||
.where(table.c.workspace_uuid == row.workspace_uuid)
|
||||
.where(table.c.plugin_author == row.plugin_author)
|
||||
.where(table.c.plugin_name == row.plugin_name)
|
||||
.values(**values)
|
||||
)
|
||||
|
||||
|
||||
def _constraint_names(conn: sa.Connection, kind: str) -> set[str]:
|
||||
inspector = sa.inspect(conn)
|
||||
getter = inspector.get_unique_constraints if kind == 'unique' else inspector.get_check_constraints
|
||||
return {str(item.get('name')) for item in getter(_TABLE) if item.get('name')}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
columns = _column_names(conn)
|
||||
if not columns:
|
||||
return
|
||||
|
||||
if 'installation_uuid' not in columns:
|
||||
op.add_column(_TABLE, sa.Column('installation_uuid', sa.String(36), nullable=True))
|
||||
if 'artifact_digest' not in columns:
|
||||
op.add_column(_TABLE, sa.Column('artifact_digest', sa.String(64), nullable=True))
|
||||
if 'runtime_revision' not in columns:
|
||||
op.add_column(_TABLE, sa.Column('runtime_revision', sa.Integer(), nullable=True))
|
||||
|
||||
rls_state = _suspend_postgres_rls(conn)
|
||||
try:
|
||||
_backfill(conn)
|
||||
finally:
|
||||
_restore_postgres_rls(conn, rls_state)
|
||||
|
||||
# Fresh databases are created from current metadata before Alembic runs;
|
||||
# guards make the revision safe for that path and for interrupted upgrades.
|
||||
indexes = {index['name'] for index in sa.inspect(conn).get_indexes(_TABLE)}
|
||||
unique_constraints = _constraint_names(conn, 'unique')
|
||||
check_constraints = _constraint_names(conn, 'check')
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
batch.alter_column('installation_uuid', existing_type=sa.String(36), nullable=False)
|
||||
batch.alter_column('artifact_digest', existing_type=sa.String(64), nullable=False)
|
||||
batch.alter_column('runtime_revision', existing_type=sa.Integer(), nullable=False)
|
||||
if _REVISION_CHECK not in check_constraints:
|
||||
batch.create_check_constraint(_REVISION_CHECK, 'runtime_revision >= 1')
|
||||
if _DIGEST_CHECK not in check_constraints:
|
||||
batch.create_check_constraint(_DIGEST_CHECK, 'length(artifact_digest) = 64')
|
||||
if _INSTALLATION_UNIQUE not in unique_constraints:
|
||||
batch.create_unique_constraint(_INSTALLATION_UNIQUE, ['installation_uuid'])
|
||||
if _INSTALLATION_INDEX not in indexes and _INSTALLATION_INDEX not in unique_constraints:
|
||||
batch.create_index(
|
||||
_INSTALLATION_INDEX,
|
||||
['workspace_uuid', 'installation_uuid'],
|
||||
unique=True,
|
||||
)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
columns = _column_names(conn)
|
||||
if not columns:
|
||||
return
|
||||
indexes = {index['name'] for index in sa.inspect(conn).get_indexes(_TABLE)}
|
||||
checks = _constraint_names(conn, 'check')
|
||||
uniques = _constraint_names(conn, 'unique')
|
||||
with op.batch_alter_table(_TABLE) as batch:
|
||||
if _INSTALLATION_INDEX in indexes:
|
||||
batch.drop_index(_INSTALLATION_INDEX)
|
||||
if _DIGEST_CHECK in checks:
|
||||
batch.drop_constraint(_DIGEST_CHECK, type_='check')
|
||||
if _REVISION_CHECK in checks:
|
||||
batch.drop_constraint(_REVISION_CHECK, type_='check')
|
||||
if _INSTALLATION_UNIQUE in uniques:
|
||||
batch.drop_constraint(_INSTALLATION_UNIQUE, type_='unique')
|
||||
for column_name in ('runtime_revision', 'artifact_digest', 'installation_uuid'):
|
||||
if column_name in columns:
|
||||
batch.drop_column(column_name)
|
||||
@@ -0,0 +1,382 @@
|
||||
"""create tenant-scoped pgvector storage in the business database
|
||||
|
||||
Revision ID: 0013_tenant_pgvector
|
||||
Revises: 0012_plugin_identity
|
||||
Create Date: 2026-07-19
|
||||
|
||||
The Cloud application role never executes this DDL. A release migration role
|
||||
installs pgvector once, creates an untyped vector column, and builds a bounded
|
||||
set of expression/partial ANN indexes. Existing legacy rows are migrated only
|
||||
when each row maps unambiguously to one knowledge base.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import typing
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
|
||||
revision = '0013_tenant_pgvector'
|
||||
down_revision = '0012_plugin_identity'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
_VECTOR_TABLE = 'langbot_vectors'
|
||||
_LEGACY_TABLE = 'langbot_vectors_legacy_0013'
|
||||
_TENANT_POLICY = 'langbot_workspace_isolation'
|
||||
_TENANT_SETTING = 'langbot.workspace_uuid'
|
||||
_KB_DIMENSION_CHECK = 'ck_knowledge_bases_embedding_dimension_positive'
|
||||
_VECTOR_DIMENSION_CHECK = 'ck_langbot_vectors_embedding_dimension'
|
||||
_VECTOR_ALLOWED_CHECK = 'ck_langbot_vectors_embedding_dimension_enabled'
|
||||
_ALLOWED_DIMENSIONS = (384, 512, 768, 1024, 1536)
|
||||
_LEGACY_SOURCE_TABLES = (
|
||||
'knowledge_bases',
|
||||
'knowledge_base_files',
|
||||
'knowledge_base_chunks',
|
||||
)
|
||||
|
||||
|
||||
def _quote(conn: sa.Connection, identifier: str) -> str:
|
||||
return conn.dialect.identifier_preparer.quote(identifier)
|
||||
|
||||
|
||||
def _columns(conn: sa.Connection, table_name: str) -> set[str]:
|
||||
inspector = sa.inspect(conn)
|
||||
if table_name not in inspector.get_table_names():
|
||||
return set()
|
||||
return {column['name'] for column in inspector.get_columns(table_name)}
|
||||
|
||||
|
||||
def _checks(conn: sa.Connection, table_name: str) -> set[str]:
|
||||
return {str(item['name']) for item in sa.inspect(conn).get_check_constraints(table_name) if item.get('name')}
|
||||
|
||||
|
||||
def _ensure_knowledge_base_dimension(conn: sa.Connection) -> None:
|
||||
columns = _columns(conn, 'knowledge_bases')
|
||||
if 'embedding_dimension' not in columns:
|
||||
op.add_column('knowledge_bases', sa.Column('embedding_dimension', sa.Integer(), nullable=True))
|
||||
if _KB_DIMENSION_CHECK not in _checks(conn, 'knowledge_bases'):
|
||||
with op.batch_alter_table('knowledge_bases') as batch:
|
||||
batch.create_check_constraint(
|
||||
_KB_DIMENSION_CHECK,
|
||||
'embedding_dimension IS NULL OR embedding_dimension > 0',
|
||||
)
|
||||
|
||||
|
||||
def _create_vector_table() -> None:
|
||||
enabled = ', '.join(str(item) for item in _ALLOWED_DIMENSIONS)
|
||||
op.create_table(
|
||||
_VECTOR_TABLE,
|
||||
sa.Column('workspace_uuid', sa.String(36), nullable=False),
|
||||
sa.Column('knowledge_base_uuid', sa.String(255), nullable=False),
|
||||
sa.Column('vector_id', sa.String(255), nullable=False),
|
||||
sa.Column('embedding_dimension', sa.Integer(), nullable=False),
|
||||
sa.Column('embedding', Vector(), nullable=False),
|
||||
sa.Column('text', sa.Text(), nullable=True),
|
||||
sa.Column('file_id', sa.String(255), nullable=True),
|
||||
sa.Column('chunk_uuid', sa.String(255), nullable=True),
|
||||
sa.PrimaryKeyConstraint(
|
||||
'workspace_uuid',
|
||||
'knowledge_base_uuid',
|
||||
'vector_id',
|
||||
name='pk_langbot_vectors',
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
['workspace_uuid', 'knowledge_base_uuid'],
|
||||
['knowledge_bases.workspace_uuid', 'knowledge_bases.uuid'],
|
||||
name='fk_langbot_vectors_workspace_kb',
|
||||
ondelete='CASCADE',
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
'vector_dims(embedding) = embedding_dimension',
|
||||
name=_VECTOR_DIMENSION_CHECK,
|
||||
),
|
||||
sa.CheckConstraint(
|
||||
f'embedding_dimension IN ({enabled})',
|
||||
name=_VECTOR_ALLOWED_CHECK,
|
||||
),
|
||||
)
|
||||
op.create_index(
|
||||
'ix_langbot_vectors_workspace_kb_file',
|
||||
_VECTOR_TABLE,
|
||||
['workspace_uuid', 'knowledge_base_uuid', 'file_id'],
|
||||
)
|
||||
op.create_index(
|
||||
'ix_langbot_vectors_workspace_kb_chunk',
|
||||
_VECTOR_TABLE,
|
||||
['workspace_uuid', 'knowledge_base_uuid', 'chunk_uuid'],
|
||||
)
|
||||
|
||||
|
||||
def _legacy_mapping_predicate() -> str:
|
||||
return """
|
||||
kb.collection_id = legacy.collection
|
||||
OR EXISTS (
|
||||
SELECT 1
|
||||
FROM knowledge_base_files AS files
|
||||
LEFT JOIN knowledge_base_chunks AS chunks
|
||||
ON chunks.workspace_uuid = files.workspace_uuid
|
||||
AND chunks.file_id = files.uuid
|
||||
WHERE files.workspace_uuid = kb.workspace_uuid
|
||||
AND files.kb_id = kb.uuid
|
||||
AND (files.uuid = legacy.file_id OR chunks.uuid = legacy.chunk_uuid)
|
||||
)
|
||||
"""
|
||||
|
||||
|
||||
def _legacy_source_rls_states(conn: sa.Connection) -> dict[str, tuple[bool, bool]]:
|
||||
rows = (
|
||||
conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT c.relname, c.relrowsecurity, c.relforcerowsecurity
|
||||
FROM pg_class AS c
|
||||
JOIN pg_namespace AS n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema()
|
||||
AND c.relname IN :table_names
|
||||
AND c.relkind IN ('r', 'p')
|
||||
"""
|
||||
).bindparams(sa.bindparam('table_names', expanding=True)),
|
||||
{'table_names': _LEGACY_SOURCE_TABLES},
|
||||
)
|
||||
.mappings()
|
||||
.all()
|
||||
)
|
||||
states = {str(row['relname']): (bool(row['relrowsecurity']), bool(row['relforcerowsecurity'])) for row in rows}
|
||||
missing = set(_LEGACY_SOURCE_TABLES) - set(states)
|
||||
if missing:
|
||||
raise RuntimeError(f'Legacy pgvector source tables are missing: {sorted(missing)!r}')
|
||||
return states
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _suspend_legacy_source_rls(conn: sa.Connection) -> typing.Iterator[None]:
|
||||
"""Temporarily let the table-owning migrator map all legacy tenant rows.
|
||||
|
||||
Revision 0011 enables and forces RLS on each source table. The release
|
||||
migrator intentionally has neither superuser nor BYPASSRLS, so even a table
|
||||
owner cannot read those rows until FORCE RLS is paused. Preserve both flags
|
||||
independently and restore them in ``finally`` so mixed pre-existing states
|
||||
survive successful, rejected, and interrupted legacy migrations.
|
||||
"""
|
||||
|
||||
states = _legacy_source_rls_states(conn)
|
||||
try:
|
||||
for table_name in _LEGACY_SOURCE_TABLES:
|
||||
table = _quote(conn, table_name)
|
||||
conn.execute(sa.text(f'ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY'))
|
||||
conn.execute(sa.text(f'ALTER TABLE {table} DISABLE ROW LEVEL SECURITY'))
|
||||
yield
|
||||
finally:
|
||||
for table_name in _LEGACY_SOURCE_TABLES:
|
||||
table = _quote(conn, table_name)
|
||||
rls_enabled, rls_forced = states[table_name]
|
||||
enabled_clause = 'ENABLE' if rls_enabled else 'DISABLE'
|
||||
forced_clause = 'FORCE' if rls_forced else 'NO FORCE'
|
||||
conn.execute(sa.text(f'ALTER TABLE {table} {enabled_clause} ROW LEVEL SECURITY'))
|
||||
conn.execute(sa.text(f'ALTER TABLE {table} {forced_clause} ROW LEVEL SECURITY'))
|
||||
|
||||
|
||||
def _migrate_legacy_rows(conn: sa.Connection) -> None:
|
||||
predicate = _legacy_mapping_predicate()
|
||||
ambiguous = conn.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
WITH candidates AS (
|
||||
SELECT legacy.id, kb.workspace_uuid, kb.uuid AS knowledge_base_uuid
|
||||
FROM {_LEGACY_TABLE} AS legacy
|
||||
JOIN knowledge_bases AS kb ON ({predicate})
|
||||
), candidate_counts AS (
|
||||
SELECT id, COUNT(*) AS count
|
||||
FROM candidates
|
||||
GROUP BY id
|
||||
)
|
||||
SELECT legacy.id, COALESCE(candidate_counts.count, 0) AS candidate_count
|
||||
FROM {_LEGACY_TABLE} AS legacy
|
||||
LEFT JOIN candidate_counts ON candidate_counts.id = legacy.id
|
||||
WHERE COALESCE(candidate_counts.count, 0) <> 1
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
).first()
|
||||
if ambiguous is not None:
|
||||
raise RuntimeError(
|
||||
'Legacy pgvector row cannot be mapped to exactly one Workspace/knowledge base: '
|
||||
f'{ambiguous.id!r} has {ambiguous.candidate_count} candidates'
|
||||
)
|
||||
|
||||
conn.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
INSERT INTO {_VECTOR_TABLE} (
|
||||
workspace_uuid,
|
||||
knowledge_base_uuid,
|
||||
vector_id,
|
||||
embedding_dimension,
|
||||
embedding,
|
||||
text,
|
||||
file_id,
|
||||
chunk_uuid
|
||||
)
|
||||
SELECT
|
||||
kb.workspace_uuid,
|
||||
kb.uuid,
|
||||
legacy.id,
|
||||
vector_dims(legacy.embedding),
|
||||
legacy.embedding,
|
||||
legacy.text,
|
||||
legacy.file_id,
|
||||
legacy.chunk_uuid
|
||||
FROM {_LEGACY_TABLE} AS legacy
|
||||
JOIN knowledge_bases AS kb ON ({predicate})
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
mixed_dimension = conn.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
SELECT workspace_uuid, knowledge_base_uuid
|
||||
FROM {_VECTOR_TABLE}
|
||||
GROUP BY workspace_uuid, knowledge_base_uuid
|
||||
HAVING MIN(embedding_dimension) <> MAX(embedding_dimension)
|
||||
LIMIT 1
|
||||
"""
|
||||
)
|
||||
).first()
|
||||
if mixed_dimension is not None:
|
||||
raise RuntimeError('Legacy knowledge base contains mixed embedding dimensions')
|
||||
|
||||
conn.execute(
|
||||
sa.text(
|
||||
f"""
|
||||
UPDATE knowledge_bases AS kb
|
||||
SET embedding_dimension = dimensions.embedding_dimension
|
||||
FROM (
|
||||
SELECT workspace_uuid, knowledge_base_uuid, MIN(embedding_dimension) AS embedding_dimension
|
||||
FROM {_VECTOR_TABLE}
|
||||
GROUP BY workspace_uuid, knowledge_base_uuid
|
||||
) AS dimensions
|
||||
WHERE kb.workspace_uuid = dimensions.workspace_uuid
|
||||
AND kb.uuid = dimensions.knowledge_base_uuid
|
||||
AND kb.embedding_dimension IS NULL
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _drop_all_policies(conn: sa.Connection) -> None:
|
||||
table = _quote(conn, _VECTOR_TABLE)
|
||||
policies = conn.execute(
|
||||
sa.text(
|
||||
"""
|
||||
SELECT p.polname
|
||||
FROM pg_policy p
|
||||
JOIN pg_class c ON c.oid = p.polrelid
|
||||
JOIN pg_namespace n ON n.oid = c.relnamespace
|
||||
WHERE n.nspname = current_schema() AND c.relname = :table_name
|
||||
"""
|
||||
),
|
||||
{'table_name': _VECTOR_TABLE},
|
||||
).scalars()
|
||||
for policy_name in policies:
|
||||
op.execute(sa.text(f'DROP POLICY {_quote(conn, policy_name)} ON {table}'))
|
||||
|
||||
|
||||
def _enable_rls(conn: sa.Connection) -> None:
|
||||
table = _quote(conn, _VECTOR_TABLE)
|
||||
policy = _quote(conn, _TENANT_POLICY)
|
||||
expression = f"workspace_uuid::text = NULLIF(current_setting('{_TENANT_SETTING}', true), '')"
|
||||
_drop_all_policies(conn)
|
||||
op.execute(sa.text(f'ALTER TABLE {table} ENABLE ROW LEVEL SECURITY'))
|
||||
op.execute(sa.text(f'ALTER TABLE {table} FORCE ROW LEVEL SECURITY'))
|
||||
op.execute(
|
||||
sa.text(
|
||||
f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR ALL TO PUBLIC '
|
||||
f'USING ({expression}) WITH CHECK ({expression})'
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _create_ann_indexes(conn: sa.Connection) -> None:
|
||||
table = _quote(conn, _VECTOR_TABLE)
|
||||
for dimension in _ALLOWED_DIMENSIONS:
|
||||
index = _quote(conn, f'ix_langbot_vectors_hnsw_cosine_{dimension}')
|
||||
op.execute(
|
||||
sa.text(
|
||||
f'CREATE INDEX {index} ON {table} USING hnsw '
|
||||
f'((embedding::vector({dimension})) vector_cosine_ops) '
|
||||
f'WHERE embedding_dimension = {dimension}'
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if 'knowledge_bases' not in sa.inspect(conn).get_table_names():
|
||||
if conn.dialect.name == 'postgresql':
|
||||
# The supported PostgreSQL release path creates the business
|
||||
# tables before stamping 0010 and reaching this migration. Missing
|
||||
# knowledge_bases therefore means the operator bypassed the
|
||||
# release bootstrap or the schema is incomplete; stamping head in
|
||||
# that state would make Cloud runtime validation unrecoverable.
|
||||
raise RuntimeError('PostgreSQL release migration requires the knowledge_bases table')
|
||||
# A direct empty SQLite Alembic walk is still used by migration tooling;
|
||||
# Core creates the complete ORM schema on its following compatibility
|
||||
# pass, including the portable embedding_dimension field.
|
||||
return
|
||||
|
||||
_ensure_knowledge_base_dimension(conn)
|
||||
|
||||
# pgvector storage is PostgreSQL-only, but ``embedding_dimension`` is an
|
||||
# ORM field used by both deployment modes. Existing OSS SQLite databases
|
||||
# must receive the column before this revision becomes a no-op.
|
||||
if conn.dialect.name != 'postgresql':
|
||||
return
|
||||
|
||||
op.execute(sa.text('CREATE EXTENSION IF NOT EXISTS vector'))
|
||||
columns = _columns(conn, _VECTOR_TABLE)
|
||||
if columns:
|
||||
scoped_columns = {
|
||||
'workspace_uuid',
|
||||
'knowledge_base_uuid',
|
||||
'vector_id',
|
||||
'embedding_dimension',
|
||||
'embedding',
|
||||
}
|
||||
if scoped_columns.issubset(columns):
|
||||
raise RuntimeError('Tenant pgvector table exists before its owning release migration')
|
||||
legacy_columns = {'id', 'collection', 'embedding'}
|
||||
if not legacy_columns.issubset(columns):
|
||||
raise RuntimeError('Existing pgvector table has an unsupported schema')
|
||||
if _LEGACY_TABLE in sa.inspect(conn).get_table_names():
|
||||
raise RuntimeError(f'Interrupted pgvector migration left {_LEGACY_TABLE!r} behind')
|
||||
op.rename_table(_VECTOR_TABLE, _LEGACY_TABLE)
|
||||
|
||||
_create_vector_table()
|
||||
if _LEGACY_TABLE in sa.inspect(conn).get_table_names():
|
||||
with _suspend_legacy_source_rls(conn):
|
||||
_migrate_legacy_rows(conn)
|
||||
op.drop_table(_LEGACY_TABLE)
|
||||
_create_ann_indexes(conn)
|
||||
_enable_rls(conn)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
conn = op.get_bind()
|
||||
if conn.dialect.name == 'postgresql' and _VECTOR_TABLE in sa.inspect(conn).get_table_names():
|
||||
_drop_all_policies(conn)
|
||||
op.drop_table(_VECTOR_TABLE)
|
||||
|
||||
columns = _columns(conn, 'knowledge_bases')
|
||||
if 'embedding_dimension' in columns:
|
||||
checks = _checks(conn, 'knowledge_bases')
|
||||
with op.batch_alter_table('knowledge_bases') as batch:
|
||||
if _KB_DIMENSION_CHECK in checks:
|
||||
batch.drop_constraint(_KB_DIMENSION_CHECK, type_='check')
|
||||
batch.drop_column('embedding_dimension')
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import abc
|
||||
|
||||
import sqlalchemy
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
|
||||
from ..core import app
|
||||
@@ -30,8 +31,14 @@ class BaseDatabaseManager(abc.ABC):
|
||||
|
||||
engine: sqlalchemy_asyncio.AsyncEngine
|
||||
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
ap: app.Application,
|
||||
*,
|
||||
url_override: sqlalchemy.engine.URL | None = None,
|
||||
) -> None:
|
||||
self.ap = ap
|
||||
self.url_override = url_override
|
||||
|
||||
@abc.abstractmethod
|
||||
async def initialize(self) -> None:
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
|
||||
from .. import database
|
||||
from ..postgresql_url import normalize_asyncpg_url
|
||||
|
||||
|
||||
@database.manager_class('postgresql')
|
||||
@@ -10,12 +12,29 @@ class PostgreSQLDatabaseManager(database.BaseDatabaseManager):
|
||||
"""PostgreSQL database manager"""
|
||||
|
||||
async def initialize(self) -> None:
|
||||
postgresql_config = self.ap.instance_config.data.get('database', {}).get('postgresql', {})
|
||||
|
||||
host = postgresql_config.get('host', '127.0.0.1')
|
||||
port = postgresql_config.get('port', 5432)
|
||||
user = postgresql_config.get('user', 'postgres')
|
||||
password = postgresql_config.get('password', 'postgres')
|
||||
database = postgresql_config.get('database', 'postgres')
|
||||
engine_url = f'postgresql+asyncpg://{user}:{password}@{host}:{port}/{database}'
|
||||
if self.url_override is not None:
|
||||
engine_url = self.url_override
|
||||
else:
|
||||
postgresql_config = self.ap.instance_config.data.get('database', {}).get('postgresql', {})
|
||||
explicit_url = postgresql_config.get('url')
|
||||
if explicit_url:
|
||||
if not isinstance(explicit_url, str):
|
||||
raise ValueError('database.postgresql.url must be a string')
|
||||
try:
|
||||
engine_url = sqlalchemy.engine.make_url(explicit_url)
|
||||
except Exception:
|
||||
raise ValueError('database.postgresql.url is invalid') from None
|
||||
try:
|
||||
engine_url = normalize_asyncpg_url(engine_url)
|
||||
except ValueError:
|
||||
raise ValueError('database.postgresql.url must use valid PostgreSQL asyncpg options') from None
|
||||
else:
|
||||
engine_url = sqlalchemy.URL.create(
|
||||
'postgresql+asyncpg',
|
||||
username=postgresql_config.get('user', 'postgres'),
|
||||
password=postgresql_config.get('password', 'postgres'),
|
||||
host=postgresql_config.get('host', '127.0.0.1'),
|
||||
port=postgresql_config.get('port', 5432),
|
||||
database=postgresql_config.get('database', 'postgres'),
|
||||
)
|
||||
self.engine = sqlalchemy_asyncio.create_async_engine(engine_url)
|
||||
|
||||
+1255
-20
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,25 @@
|
||||
"""Safe PostgreSQL URL normalization shared by runtime and migration jobs."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
|
||||
def normalize_asyncpg_url(url: sqlalchemy.engine.URL) -> sqlalchemy.engine.URL:
|
||||
"""Select asyncpg and translate the common libpq TLS query spelling."""
|
||||
|
||||
if url.drivername == 'postgresql':
|
||||
url = url.set(drivername='postgresql+asyncpg')
|
||||
elif url.drivername != 'postgresql+asyncpg':
|
||||
raise ValueError('PostgreSQL URL must use PostgreSQL with the asyncpg driver')
|
||||
|
||||
query = dict(url.query)
|
||||
sslmode = query.pop('sslmode', None)
|
||||
if sslmode is not None:
|
||||
if 'ssl' in query and query['ssl'] != sslmode:
|
||||
raise ValueError('PostgreSQL URL cannot specify conflicting ssl and sslmode options')
|
||||
# SQLAlchemy expands URL query keys into asyncpg keyword arguments.
|
||||
# asyncpg calls this keyword ``ssl`` even though PostgreSQL DSNs
|
||||
# conventionally spell the same mode ``sslmode``.
|
||||
query['ssl'] = sslmode
|
||||
return url.set(query=query)
|
||||
@@ -0,0 +1,193 @@
|
||||
"""One-shot, operator-only Cloud PostgreSQL release migration entrypoint."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
from collections.abc import Mapping
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from ..cloud.bootstrap import SUPPORTED_PGVECTOR_DIMENSIONS
|
||||
from ..core import app as core_app
|
||||
from ..core.stages.load_config import LoadConfigStage
|
||||
from ..core.stages.setup_logger import SetupLoggerStage
|
||||
from .mgr import PersistenceManager, PersistenceMode
|
||||
from .postgresql_url import normalize_asyncpg_url
|
||||
|
||||
|
||||
DEFAULT_OPERATOR_DSN_ENV = 'LANGBOT_CLOUD_MIGRATION_DSN'
|
||||
_ENV_NAME = re.compile(r'^[A-Z_][A-Z0-9_]*$')
|
||||
|
||||
|
||||
class CloudReleaseMigrationConfigurationError(RuntimeError):
|
||||
"""Raised before any database operation when migration input is unsafe."""
|
||||
|
||||
|
||||
def _url_endpoint(url: sqlalchemy.engine.URL, *, label: str) -> tuple[str, int]:
|
||||
"""Return a comparison-safe PostgreSQL endpoint without leaking its DSN."""
|
||||
|
||||
try:
|
||||
host = (url.host or '').strip().casefold()
|
||||
port = url.port or 5432
|
||||
except (TypeError, ValueError):
|
||||
raise CloudReleaseMigrationConfigurationError(f'{label} PostgreSQL host or port is invalid') from None
|
||||
if not host or isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535:
|
||||
raise CloudReleaseMigrationConfigurationError(f'{label} PostgreSQL host or port is invalid')
|
||||
return host, port
|
||||
|
||||
|
||||
def _operator_database_url(
|
||||
instance_config: dict,
|
||||
*,
|
||||
environ: Mapping[str, str],
|
||||
) -> sqlalchemy.engine.URL:
|
||||
database_config = instance_config.get('database')
|
||||
if not isinstance(database_config, dict) or database_config.get('use') != 'postgresql':
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration requires explicit database.use=postgresql; SQLite fallback is forbidden'
|
||||
)
|
||||
|
||||
runtime_config = database_config.get('postgresql')
|
||||
if not isinstance(runtime_config, dict):
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud runtime PostgreSQL configuration is missing')
|
||||
|
||||
migration_config = database_config.get('cloud_migration', {})
|
||||
if not isinstance(migration_config, dict):
|
||||
raise CloudReleaseMigrationConfigurationError('database.cloud_migration must be a mapping')
|
||||
dsn_env = migration_config.get('operator_dsn_env', DEFAULT_OPERATOR_DSN_ENV)
|
||||
if not isinstance(dsn_env, str) or not _ENV_NAME.fullmatch(dsn_env):
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'database.cloud_migration.operator_dsn_env must name an uppercase environment variable'
|
||||
)
|
||||
|
||||
raw_dsn = environ.get(dsn_env, '').strip()
|
||||
if not raw_dsn:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
f'Cloud release migration requires the operator DSN in environment variable {dsn_env}'
|
||||
)
|
||||
try:
|
||||
operator_url = sqlalchemy.engine.make_url(raw_dsn)
|
||||
except Exception:
|
||||
# Never echo a malformed DSN because it may contain an unescaped secret.
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud release migration operator DSN is invalid') from None
|
||||
|
||||
try:
|
||||
operator_url = normalize_asyncpg_url(operator_url)
|
||||
except ValueError:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration operator DSN must use valid PostgreSQL asyncpg options'
|
||||
) from None
|
||||
|
||||
operator_user = (operator_url.username or '').strip()
|
||||
operator_database = (operator_url.database or '').strip()
|
||||
operator_host, operator_port = _url_endpoint(operator_url, label='Cloud release migration operator')
|
||||
runtime_url_value = runtime_config.get('url')
|
||||
if runtime_url_value:
|
||||
if not isinstance(runtime_url_value, str):
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud runtime PostgreSQL URL must be a string')
|
||||
try:
|
||||
runtime_url = sqlalchemy.engine.make_url(runtime_url_value)
|
||||
except Exception:
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud runtime PostgreSQL URL is invalid') from None
|
||||
if runtime_url.drivername not in {'postgresql', 'postgresql+asyncpg'}:
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud runtime database URL must use PostgreSQL')
|
||||
runtime_user = (runtime_url.username or '').strip()
|
||||
runtime_database = (runtime_url.database or '').strip()
|
||||
runtime_host, runtime_port = _url_endpoint(runtime_url, label='Cloud runtime')
|
||||
else:
|
||||
runtime_user = str(runtime_config.get('user', 'postgres') or '').strip()
|
||||
runtime_database = str(runtime_config.get('database', 'postgres') or '').strip()
|
||||
runtime_host = str(runtime_config.get('host', '') or '').strip().casefold()
|
||||
runtime_port = runtime_config.get('port', 5432)
|
||||
if not operator_user or not operator_database:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration operator DSN must include a user, host, and database'
|
||||
)
|
||||
if (
|
||||
not runtime_user
|
||||
or not runtime_database
|
||||
or not runtime_host
|
||||
or isinstance(runtime_port, bool)
|
||||
or not isinstance(runtime_port, int)
|
||||
or not 1 <= runtime_port <= 65535
|
||||
):
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud runtime PostgreSQL user, host, port, and database are required'
|
||||
)
|
||||
if operator_user == runtime_user:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration requires a distinct operator role from the runtime PostgreSQL role'
|
||||
)
|
||||
if operator_database != runtime_database:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration operator DSN must target the configured runtime database'
|
||||
)
|
||||
if operator_host != runtime_host or operator_port != runtime_port:
|
||||
# The first Cloud release intentionally requires the migrator and
|
||||
# runtime to use the same PostgreSQL endpoint. Supporting a direct
|
||||
# operator endpoint plus a runtime pooler requires a database-backed
|
||||
# immutable cluster identity check; accepting aliases here would turn a
|
||||
# same-named database on another cluster into a silent migration target.
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration operator DSN must target the configured runtime PostgreSQL endpoint'
|
||||
)
|
||||
|
||||
vdb_config = instance_config.get('vdb')
|
||||
if not isinstance(vdb_config, dict) or vdb_config.get('use') != 'pgvector':
|
||||
raise CloudReleaseMigrationConfigurationError('Cloud release migration requires vdb.use=pgvector')
|
||||
pgvector_config = vdb_config.get('pgvector')
|
||||
if not isinstance(pgvector_config, dict) or pgvector_config.get('use_business_database') is not True:
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration requires vdb.pgvector.use_business_database=true'
|
||||
)
|
||||
dimensions = pgvector_config.get('allowed_dimensions')
|
||||
if (
|
||||
not isinstance(dimensions, list)
|
||||
or not dimensions
|
||||
or any(isinstance(item, bool) or not isinstance(item, int) for item in dimensions)
|
||||
or not set(dimensions).issubset(SUPPORTED_PGVECTOR_DIMENSIONS)
|
||||
):
|
||||
raise CloudReleaseMigrationConfigurationError(
|
||||
'Cloud release migration pgvector dimensions are outside the release-created index set'
|
||||
)
|
||||
|
||||
return operator_url
|
||||
|
||||
|
||||
async def run_cloud_release_migration(
|
||||
ap: core_app.Application,
|
||||
*,
|
||||
environ: Mapping[str, str] | None = None,
|
||||
) -> None:
|
||||
"""Run and validate one release migration with an isolated operator DSN."""
|
||||
|
||||
operator_url = _operator_database_url(
|
||||
ap.instance_config.data,
|
||||
environ=os.environ if environ is None else environ,
|
||||
)
|
||||
manager = PersistenceManager(
|
||||
ap,
|
||||
mode=PersistenceMode.RELEASE_MIGRATION,
|
||||
database_url=operator_url,
|
||||
)
|
||||
ap.persistence_mgr = manager
|
||||
try:
|
||||
await manager.initialize()
|
||||
ap.logger.info('Cloud PostgreSQL release migration reached and validated the exact release head.')
|
||||
finally:
|
||||
db = getattr(manager, 'db', None)
|
||||
engine = getattr(db, 'engine', None)
|
||||
if engine is not None:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
async def run_cloud_release_migration_from_config(loop: asyncio.AbstractEventLoop) -> None:
|
||||
"""Load only process configuration/logging, then run the one-shot job."""
|
||||
|
||||
ap = core_app.Application()
|
||||
ap.event_loop = loop
|
||||
await LoadConfigStage().run(ap)
|
||||
await SetupLoggerStage().run(ap)
|
||||
await run_cloud_release_migration(ap)
|
||||
@@ -1,5 +1,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextvars
|
||||
import dataclasses
|
||||
import enum
|
||||
import types
|
||||
import typing
|
||||
|
||||
@@ -8,7 +12,17 @@ import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
|
||||
|
||||
|
||||
TENANT_SETTING = 'langbot.workspace_uuid'
|
||||
ACCOUNT_SETTING = 'langbot.account_uuid'
|
||||
API_KEY_HASH_SETTING = 'langbot.api_key_hash'
|
||||
INVITATION_HASH_SETTING = 'langbot.invitation_hash'
|
||||
INSTANCE_SETTING = 'langbot.instance_uuid'
|
||||
IDENTITY_DIGEST_SETTING = 'langbot.identity_digest'
|
||||
|
||||
TENANT_POLICY_NAME = 'langbot_workspace_isolation'
|
||||
ACCOUNT_DISCOVERY_POLICY_NAME = 'langbot_account_discovery'
|
||||
API_KEY_DISCOVERY_POLICY_NAME = 'langbot_api_key_discovery'
|
||||
INVITATION_DISCOVERY_POLICY_NAME = 'langbot_invitation_discovery'
|
||||
INSTANCE_DISCOVERY_POLICY_NAME = 'langbot_instance_discovery'
|
||||
|
||||
# Keep this contract explicit. A new tenant-owned table must be added to both
|
||||
# this runtime list and the corresponding Alembic migration before release.
|
||||
@@ -41,26 +55,269 @@ TENANT_TABLE_COLUMNS: dict[str, str] = {
|
||||
'monitoring_errors': 'workspace_uuid',
|
||||
'monitoring_embedding_calls': 'workspace_uuid',
|
||||
'monitoring_feedback': 'workspace_uuid',
|
||||
# Created by 0013 rather than ORM metadata; it is still part of the same
|
||||
# business-database RLS contract and permits no discovery policies.
|
||||
'langbot_vectors': 'workspace_uuid',
|
||||
}
|
||||
|
||||
|
||||
class TenantUnitOfWork:
|
||||
"""Bind one Workspace scope to one database transaction.
|
||||
class PersistenceScopeKind(enum.StrEnum):
|
||||
WORKSPACE = 'workspace'
|
||||
ACCOUNT_DISCOVERY = 'account_discovery'
|
||||
API_KEY_DISCOVERY = 'api_key_discovery'
|
||||
INVITATION_DISCOVERY = 'invitation_discovery'
|
||||
INSTANCE_DISCOVERY = 'instance_discovery'
|
||||
IDENTITY_DISCOVERY = 'identity_discovery'
|
||||
|
||||
PostgreSQL receives a transaction-local setting used by RLS policies.
|
||||
SQLite keeps the same transaction boundary without a database setting so
|
||||
OSS callers can adopt this API without changing database engines.
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class PersistenceScope:
|
||||
"""A trusted, transaction-local database visibility scope."""
|
||||
|
||||
kind: PersistenceScopeKind
|
||||
settings: tuple[tuple[str, str], ...]
|
||||
|
||||
@classmethod
|
||||
def workspace(cls, workspace_uuid: str) -> PersistenceScope:
|
||||
return cls._one(PersistenceScopeKind.WORKSPACE, TENANT_SETTING, workspace_uuid)
|
||||
|
||||
@classmethod
|
||||
def account(cls, account_uuid: str) -> PersistenceScope:
|
||||
return cls._one(PersistenceScopeKind.ACCOUNT_DISCOVERY, ACCOUNT_SETTING, account_uuid)
|
||||
|
||||
@classmethod
|
||||
def api_key(cls, key_hash: str) -> PersistenceScope:
|
||||
return cls._one(PersistenceScopeKind.API_KEY_DISCOVERY, API_KEY_HASH_SETTING, key_hash)
|
||||
|
||||
@classmethod
|
||||
def invitation(cls, token_hash: str) -> PersistenceScope:
|
||||
return cls._one(PersistenceScopeKind.INVITATION_DISCOVERY, INVITATION_HASH_SETTING, token_hash)
|
||||
|
||||
@classmethod
|
||||
def instance(cls, instance_uuid: str) -> PersistenceScope:
|
||||
return cls._one(PersistenceScopeKind.INSTANCE_DISCOVERY, INSTANCE_SETTING, instance_uuid)
|
||||
|
||||
@classmethod
|
||||
def identity(cls, identity_digest: str) -> PersistenceScope:
|
||||
return cls._one(PersistenceScopeKind.IDENTITY_DISCOVERY, IDENTITY_DIGEST_SETTING, identity_digest)
|
||||
|
||||
@classmethod
|
||||
def _one(cls, kind: PersistenceScopeKind, setting: str, value: str) -> PersistenceScope:
|
||||
return cls(kind, ((setting, cls._normalize(value, kind.value)),))
|
||||
|
||||
@staticmethod
|
||||
def _normalize(value: str, label: str) -> str:
|
||||
if not isinstance(value, str):
|
||||
raise TypeError(f'{label} must be a string')
|
||||
normalized = value.strip()
|
||||
if not normalized:
|
||||
raise ValueError(f'{label} must not be empty')
|
||||
if len(normalized) > 512:
|
||||
raise ValueError(f'{label} exceeds the database scope limit')
|
||||
return normalized
|
||||
|
||||
|
||||
class TenantScopeRequiredError(RuntimeError):
|
||||
"""Raised when Cloud business data is accessed without a trusted scope."""
|
||||
|
||||
|
||||
class CrossScopeTransactionError(RuntimeError):
|
||||
"""Raised when code tries to change scope inside an active transaction."""
|
||||
|
||||
|
||||
class TransactionRollbackOnlyError(RuntimeError):
|
||||
"""Raised when a caught failure made the outer transaction unsafe to commit."""
|
||||
|
||||
|
||||
@dataclasses.dataclass(slots=True)
|
||||
class ActiveScopedTransaction:
|
||||
scope: PersistenceScope
|
||||
session: sqlalchemy_asyncio.AsyncSession
|
||||
engine: sqlalchemy_asyncio.AsyncEngine
|
||||
owner_task: asyncio.Task[typing.Any] | None
|
||||
depth: int = 1
|
||||
rollback_only: bool = False
|
||||
rollback_only_cause: BaseException | None = None
|
||||
after_commit_waiters: list[asyncio.Future[None]] = dataclasses.field(default_factory=list)
|
||||
|
||||
def mark_rollback_only(self, cause: BaseException | None = None) -> None:
|
||||
"""Remember that this transaction must never release commit-gated work."""
|
||||
|
||||
self.rollback_only = True
|
||||
if self.rollback_only_cause is None:
|
||||
self.rollback_only_cause = cause
|
||||
|
||||
|
||||
ActiveTransactionVar = contextvars.ContextVar[ActiveScopedTransaction | None]
|
||||
|
||||
# ``AsyncSession`` delegates database I/O to a greenlet-backed synchronous
|
||||
# Engine, but Python context variables are preserved across that boundary. A
|
||||
# per-engine ``handle_error`` listener therefore covers DBAPI failures from all
|
||||
# direct Session APIs (execute/scalar/get/flush) as well as callers which obtain
|
||||
# the transaction-bound Connection. The owner-task and engine checks prevent
|
||||
# copied child-task contexts or unrelated database engines from poisoning this
|
||||
# transaction.
|
||||
_DATABASE_OPERATION_TRANSACTION: contextvars.ContextVar[ActiveScopedTransaction | None] = contextvars.ContextVar(
|
||||
'langbot_database_operation_transaction',
|
||||
default=None,
|
||||
)
|
||||
|
||||
|
||||
def _mark_current_transaction_rollback_only(exception_context: typing.Any) -> None:
|
||||
state = _DATABASE_OPERATION_TRANSACTION.get()
|
||||
if state is None:
|
||||
return
|
||||
try:
|
||||
current_task = asyncio.current_task()
|
||||
except RuntimeError: # pragma: no cover - async engines always run in an event loop
|
||||
return
|
||||
if state.owner_task is not current_task or exception_context.engine is not state.engine.sync_engine:
|
||||
return
|
||||
cause = exception_context.sqlalchemy_exception or exception_context.original_exception
|
||||
state.mark_rollback_only(cause)
|
||||
|
||||
|
||||
def _install_rollback_only_error_listener(engine: sqlalchemy_asyncio.AsyncEngine) -> None:
|
||||
sync_engine = engine.sync_engine
|
||||
if not sqlalchemy.event.contains(sync_engine, 'handle_error', _mark_current_transaction_rollback_only):
|
||||
sqlalchemy.event.listen(sync_engine, 'handle_error', _mark_current_transaction_rollback_only)
|
||||
|
||||
|
||||
@dataclasses.dataclass(slots=True)
|
||||
class ActivePersistenceScope:
|
||||
"""A trusted visibility scope without an open database transaction."""
|
||||
|
||||
scope: PersistenceScope
|
||||
owner_task: asyncio.Task[typing.Any] | None
|
||||
depth: int = 1
|
||||
|
||||
|
||||
ActivePersistenceScopeVar = contextvars.ContextVar[ActivePersistenceScope | None]
|
||||
|
||||
|
||||
class PersistenceScopeBoundary:
|
||||
"""Carry a trusted persistence scope across long-running async work.
|
||||
|
||||
Unlike :class:`TenantUnitOfWork`, this boundary never opens a database
|
||||
session. ``PersistenceManager.execute_async`` materializes the scope as a
|
||||
short transaction for each database call. This makes the boundary suitable
|
||||
for request and pipeline lifetimes which can spend substantial time waiting
|
||||
on model providers, tools, or streaming clients.
|
||||
|
||||
Context variables are copied into child tasks, so implicit use from a child
|
||||
task is rejected by the manager. A child may establish its own explicit
|
||||
boundary because the scope value still comes from trusted application code.
|
||||
"""
|
||||
|
||||
def __init__(self, engine: sqlalchemy_asyncio.AsyncEngine, workspace_uuid: str) -> None:
|
||||
workspace_uuid = workspace_uuid.strip()
|
||||
if not workspace_uuid:
|
||||
raise ValueError('TenantUnitOfWork requires a non-empty workspace_uuid')
|
||||
def __init__(
|
||||
self,
|
||||
scope: PersistenceScope,
|
||||
*,
|
||||
active_scope: ActivePersistenceScopeVar,
|
||||
active_transaction: ActiveTransactionVar,
|
||||
) -> None:
|
||||
self.scope = scope
|
||||
self._active_scope = active_scope
|
||||
self._active_transaction = active_transaction
|
||||
self._active_state: ActivePersistenceScope | None = None
|
||||
self._context_token: contextvars.Token[ActivePersistenceScope | None] | None = None
|
||||
self._used = False
|
||||
self._owns_scope = False
|
||||
|
||||
async def __aenter__(self) -> PersistenceScopeBoundary:
|
||||
if self._used:
|
||||
raise RuntimeError('PersistenceScopeBoundary instances cannot be reused')
|
||||
self._used = True
|
||||
|
||||
transaction = self._active_transaction.get()
|
||||
if transaction is not None:
|
||||
if transaction.owner_task is not asyncio.current_task():
|
||||
raise CrossScopeTransactionError(
|
||||
'Scoped database transactions cannot be inherited by child tasks; open an explicit task scope'
|
||||
)
|
||||
if transaction.scope != self.scope:
|
||||
raise CrossScopeTransactionError(
|
||||
f'Cannot enter {self.scope.kind.value} scope while {transaction.scope.kind.value} scope is active'
|
||||
)
|
||||
|
||||
active = self._active_scope.get()
|
||||
if active is not None and active.owner_task is asyncio.current_task():
|
||||
if active.scope != self.scope:
|
||||
raise CrossScopeTransactionError(
|
||||
f'Cannot enter {self.scope.kind.value} scope while {active.scope.kind.value} scope is active'
|
||||
)
|
||||
active.depth += 1
|
||||
self._active_state = active
|
||||
return self
|
||||
|
||||
state = ActivePersistenceScope(
|
||||
scope=self.scope,
|
||||
owner_task=asyncio.current_task(),
|
||||
)
|
||||
self._active_state = state
|
||||
self._context_token = self._active_scope.set(state)
|
||||
self._owns_scope = True
|
||||
return self
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc_value: BaseException | None,
|
||||
traceback: types.TracebackType | None,
|
||||
) -> bool:
|
||||
del exc_type, exc_value, traceback
|
||||
state = self._active_state
|
||||
if state is None:
|
||||
raise RuntimeError('PersistenceScopeBoundary has no active state')
|
||||
if state.owner_task is not asyncio.current_task():
|
||||
raise CrossScopeTransactionError('Scoped persistence boundaries cannot be exited by a different task')
|
||||
|
||||
if not self._owns_scope:
|
||||
state.depth -= 1
|
||||
self._active_state = None
|
||||
return False
|
||||
|
||||
if self._context_token is None:
|
||||
raise RuntimeError('PersistenceScopeBoundary has no context token')
|
||||
self._active_scope.reset(self._context_token)
|
||||
state.depth = 0
|
||||
self._active_state = None
|
||||
return False
|
||||
|
||||
|
||||
class TenantUnitOfWork:
|
||||
"""Bind one trusted visibility scope to one database transaction.
|
||||
|
||||
PostgreSQL receives transaction-local settings consumed by RLS policies.
|
||||
SQLite keeps the same transaction boundary so OSS code exercises the same
|
||||
request/task ownership and rollback semantics. A manager-owned ContextVar
|
||||
lets all legacy ``execute_async`` helpers reuse this exact session.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
engine: sqlalchemy_asyncio.AsyncEngine,
|
||||
workspace_uuid: str | None = None,
|
||||
*,
|
||||
scope: PersistenceScope | None = None,
|
||||
active_transaction: ActiveTransactionVar | None = None,
|
||||
active_scope: ActivePersistenceScopeVar | None = None,
|
||||
) -> None:
|
||||
if (workspace_uuid is None) == (scope is None):
|
||||
raise ValueError('TenantUnitOfWork requires exactly one Workspace or persistence scope')
|
||||
|
||||
self._engine = engine
|
||||
self.workspace_uuid = workspace_uuid
|
||||
self.scope = scope or PersistenceScope.workspace(typing.cast(str, workspace_uuid))
|
||||
self.workspace_uuid = self.scope.settings[0][1] if self.scope.kind == PersistenceScopeKind.WORKSPACE else None
|
||||
self._active_transaction = active_transaction
|
||||
self._active_scope = active_scope
|
||||
self._session: sqlalchemy_asyncio.AsyncSession | None = None
|
||||
self._active_state: ActiveScopedTransaction | None = None
|
||||
self._context_token: contextvars.Token[ActiveScopedTransaction | None] | None = None
|
||||
self._database_operation_token: contextvars.Token[ActiveScopedTransaction | None] | None = None
|
||||
self._used = False
|
||||
self._owns_transaction = False
|
||||
_install_rollback_only_error_listener(engine)
|
||||
|
||||
@property
|
||||
def session(self) -> sqlalchemy_asyncio.AsyncSession:
|
||||
@@ -73,16 +330,67 @@ class TenantUnitOfWork:
|
||||
raise RuntimeError('TenantUnitOfWork instances cannot be reused')
|
||||
self._used = True
|
||||
|
||||
active_scope = self._active_scope.get() if self._active_scope is not None else None
|
||||
if active_scope is not None and active_scope.owner_task is asyncio.current_task():
|
||||
if active_scope.scope != self.scope:
|
||||
# A request/runtime Workspace boundary is transaction-free and
|
||||
# may temporarily enter a narrower trusted discovery UoW (for
|
||||
# example, resolving an account record by its verified UUID).
|
||||
# It must still never switch directly to another Workspace.
|
||||
workspace_to_discovery = (
|
||||
active_scope.scope.kind == PersistenceScopeKind.WORKSPACE
|
||||
and self.scope.kind != PersistenceScopeKind.WORKSPACE
|
||||
)
|
||||
if not workspace_to_discovery:
|
||||
raise CrossScopeTransactionError(
|
||||
f'Cannot enter {self.scope.kind.value} scope '
|
||||
f'while {active_scope.scope.kind.value} scope is active'
|
||||
)
|
||||
|
||||
active = self._active_transaction.get() if self._active_transaction is not None else None
|
||||
if active is not None:
|
||||
if active.owner_task is asyncio.current_task():
|
||||
if active.scope != self.scope:
|
||||
raise CrossScopeTransactionError(
|
||||
f'Cannot enter {self.scope.kind.value} scope while {active.scope.kind.value} scope is active'
|
||||
)
|
||||
active.depth += 1
|
||||
self._active_state = active
|
||||
self._session = active.session
|
||||
return self
|
||||
# ContextVars are copied into child tasks. Merely calling a helper
|
||||
# there must fail (PersistenceManager enforces that), but an
|
||||
# explicit UoW is allowed to replace the inherited pointer with an
|
||||
# independent transaction owned by the child task.
|
||||
|
||||
session = sqlalchemy_asyncio.AsyncSession(self._engine, expire_on_commit=False)
|
||||
self._session = session
|
||||
try:
|
||||
await session.begin()
|
||||
if self._engine.dialect.name == 'postgresql':
|
||||
await session.execute(
|
||||
sqlalchemy.text(f"SELECT set_config('{TENANT_SETTING}', :workspace_uuid, true)"),
|
||||
{'workspace_uuid': self.workspace_uuid},
|
||||
)
|
||||
for setting_name, setting_value in self.scope.settings:
|
||||
await session.execute(
|
||||
sqlalchemy.text('SELECT set_config(:setting_name, :setting_value, true)'),
|
||||
{'setting_name': setting_name, 'setting_value': setting_value},
|
||||
)
|
||||
state = ActiveScopedTransaction(
|
||||
scope=self.scope,
|
||||
session=session,
|
||||
engine=self._engine,
|
||||
owner_task=asyncio.current_task(),
|
||||
)
|
||||
self._active_state = state
|
||||
if self._active_transaction is not None:
|
||||
self._context_token = self._active_transaction.set(state)
|
||||
self._database_operation_token = _DATABASE_OPERATION_TRANSACTION.set(state)
|
||||
self._owns_transaction = True
|
||||
except BaseException:
|
||||
if self._database_operation_token is not None:
|
||||
_DATABASE_OPERATION_TRANSACTION.reset(self._database_operation_token)
|
||||
self._database_operation_token = None
|
||||
if self._active_transaction is not None and self._context_token is not None:
|
||||
self._active_transaction.reset(self._context_token)
|
||||
self._context_token = None
|
||||
await session.rollback()
|
||||
await session.close()
|
||||
self._session = None
|
||||
@@ -95,19 +403,88 @@ class TenantUnitOfWork:
|
||||
exc_value: BaseException | None,
|
||||
traceback: types.TracebackType | None,
|
||||
) -> bool:
|
||||
state = self._active_state
|
||||
if state is None:
|
||||
raise RuntimeError('TenantUnitOfWork has no active transaction state')
|
||||
self._require_owner_task(state)
|
||||
|
||||
if not self._owns_transaction:
|
||||
if exc_type is not None:
|
||||
state.mark_rollback_only(exc_value)
|
||||
state.depth -= 1
|
||||
self._session = None
|
||||
self._active_state = None
|
||||
return False
|
||||
|
||||
session = self.session
|
||||
if exc_type is not None:
|
||||
state.mark_rollback_only(exc_value)
|
||||
rollback_only = state.rollback_only
|
||||
committed = False
|
||||
try:
|
||||
if exc_type is None:
|
||||
if exc_type is None and not rollback_only:
|
||||
await session.commit()
|
||||
committed = True
|
||||
else:
|
||||
await session.rollback()
|
||||
finally:
|
||||
await session.close()
|
||||
self._session = None
|
||||
try:
|
||||
if self._active_transaction is not None and self._context_token is not None:
|
||||
self._active_transaction.reset(self._context_token)
|
||||
await session.close()
|
||||
finally:
|
||||
if self._database_operation_token is not None:
|
||||
_DATABASE_OPERATION_TRANSACTION.reset(self._database_operation_token)
|
||||
self._database_operation_token = None
|
||||
self._session = None
|
||||
self._active_state = None
|
||||
state.depth = 0
|
||||
self._complete_after_commit_waiters(state, committed=committed)
|
||||
|
||||
if exc_type is None and rollback_only:
|
||||
raise TransactionRollbackOnlyError(
|
||||
'A scoped database operation failed or rolled back; '
|
||||
'the transaction was rolled back and after-commit work was cancelled'
|
||||
) from state.rollback_only_cause
|
||||
return False
|
||||
|
||||
async def execute(self, *args: typing.Any, **kwargs: typing.Any) -> sqlalchemy.engine.Result[typing.Any]:
|
||||
return await self.session.execute(*args, **kwargs)
|
||||
try:
|
||||
return await self.session.execute(*args, **kwargs)
|
||||
except BaseException as exc:
|
||||
self._mark_rollback_only(exc)
|
||||
raise
|
||||
|
||||
async def flush(self) -> None:
|
||||
await self.session.flush()
|
||||
try:
|
||||
await self.session.flush()
|
||||
except BaseException as exc:
|
||||
self._mark_rollback_only(exc)
|
||||
raise
|
||||
|
||||
def _mark_rollback_only(self, cause: BaseException) -> None:
|
||||
state = self._active_state
|
||||
if state is not None:
|
||||
state.mark_rollback_only(cause)
|
||||
|
||||
@staticmethod
|
||||
def _require_owner_task(state: ActiveScopedTransaction) -> None:
|
||||
if state.owner_task is not asyncio.current_task():
|
||||
raise CrossScopeTransactionError(
|
||||
'Scoped database transactions cannot be inherited by child tasks; open an explicit task scope'
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _complete_after_commit_waiters(
|
||||
state: ActiveScopedTransaction,
|
||||
*,
|
||||
committed: bool,
|
||||
) -> None:
|
||||
for waiter in state.after_commit_waiters:
|
||||
if waiter.done():
|
||||
continue
|
||||
if committed:
|
||||
waiter.set_result(None)
|
||||
else:
|
||||
waiter.cancel()
|
||||
state.after_commit_waiters.clear()
|
||||
|
||||
@@ -13,6 +13,7 @@ import langbot_plugin.api.entities.builtin.platform.message as platform_message
|
||||
import langbot_plugin.api.entities.builtin.provider.session as provider_session
|
||||
|
||||
from ..api.http.context import ExecutionContext
|
||||
from ..core.task_boundary import create_detached_task, run_in_workspace_uow
|
||||
from .pool import ExecutionContextMismatchError
|
||||
from ..workspace.errors import WorkspaceError, WorkspaceInvariantError
|
||||
|
||||
@@ -203,7 +204,10 @@ class MessageAggregator:
|
||||
if len(buffer.messages) >= MAX_BUFFER_MESSAGES:
|
||||
force_flush = True
|
||||
else:
|
||||
buffer.timer_task = asyncio.create_task(self._delayed_flush(aggregation_key, delay, execution_context))
|
||||
buffer.timer_task = create_detached_task(
|
||||
self._delayed_flush(aggregation_key, delay, execution_context),
|
||||
after_commit_manager=getattr(self.ap, 'persistence_mgr', None),
|
||||
)
|
||||
|
||||
if force_flush:
|
||||
await self._flush_buffer(aggregation_key, execution_context)
|
||||
@@ -218,7 +222,11 @@ class MessageAggregator:
|
||||
|
||||
try:
|
||||
await asyncio.sleep(delay)
|
||||
await self._flush_buffer(aggregation_key, execution_context)
|
||||
await run_in_workspace_uow(
|
||||
self.ap,
|
||||
execution_context.workspace_uuid,
|
||||
lambda: self._flush_buffer(aggregation_key, execution_context),
|
||||
)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
except WorkspaceError as exc:
|
||||
|
||||
@@ -43,22 +43,37 @@ class Controller:
|
||||
|
||||
try:
|
||||
async with self.semaphore:
|
||||
execution_context = await self._assert_query_execution_active(selected_query)
|
||||
pipeline_uuid = selected_query.pipeline_uuid
|
||||
queued_context = get_query_execution_context(selected_query)
|
||||
|
||||
if pipeline_uuid:
|
||||
pipeline = await self.ap.pipeline_mgr.get_pipeline_by_uuid(
|
||||
execution_context,
|
||||
pipeline_uuid,
|
||||
)
|
||||
if pipeline:
|
||||
await pipeline.run(selected_query)
|
||||
else:
|
||||
self.ap.logger.warning(
|
||||
f'Pipeline {pipeline_uuid} not found for query {selected_query.query_id}, query dropped'
|
||||
async def run_scoped_query() -> None:
|
||||
execution_context = await self._assert_query_execution_active(selected_query)
|
||||
pipeline_uuid = selected_query.pipeline_uuid
|
||||
|
||||
if pipeline_uuid:
|
||||
pipeline = await self.ap.pipeline_mgr.get_pipeline_by_uuid(
|
||||
execution_context,
|
||||
pipeline_uuid,
|
||||
)
|
||||
if pipeline:
|
||||
await pipeline.run(selected_query)
|
||||
else:
|
||||
self.ap.logger.warning(
|
||||
f'Pipeline {pipeline_uuid} not found for query {selected_query.query_id}, query dropped'
|
||||
)
|
||||
else:
|
||||
self.ap.logger.warning(f'No pipeline_uuid for query {selected_query.query_id}, query dropped')
|
||||
|
||||
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
|
||||
cloud_runtime = (
|
||||
getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
)
|
||||
if cloud_runtime:
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('Cloud query processing requires an explicit tenant scope')
|
||||
async with tenant_scope(queued_context.workspace_uuid):
|
||||
await run_scoped_query()
|
||||
else:
|
||||
self.ap.logger.warning(f'No pipeline_uuid for query {selected_query.query_id}, query dropped')
|
||||
await run_scoped_query()
|
||||
except WorkspaceError as exc:
|
||||
self.ap.logger.info(
|
||||
f'Dropped query {selected_query.query_id} because its Workspace execution binding is stale: {exc}'
|
||||
|
||||
@@ -506,6 +506,34 @@ class PipelineManager:
|
||||
async def load_pipelines_from_db(self):
|
||||
self.ap.logger.info('Loading pipelines from db...')
|
||||
|
||||
self.pipelines = []
|
||||
list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
if not callable(list_bindings) or not callable(tenant_uow):
|
||||
raise RuntimeError('Cloud pipeline loading requires explicit instance discovery and tenant UoWs')
|
||||
for binding in await list_bindings():
|
||||
async with tenant_uow(binding.workspace_uuid):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_pipeline.LegacyPipeline)
|
||||
.where(persistence_pipeline.LegacyPipeline.workspace_uuid == binding.workspace_uuid)
|
||||
.order_by(persistence_pipeline.LegacyPipeline.uuid)
|
||||
)
|
||||
for pipeline in result.all():
|
||||
await self.load_pipeline(
|
||||
ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
pipeline_uuid=pipeline.uuid,
|
||||
trigger_principal=PrincipalContext(PrincipalType.SYSTEM),
|
||||
),
|
||||
pipeline,
|
||||
)
|
||||
return
|
||||
|
||||
# Compatibility path for isolated manager tests and older embedders.
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
|
||||
|
||||
pipelines = result.all()
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import dataclasses
|
||||
import functools
|
||||
import json
|
||||
import re
|
||||
import traceback
|
||||
@@ -14,6 +15,7 @@ from ..discover import engine
|
||||
|
||||
from ..entity.persistence import bot as persistence_bot
|
||||
from ..entity.persistence import pipeline as persistence_pipeline
|
||||
from ..entity.persistence import workspace as persistence_workspace
|
||||
|
||||
from ..entity.errors import platform as platform_errors
|
||||
from ..api.http.context import ExecutionContext, PrincipalContext, PrincipalType, RequestContext
|
||||
@@ -256,6 +258,24 @@ class RuntimeBot:
|
||||
await self.logger.error(f'Failed to record discarded message: {e}')
|
||||
|
||||
async def initialize(self):
|
||||
def tenant_scoped_listener(listener):
|
||||
"""Bind adapter callbacks to a Workspace without holding a DB transaction."""
|
||||
|
||||
@functools.wraps(listener)
|
||||
async def wrapped(*args, **kwargs):
|
||||
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
|
||||
cloud_runtime = (
|
||||
getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
)
|
||||
if cloud_runtime:
|
||||
if not callable(tenant_scope):
|
||||
raise RuntimeError('Cloud platform callbacks require an explicit tenant scope')
|
||||
async with tenant_scope(self.workspace_uuid):
|
||||
return await listener(*args, **kwargs)
|
||||
return await listener(*args, **kwargs)
|
||||
|
||||
return wrapped
|
||||
|
||||
async def on_friend_message(
|
||||
event: platform_events.FriendMessage,
|
||||
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
|
||||
@@ -396,8 +416,8 @@ class RuntimeBot:
|
||||
else:
|
||||
await self.logger.info('Pipeline skipped for group message due to webhook response')
|
||||
|
||||
self.adapter.register_listener(platform_events.FriendMessage, on_friend_message)
|
||||
self.adapter.register_listener(platform_events.GroupMessage, on_group_message)
|
||||
self.adapter.register_listener(platform_events.FriendMessage, tenant_scoped_listener(on_friend_message))
|
||||
self.adapter.register_listener(platform_events.GroupMessage, tenant_scoped_listener(on_group_message))
|
||||
|
||||
# Register feedback listener (only effective on adapters that support it)
|
||||
async def on_feedback(
|
||||
@@ -444,7 +464,7 @@ class RuntimeBot:
|
||||
except Exception:
|
||||
await self.logger.error(f'Failed to record feedback: {traceback.format_exc()}')
|
||||
|
||||
self.adapter.register_listener(platform_events.FeedbackEvent, on_feedback)
|
||||
self.adapter.register_listener(platform_events.FeedbackEvent, tenant_scoped_listener(on_feedback))
|
||||
|
||||
async def run(self):
|
||||
async def exception_wrapper():
|
||||
@@ -642,14 +662,51 @@ class PlatformManager:
|
||||
|
||||
self.bots = []
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_bot.Bot))
|
||||
instance_uow = getattr(self.ap.persistence_mgr, 'instance_discovery_uow', None)
|
||||
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
|
||||
if callable(instance_uow) and callable(tenant_scope):
|
||||
async with instance_uow(self.ap.workspace_service.instance_uuid) as discovery:
|
||||
workspace_uuids = list(
|
||||
(
|
||||
await discovery.session.scalars(
|
||||
sqlalchemy.select(persistence_workspace.WorkspaceExecutionState.workspace_uuid)
|
||||
.where(
|
||||
persistence_workspace.WorkspaceExecutionState.instance_uuid
|
||||
== self.ap.workspace_service.instance_uuid,
|
||||
persistence_workspace.WorkspaceExecutionState.state
|
||||
== persistence_workspace.WorkspaceExecutionStatus.ACTIVE.value,
|
||||
persistence_workspace.WorkspaceExecutionState.write_fenced.is_(False),
|
||||
)
|
||||
.order_by(persistence_workspace.WorkspaceExecutionState.workspace_uuid)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
else:
|
||||
# Compatibility for lightweight tests and pre-tenancy managers.
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_bot.Bot))
|
||||
workspace_uuids = sorted({bot.workspace_uuid for bot in result.all()})
|
||||
|
||||
bots = result.all()
|
||||
|
||||
for bot in bots:
|
||||
# load all bots here, enable or disable will be handled in runtime
|
||||
for workspace_uuid in workspace_uuids:
|
||||
try:
|
||||
binding = await self.ap.workspace_service.get_execution_binding(bot.workspace_uuid)
|
||||
if callable(tenant_scope):
|
||||
async with tenant_scope(workspace_uuid):
|
||||
await self._load_workspace_bots(workspace_uuid)
|
||||
else:
|
||||
await self._load_workspace_bots(workspace_uuid)
|
||||
except Exception as e:
|
||||
self.ap.logger.error(
|
||||
f'Failed to load Workspace bots for {workspace_uuid}: {e}\n{traceback.format_exc()}'
|
||||
)
|
||||
|
||||
async def _load_workspace_bots(self, workspace_uuid: str) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_bot.Bot)
|
||||
.where(persistence_bot.Bot.workspace_uuid == workspace_uuid)
|
||||
.order_by(persistence_bot.Bot.uuid)
|
||||
)
|
||||
for bot in result.all():
|
||||
try:
|
||||
binding = await self.ap.workspace_service.get_execution_binding(workspace_uuid)
|
||||
execution_context = ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
|
||||
@@ -286,19 +286,30 @@ class OpenClawWeixinAdapter(abstract_platform_adapter.AbstractMessagePlatformAda
|
||||
if execution_context.bot_uuid != self._bot_uuid:
|
||||
raise RuntimeError('Weixin Bot UUID does not match its ExecutionContext')
|
||||
|
||||
binding = await ap.workspace_service.get_execution_binding(
|
||||
execution_context.workspace_uuid,
|
||||
expected_generation=execution_context.placement_generation,
|
||||
)
|
||||
if binding.instance_uuid != execution_context.instance_uuid:
|
||||
raise RuntimeError('Weixin Bot Workspace belongs to another LangBot instance')
|
||||
async def persist() -> None:
|
||||
binding = await ap.workspace_service.get_execution_binding(
|
||||
execution_context.workspace_uuid,
|
||||
expected_generation=execution_context.placement_generation,
|
||||
)
|
||||
if binding.instance_uuid != execution_context.instance_uuid:
|
||||
raise RuntimeError('Weixin Bot Workspace belongs to another LangBot instance')
|
||||
|
||||
await ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_bot.Bot)
|
||||
.where(persistence_bot.Bot.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_bot.Bot.uuid == self._bot_uuid)
|
||||
.values(adapter_config=self.config)
|
||||
)
|
||||
await ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_bot.Bot)
|
||||
.where(persistence_bot.Bot.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_bot.Bot.uuid == self._bot_uuid)
|
||||
.values(adapter_config=self.config)
|
||||
)
|
||||
|
||||
cloud_runtime = getattr(getattr(ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
tenant_uow = getattr(ap.persistence_mgr, 'tenant_uow', None)
|
||||
if not callable(tenant_uow):
|
||||
raise RuntimeError('Cloud adapter persistence requires an explicit tenant UoW')
|
||||
async with tenant_uow(execution_context.workspace_uuid):
|
||||
await persist()
|
||||
else:
|
||||
await persist()
|
||||
except Exception as e:
|
||||
await self.logger.warning(f'Failed to persist adapter config: {e}')
|
||||
|
||||
|
||||
+1351
-375
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,97 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
from urllib.parse import unquote, urlparse
|
||||
|
||||
|
||||
_GITHUB_OWNER_PATTERN = re.compile(r'^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$')
|
||||
_GITHUB_REPO_PATTERN = re.compile(r'^[A-Za-z0-9._-]{1,100}$')
|
||||
|
||||
|
||||
def _positive_github_id(value: object, field_name: str) -> int:
|
||||
if isinstance(value, bool):
|
||||
raise ValueError(f'{field_name} must be a positive GitHub identifier')
|
||||
try:
|
||||
identifier = int(value)
|
||||
except (TypeError, ValueError) as exc:
|
||||
raise ValueError(f'{field_name} must be a positive GitHub identifier') from exc
|
||||
if identifier <= 0 or str(value).strip() != str(identifier):
|
||||
raise ValueError(f'{field_name} must be a positive GitHub identifier')
|
||||
return identifier
|
||||
|
||||
|
||||
def validate_github_release_asset_url(
|
||||
asset_url: object,
|
||||
*,
|
||||
owner: str,
|
||||
repo: str,
|
||||
release_tag: str,
|
||||
) -> str:
|
||||
"""Accept only a GitHub browser release URL tied to the requested release."""
|
||||
|
||||
normalized_url = str(asset_url or '').strip()
|
||||
parsed = urlparse(normalized_url)
|
||||
try:
|
||||
port = parsed.port
|
||||
except ValueError as exc:
|
||||
raise ValueError('asset_url has an invalid port') from exc
|
||||
if (
|
||||
parsed.scheme != 'https'
|
||||
or (parsed.hostname or '').lower() != 'github.com'
|
||||
or parsed.username is not None
|
||||
or parsed.password is not None
|
||||
or port not in {None, 443}
|
||||
or parsed.fragment
|
||||
):
|
||||
raise ValueError('asset_url must be an HTTPS GitHub release asset URL')
|
||||
decoded_path = unquote(parsed.path)
|
||||
expected_prefix = f'/{owner}/{repo}/releases/download/{release_tag}/'
|
||||
if not decoded_path.casefold().startswith(
|
||||
f'/{owner}/{repo}/releases/download/'.casefold()
|
||||
) or not decoded_path.startswith(expected_prefix):
|
||||
raise ValueError('asset_url does not match the requested GitHub release')
|
||||
if decoded_path == expected_prefix or decoded_path.endswith('/'):
|
||||
raise ValueError('asset_url must identify a GitHub release asset')
|
||||
return normalized_url
|
||||
|
||||
|
||||
def validate_github_plugin_install_info(install_info: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Normalize a GitHub install request without trusting a tenant-provided URL."""
|
||||
|
||||
owner = str(install_info.get('owner') or '').strip()
|
||||
repo = str(install_info.get('repo') or '').strip()
|
||||
release_tag = str(install_info.get('release_tag') or '').strip()
|
||||
if _GITHUB_OWNER_PATTERN.fullmatch(owner) is None:
|
||||
raise ValueError('owner must be a valid GitHub repository owner')
|
||||
if _GITHUB_REPO_PATTERN.fullmatch(repo) is None:
|
||||
raise ValueError('repo must be a valid GitHub repository name')
|
||||
if not release_tag or '\x00' in release_tag or len(release_tag) > 255:
|
||||
raise ValueError('release_tag must identify a GitHub release')
|
||||
|
||||
release_id_value = install_info.get('release_id')
|
||||
asset_id_value = install_info.get('asset_id')
|
||||
normalized = dict(install_info)
|
||||
normalized.update(
|
||||
{
|
||||
'owner': owner,
|
||||
'repo': repo,
|
||||
'release_tag': release_tag,
|
||||
'github_url': f'https://github.com/{owner}/{repo}',
|
||||
}
|
||||
)
|
||||
if release_id_value is not None or asset_id_value is not None:
|
||||
if release_id_value is None or asset_id_value is None:
|
||||
raise ValueError('release_id and asset_id must be provided together')
|
||||
normalized['release_id'] = _positive_github_id(release_id_value, 'release_id')
|
||||
normalized['asset_id'] = _positive_github_id(asset_id_value, 'asset_id')
|
||||
normalized.pop('asset_url', None)
|
||||
return normalized
|
||||
|
||||
normalized['asset_url'] = validate_github_release_asset_url(
|
||||
install_info.get('asset_url'),
|
||||
owner=owner,
|
||||
repo=repo,
|
||||
release_tag=release_tag,
|
||||
)
|
||||
return normalized
|
||||
+362
-162
@@ -4,15 +4,26 @@ import inspect
|
||||
import typing
|
||||
from typing import Any
|
||||
import base64
|
||||
import contextlib
|
||||
import contextvars
|
||||
import traceback
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
|
||||
import sqlalchemy
|
||||
|
||||
from langbot_plugin.runtime.io import handler
|
||||
from langbot_plugin.runtime.io.connection import Connection
|
||||
from langbot_plugin.entities.io.context import ActionContext
|
||||
from langbot_plugin.entities.io.context import (
|
||||
ActionContext,
|
||||
ApplyPluginInstallationRequest,
|
||||
InstallationBinding,
|
||||
PluginInstallationDesiredState,
|
||||
PluginWorkerPolicy,
|
||||
ReconcilePluginInstallationsRequest,
|
||||
RemovePluginInstallationRequest,
|
||||
RuntimeConfig,
|
||||
RuntimeIdentity,
|
||||
)
|
||||
from langbot_plugin.entities.io.actions.enums import (
|
||||
CommonAction,
|
||||
RuntimeToLangBotAction,
|
||||
@@ -61,6 +72,9 @@ class _PluginInstallationIdentity:
|
||||
workspace_uuid: str
|
||||
plugin_author: str
|
||||
plugin_name: str
|
||||
installation_uuid: str
|
||||
runtime_revision: int
|
||||
artifact_digest: str
|
||||
|
||||
|
||||
_UNTRUSTED_SCOPE_FIELDS = frozenset(
|
||||
@@ -71,12 +85,13 @@ _UNTRUSTED_SCOPE_FIELDS = frozenset(
|
||||
'workspace_uuid',
|
||||
'placement_generation',
|
||||
'installation_uuid',
|
||||
'runtime_revision',
|
||||
'artifact_digest',
|
||||
}
|
||||
)
|
||||
|
||||
_RUNTIME_SCOPED_ACTIONS = frozenset(
|
||||
{
|
||||
CommonAction.PING.value,
|
||||
CommonAction.FILE_CHUNK.value,
|
||||
RuntimeToLangBotAction.INITIALIZE_PLUGIN_SETTINGS.value,
|
||||
RuntimeToLangBotAction.GET_PLUGIN_SETTINGS.value,
|
||||
@@ -89,113 +104,129 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
|
||||
ap: app.Application
|
||||
|
||||
@staticmethod
|
||||
def derive_installation_uuid(
|
||||
action_context: ActionContext,
|
||||
plugin_author: str,
|
||||
plugin_name: str,
|
||||
) -> str:
|
||||
"""Derive the current stable installation capability without trusting plugin data."""
|
||||
|
||||
return str(
|
||||
uuid.uuid5(
|
||||
uuid.NAMESPACE_URL,
|
||||
'langbot:plugin-installation:'
|
||||
f'{action_context.instance_uuid}:{action_context.workspace_uuid}:'
|
||||
f'{plugin_author}/{plugin_name}',
|
||||
)
|
||||
)
|
||||
|
||||
def validate_inbound_action_context(
|
||||
self,
|
||||
action: str,
|
||||
action_context: ActionContext | None,
|
||||
) -> ActionContext | None:
|
||||
"""Keep a validated child installation capability from the request envelope."""
|
||||
"""Require a complete installation tuple on every tenant action."""
|
||||
|
||||
validated = super().validate_inbound_action_context(action, action_context)
|
||||
if action_context is not None and action_context.installation_uuid is not None:
|
||||
# The base handler already checked instance, Workspace and placement
|
||||
# generation against this connector's immutable binding. The
|
||||
# installation is resolved against trusted Host state asynchronously
|
||||
# before dispatch.
|
||||
if action == CommonAction.PING.value:
|
||||
if action_context is not None:
|
||||
raise ValueError('PING does not accept an installation binding')
|
||||
return None
|
||||
if isinstance(action_context, InstallationBinding):
|
||||
return action_context
|
||||
return validated
|
||||
if self._allow_legacy_oss_context(action, action_context):
|
||||
return action_context
|
||||
raise ValueError(f'{action} requires a complete InstallationBinding context')
|
||||
|
||||
def _allow_legacy_oss_context(self, action: str, action_context: ActionContext | None) -> bool:
|
||||
"""Keep pre-v4 local plugins usable without weakening shared Runtime."""
|
||||
|
||||
if not (
|
||||
getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') == 'oss'
|
||||
and isinstance(action_context, ActionContext)
|
||||
and not isinstance(action_context, InstallationBinding)
|
||||
):
|
||||
return False
|
||||
if action in {
|
||||
RuntimeToLangBotAction.INITIALIZE_PLUGIN_SETTINGS.value,
|
||||
RuntimeToLangBotAction.GET_PLUGIN_SETTINGS.value,
|
||||
CommonAction.FILE_CHUNK.value,
|
||||
}:
|
||||
return True
|
||||
# Pre-v4 OSS workers receive this capability from Core's settings
|
||||
# response, but their pinned SDK cannot carry revision/digest fields.
|
||||
# Shared Runtime never enters this compatibility branch.
|
||||
return bool(action_context.installation_uuid)
|
||||
|
||||
def _require_runtime_action_context(self) -> ActionContext:
|
||||
action_context = self.current_action_context or self.bound_action_context
|
||||
action_context = self.current_action_context
|
||||
if action_context is None:
|
||||
raise ValueError('Plugin Runtime action is missing a trusted Workspace context')
|
||||
|
||||
bound = self.require_bound_action_context()
|
||||
if not bound.same_workspace(action_context):
|
||||
raise ValueError('Plugin Runtime action context does not match its connector binding')
|
||||
return action_context
|
||||
|
||||
def _remember_installation(
|
||||
self,
|
||||
action_context: ActionContext,
|
||||
plugin_author: str,
|
||||
plugin_name: str,
|
||||
) -> str:
|
||||
installation_uuid = self.derive_installation_uuid(
|
||||
action_context,
|
||||
plugin_author,
|
||||
plugin_name,
|
||||
)
|
||||
self._installation_bindings[installation_uuid] = _PluginInstallationIdentity(
|
||||
workspace_uuid=action_context.workspace_uuid,
|
||||
plugin_author=plugin_author,
|
||||
plugin_name=plugin_name,
|
||||
)
|
||||
return installation_uuid
|
||||
|
||||
async def _resolve_installation_identity(
|
||||
self,
|
||||
action_context: ActionContext,
|
||||
action_context: InstallationBinding,
|
||||
) -> _PluginInstallationIdentity:
|
||||
installation_uuid = action_context.installation_uuid
|
||||
if installation_uuid is None:
|
||||
raise ValueError('Plugin action is missing installation_uuid')
|
||||
|
||||
identity = self._installation_bindings.get(installation_uuid)
|
||||
if identity is not None:
|
||||
if identity.workspace_uuid != action_context.workspace_uuid:
|
||||
raise ValueError('Plugin installation belongs to another Workspace')
|
||||
cached = self._installation_bindings.get(action_context.installation_uuid)
|
||||
if cached is not None:
|
||||
cached_binding, identity = cached
|
||||
if cached_binding != action_context:
|
||||
raise ValueError('Plugin installation revision or artifact is stale')
|
||||
return identity
|
||||
|
||||
# A control connection may reconnect while the Runtime keeps plugin
|
||||
# processes alive. Rebuild the capability map from Workspace-scoped
|
||||
# settings instead of accepting an installation asserted by the peer.
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
persistence_plugin.PluginSetting.plugin_author,
|
||||
persistence_plugin.PluginSetting.plugin_name,
|
||||
).where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
|
||||
)
|
||||
for setting in result.all():
|
||||
candidate_uuid = self.derive_installation_uuid(
|
||||
action_context,
|
||||
setting.plugin_author,
|
||||
setting.plugin_name,
|
||||
persistence_plugin.PluginSetting.installation_uuid,
|
||||
persistence_plugin.PluginSetting.runtime_revision,
|
||||
persistence_plugin.PluginSetting.artifact_digest,
|
||||
)
|
||||
if candidate_uuid == installation_uuid:
|
||||
return self._installation_bindings.setdefault(
|
||||
installation_uuid,
|
||||
_PluginInstallationIdentity(
|
||||
workspace_uuid=action_context.workspace_uuid,
|
||||
plugin_author=setting.plugin_author,
|
||||
plugin_name=setting.plugin_name,
|
||||
),
|
||||
)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
|
||||
.where(persistence_plugin.PluginSetting.installation_uuid == action_context.installation_uuid)
|
||||
)
|
||||
setting = result.first()
|
||||
if setting is None:
|
||||
raise ValueError('Plugin installation is not registered in this Workspace')
|
||||
if (
|
||||
setting.runtime_revision != action_context.runtime_revision
|
||||
or setting.artifact_digest != action_context.artifact_digest
|
||||
):
|
||||
raise ValueError('Plugin installation revision or artifact is stale')
|
||||
identity = _PluginInstallationIdentity(
|
||||
workspace_uuid=action_context.workspace_uuid,
|
||||
plugin_author=setting.plugin_author,
|
||||
plugin_name=setting.plugin_name,
|
||||
installation_uuid=setting.installation_uuid,
|
||||
runtime_revision=setting.runtime_revision,
|
||||
artifact_digest=setting.artifact_digest,
|
||||
)
|
||||
self._installation_bindings[action_context.installation_uuid] = (action_context, identity)
|
||||
return identity
|
||||
|
||||
raise ValueError('Plugin installation is not registered in this Workspace')
|
||||
async def _resolve_legacy_oss_installation_identity(
|
||||
self,
|
||||
action_context: ActionContext,
|
||||
) -> _PluginInstallationIdentity:
|
||||
if not action_context.installation_uuid:
|
||||
raise ValueError('Legacy OSS plugin action is missing its installation capability')
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
persistence_plugin.PluginSetting.plugin_author,
|
||||
persistence_plugin.PluginSetting.plugin_name,
|
||||
persistence_plugin.PluginSetting.installation_uuid,
|
||||
persistence_plugin.PluginSetting.runtime_revision,
|
||||
persistence_plugin.PluginSetting.artifact_digest,
|
||||
)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
|
||||
.where(persistence_plugin.PluginSetting.installation_uuid == action_context.installation_uuid)
|
||||
)
|
||||
setting = result.first()
|
||||
if setting is None:
|
||||
raise ValueError('Plugin installation is not registered in this Workspace')
|
||||
return _PluginInstallationIdentity(
|
||||
workspace_uuid=action_context.workspace_uuid,
|
||||
plugin_author=setting.plugin_author,
|
||||
plugin_name=setting.plugin_name,
|
||||
installation_uuid=setting.installation_uuid,
|
||||
runtime_revision=setting.runtime_revision,
|
||||
artifact_digest=setting.artifact_digest,
|
||||
)
|
||||
|
||||
async def _require_plugin_action_context(
|
||||
self,
|
||||
) -> tuple[ActionContext, _PluginInstallationIdentity]:
|
||||
action_context = self._require_runtime_action_context()
|
||||
identity = await self._resolve_installation_identity(action_context)
|
||||
if isinstance(action_context, InstallationBinding):
|
||||
identity = await self._resolve_installation_identity(action_context)
|
||||
elif self._allow_legacy_oss_context('plugin_action', action_context):
|
||||
identity = await self._resolve_legacy_oss_installation_identity(action_context)
|
||||
else:
|
||||
raise ValueError('Plugin action requires a complete InstallationBinding context')
|
||||
return action_context, identity
|
||||
|
||||
async def _require_active_action_context(self, action_context: ActionContext) -> None:
|
||||
@@ -215,25 +246,53 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
):
|
||||
raise ValueError('Plugin Runtime action uses a stale Workspace execution binding')
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _tenant_action_scope(self, action_context: ActionContext):
|
||||
"""Bind Runtime-origin work to the trusted Workspace database scope.
|
||||
|
||||
Cloud persistence deliberately rejects unscoped access and PostgreSQL
|
||||
RLS reads the Workspace id from each short database transaction. The
|
||||
wire envelope is validated before this helper is entered, so plugin
|
||||
payload fields can never select the scope. A transaction-free boundary
|
||||
avoids reserving one pooled connection across provider and network waits.
|
||||
"""
|
||||
|
||||
persistence_mgr = getattr(self.ap, 'persistence_mgr', None)
|
||||
if persistence_mgr is None:
|
||||
yield
|
||||
return
|
||||
tenant_scope_descriptor = getattr(type(persistence_mgr), 'tenant_scope', None)
|
||||
if not callable(tenant_scope_descriptor):
|
||||
# Lightweight test doubles and older OSS persistence managers do
|
||||
# not expose the transaction-free scope API.
|
||||
yield
|
||||
return
|
||||
async with persistence_mgr.tenant_scope(action_context.workspace_uuid):
|
||||
yield
|
||||
|
||||
def _secure_plugin_actions(self) -> None:
|
||||
"""Wrap plugin-origin actions with installation validation and payload scrubbing."""
|
||||
|
||||
for action_name, action_handler in list(self.actions.items()):
|
||||
if action_name in _RUNTIME_SCOPED_ACTIONS:
|
||||
if action_name == CommonAction.PING.value:
|
||||
continue
|
||||
|
||||
async def secured_action(
|
||||
data: dict[str, Any],
|
||||
*,
|
||||
_action_handler=action_handler,
|
||||
_runtime_scoped=action_name in _RUNTIME_SCOPED_ACTIONS,
|
||||
) -> handler.ActionResponse:
|
||||
action_context, _ = await self._require_plugin_action_context()
|
||||
await self._require_active_action_context(action_context)
|
||||
safe_data = {key: value for key, value in data.items() if key not in _UNTRUSTED_SCOPE_FIELDS}
|
||||
response = _action_handler(safe_data)
|
||||
if inspect.isawaitable(response):
|
||||
response = await response
|
||||
return response
|
||||
action_context = self._require_runtime_action_context()
|
||||
async with self._tenant_action_scope(action_context):
|
||||
if not _runtime_scoped:
|
||||
action_context, _ = await self._require_plugin_action_context()
|
||||
await self._require_active_action_context(action_context)
|
||||
safe_data = {key: value for key, value in data.items() if key not in _UNTRUSTED_SCOPE_FIELDS}
|
||||
response = _action_handler(safe_data)
|
||||
if inspect.isawaitable(response):
|
||||
response = await response
|
||||
return response
|
||||
|
||||
self.actions[action_name] = secured_action
|
||||
|
||||
@@ -253,6 +312,31 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
raise ValueError('Plugin installation setting was not found')
|
||||
return setting
|
||||
|
||||
async def _get_plugin_setting_by_name(
|
||||
self,
|
||||
action_context: ActionContext,
|
||||
plugin_author: str,
|
||||
plugin_name: str,
|
||||
):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_plugin.PluginSetting)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
|
||||
.where(persistence_plugin.PluginSetting.plugin_author == plugin_author)
|
||||
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
|
||||
)
|
||||
return result.first()
|
||||
|
||||
@staticmethod
|
||||
def _require_setting_binding(setting, action_context: InstallationBinding) -> None:
|
||||
if setting is None:
|
||||
raise ValueError('Plugin installation setting was not found')
|
||||
if (
|
||||
setting.installation_uuid != action_context.installation_uuid
|
||||
or setting.runtime_revision != action_context.runtime_revision
|
||||
or setting.artifact_digest != action_context.artifact_digest
|
||||
):
|
||||
raise ValueError('Plugin installation binding does not match the active setting')
|
||||
|
||||
async def _resolve_query(
|
||||
self,
|
||||
data: dict[str, Any],
|
||||
@@ -355,18 +439,24 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
connection: Connection,
|
||||
disconnect_callback: typing.Callable[[], typing.Coroutine[typing.Any, typing.Any, bool]],
|
||||
ap: app.Application,
|
||||
action_context: ActionContext,
|
||||
):
|
||||
super().__init__(connection, disconnect_callback)
|
||||
self.ap = ap
|
||||
self.bind_action_context(action_context.without_installation())
|
||||
self._installation_bindings: dict[str, _PluginInstallationIdentity] = {}
|
||||
self._outbound_installation_context: contextvars.ContextVar[InstallationBinding | None] = (
|
||||
contextvars.ContextVar(
|
||||
f'{self.__class__.__name__}_{id(self)}_outbound_installation',
|
||||
default=None,
|
||||
)
|
||||
)
|
||||
self._installation_bindings: dict[
|
||||
str,
|
||||
tuple[InstallationBinding, _PluginInstallationIdentity],
|
||||
] = {}
|
||||
|
||||
@self.action(RuntimeToLangBotAction.INITIALIZE_PLUGIN_SETTINGS)
|
||||
async def initialize_plugin_settings(data: dict[str, Any]) -> handler.ActionResponse:
|
||||
"""Initialize plugin settings"""
|
||||
action_context = self._require_runtime_action_context()
|
||||
await self._require_active_action_context(action_context)
|
||||
# check if exists plugin setting
|
||||
plugin_author = data['plugin_author']
|
||||
plugin_name = data['plugin_name']
|
||||
@@ -374,39 +464,37 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
install_info = data['install_info']
|
||||
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_plugin.PluginSetting)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
|
||||
.where(persistence_plugin.PluginSetting.plugin_author == plugin_author)
|
||||
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
|
||||
setting = await self._get_plugin_setting_by_name(
|
||||
action_context,
|
||||
plugin_author,
|
||||
plugin_name,
|
||||
)
|
||||
|
||||
setting = result.first()
|
||||
|
||||
if setting is not None:
|
||||
# delete plugin setting
|
||||
if isinstance(action_context, InstallationBinding):
|
||||
self._require_setting_binding(setting, action_context)
|
||||
elif setting is None:
|
||||
# OSS debug and pre-v4 data/plugins are the only callers
|
||||
# allowed to create settings without a desired-state row.
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_plugin.PluginSetting)
|
||||
sqlalchemy.insert(persistence_plugin.PluginSetting).values(
|
||||
workspace_uuid=action_context.workspace_uuid,
|
||||
plugin_author=plugin_author,
|
||||
plugin_name=plugin_name,
|
||||
install_source=install_source,
|
||||
install_info=install_info,
|
||||
enabled=True,
|
||||
priority=0,
|
||||
config={},
|
||||
)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_plugin.PluginSetting)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
|
||||
.where(persistence_plugin.PluginSetting.plugin_author == plugin_author)
|
||||
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
|
||||
.values(install_source=install_source, install_info=install_info)
|
||||
)
|
||||
|
||||
# create plugin setting
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(persistence_plugin.PluginSetting).values(
|
||||
workspace_uuid=action_context.workspace_uuid,
|
||||
plugin_author=plugin_author,
|
||||
plugin_name=plugin_name,
|
||||
install_source=install_source,
|
||||
install_info=install_info,
|
||||
# inherit from existing setting
|
||||
enabled=setting.enabled if setting is not None else True,
|
||||
priority=setting.priority if setting is not None else 0,
|
||||
config=setting.config if setting is not None else {}, # noqa: F821
|
||||
)
|
||||
)
|
||||
|
||||
return handler.ActionResponse.success(
|
||||
data={},
|
||||
)
|
||||
@@ -421,16 +509,16 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
"""Get plugin settings"""
|
||||
|
||||
action_context = self._require_runtime_action_context()
|
||||
await self._require_active_action_context(action_context)
|
||||
plugin_author = data['plugin_author']
|
||||
plugin_name = data['plugin_name']
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_plugin.PluginSetting)
|
||||
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
|
||||
.where(persistence_plugin.PluginSetting.plugin_author == plugin_author)
|
||||
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
|
||||
setting = await self._get_plugin_setting_by_name(
|
||||
action_context,
|
||||
plugin_author,
|
||||
plugin_name,
|
||||
)
|
||||
if isinstance(action_context, InstallationBinding):
|
||||
self._require_setting_binding(setting, action_context)
|
||||
|
||||
data = {
|
||||
'enabled': True,
|
||||
@@ -438,21 +526,20 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
'plugin_config': {},
|
||||
'install_source': 'local',
|
||||
'install_info': {},
|
||||
'installation_uuid': self._remember_installation(
|
||||
action_context,
|
||||
plugin_author,
|
||||
plugin_name,
|
||||
),
|
||||
'installation_uuid': None,
|
||||
'runtime_revision': None,
|
||||
'artifact_digest': None,
|
||||
}
|
||||
|
||||
setting = result.first()
|
||||
|
||||
if setting is not None:
|
||||
data['enabled'] = setting.enabled
|
||||
data['priority'] = setting.priority
|
||||
data['plugin_config'] = setting.config
|
||||
data['install_source'] = setting.install_source
|
||||
data['install_info'] = setting.install_info
|
||||
data['installation_uuid'] = setting.installation_uuid
|
||||
data['runtime_revision'] = setting.runtime_revision
|
||||
data['artifact_digest'] = setting.artifact_digest
|
||||
|
||||
return handler.ActionResponse.success(
|
||||
data=data,
|
||||
@@ -1369,24 +1456,144 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
|
||||
self._secure_plugin_actions()
|
||||
|
||||
async def ping(self) -> dict[str, Any]:
|
||||
"""Ping the runtime"""
|
||||
return await self.call_action(
|
||||
CommonAction.PING,
|
||||
{},
|
||||
timeout=10,
|
||||
@contextlib.contextmanager
|
||||
def installation_scope(self, binding: InstallationBinding | None):
|
||||
"""Attach one immutable installation tuple to nested wire actions."""
|
||||
|
||||
token = self._outbound_installation_context.set(binding)
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
self._outbound_installation_context.reset(token)
|
||||
|
||||
def register_installation_binding(
|
||||
self,
|
||||
binding: InstallationBinding,
|
||||
*,
|
||||
plugin_author: str,
|
||||
plugin_name: str,
|
||||
) -> None:
|
||||
"""Install Core's current desired-state fence for inbound actions."""
|
||||
|
||||
existing = self._installation_bindings.get(binding.installation_uuid)
|
||||
if existing is not None:
|
||||
existing_binding = existing[0]
|
||||
if existing_binding.workspace_uuid != binding.workspace_uuid:
|
||||
raise ValueError('Plugin installation cannot move between Workspaces')
|
||||
if binding.placement_generation < existing_binding.placement_generation or (
|
||||
binding.placement_generation == existing_binding.placement_generation
|
||||
and binding.runtime_revision < existing_binding.runtime_revision
|
||||
):
|
||||
raise ValueError('Cannot register a stale plugin installation binding')
|
||||
self._installation_bindings[binding.installation_uuid] = (
|
||||
binding,
|
||||
_PluginInstallationIdentity(
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
plugin_author=plugin_author,
|
||||
plugin_name=plugin_name,
|
||||
installation_uuid=binding.installation_uuid,
|
||||
runtime_revision=binding.runtime_revision,
|
||||
artifact_digest=binding.artifact_digest,
|
||||
),
|
||||
)
|
||||
|
||||
async def set_runtime_config(self, cloud_service_url: str | None) -> dict[str, Any]:
|
||||
"""Push runtime configuration (e.g. marketplace URL) to the runtime."""
|
||||
data = {}
|
||||
if cloud_service_url:
|
||||
data['cloud_service_url'] = cloud_service_url
|
||||
return await self.call_action(
|
||||
LangBotToRuntimeAction.SET_RUNTIME_CONFIG,
|
||||
data,
|
||||
timeout=10,
|
||||
def unregister_installation_binding(self, binding: InstallationBinding) -> None:
|
||||
existing = self._installation_bindings.get(binding.installation_uuid)
|
||||
if existing is not None and existing[0] == binding:
|
||||
self._installation_bindings.pop(binding.installation_uuid, None)
|
||||
|
||||
def resolve_outbound_action_context(
|
||||
self,
|
||||
action_context: InstallationBinding | ActionContext | dict[str, Any] | None,
|
||||
) -> InstallationBinding | ActionContext | None:
|
||||
if action_context is not None:
|
||||
return super().resolve_outbound_action_context(action_context)
|
||||
inbound_context = self.current_action_context
|
||||
if inbound_context is not None:
|
||||
return inbound_context
|
||||
return self._outbound_installation_context.get()
|
||||
|
||||
def require_outbound_installation_context(self) -> InstallationBinding:
|
||||
binding = self._outbound_installation_context.get()
|
||||
if not isinstance(binding, InstallationBinding):
|
||||
raise ValueError('Host plugin action requires an InstallationBinding scope')
|
||||
return binding
|
||||
|
||||
async def ping(self) -> dict[str, Any]:
|
||||
"""Ping the runtime"""
|
||||
with self.installation_scope(None):
|
||||
return await self.call_action(
|
||||
CommonAction.PING,
|
||||
{},
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
async def set_runtime_config(
|
||||
self,
|
||||
*,
|
||||
runtime_identity: RuntimeIdentity,
|
||||
worker_policy: PluginWorkerPolicy,
|
||||
runtime_profile: typing.Literal['oss_dev', 'shared'],
|
||||
cloud_service_url: str | None,
|
||||
) -> dict[str, Any]:
|
||||
"""Push the instance-scoped, immutable Runtime handshake."""
|
||||
|
||||
runtime_config = RuntimeConfig(
|
||||
runtime_identity=runtime_identity,
|
||||
worker_policy=worker_policy,
|
||||
runtime_profile=runtime_profile,
|
||||
cloud_service_url=cloud_service_url,
|
||||
)
|
||||
with self.installation_scope(None):
|
||||
return await self.call_action(
|
||||
LangBotToRuntimeAction.SET_RUNTIME_CONFIG,
|
||||
runtime_config.model_dump(exclude_none=True),
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
async def reconcile_plugin_installations(
|
||||
self,
|
||||
installations: tuple[PluginInstallationDesiredState, ...],
|
||||
) -> dict[str, Any]:
|
||||
request = ReconcilePluginInstallationsRequest(installations=installations)
|
||||
with self.installation_scope(None):
|
||||
return await self.call_action(
|
||||
LangBotToRuntimeAction.RECONCILE_PLUGIN_INSTALLATIONS,
|
||||
request.model_dump(),
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
async def apply_plugin_installation(
|
||||
self,
|
||||
binding: InstallationBinding,
|
||||
*,
|
||||
artifact_package: bytes | None,
|
||||
enabled: bool,
|
||||
) -> dict[str, Any]:
|
||||
with self.installation_scope(binding):
|
||||
artifact_file_key = None
|
||||
if artifact_package is not None:
|
||||
artifact_file_key = await self.send_file(artifact_package, 'lbpkg')
|
||||
request = ApplyPluginInstallationRequest(
|
||||
artifact_file_key=artifact_file_key,
|
||||
enabled=enabled,
|
||||
)
|
||||
return await self.call_action(
|
||||
LangBotToRuntimeAction.APPLY_PLUGIN_INSTALLATION,
|
||||
request.model_dump(exclude_none=True),
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
async def remove_plugin_installation(
|
||||
self,
|
||||
binding: InstallationBinding,
|
||||
) -> dict[str, Any]:
|
||||
with self.installation_scope(binding):
|
||||
return await self.call_action(
|
||||
LangBotToRuntimeAction.REMOVE_PLUGIN_INSTALLATION,
|
||||
RemovePluginInstallationRequest().model_dump(),
|
||||
timeout=120,
|
||||
)
|
||||
|
||||
async def install_plugin(
|
||||
self, install_source: str, install_info: dict[str, Any]
|
||||
@@ -1455,7 +1662,7 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
|
||||
async def set_plugin_config(self, plugin_author: str, plugin_name: str, config: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Set plugin config"""
|
||||
action_context = self.require_bound_action_context()
|
||||
action_context = self.require_outbound_installation_context()
|
||||
# update plugin setting
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_plugin.PluginSetting)
|
||||
@@ -1642,12 +1849,7 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
|
||||
async def cleanup_plugin_data(self, plugin_author: str, plugin_name: str) -> None:
|
||||
"""Cleanup plugin settings and binary storage"""
|
||||
action_context = self.require_bound_action_context()
|
||||
installation_uuid = self.derive_installation_uuid(
|
||||
action_context,
|
||||
plugin_author,
|
||||
plugin_name,
|
||||
)
|
||||
action_context = self.require_outbound_installation_context()
|
||||
# Delete plugin settings
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.delete(persistence_plugin.PluginSetting)
|
||||
@@ -1664,9 +1866,6 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
.where(persistence_bstorage.BinaryStorage.owner_type == 'plugin')
|
||||
.where(persistence_bstorage.BinaryStorage.owner == owner)
|
||||
)
|
||||
installation_bindings = getattr(self, '_installation_bindings', None)
|
||||
if installation_bindings is not None:
|
||||
installation_bindings.pop(installation_uuid, None)
|
||||
|
||||
async def call_tool(
|
||||
self,
|
||||
@@ -1744,11 +1943,12 @@ class RuntimeConnectionHandler(handler.Handler):
|
||||
|
||||
async def get_debug_info(self) -> dict[str, Any]:
|
||||
"""Get debug information including debug key and WS URL"""
|
||||
result = await self.call_action(
|
||||
LangBotToRuntimeAction.GET_DEBUG_INFO,
|
||||
{},
|
||||
timeout=10,
|
||||
)
|
||||
with self.installation_scope(None):
|
||||
result = await self.call_action(
|
||||
LangBotToRuntimeAction.GET_DEBUG_INFO,
|
||||
{},
|
||||
timeout=10,
|
||||
)
|
||||
return result
|
||||
|
||||
# ================= RAG Capability Callers (LangBot -> Runtime) =================
|
||||
|
||||
@@ -197,6 +197,23 @@ class ModelManager:
|
||||
self.llm_model_dict = {}
|
||||
self.embedding_model_dict = {}
|
||||
self.rerank_model_dict = {}
|
||||
|
||||
list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
if not callable(list_bindings) or not callable(tenant_uow):
|
||||
raise RuntimeError('Cloud model loading requires explicit instance discovery and tenant UoWs')
|
||||
for binding in await list_bindings():
|
||||
context = self._context_from_binding(
|
||||
binding,
|
||||
trigger_principal=PrincipalContext(principal_type=PrincipalType.SYSTEM),
|
||||
)
|
||||
async with tenant_uow(binding.workspace_uuid):
|
||||
await self._load_workspace_models(context)
|
||||
return
|
||||
|
||||
# Compatibility path for isolated manager tests and older embedders.
|
||||
contexts: dict[str, ExecutionContext] = {}
|
||||
|
||||
async def context_for(workspace_uuid: str | None) -> ExecutionContext:
|
||||
@@ -263,6 +280,61 @@ class ModelManager:
|
||||
except Exception as exc:
|
||||
self.ap.logger.error(f'Failed to load model {model_entity.uuid}: {exc}\n{traceback.format_exc()}')
|
||||
|
||||
async def _load_workspace_models(self, context: ExecutionContext) -> None:
|
||||
"""Load one Workspace while its tenant transaction is active."""
|
||||
|
||||
providers_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_model.ModelProvider).where(
|
||||
persistence_model.ModelProvider.workspace_uuid == context.workspace_uuid
|
||||
)
|
||||
)
|
||||
for provider_entity in providers_result.all():
|
||||
try:
|
||||
runtime_provider = await self._build_provider(context, provider_entity)
|
||||
self.provider_dict[self._cache_key(context, provider_entity.uuid)] = runtime_provider
|
||||
except provider_errors.RequesterNotFoundError as exc:
|
||||
self.ap.logger.warning(
|
||||
f'Requester {exc.requester_name} not found, skipping provider {provider_entity.uuid}'
|
||||
)
|
||||
except Exception as exc:
|
||||
self.ap.logger.error(f'Failed to load provider {provider_entity.uuid}: {exc}\n{traceback.format_exc()}')
|
||||
|
||||
await self._load_workspace_model_kind(
|
||||
context,
|
||||
persistence_model.LLMModel,
|
||||
self.llm_model_dict,
|
||||
self._build_llm_model,
|
||||
)
|
||||
await self._load_workspace_model_kind(
|
||||
context,
|
||||
persistence_model.EmbeddingModel,
|
||||
self.embedding_model_dict,
|
||||
self._build_embedding_model,
|
||||
)
|
||||
await self._load_workspace_model_kind(
|
||||
context,
|
||||
persistence_model.RerankModel,
|
||||
self.rerank_model_dict,
|
||||
self._build_rerank_model,
|
||||
)
|
||||
|
||||
async def _load_workspace_model_kind(self, context, entity_type, cache: dict, builder) -> None:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(entity_type).where(entity_type.workspace_uuid == context.workspace_uuid)
|
||||
)
|
||||
for model_entity in result.all():
|
||||
try:
|
||||
provider = self.provider_dict.get(self._cache_key(context, model_entity.provider_uuid))
|
||||
if provider is None:
|
||||
self.ap.logger.warning(
|
||||
f'Provider {model_entity.provider_uuid} not found for model {model_entity.uuid}'
|
||||
)
|
||||
continue
|
||||
runtime_model = builder(context, model_entity, provider)
|
||||
cache[self._cache_key(context, model_entity.uuid)] = runtime_model
|
||||
except Exception as exc:
|
||||
self.ap.logger.error(f'Failed to load model {model_entity.uuid}: {exc}\n{traceback.format_exc()}')
|
||||
|
||||
async def sync_new_models_from_space(self, context: ExecutionContext) -> None:
|
||||
"""Sync legacy Space models for the explicitly selected OSS Workspace."""
|
||||
|
||||
|
||||
@@ -211,7 +211,7 @@ class LocalAgentRunner(runner.RequestRunner):
|
||||
req_messages.append(
|
||||
provider_message.Message(
|
||||
role='system',
|
||||
content=self.ap.box_service.get_system_guidance(query.query_id),
|
||||
content=self.ap.box_service.get_system_guidance(query),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ from pydantic import AnyUrl
|
||||
|
||||
from .. import loader
|
||||
from ....core import app
|
||||
from ....core.task_boundary import create_detached_task, run_in_workspace_uow
|
||||
from ....api.http.context import ExecutionContext
|
||||
from ....api.http.service.tenant import TenantContext, require_workspace_uuid
|
||||
from ....workspace.errors import WorkspaceError, WorkspaceInvariantError
|
||||
@@ -1437,11 +1438,41 @@ class MCPLoader(loader.ToolLoader):
|
||||
|
||||
self.sessions = {}
|
||||
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_mcp.MCPServer))
|
||||
servers = result.all()
|
||||
server_configs: list[tuple[typing.Any, typing.Any, dict]] = []
|
||||
list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
if not callable(list_bindings) or not callable(tenant_uow):
|
||||
raise RuntimeError('Cloud MCP loading requires explicit instance discovery and tenant UoWs')
|
||||
for binding in await list_bindings():
|
||||
async with tenant_uow(binding.workspace_uuid):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_mcp.MCPServer)
|
||||
.where(persistence_mcp.MCPServer.workspace_uuid == binding.workspace_uuid)
|
||||
.order_by(persistence_mcp.MCPServer.uuid)
|
||||
)
|
||||
for server in result.all():
|
||||
server_configs.append(
|
||||
(
|
||||
binding,
|
||||
server,
|
||||
self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server),
|
||||
)
|
||||
)
|
||||
else:
|
||||
# Compatibility path for isolated loader tests and older embedders.
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_mcp.MCPServer))
|
||||
for server in result.all():
|
||||
server_configs.append(
|
||||
(
|
||||
None,
|
||||
server,
|
||||
self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server),
|
||||
)
|
||||
)
|
||||
|
||||
for server in servers:
|
||||
config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server)
|
||||
for binding, server, config in server_configs:
|
||||
if config.get('mode') == 'stdio' and not stdio_mcp_enabled(self.ap):
|
||||
self.ap.logger.info(
|
||||
f'Skipping disabled stdio MCP server {server.uuid}; '
|
||||
@@ -1449,7 +1480,8 @@ class MCPLoader(loader.ToolLoader):
|
||||
)
|
||||
continue
|
||||
try:
|
||||
binding = await self.ap.workspace_service.get_execution_binding(server.workspace_uuid)
|
||||
if binding is None:
|
||||
binding = await self.ap.workspace_service.get_execution_binding(server.workspace_uuid)
|
||||
execution_context = ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
@@ -1461,7 +1493,10 @@ class MCPLoader(loader.ToolLoader):
|
||||
)
|
||||
continue
|
||||
|
||||
task = asyncio.create_task(self.host_mcp_server(execution_context, config))
|
||||
task = create_detached_task(
|
||||
self.host_mcp_server(execution_context, config),
|
||||
after_commit_manager=getattr(self.ap, 'persistence_mgr', None),
|
||||
)
|
||||
self._hosted_mcp_tasks.append(task)
|
||||
|
||||
@staticmethod
|
||||
@@ -1482,7 +1517,12 @@ class MCPLoader(loader.ToolLoader):
|
||||
return [session for key, session in self.sessions.items() if key[:3] == scope_key]
|
||||
|
||||
async def host_mcp_server(self, context: TenantContext, server_config: dict):
|
||||
execution_context = await self._assert_execution_active(context)
|
||||
requested_context = _execution_context_from_tenant(context)
|
||||
execution_context = await run_in_workspace_uow(
|
||||
self.ap,
|
||||
requested_context.workspace_uuid,
|
||||
lambda: self._assert_execution_active(requested_context),
|
||||
)
|
||||
configured_workspace = str(server_config.get('workspace_uuid') or '').strip()
|
||||
if configured_workspace and configured_workspace != execution_context.workspace_uuid:
|
||||
raise ValueError('MCP server configuration belongs to another Workspace')
|
||||
|
||||
@@ -1,8 +1,15 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import errno
|
||||
import json
|
||||
import os
|
||||
import posixpath
|
||||
import stat
|
||||
from collections.abc import Iterator
|
||||
from dataclasses import dataclass
|
||||
from pathlib import PurePosixPath
|
||||
|
||||
import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
|
||||
from langbot_plugin.api.entities.events import pipeline_query
|
||||
@@ -34,6 +41,158 @@ _GREP_MAX_MATCHES = 200
|
||||
_GREP_MAX_FILES = 5000
|
||||
_GREP_MAX_LINE_CHARS = 500
|
||||
|
||||
_DIRECTORY_OPEN_FLAGS = (
|
||||
os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
|
||||
)
|
||||
_FILE_OPEN_FLAGS = getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0) | getattr(os, 'O_NONBLOCK', 0)
|
||||
_SECURE_HOST_FILE_OPS_AVAILABLE = bool(
|
||||
getattr(os, 'O_NOFOLLOW', 0)
|
||||
and os.open in os.supports_dir_fd
|
||||
and os.stat in os.supports_dir_fd
|
||||
and os.mkdir in os.supports_dir_fd
|
||||
and os.listdir in os.supports_fd
|
||||
and os.scandir in os.supports_fd
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _HostLocation:
|
||||
root: str
|
||||
relative_parts: tuple[str, ...]
|
||||
selected_skill: dict | None
|
||||
workspace_anchor: str | None = None
|
||||
|
||||
|
||||
def _unsafe_host_path(path: str, exc: BaseException | None = None) -> ValueError:
|
||||
error = ValueError(f'Path escapes the workspace boundary or contains a symbolic link: {path}')
|
||||
if exc is not None:
|
||||
error.__cause__ = exc
|
||||
return error
|
||||
|
||||
|
||||
def _relative_workspace_parts(path: str) -> tuple[str, ...]:
|
||||
normalized = posixpath.normpath(str(path or '/workspace').strip() or '/workspace')
|
||||
if normalized == '/workspace':
|
||||
return ()
|
||||
if not normalized.startswith('/workspace/'):
|
||||
raise ValueError('Path escapes the workspace boundary.')
|
||||
|
||||
parts = tuple(part for part in normalized.removeprefix('/workspace/').split('/') if part)
|
||||
if any(part in {'.', '..'} or '\x00' in part for part in parts):
|
||||
raise ValueError('Path escapes the workspace boundary.')
|
||||
return parts
|
||||
|
||||
|
||||
def _is_symlink_at(parent_fd: int, name: str) -> bool:
|
||||
try:
|
||||
return stat.S_ISLNK(os.stat(name, dir_fd=parent_fd, follow_symlinks=False).st_mode)
|
||||
except FileNotFoundError:
|
||||
return False
|
||||
|
||||
|
||||
def _open_directory_at(parent_fd: int, name: str, *, create: bool) -> int:
|
||||
if create:
|
||||
try:
|
||||
os.mkdir(name, mode=0o777, dir_fd=parent_fd)
|
||||
except FileExistsError:
|
||||
pass
|
||||
|
||||
try:
|
||||
directory_fd = os.open(name, _DIRECTORY_OPEN_FLAGS, dir_fd=parent_fd)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ELOOP or _is_symlink_at(parent_fd, name):
|
||||
raise _unsafe_host_path(name, exc)
|
||||
raise
|
||||
if not stat.S_ISDIR(os.fstat(directory_fd).st_mode):
|
||||
os.close(directory_fd)
|
||||
raise NotADirectoryError(name)
|
||||
return directory_fd
|
||||
|
||||
|
||||
def _open_directory_parts(root_fd: int, parts: tuple[str, ...], *, create: bool) -> int:
|
||||
current_fd = os.dup(root_fd)
|
||||
try:
|
||||
for part in parts:
|
||||
next_fd = _open_directory_at(current_fd, part, create=create)
|
||||
os.close(current_fd)
|
||||
current_fd = next_fd
|
||||
return current_fd
|
||||
except BaseException:
|
||||
os.close(current_fd)
|
||||
raise
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _open_host_root(location: _HostLocation, *, create: bool) -> Iterator[int]:
|
||||
"""Open and pin the tenant root before resolving tenant-controlled names."""
|
||||
|
||||
if location.workspace_anchor is None:
|
||||
root_path = os.path.realpath(location.root)
|
||||
try:
|
||||
root_fd = os.open(root_path, _DIRECTORY_OPEN_FLAGS)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ELOOP:
|
||||
raise _unsafe_host_path(location.root, exc)
|
||||
raise
|
||||
else:
|
||||
anchor_path = os.path.abspath(location.workspace_anchor)
|
||||
root_path = os.path.abspath(location.root)
|
||||
try:
|
||||
if os.path.commonpath((anchor_path, root_path)) != anchor_path:
|
||||
raise _unsafe_host_path(location.root)
|
||||
except ValueError as exc:
|
||||
raise _unsafe_host_path(location.root, exc)
|
||||
|
||||
anchor_real_path = os.path.realpath(anchor_path)
|
||||
try:
|
||||
anchor_fd = os.open(anchor_real_path, _DIRECTORY_OPEN_FLAGS)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ELOOP:
|
||||
raise _unsafe_host_path(location.workspace_anchor, exc)
|
||||
raise
|
||||
try:
|
||||
root_relative = os.path.relpath(root_path, anchor_path)
|
||||
root_parts = () if root_relative == '.' else tuple(root_relative.split(os.sep))
|
||||
root_fd = _open_directory_parts(anchor_fd, root_parts, create=create)
|
||||
finally:
|
||||
os.close(anchor_fd)
|
||||
|
||||
try:
|
||||
if not stat.S_ISDIR(os.fstat(root_fd).st_mode):
|
||||
raise _unsafe_host_path(location.root)
|
||||
yield root_fd
|
||||
finally:
|
||||
os.close(root_fd)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _open_location_fd(
|
||||
root_fd: int,
|
||||
relative_parts: tuple[str, ...],
|
||||
flags: int,
|
||||
*,
|
||||
create_parents: bool = False,
|
||||
mode: int = 0o666,
|
||||
) -> Iterator[int]:
|
||||
if not relative_parts:
|
||||
target_fd = os.dup(root_fd)
|
||||
else:
|
||||
parent_fd = _open_directory_parts(root_fd, relative_parts[:-1], create=create_parents)
|
||||
try:
|
||||
try:
|
||||
target_fd = os.open(relative_parts[-1], flags | _FILE_OPEN_FLAGS, mode, dir_fd=parent_fd)
|
||||
except OSError as exc:
|
||||
if exc.errno == errno.ELOOP or _is_symlink_at(parent_fd, relative_parts[-1]):
|
||||
raise _unsafe_host_path(relative_parts[-1], exc)
|
||||
raise
|
||||
finally:
|
||||
os.close(parent_fd)
|
||||
|
||||
try:
|
||||
yield target_fd
|
||||
finally:
|
||||
os.close(target_fd)
|
||||
|
||||
|
||||
class NativeToolLoader(loader.ToolLoader):
|
||||
def __init__(self, ap):
|
||||
@@ -59,6 +218,9 @@ class NativeToolLoader(loader.ToolLoader):
|
||||
|
||||
@staticmethod
|
||||
def _execution_context(query: pipeline_query.Query) -> ExecutionContext:
|
||||
attached_context = getattr(query, '_execution_context', None)
|
||||
if isinstance(attached_context, ExecutionContext):
|
||||
return attached_context
|
||||
return ExecutionContext(
|
||||
instance_uuid=str(getattr(query, 'instance_uuid', '') or ''),
|
||||
workspace_uuid=str(getattr(query, 'workspace_uuid', '') or ''),
|
||||
@@ -66,6 +228,7 @@ class NativeToolLoader(loader.ToolLoader):
|
||||
bot_uuid=getattr(query, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(query, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(query, 'query_uuid', None),
|
||||
entitlement_revision=getattr(query, 'entitlement_revision', 0),
|
||||
)
|
||||
|
||||
async def get_tools(self, bound_plugins: list[str] | None = None) -> list[resource_tool.LLMTool]:
|
||||
@@ -86,6 +249,13 @@ class NativeToolLoader(loader.ToolLoader):
|
||||
return name in _ALL_TOOL_NAMES and self._is_sandbox_available()
|
||||
|
||||
async def invoke_tool(self, name: str, parameters: dict, query: pipeline_query.Query):
|
||||
require_sandbox = getattr(
|
||||
getattr(self.ap, 'box_service', None),
|
||||
'require_workspace_sandbox',
|
||||
None,
|
||||
)
|
||||
if callable(require_sandbox):
|
||||
await require_sandbox(self._execution_context(query))
|
||||
if name == EXEC_TOOL_NAME:
|
||||
self.ap.logger.info(
|
||||
'exec tool invoked: '
|
||||
@@ -111,6 +281,7 @@ class NativeToolLoader(loader.ToolLoader):
|
||||
async def _invoke_exec(self, parameters: dict, query: pipeline_query.Query) -> dict:
|
||||
command = str(parameters['command'])
|
||||
workdir = str(parameters.get('workdir', '/workspace') or '/workspace')
|
||||
selected_skill_name: str | None = None
|
||||
|
||||
# Validate that skill references target activated skills.
|
||||
selected_skill, _ = skill_loader.resolve_virtual_skill_path(
|
||||
@@ -140,31 +311,51 @@ class NativeToolLoader(loader.ToolLoader):
|
||||
if not package_root:
|
||||
raise ValueError(f'Activated skill "{selected_skill_name}" has no package_root.')
|
||||
|
||||
# Pass only the logical name across the authenticated Core→Runtime
|
||||
# boundary. In Cloud mode the shared Box Runtime resolves the
|
||||
# Workspace-scoped package root and constructs the read-only mount;
|
||||
# Core host paths are never accepted as mount authority.
|
||||
# Wrap command with Python venv bootstrap if the skill has a Python project.
|
||||
# The venv is created inside the skill's mount path.
|
||||
skill_mount = f'/workspace/.skills/{selected_skill_name}'
|
||||
if skill_loader.should_prepare_skill_python_env(package_root):
|
||||
python_project = selected_skill.get('python_project') is True
|
||||
if 'python_project' not in selected_skill and bool(
|
||||
getattr(self.ap.box_service, 'shares_filesystem_with_box', False)
|
||||
):
|
||||
# Backward compatibility for a same-process OSS Runtime that
|
||||
# predates trusted Box metadata. Never probe a path reported by
|
||||
# an external Runtime from the Core filesystem.
|
||||
python_project = skill_loader.should_prepare_skill_python_env(package_root)
|
||||
if python_project:
|
||||
parameters = dict(parameters)
|
||||
parameters['command'] = skill_loader.wrap_skill_command_with_python_env(command, mount_path=skill_mount)
|
||||
parameters['command'] = skill_loader.wrap_skill_command_with_python_env(
|
||||
command,
|
||||
mount_path=skill_mount,
|
||||
state_path=f'/workspace/.skill-envs/{selected_skill_name}',
|
||||
)
|
||||
|
||||
# All exec calls (with or without skills) go through the same container
|
||||
# via execute_tool. Skills are mounted at /workspace/.skills/{name}/
|
||||
# via extra_mounts built by BoxService.
|
||||
result = await self.ap.box_service.execute_tool(parameters, query)
|
||||
result = await self.ap.box_service.execute_tool(
|
||||
parameters,
|
||||
query,
|
||||
skill_name=selected_skill_name,
|
||||
)
|
||||
result = self._normalize_exec_result(result)
|
||||
|
||||
if selected_skill is not None:
|
||||
self._refresh_skill_from_disk(query, selected_skill)
|
||||
return result
|
||||
|
||||
def _resolve_host_path(
|
||||
def _resolve_host_location(
|
||||
self,
|
||||
query: pipeline_query.Query,
|
||||
sandbox_path: str,
|
||||
*,
|
||||
include_visible: bool,
|
||||
include_activated: bool,
|
||||
) -> tuple[str, dict | None]:
|
||||
) -> _HostLocation:
|
||||
selected_skill, rewritten_path = skill_loader.resolve_virtual_skill_path(
|
||||
self.ap,
|
||||
query,
|
||||
@@ -174,26 +365,26 @@ class NativeToolLoader(loader.ToolLoader):
|
||||
)
|
||||
|
||||
box_service = self.ap.box_service
|
||||
host_root = (
|
||||
selected_skill.get('package_root')
|
||||
if selected_skill is not None
|
||||
else box_service._tenant_workspace(self._execution_context(query))
|
||||
)
|
||||
if selected_skill is not None:
|
||||
if not self._can_interpret_skill_host_paths():
|
||||
raise ValueError(
|
||||
'Skill package paths are owned by the Box Runtime; '
|
||||
'this operation requires a Runtime skill-file API.'
|
||||
)
|
||||
host_root = selected_skill.get('package_root')
|
||||
workspace_anchor = None
|
||||
else:
|
||||
host_root = box_service._tenant_workspace(self._execution_context(query))
|
||||
workspace_anchor = getattr(box_service, 'default_workspace', None)
|
||||
if not host_root:
|
||||
raise ValueError('No host workspace configured for file operations.')
|
||||
|
||||
mount_path = '/workspace'
|
||||
if not rewritten_path.startswith(mount_path):
|
||||
raise ValueError(f'Path must be under {mount_path}.')
|
||||
|
||||
relative = rewritten_path[len(mount_path) :].lstrip('/')
|
||||
host_path = os.path.realpath(os.path.join(host_root, relative))
|
||||
host_root = os.path.realpath(host_root)
|
||||
|
||||
if not (host_path == host_root or host_path.startswith(host_root + os.sep)):
|
||||
raise ValueError('Path escapes the workspace boundary.')
|
||||
|
||||
return host_path, selected_skill
|
||||
return _HostLocation(
|
||||
root=str(host_root),
|
||||
relative_parts=_relative_workspace_parts(rewritten_path),
|
||||
selected_skill=selected_skill,
|
||||
workspace_anchor=str(workspace_anchor) if workspace_anchor else None,
|
||||
)
|
||||
|
||||
def _resolve_skill_relative_path(
|
||||
self,
|
||||
@@ -213,21 +404,259 @@ class NativeToolLoader(loader.ToolLoader):
|
||||
if selected_skill is None:
|
||||
return None
|
||||
|
||||
mount_path = '/workspace'
|
||||
if not rewritten_path.startswith(mount_path):
|
||||
raise ValueError(f'Path must be under {mount_path}.')
|
||||
relative = rewritten_path[len(mount_path) :].lstrip('/') or '.'
|
||||
relative = '/'.join(_relative_workspace_parts(rewritten_path)) or '.'
|
||||
return selected_skill, relative
|
||||
|
||||
def _can_interpret_skill_host_paths(self) -> bool:
|
||||
"""Require an explicitly proven shared Core/Runtime filesystem view."""
|
||||
|
||||
return _SECURE_HOST_FILE_OPS_AVAILABLE and bool(
|
||||
getattr(self.ap.box_service, 'shares_filesystem_with_box', False)
|
||||
)
|
||||
|
||||
def _should_use_box_workspace_files(self, selected_skill: dict | None) -> bool:
|
||||
if selected_skill is not None:
|
||||
return False
|
||||
box_service = getattr(self.ap, 'box_service', None)
|
||||
if box_service is None or not hasattr(box_service, 'execute_tool'):
|
||||
return False
|
||||
if not _SECURE_HOST_FILE_OPS_AVAILABLE:
|
||||
# Preserve the OSS API on platforms without openat/O_NOFOLLOW by
|
||||
# running inside the tenant-scoped Box mount, never via a racy
|
||||
# host-path fallback.
|
||||
return True
|
||||
default_workspace = getattr(box_service, 'default_workspace', None)
|
||||
return bool(default_workspace and not os.path.isdir(os.path.realpath(default_workspace)))
|
||||
|
||||
def _read_host_location(self, location: _HostLocation, parameters: dict) -> dict:
|
||||
with _open_host_root(location, create=False) as root_fd:
|
||||
with _open_location_fd(root_fd, location.relative_parts, os.O_RDONLY) as target_fd:
|
||||
metadata = os.fstat(target_fd)
|
||||
if stat.S_ISDIR(metadata.st_mode):
|
||||
return self._build_directory_result(os.listdir(target_fd))
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise ValueError('Path must reference a regular file or directory.')
|
||||
return self._read_text_file_preview(target_fd, parameters, metadata=metadata)
|
||||
|
||||
def _write_host_location(self, location: _HostLocation, content: str, parameters: dict) -> None:
|
||||
if not location.relative_parts:
|
||||
raise ValueError('Path must reference a file under /workspace.')
|
||||
|
||||
encoding, mode = self._write_options(parameters)
|
||||
if encoding == 'base64':
|
||||
try:
|
||||
payload = base64.b64decode(content, validate=True)
|
||||
except Exception as exc:
|
||||
raise ValueError(f'invalid base64 content: {exc}') from exc
|
||||
else:
|
||||
payload = content.encode('utf-8')
|
||||
|
||||
flags = os.O_WRONLY | os.O_CREAT
|
||||
if mode == 'append':
|
||||
flags |= os.O_APPEND
|
||||
with _open_host_root(location, create=True) as root_fd:
|
||||
with _open_location_fd(
|
||||
root_fd,
|
||||
location.relative_parts,
|
||||
flags,
|
||||
create_parents=True,
|
||||
) as target_fd:
|
||||
if not stat.S_ISREG(os.fstat(target_fd).st_mode):
|
||||
raise ValueError('Path must reference a regular file.')
|
||||
if mode != 'append':
|
||||
os.ftruncate(target_fd, 0)
|
||||
os.lseek(target_fd, 0, os.SEEK_SET)
|
||||
self._write_all(target_fd, payload)
|
||||
|
||||
def _edit_host_location(
|
||||
self,
|
||||
location: _HostLocation,
|
||||
old_string: str,
|
||||
new_string: str,
|
||||
) -> tuple[bool, str | None]:
|
||||
if not location.relative_parts:
|
||||
raise ValueError('Path must reference a file under /workspace.')
|
||||
|
||||
with _open_host_root(location, create=False) as root_fd:
|
||||
with _open_location_fd(root_fd, location.relative_parts, os.O_RDWR) as target_fd:
|
||||
if not stat.S_ISREG(os.fstat(target_fd).st_mode):
|
||||
return False, 'File not found.'
|
||||
with os.fdopen(os.dup(target_fd), 'r', encoding='utf-8', errors='replace') as file_obj:
|
||||
content = file_obj.read()
|
||||
count = content.count(old_string)
|
||||
if count == 0:
|
||||
return False, 'old_string not found in file.'
|
||||
if count > 1:
|
||||
return False, f'old_string matches {count} locations; provide a more unique string.'
|
||||
|
||||
payload = content.replace(old_string, new_string, 1).encode('utf-8')
|
||||
os.ftruncate(target_fd, 0)
|
||||
os.lseek(target_fd, 0, os.SEEK_SET)
|
||||
self._write_all(target_fd, payload)
|
||||
return True, None
|
||||
|
||||
@staticmethod
|
||||
def _write_all(file_fd: int, payload: bytes) -> None:
|
||||
view = memoryview(payload)
|
||||
while view:
|
||||
written = os.write(file_fd, view)
|
||||
if written <= 0:
|
||||
raise OSError('Could not write the complete workspace file.')
|
||||
view = view[written:]
|
||||
|
||||
@staticmethod
|
||||
def _rglob_matches(relative_path: str, pattern: str) -> bool:
|
||||
candidates = {pattern}
|
||||
pending = [pattern]
|
||||
while pending:
|
||||
candidate = pending.pop()
|
||||
marker = candidate.find('**/')
|
||||
while marker >= 0:
|
||||
without_recursive_segment = candidate[:marker] + candidate[marker + 3 :]
|
||||
if without_recursive_segment not in candidates:
|
||||
candidates.add(without_recursive_segment)
|
||||
pending.append(without_recursive_segment)
|
||||
marker = candidate.find('**/', marker + 3)
|
||||
return any(candidate and PurePosixPath(relative_path).match(candidate) for candidate in candidates)
|
||||
|
||||
def _glob_host_location(self, location: _HostLocation, pattern: str, sandbox_base: str) -> dict:
|
||||
hits: list[tuple[str, float]] = []
|
||||
|
||||
def walk(directory_fd: int, prefix: str) -> None:
|
||||
with os.scandir(directory_fd) as entries:
|
||||
for entry in entries:
|
||||
name = entry.name
|
||||
if name in _SKIP_DIRS:
|
||||
continue
|
||||
try:
|
||||
child_fd = os.open(name, os.O_RDONLY | _FILE_OPEN_FLAGS, dir_fd=directory_fd)
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
metadata = os.fstat(child_fd)
|
||||
relative = f'{prefix}/{name}' if prefix else name
|
||||
if self._rglob_matches(relative, pattern):
|
||||
hits.append((relative, metadata.st_mtime))
|
||||
if stat.S_ISDIR(metadata.st_mode):
|
||||
walk(child_fd, relative)
|
||||
finally:
|
||||
os.close(child_fd)
|
||||
|
||||
with _open_host_root(location, create=False) as root_fd:
|
||||
with _open_location_fd(root_fd, location.relative_parts, os.O_RDONLY) as target_fd:
|
||||
if not stat.S_ISDIR(os.fstat(target_fd).st_mode):
|
||||
return {'ok': False, 'error': f'Path is not a directory: {sandbox_base}'}
|
||||
walk(target_fd, '')
|
||||
|
||||
hits.sort(key=lambda item: item[1], reverse=True)
|
||||
total = len(hits)
|
||||
sandbox_paths: list[str] = []
|
||||
output_bytes = 0
|
||||
truncated_by_bytes = False
|
||||
for relative, _mtime in hits[:_GLOB_MAX_MATCHES]:
|
||||
sandbox_path = self._sandbox_child_path(sandbox_base, relative)
|
||||
entry_bytes = len(sandbox_path.encode('utf-8')) + (1 if sandbox_paths else 0)
|
||||
if output_bytes + entry_bytes > _DEFAULT_TOOL_RESULT_MAX_BYTES:
|
||||
truncated_by_bytes = True
|
||||
break
|
||||
sandbox_paths.append(sandbox_path)
|
||||
output_bytes += entry_bytes
|
||||
|
||||
return {
|
||||
'ok': True,
|
||||
'matches': sandbox_paths,
|
||||
'preview': '\n'.join(sandbox_paths),
|
||||
'total': total,
|
||||
'truncated': total > len(sandbox_paths) or truncated_by_bytes,
|
||||
'truncated_by': 'bytes' if truncated_by_bytes else ('matches' if total > len(sandbox_paths) else None),
|
||||
}
|
||||
|
||||
def _grep_host_location(
|
||||
self,
|
||||
location: _HostLocation,
|
||||
regex,
|
||||
include: str | None,
|
||||
sandbox_base: str,
|
||||
) -> dict:
|
||||
matches: list[dict] = []
|
||||
output_bytes = 0
|
||||
truncated_by: str | None = None
|
||||
files_seen = 0
|
||||
|
||||
def grep_file(file_fd: int, sandbox_path: str) -> bool:
|
||||
nonlocal output_bytes, truncated_by
|
||||
with os.fdopen(os.dup(file_fd), 'r', encoding='utf-8', errors='ignore') as handle:
|
||||
for lineno, line in enumerate(handle, 1):
|
||||
if not regex.search(line):
|
||||
continue
|
||||
content, line_truncated = self._truncate_grep_line(line.rstrip())
|
||||
entry = {'file': sandbox_path, 'line': lineno, 'content': content}
|
||||
entry_bytes = len(json.dumps(entry, ensure_ascii=False).encode('utf-8')) + 1
|
||||
if output_bytes + entry_bytes > _DEFAULT_TOOL_RESULT_MAX_BYTES:
|
||||
truncated_by = 'bytes'
|
||||
return True
|
||||
if line_truncated and truncated_by is None:
|
||||
truncated_by = 'line'
|
||||
matches.append(entry)
|
||||
output_bytes += entry_bytes
|
||||
if len(matches) >= _GREP_MAX_MATCHES:
|
||||
truncated_by = truncated_by or 'matches'
|
||||
return True
|
||||
return False
|
||||
|
||||
def walk(directory_fd: int, prefix: str) -> bool:
|
||||
nonlocal files_seen
|
||||
with os.scandir(directory_fd) as entries:
|
||||
for entry in entries:
|
||||
name = entry.name
|
||||
if name in _SKIP_DIRS:
|
||||
continue
|
||||
try:
|
||||
child_fd = os.open(name, os.O_RDONLY | _FILE_OPEN_FLAGS, dir_fd=directory_fd)
|
||||
except OSError:
|
||||
continue
|
||||
try:
|
||||
metadata = os.fstat(child_fd)
|
||||
relative = f'{prefix}/{name}' if prefix else name
|
||||
if stat.S_ISDIR(metadata.st_mode):
|
||||
if walk(child_fd, relative):
|
||||
return True
|
||||
continue
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
continue
|
||||
if include and not self._rglob_matches(relative, include):
|
||||
continue
|
||||
files_seen += 1
|
||||
if grep_file(child_fd, self._sandbox_child_path(sandbox_base, relative)):
|
||||
return True
|
||||
if files_seen >= _GREP_MAX_FILES:
|
||||
return True
|
||||
finally:
|
||||
os.close(child_fd)
|
||||
return False
|
||||
|
||||
with _open_host_root(location, create=False) as root_fd:
|
||||
with _open_location_fd(root_fd, location.relative_parts, os.O_RDONLY) as target_fd:
|
||||
metadata = os.fstat(target_fd)
|
||||
if stat.S_ISREG(metadata.st_mode):
|
||||
grep_file(target_fd, sandbox_base)
|
||||
elif stat.S_ISDIR(metadata.st_mode):
|
||||
walk(target_fd, '')
|
||||
else:
|
||||
return {'ok': False, 'error': f'Path not found: {sandbox_base}'}
|
||||
|
||||
return {
|
||||
'ok': True,
|
||||
'matches': matches,
|
||||
'total': len(matches),
|
||||
'truncated': truncated_by is not None,
|
||||
'truncated_by': truncated_by,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _sandbox_child_path(base: str, relative: str) -> str:
|
||||
return f'{str(base).rstrip("/")}/{relative}'
|
||||
|
||||
async def _run_workspace_file_script(self, script: str, query: pipeline_query.Query) -> dict:
|
||||
result = await self.ap.box_service.execute_tool(
|
||||
{
|
||||
@@ -531,11 +960,15 @@ else:
|
||||
)
|
||||
if skill_request is not None and hasattr(self.ap.box_service, 'read_skill_file'):
|
||||
selected_skill, relative = skill_request
|
||||
host_path = self._resolve_skill_host_path(selected_skill, relative)
|
||||
if host_path and os.path.exists(host_path):
|
||||
if os.path.isdir(host_path):
|
||||
return self._build_directory_result(os.listdir(host_path))
|
||||
return self._read_text_file_preview(host_path, parameters)
|
||||
if self._can_interpret_skill_host_paths():
|
||||
host_location = self._resolve_skill_host_location(selected_skill, relative)
|
||||
else:
|
||||
host_location = None
|
||||
if host_location is not None:
|
||||
try:
|
||||
return self._read_host_location(host_location, parameters)
|
||||
except FileNotFoundError:
|
||||
pass
|
||||
|
||||
try:
|
||||
result = await self.ap.box_service.read_skill_file(
|
||||
@@ -556,20 +989,18 @@ else:
|
||||
except Exception as exc:
|
||||
return {'ok': False, 'error': str(exc)}
|
||||
|
||||
host_path, selected_skill = self._resolve_host_path(
|
||||
host_location = self._resolve_host_location(
|
||||
query,
|
||||
path,
|
||||
include_visible=True,
|
||||
include_activated=True,
|
||||
)
|
||||
if self._should_use_box_workspace_files(selected_skill):
|
||||
if self._should_use_box_workspace_files(host_location.selected_skill):
|
||||
return await self._read_workspace_via_box(path, parameters, query)
|
||||
if not os.path.exists(host_path):
|
||||
try:
|
||||
return self._read_host_location(host_location, parameters)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return {'ok': False, 'error': f'File not found: {path}'}
|
||||
if os.path.isdir(host_path):
|
||||
entries = os.listdir(host_path)
|
||||
return self._build_directory_result(entries)
|
||||
return self._read_text_file_preview(host_path, parameters)
|
||||
|
||||
async def _invoke_write(self, parameters: dict, query: pipeline_query.Query) -> dict:
|
||||
path = parameters['path']
|
||||
@@ -591,20 +1022,19 @@ else:
|
||||
await self.ap.skill_mgr.reload_skills(execution_context)
|
||||
return {'ok': True, 'path': path}
|
||||
|
||||
host_path, selected_skill = self._resolve_host_path(
|
||||
host_location = self._resolve_host_location(
|
||||
query,
|
||||
path,
|
||||
include_visible=False,
|
||||
include_activated=True,
|
||||
)
|
||||
if self._should_use_box_workspace_files(selected_skill):
|
||||
if self._should_use_box_workspace_files(host_location.selected_skill):
|
||||
return await self._write_workspace_via_box(path, content, parameters, query)
|
||||
os.makedirs(os.path.dirname(host_path), exist_ok=True)
|
||||
try:
|
||||
self._write_host_file(host_path, content, parameters)
|
||||
self._write_host_location(host_location, content, parameters)
|
||||
except ValueError as exc:
|
||||
return {'ok': False, 'error': str(exc)}
|
||||
self._refresh_skill_from_disk(query, selected_skill)
|
||||
self._refresh_skill_from_disk(query, host_location.selected_skill)
|
||||
return {'ok': True, 'path': path}
|
||||
|
||||
async def _invoke_edit(self, parameters: dict, query: pipeline_query.Query) -> dict:
|
||||
@@ -652,27 +1082,21 @@ else:
|
||||
await self.ap.skill_mgr.reload_skills(execution_context)
|
||||
return {'ok': True, 'path': path}
|
||||
|
||||
host_path, selected_skill = self._resolve_host_path(
|
||||
host_location = self._resolve_host_location(
|
||||
query,
|
||||
path,
|
||||
include_visible=False,
|
||||
include_activated=True,
|
||||
)
|
||||
if self._should_use_box_workspace_files(selected_skill):
|
||||
if self._should_use_box_workspace_files(host_location.selected_skill):
|
||||
return await self._edit_workspace_via_box(path, old_string, new_string, query)
|
||||
if not os.path.isfile(host_path):
|
||||
try:
|
||||
changed, error = self._edit_host_location(host_location, old_string, new_string)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return {'ok': False, 'error': f'File not found: {path}'}
|
||||
with open(host_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
content = f.read()
|
||||
count = content.count(old_string)
|
||||
if count == 0:
|
||||
return {'ok': False, 'error': 'old_string not found in file.'}
|
||||
if count > 1:
|
||||
return {'ok': False, 'error': f'old_string matches {count} locations; provide a more unique string.'}
|
||||
new_content = content.replace(old_string, new_string, 1)
|
||||
with open(host_path, 'w', encoding='utf-8') as f:
|
||||
f.write(new_content)
|
||||
self._refresh_skill_from_disk(query, selected_skill)
|
||||
if not changed:
|
||||
return {'ok': False, 'error': error or f'File not found: {path}'}
|
||||
self._refresh_skill_from_disk(query, host_location.selected_skill)
|
||||
return {'ok': True, 'path': path}
|
||||
|
||||
def _refresh_skill_from_disk(self, query: pipeline_query.Query, selected_skill: dict | None) -> None:
|
||||
@@ -934,55 +1358,19 @@ else:
|
||||
path = str(parameters.get('path', '/workspace') or '/workspace')
|
||||
self.ap.logger.info(f'glob tool invoked: query_id={query.query_id} pattern={pattern} path={path}')
|
||||
|
||||
host_path, selected_skill = self._resolve_host_path(
|
||||
host_location = self._resolve_host_location(
|
||||
query,
|
||||
path,
|
||||
include_visible=True,
|
||||
include_activated=True,
|
||||
)
|
||||
if self._should_use_box_workspace_files(selected_skill):
|
||||
if self._should_use_box_workspace_files(host_location.selected_skill):
|
||||
return await self._glob_workspace_via_box(path, pattern, query)
|
||||
|
||||
if not os.path.isdir(host_path):
|
||||
try:
|
||||
return self._glob_host_location(host_location, pattern, path)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return {'ok': False, 'error': f'Path is not a directory: {path}'}
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
base = Path(host_path)
|
||||
hits = list(base.rglob(pattern))
|
||||
|
||||
# Filter out skipped directories
|
||||
hits = [h for h in hits if not any(skip in h.parts for skip in _SKIP_DIRS)]
|
||||
|
||||
# Sort by mtime, newest first
|
||||
hits.sort(key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True)
|
||||
|
||||
total = len(hits)
|
||||
shown = hits[:_GLOB_MAX_MATCHES]
|
||||
|
||||
# Convert back to sandbox paths
|
||||
sandbox_paths = []
|
||||
output_bytes = 0
|
||||
truncated_by_bytes = False
|
||||
for h in shown:
|
||||
rel = os.path.relpath(str(h), host_path)
|
||||
sandbox_path = os.path.join(path, rel)
|
||||
entry_bytes = len(sandbox_path.encode('utf-8')) + (1 if sandbox_paths else 0)
|
||||
if output_bytes + entry_bytes > _DEFAULT_TOOL_RESULT_MAX_BYTES:
|
||||
truncated_by_bytes = True
|
||||
break
|
||||
sandbox_paths.append(sandbox_path)
|
||||
output_bytes += entry_bytes
|
||||
|
||||
return {
|
||||
'ok': True,
|
||||
'matches': sandbox_paths,
|
||||
'preview': '\n'.join(sandbox_paths),
|
||||
'total': total,
|
||||
'truncated': total > len(sandbox_paths) or truncated_by_bytes,
|
||||
'truncated_by': 'bytes' if truncated_by_bytes else ('matches' if total > len(sandbox_paths) else None),
|
||||
}
|
||||
|
||||
async def _invoke_grep(self, parameters: dict, query: pipeline_query.Query) -> dict:
|
||||
pattern = parameters['pattern']
|
||||
path = str(parameters.get('path', '/workspace') or '/workspace')
|
||||
@@ -990,99 +1378,36 @@ else:
|
||||
self.ap.logger.info(f'grep tool invoked: query_id={query.query_id} pattern={pattern} path={path}')
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
try:
|
||||
regex = re.compile(pattern)
|
||||
except re.error as e:
|
||||
return {'ok': False, 'error': f'Invalid regex: {e}'}
|
||||
|
||||
host_path, selected_skill = self._resolve_host_path(
|
||||
host_location = self._resolve_host_location(
|
||||
query,
|
||||
path,
|
||||
include_visible=True,
|
||||
include_activated=True,
|
||||
)
|
||||
if self._should_use_box_workspace_files(selected_skill):
|
||||
if self._should_use_box_workspace_files(host_location.selected_skill):
|
||||
return await self._grep_workspace_via_box(path, pattern, include, query)
|
||||
|
||||
if not os.path.exists(host_path):
|
||||
try:
|
||||
return self._grep_host_location(host_location, regex, include, path)
|
||||
except (FileNotFoundError, NotADirectoryError):
|
||||
return {'ok': False, 'error': f'Path not found: {path}'}
|
||||
|
||||
base = Path(host_path)
|
||||
|
||||
if base.is_file():
|
||||
files = [base]
|
||||
else:
|
||||
files = self._grep_walk(base, include)
|
||||
|
||||
matches = []
|
||||
output_bytes = 0
|
||||
truncated_by = None
|
||||
for fp in files:
|
||||
try:
|
||||
handle = fp.open('r', encoding='utf-8', errors='ignore')
|
||||
except OSError:
|
||||
continue
|
||||
with handle:
|
||||
for lineno, line in enumerate(handle, 1):
|
||||
if regex.search(line):
|
||||
rel = os.path.relpath(str(fp), host_path)
|
||||
sandbox_path = os.path.join(path, rel)
|
||||
content, line_truncated = self._truncate_grep_line(line.rstrip())
|
||||
entry = {
|
||||
'file': sandbox_path,
|
||||
'line': lineno,
|
||||
'content': content,
|
||||
}
|
||||
entry_bytes = len(json.dumps(entry, ensure_ascii=False).encode('utf-8')) + 1
|
||||
if output_bytes + entry_bytes > _DEFAULT_TOOL_RESULT_MAX_BYTES:
|
||||
truncated_by = 'bytes'
|
||||
break
|
||||
if line_truncated and truncated_by is None:
|
||||
truncated_by = 'line'
|
||||
matches.append(entry)
|
||||
output_bytes += entry_bytes
|
||||
if len(matches) >= _GREP_MAX_MATCHES:
|
||||
truncated_by = truncated_by or 'matches'
|
||||
break
|
||||
if truncated_by == 'bytes' or len(matches) >= _GREP_MAX_MATCHES:
|
||||
break
|
||||
if truncated_by == 'bytes' or len(matches) >= _GREP_MAX_MATCHES:
|
||||
break
|
||||
|
||||
return {
|
||||
'ok': True,
|
||||
'matches': matches,
|
||||
'total': len(matches),
|
||||
'truncated': truncated_by is not None,
|
||||
'truncated_by': truncated_by,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _grep_walk(root, include: str | None) -> list:
|
||||
"""Walk dir tree for grep, skipping junk dirs."""
|
||||
results = []
|
||||
for item in root.rglob(include or '*'):
|
||||
if any(skip in item.parts for skip in _SKIP_DIRS):
|
||||
continue
|
||||
if item.is_file():
|
||||
results.append(item)
|
||||
if len(results) >= _GREP_MAX_FILES:
|
||||
break
|
||||
return results
|
||||
|
||||
@staticmethod
|
||||
def _resolve_skill_host_path(selected_skill: dict, relative: str) -> str | None:
|
||||
def _resolve_skill_host_location(selected_skill: dict, relative: str) -> _HostLocation | None:
|
||||
package_root = str(selected_skill.get('package_root', '') or '').strip()
|
||||
if not package_root:
|
||||
return None
|
||||
|
||||
host_root = os.path.realpath(package_root)
|
||||
host_path = os.path.realpath(os.path.join(host_root, relative))
|
||||
if not (host_path == host_root or host_path.startswith(host_root + os.sep)):
|
||||
raise ValueError('Path escapes the skill package boundary.')
|
||||
return host_path
|
||||
relative_path = '/workspace' if relative in {'', '.'} else f'/workspace/{relative}'
|
||||
return _HostLocation(
|
||||
root=package_root,
|
||||
relative_parts=_relative_workspace_parts(relative_path),
|
||||
selected_skill=selected_skill,
|
||||
)
|
||||
|
||||
def _normalize_exec_result(self, result: dict) -> dict:
|
||||
normalized = dict(result)
|
||||
@@ -1122,9 +1447,9 @@ else:
|
||||
'truncated_by': 'bytes' if truncated else None,
|
||||
}
|
||||
|
||||
def _read_text_file_preview(self, host_path: str, parameters: dict) -> dict:
|
||||
def _read_text_file_preview(self, file_fd: int, parameters: dict, *, metadata: os.stat_result) -> dict:
|
||||
if self._read_encoding(parameters) == 'base64':
|
||||
return self._read_binary_file_chunk(host_path, parameters)
|
||||
return self._read_binary_file_chunk(file_fd, parameters, metadata=metadata)
|
||||
|
||||
offset = self._positive_int(parameters.get('offset'), default=1)
|
||||
max_lines = self._positive_int(
|
||||
@@ -1144,7 +1469,7 @@ else:
|
||||
truncated_by: str | None = None
|
||||
next_offset: int | None = None
|
||||
|
||||
with open(host_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
with os.fdopen(os.dup(file_fd), 'r', encoding='utf-8', errors='replace') as f:
|
||||
for line_number, line in enumerate(f, 1):
|
||||
if line_number < offset:
|
||||
continue
|
||||
@@ -1185,15 +1510,15 @@ else:
|
||||
'max_bytes': max_bytes,
|
||||
}
|
||||
|
||||
def _read_binary_file_chunk(self, host_path: str, parameters: dict) -> dict:
|
||||
def _read_binary_file_chunk(self, file_fd: int, parameters: dict, *, metadata: os.stat_result) -> dict:
|
||||
byte_offset = self._non_negative_int(parameters.get('byte_offset'), default=0)
|
||||
max_bytes = self._positive_int(
|
||||
parameters.get('max_bytes'),
|
||||
default=_DEFAULT_TOOL_RESULT_MAX_BYTES,
|
||||
max_value=_DEFAULT_TOOL_RESULT_MAX_BYTES,
|
||||
)
|
||||
size_bytes = os.path.getsize(host_path)
|
||||
with open(host_path, 'rb') as f:
|
||||
size_bytes = metadata.st_size
|
||||
with os.fdopen(os.dup(file_fd), 'rb') as f:
|
||||
f.seek(byte_offset)
|
||||
data = f.read(max_bytes + 1)
|
||||
chunk = data[:max_bytes]
|
||||
@@ -1210,19 +1535,6 @@ else:
|
||||
'max_bytes': max_bytes,
|
||||
}
|
||||
|
||||
def _write_host_file(self, host_path: str, content: str, parameters: dict) -> None:
|
||||
encoding, mode = self._write_options(parameters)
|
||||
if encoding == 'base64':
|
||||
try:
|
||||
data = base64.b64decode(content, validate=True)
|
||||
except Exception as exc:
|
||||
raise ValueError(f'invalid base64 content: {exc}') from exc
|
||||
with open(host_path, 'ab' if mode == 'append' else 'wb') as f:
|
||||
f.write(data)
|
||||
return
|
||||
with open(host_path, 'a' if mode == 'append' else 'w', encoding='utf-8') as f:
|
||||
f.write(content)
|
||||
|
||||
@staticmethod
|
||||
def _read_encoding(parameters: dict) -> str:
|
||||
return 'base64' if parameters.get('encoding') == 'base64' else 'text'
|
||||
|
||||
@@ -201,5 +201,14 @@ def should_prepare_skill_python_env(package_root: str | None) -> bool:
|
||||
return box_workspace.should_prepare_python_env(package_root)
|
||||
|
||||
|
||||
def wrap_skill_command_with_python_env(command: str, *, mount_path: str = '/workspace') -> str:
|
||||
return box_workspace.wrap_python_command_with_env(command, mount_path=mount_path).rstrip()
|
||||
def wrap_skill_command_with_python_env(
|
||||
command: str,
|
||||
*,
|
||||
mount_path: str = '/workspace',
|
||||
state_path: str | None = None,
|
||||
) -> str:
|
||||
return box_workspace.wrap_python_command_with_env(
|
||||
command,
|
||||
mount_path=mount_path,
|
||||
state_path=state_path,
|
||||
).rstrip()
|
||||
|
||||
@@ -65,12 +65,34 @@ class SkillToolLoader(loader.ToolLoader):
|
||||
return self._has_skill_manager() and self._sandbox_available
|
||||
|
||||
async def invoke_tool(self, name: str, parameters: dict, query) -> typing.Any:
|
||||
require_sandbox = getattr(
|
||||
getattr(self.ap, 'box_service', None),
|
||||
'require_workspace_sandbox',
|
||||
None,
|
||||
)
|
||||
if callable(require_sandbox):
|
||||
await require_sandbox(self._execution_context(query))
|
||||
if name == ACTIVATE_SKILL_TOOL_NAME:
|
||||
return await self._invoke_activate_skill(parameters, query)
|
||||
if name == REGISTER_SKILL_TOOL_NAME:
|
||||
return await self._invoke_register_skill(parameters, query)
|
||||
raise ValueError(f'Unknown skill tool: {name}')
|
||||
|
||||
@staticmethod
|
||||
def _execution_context(query) -> ExecutionContext:
|
||||
attached_context = getattr(query, '_execution_context', None)
|
||||
if isinstance(attached_context, ExecutionContext):
|
||||
return attached_context
|
||||
return ExecutionContext(
|
||||
instance_uuid=str(getattr(query, 'instance_uuid', '') or ''),
|
||||
workspace_uuid=str(getattr(query, 'workspace_uuid', '') or ''),
|
||||
placement_generation=getattr(query, 'placement_generation', 0) or 0,
|
||||
bot_uuid=getattr(query, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(query, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(query, 'query_uuid', None),
|
||||
entitlement_revision=getattr(query, 'entitlement_revision', 0),
|
||||
)
|
||||
|
||||
async def shutdown(self):
|
||||
pass
|
||||
|
||||
@@ -128,7 +150,8 @@ class SkillToolLoader(loader.ToolLoader):
|
||||
raise ValueError('path is required')
|
||||
|
||||
# Resolve sandbox path to host path
|
||||
host_path = self._resolve_workspace_directory(sandbox_path)
|
||||
execution_context = self._execution_context(query)
|
||||
host_path = self._resolve_workspace_directory(sandbox_path, execution_context)
|
||||
|
||||
# Get or create skill service
|
||||
skill_service = getattr(self.ap, 'skill_service', None)
|
||||
@@ -136,14 +159,6 @@ class SkillToolLoader(loader.ToolLoader):
|
||||
raise ValueError('Skill service not available')
|
||||
|
||||
# Scan and register the skill
|
||||
execution_context = ExecutionContext(
|
||||
instance_uuid=str(getattr(query, 'instance_uuid', '') or ''),
|
||||
workspace_uuid=str(getattr(query, 'workspace_uuid', '') or ''),
|
||||
placement_generation=getattr(query, 'placement_generation', 0) or 0,
|
||||
bot_uuid=getattr(query, 'bot_uuid', None),
|
||||
pipeline_uuid=getattr(query, 'pipeline_uuid', None),
|
||||
query_uuid=getattr(query, 'query_uuid', None),
|
||||
)
|
||||
scanned = await skill_service.scan_directory_async(execution_context, host_path)
|
||||
|
||||
# Override name if provided
|
||||
@@ -170,10 +185,19 @@ class SkillToolLoader(loader.ToolLoader):
|
||||
'skill': created,
|
||||
}
|
||||
|
||||
def _resolve_workspace_directory(self, sandbox_path: str) -> str:
|
||||
def _resolve_workspace_directory(
|
||||
self,
|
||||
sandbox_path: str,
|
||||
execution_context: ExecutionContext,
|
||||
) -> str:
|
||||
"""Resolve sandbox path to host filesystem path."""
|
||||
box_service = getattr(self.ap, 'box_service', None)
|
||||
workspace_root = getattr(box_service, 'default_workspace', None)
|
||||
tenant_workspace = getattr(box_service, '_tenant_workspace', None)
|
||||
workspace_root = (
|
||||
tenant_workspace(execution_context)
|
||||
if callable(tenant_workspace)
|
||||
else getattr(box_service, 'default_workspace', None)
|
||||
)
|
||||
if not workspace_root:
|
||||
raise ValueError('No default workspace configured')
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import typing
|
||||
import time
|
||||
import inspect
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
|
||||
@@ -35,6 +36,36 @@ class ToolManager:
|
||||
def __init__(self, ap: app.Application):
|
||||
self.ap = ap
|
||||
|
||||
async def _bind_plugin_workspace(self, context: TenantContext) -> None:
|
||||
"""Select the tenant before any plugin catalog lookup.
|
||||
|
||||
Tool discovery happens before invocation, so relying on ``call_tool``
|
||||
to bind the Workspace is too late and can expose another task's
|
||||
catalog in a shared Runtime.
|
||||
"""
|
||||
|
||||
connector = getattr(self.ap, 'plugin_connector', None)
|
||||
require_context = getattr(connector, 'require_workspace_context', None)
|
||||
if require_context is None:
|
||||
return
|
||||
result = require_context(context)
|
||||
if inspect.isawaitable(result):
|
||||
await result
|
||||
|
||||
async def _workspace_sandbox_available(self, context: TenantContext) -> bool:
|
||||
"""Resolve the Workspace capability before exposing sandbox tools."""
|
||||
|
||||
box_service = getattr(self.ap, 'box_service', None)
|
||||
checker = getattr(box_service, 'is_workspace_sandbox_available', None)
|
||||
if not callable(checker):
|
||||
# Compatibility for OSS embedders and isolated manager tests. The
|
||||
# BoxService execution path remains the final authority.
|
||||
return True
|
||||
try:
|
||||
return bool(await checker(context))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
async def initialize(self):
|
||||
from langbot.pkg.utils import importutil
|
||||
from langbot.pkg.provider.tools import loaders
|
||||
@@ -65,10 +96,13 @@ class ToolManager:
|
||||
include_skill_authoring: bool = False,
|
||||
include_mcp_resource_tools: bool = True,
|
||||
) -> list[resource_tool.LLMTool]:
|
||||
await self._bind_plugin_workspace(context)
|
||||
all_functions: list[resource_tool.LLMTool] = []
|
||||
|
||||
all_functions.extend(await self.native_tool_loader.get_tools())
|
||||
if include_skill_authoring:
|
||||
sandbox_available = await self._workspace_sandbox_available(context)
|
||||
if sandbox_available:
|
||||
all_functions.extend(await self.native_tool_loader.get_tools())
|
||||
if include_skill_authoring and sandbox_available:
|
||||
all_functions.extend(await self.skill_tool_loader.get_tools())
|
||||
all_functions.extend(await self.plugin_tool_loader.get_tools(bound_plugins))
|
||||
all_functions.extend(
|
||||
@@ -89,6 +123,7 @@ class ToolManager:
|
||||
include_skill_authoring: bool = False,
|
||||
include_mcp_resource_tools: bool = False,
|
||||
) -> list[dict[str, typing.Any]]:
|
||||
await self._bind_plugin_workspace(context)
|
||||
catalog: list[dict[str, typing.Any]] = []
|
||||
|
||||
def append_tools(source: str, source_name: str, tools: list[resource_tool.LLMTool]) -> None:
|
||||
@@ -104,8 +139,10 @@ class ToolManager:
|
||||
}
|
||||
)
|
||||
|
||||
append_tools('builtin', 'LangBot', await self.native_tool_loader.get_tools())
|
||||
if include_skill_authoring:
|
||||
sandbox_available = await self._workspace_sandbox_available(context)
|
||||
if sandbox_available:
|
||||
append_tools('builtin', 'LangBot', await self.native_tool_loader.get_tools())
|
||||
if include_skill_authoring and sandbox_available:
|
||||
append_tools('skill', 'LangBot', await self.skill_tool_loader.get_tools())
|
||||
catalog.extend(await self.plugin_tool_loader.get_tool_catalog(bound_plugins))
|
||||
|
||||
@@ -121,14 +158,20 @@ class ToolManager:
|
||||
|
||||
async def get_tool_by_name(self, context: TenantContext, name: str) -> tool_loader.ToolLookupResult | None:
|
||||
"""Get tool by name from any active loader."""
|
||||
for active_loader in (
|
||||
self.native_tool_loader,
|
||||
self.plugin_tool_loader,
|
||||
self.skill_tool_loader,
|
||||
):
|
||||
await self._bind_plugin_workspace(context)
|
||||
sandbox_available = await self._workspace_sandbox_available(context)
|
||||
if sandbox_available:
|
||||
tool = await self.native_tool_loader.get_tool(name)
|
||||
if tool:
|
||||
return tool
|
||||
for active_loader in (self.plugin_tool_loader,):
|
||||
tool = await active_loader.get_tool(name)
|
||||
if tool:
|
||||
return tool
|
||||
if sandbox_available:
|
||||
tool = await self.skill_tool_loader.get_tool(name)
|
||||
if tool:
|
||||
return tool
|
||||
|
||||
return await self.mcp_tool_loader.get_tool(context, name)
|
||||
|
||||
@@ -237,7 +280,10 @@ class ToolManager:
|
||||
async def execute_func_call(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
|
||||
from langbot.pkg.telemetry import features as telemetry_features
|
||||
|
||||
if await self.native_tool_loader.has_tool(name):
|
||||
execution_context = get_query_execution_context(query)
|
||||
await self._bind_plugin_workspace(execution_context)
|
||||
sandbox_available = await self._workspace_sandbox_available(execution_context)
|
||||
if sandbox_available and await self.native_tool_loader.has_tool(name):
|
||||
telemetry_features.increment(query, 'tool_calls', 'native')
|
||||
return await self._invoke_tool_with_monitoring(
|
||||
source='native',
|
||||
@@ -255,7 +301,6 @@ class ToolManager:
|
||||
query=query,
|
||||
invoke=lambda: self.plugin_tool_loader.invoke_tool(name, parameters, query),
|
||||
)
|
||||
execution_context = get_query_execution_context(query)
|
||||
if await self.mcp_tool_loader.has_tool(execution_context, name):
|
||||
telemetry_features.increment(query, 'tool_calls', 'mcp')
|
||||
return await self._invoke_tool_with_monitoring(
|
||||
@@ -265,7 +310,7 @@ class ToolManager:
|
||||
query=query,
|
||||
invoke=lambda: self.mcp_tool_loader.invoke_tool(name, parameters, query),
|
||||
)
|
||||
if await self.skill_tool_loader.has_tool(name):
|
||||
if sandbox_available and await self.skill_tool_loader.has_tool(name):
|
||||
telemetry_features.increment(query, 'tool_calls', 'skill')
|
||||
return await self._invoke_tool_with_monitoring(
|
||||
source='skill',
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import io
|
||||
import mimetypes
|
||||
import os.path
|
||||
@@ -14,6 +15,7 @@ from langbot.pkg.api.http.authz import WorkspaceRequiredError
|
||||
from langbot.pkg.api.http.context import ExecutionContext, RequestContext
|
||||
from langbot.pkg.api.http.service.tenant import TenantContext, require_workspace_uuid
|
||||
from langbot.pkg.core import app, taskmgr
|
||||
from langbot.pkg.core.task_boundary import run_in_workspace_uow
|
||||
from langbot.pkg.entity.persistence import rag as persistence_rag
|
||||
from langbot.pkg.workspace.errors import WorkspaceNotFoundError
|
||||
|
||||
@@ -90,16 +92,23 @@ class RuntimeKnowledgeBase(KnowledgeBaseInterface):
|
||||
task_context: taskmgr.TaskContext,
|
||||
parser_plugin_id: str | None = None,
|
||||
):
|
||||
await self._assert_execution_context(execution_context)
|
||||
await run_in_workspace_uow(
|
||||
self.ap,
|
||||
execution_context.workspace_uuid,
|
||||
lambda: self._assert_execution_context(execution_context),
|
||||
)
|
||||
self._require_upload_object_key(execution_context, file.file_name)
|
||||
try:
|
||||
# set file status to processing
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.File.uuid == file.uuid)
|
||||
.values(status='processing')
|
||||
)
|
||||
status_visible = False
|
||||
for retry_delay in (0.0, 0.01, 0.05, 0.1):
|
||||
if retry_delay:
|
||||
await asyncio.sleep(retry_delay)
|
||||
if await self._set_file_status(execution_context, file.uuid, 'processing'):
|
||||
status_visible = True
|
||||
break
|
||||
if not status_visible:
|
||||
raise WorkspaceNotFoundError('Knowledge file was not committed before its background task started')
|
||||
|
||||
task_context.set_current_action('Processing file')
|
||||
|
||||
@@ -152,13 +161,8 @@ class RuntimeKnowledgeBase(KnowledgeBaseInterface):
|
||||
raise Exception(error_msg)
|
||||
|
||||
# set file status to completed
|
||||
await self._assert_execution_context(execution_context)
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.File.uuid == file.uuid)
|
||||
.values(status='completed')
|
||||
)
|
||||
if not await self._set_file_status(execution_context, file.uuid, 'completed'):
|
||||
raise WorkspaceNotFoundError('Knowledge file not found')
|
||||
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Error storing file {file.uuid}: {e}')
|
||||
@@ -166,16 +170,10 @@ class RuntimeKnowledgeBase(KnowledgeBaseInterface):
|
||||
# A stale placement is fenced from all writes, including failure
|
||||
# status updates from an old background task.
|
||||
try:
|
||||
await self._assert_execution_context(execution_context)
|
||||
if not await self._set_file_status(execution_context, file.uuid, 'failed'):
|
||||
raise WorkspaceNotFoundError('Knowledge file not found')
|
||||
except Exception:
|
||||
self.ap.logger.warning(f'Skipping stale RAG task status update for file {file.uuid}')
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.File.uuid == file.uuid)
|
||||
.values(status='failed')
|
||||
)
|
||||
|
||||
raise
|
||||
finally:
|
||||
@@ -191,6 +189,37 @@ class RuntimeKnowledgeBase(KnowledgeBaseInterface):
|
||||
except (WorkspaceRequiredError, WorkspaceNotFoundError):
|
||||
self.ap.logger.warning(f'Skipping stale RAG upload cleanup for file {file.uuid}')
|
||||
|
||||
async def _set_file_status(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
file_uuid: str,
|
||||
status: str,
|
||||
) -> bool:
|
||||
"""Commit one detached-task status transition in its own tenant UoW."""
|
||||
|
||||
async def update() -> bool:
|
||||
await self._assert_execution_context(execution_context)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.File)
|
||||
.where(persistence_rag.File.workspace_uuid == execution_context.workspace_uuid)
|
||||
.where(persistence_rag.File.uuid == file_uuid)
|
||||
.values(status=status)
|
||||
)
|
||||
return getattr(result, 'rowcount', 0) > 0
|
||||
|
||||
persistence_mgr = self.ap.persistence_mgr
|
||||
managed_mode = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) in {
|
||||
'cloud_runtime',
|
||||
'oss_compat',
|
||||
}
|
||||
tenant_uow = getattr(persistence_mgr, 'tenant_uow', None)
|
||||
if managed_mode:
|
||||
if not callable(tenant_uow):
|
||||
raise RuntimeError('Knowledge tasks require an explicit tenant UoW')
|
||||
async with tenant_uow(execution_context.workspace_uuid):
|
||||
return await update()
|
||||
return await update()
|
||||
|
||||
async def store_file(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
@@ -735,7 +764,36 @@ class RAGManager:
|
||||
|
||||
self.knowledge_bases = {}
|
||||
|
||||
# Load knowledge bases
|
||||
list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
if not callable(list_bindings) or not callable(tenant_uow):
|
||||
raise RuntimeError('Cloud knowledge loading requires explicit instance discovery and tenant UoWs')
|
||||
for binding in await list_bindings():
|
||||
async with tenant_uow(binding.workspace_uuid):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(persistence_rag.KnowledgeBase)
|
||||
.where(persistence_rag.KnowledgeBase.workspace_uuid == binding.workspace_uuid)
|
||||
.order_by(persistence_rag.KnowledgeBase.uuid)
|
||||
)
|
||||
for knowledge_base in result.all():
|
||||
try:
|
||||
await self.load_knowledge_base(
|
||||
ExecutionContext(
|
||||
instance_uuid=binding.instance_uuid,
|
||||
workspace_uuid=binding.workspace_uuid,
|
||||
placement_generation=binding.placement_generation,
|
||||
),
|
||||
knowledge_base,
|
||||
)
|
||||
except Exception as e:
|
||||
self.ap.logger.error(
|
||||
f'Error loading knowledge base {knowledge_base.uuid}: {e}\n{traceback.format_exc()}'
|
||||
)
|
||||
return
|
||||
|
||||
# Compatibility path for isolated manager tests and older embedders.
|
||||
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_rag.KnowledgeBase))
|
||||
knowledge_bases = result.all()
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import contextlib
|
||||
import json
|
||||
import typing
|
||||
import httpx
|
||||
@@ -10,6 +11,7 @@ import sqlalchemy
|
||||
|
||||
from ..core import app as core_app
|
||||
from ..entity.persistence.metadata import Metadata
|
||||
from ..persistence.tenant_uow import CrossScopeTransactionError
|
||||
from ..utils import constants
|
||||
|
||||
SURVEY_TRIGGERED_KEY = 'survey_triggered_events'
|
||||
@@ -36,15 +38,41 @@ class SurveyManager:
|
||||
await self._load_triggered_events()
|
||||
await self._load_bot_response_count()
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _instance_transaction(self):
|
||||
"""Bind instance-global survey metadata to an explicit Cloud transaction."""
|
||||
|
||||
persistence_mgr = self.ap.persistence_mgr
|
||||
try:
|
||||
active_session = getattr(persistence_mgr, 'current_session', lambda: None)()
|
||||
except CrossScopeTransactionError:
|
||||
# A newly-created child task inherited its parent's ContextVar;
|
||||
# opening an explicit UoW below gives it an independent session.
|
||||
active_session = None
|
||||
if active_session is not None:
|
||||
yield
|
||||
return
|
||||
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
instance_uow = getattr(persistence_mgr, 'instance_discovery_uow', None)
|
||||
if not callable(instance_uow):
|
||||
raise RuntimeError('Cloud survey metadata requires an explicit instance UoW')
|
||||
async with instance_uow(self.ap.workspace_service.instance_uuid):
|
||||
yield
|
||||
return
|
||||
yield
|
||||
|
||||
async def _load_triggered_events(self):
|
||||
"""Load previously triggered events from metadata table."""
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == SURVEY_TRIGGERED_KEY)
|
||||
)
|
||||
row = result.first()
|
||||
if row:
|
||||
self._triggered_events = set(json.loads(row[0].value))
|
||||
async with self._instance_transaction():
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata.value).where(Metadata.key == SURVEY_TRIGGERED_KEY)
|
||||
)
|
||||
value = result.scalar_one_or_none()
|
||||
if value is not None:
|
||||
self._triggered_events = set(json.loads(value))
|
||||
except Exception:
|
||||
self._triggered_events = set()
|
||||
|
||||
@@ -52,17 +80,18 @@ class SurveyManager:
|
||||
"""Persist triggered events to metadata table."""
|
||||
try:
|
||||
value = json.dumps(list(self._triggered_events))
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == SURVEY_TRIGGERED_KEY)
|
||||
)
|
||||
if result.first():
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == SURVEY_TRIGGERED_KEY).values(value=value)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key=SURVEY_TRIGGERED_KEY, value=value)
|
||||
async with self._instance_transaction():
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata.value).where(Metadata.key == SURVEY_TRIGGERED_KEY)
|
||||
)
|
||||
if result.scalar_one_or_none() is not None:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == SURVEY_TRIGGERED_KEY).values(value=value)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key=SURVEY_TRIGGERED_KEY, value=value)
|
||||
)
|
||||
except Exception as e:
|
||||
self.ap.logger.debug(f'Failed to save survey triggered events: {e}')
|
||||
|
||||
@@ -75,12 +104,13 @@ class SurveyManager:
|
||||
async def _load_bot_response_count(self):
|
||||
"""Load the persisted successful bot response count from metadata table."""
|
||||
try:
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY)
|
||||
)
|
||||
row = result.first()
|
||||
if row:
|
||||
self._bot_response_count = int(row[0].value)
|
||||
async with self._instance_transaction():
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata.value).where(Metadata.key == BOT_RESPONSE_COUNT_KEY)
|
||||
)
|
||||
value = result.scalar_one_or_none()
|
||||
if value is not None:
|
||||
self._bot_response_count = int(value)
|
||||
except Exception:
|
||||
self._bot_response_count = 0
|
||||
|
||||
@@ -88,17 +118,18 @@ class SurveyManager:
|
||||
"""Persist the successful bot response count to metadata table."""
|
||||
try:
|
||||
value = str(self._bot_response_count)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY)
|
||||
)
|
||||
if result.first():
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY).values(value=value)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key=BOT_RESPONSE_COUNT_KEY, value=value)
|
||||
async with self._instance_transaction():
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(Metadata.value).where(Metadata.key == BOT_RESPONSE_COUNT_KEY)
|
||||
)
|
||||
if result.scalar_one_or_none() is not None:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY).values(value=value)
|
||||
)
|
||||
else:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.insert(Metadata).values(key=BOT_RESPONSE_COUNT_KEY, value=value)
|
||||
)
|
||||
except Exception as e:
|
||||
self.ap.logger.debug(f'Failed to save survey bot response count: {e}')
|
||||
|
||||
|
||||
@@ -30,6 +30,20 @@ HEARTBEAT_INTERVAL_SECONDS = 24 * 3600
|
||||
async def _count(ap: core_app.Application, table) -> int:
|
||||
"""Count rows in a persistence table; -1 when unavailable."""
|
||||
try:
|
||||
persistence_mgr = ap.persistence_mgr
|
||||
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
|
||||
if cloud_runtime:
|
||||
tenant_uow = getattr(persistence_mgr, 'tenant_uow', None)
|
||||
if not callable(tenant_uow):
|
||||
return -1
|
||||
total = 0
|
||||
for binding in await ap.workspace_service.list_active_execution_bindings():
|
||||
async with tenant_uow(binding.workspace_uuid):
|
||||
result = await persistence_mgr.execute_async(
|
||||
sqlalchemy.select(sqlalchemy.func.count()).select_from(table)
|
||||
)
|
||||
total += int(result.scalar() or 0)
|
||||
return total
|
||||
result = await ap.persistence_mgr.execute_async(sqlalchemy.select(sqlalchemy.func.count()).select_from(table))
|
||||
return int(result.scalar() or 0)
|
||||
except Exception:
|
||||
|
||||
+183
-29
@@ -64,9 +64,26 @@ class VectorDBManager:
|
||||
|
||||
# Get pgvector configuration
|
||||
pgvector_config = kb_config.get('pgvector', {})
|
||||
use_business_database = pgvector_config.get('use_business_database', False)
|
||||
allowed_dimensions = pgvector_config.get(
|
||||
'allowed_dimensions',
|
||||
[384, 512, 768, 1024, 1536],
|
||||
)
|
||||
common_options = {
|
||||
'use_business_database': use_business_database,
|
||||
'allowed_dimensions': allowed_dimensions,
|
||||
}
|
||||
if use_business_database:
|
||||
self.vector_db = PgVectorDatabase(self.ap, **common_options)
|
||||
self.ap.logger.info('Initialized pgvector on the shared business PostgreSQL database.')
|
||||
return
|
||||
connection_string = pgvector_config.get('connection_string')
|
||||
if connection_string:
|
||||
self.vector_db = PgVectorDatabase(self.ap, connection_string=connection_string)
|
||||
self.vector_db = PgVectorDatabase(
|
||||
self.ap,
|
||||
connection_string=connection_string,
|
||||
**common_options,
|
||||
)
|
||||
else:
|
||||
# Use individual parameters
|
||||
host = pgvector_config.get('host', 'localhost')
|
||||
@@ -75,7 +92,13 @@ class VectorDBManager:
|
||||
user = pgvector_config.get('user', 'postgres')
|
||||
password = pgvector_config.get('password', 'postgres')
|
||||
self.vector_db = PgVectorDatabase(
|
||||
self.ap, host=host, port=port, database=database, user=user, password=password
|
||||
self.ap,
|
||||
host=host,
|
||||
port=port,
|
||||
database=database,
|
||||
user=user,
|
||||
password=password,
|
||||
**common_options,
|
||||
)
|
||||
self.ap.logger.info('Initialized pgvector database backend.')
|
||||
|
||||
@@ -156,23 +179,24 @@ class VectorDBManager:
|
||||
"""
|
||||
|
||||
await self._validate_execution_context(execution_context)
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
persistence_rag.KnowledgeBase.collection_id,
|
||||
persistence_rag.KnowledgeBase.legacy_vector_collection,
|
||||
persistence_workspace.Workspace.source,
|
||||
async with self.ap.persistence_mgr.tenant_uow(execution_context.workspace_uuid):
|
||||
result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
persistence_rag.KnowledgeBase.collection_id,
|
||||
persistence_rag.KnowledgeBase.legacy_vector_collection,
|
||||
persistence_workspace.Workspace.source,
|
||||
)
|
||||
.join(
|
||||
persistence_workspace.Workspace,
|
||||
persistence_workspace.Workspace.uuid == persistence_rag.KnowledgeBase.workspace_uuid,
|
||||
)
|
||||
.where(
|
||||
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
|
||||
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
|
||||
persistence_workspace.Workspace.instance_uuid == execution_context.instance_uuid,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
.join(
|
||||
persistence_workspace.Workspace,
|
||||
persistence_workspace.Workspace.uuid == persistence_rag.KnowledgeBase.workspace_uuid,
|
||||
)
|
||||
.where(
|
||||
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
|
||||
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
|
||||
persistence_workspace.Workspace.instance_uuid == execution_context.instance_uuid,
|
||||
)
|
||||
.limit(1)
|
||||
)
|
||||
row = result.first()
|
||||
if row is None:
|
||||
raise WorkspaceNotFoundError('Knowledge base not found')
|
||||
@@ -194,6 +218,58 @@ class VectorDBManager:
|
||||
|
||||
return self.physical_collection_name(execution_context, knowledge_base_uuid)
|
||||
|
||||
def _pgvector_database(self):
|
||||
from .vdbs.pgvector_db import PgVectorDatabase
|
||||
|
||||
return self.vector_db if isinstance(self.vector_db, PgVectorDatabase) else None
|
||||
|
||||
async def _resolve_pgvector_scope(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
knowledge_base_uuid: str,
|
||||
*,
|
||||
expected_dimension: int | None,
|
||||
initialize_dimension: bool,
|
||||
):
|
||||
"""Bind and verify the server-owned knowledge-base vector dimension."""
|
||||
|
||||
from .vdbs.pgvector_db import PgVectorScope
|
||||
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is None: # pragma: no cover - private call invariant
|
||||
raise RuntimeError('pgvector scope requested for another vector backend')
|
||||
if expected_dimension is not None and expected_dimension not in pgvector.allowed_dimensions:
|
||||
raise ValueError(f'Embedding dimension {expected_dimension} is not enabled for this deployment')
|
||||
|
||||
async with self.ap.persistence_mgr.tenant_uow(execution_context.workspace_uuid):
|
||||
query = sqlalchemy.select(persistence_rag.KnowledgeBase.embedding_dimension).where(
|
||||
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
|
||||
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
|
||||
)
|
||||
current_dimension = (await self.ap.persistence_mgr.execute_async(query)).scalar_one_or_none()
|
||||
if current_dimension is None and expected_dimension is not None and initialize_dimension:
|
||||
await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.update(persistence_rag.KnowledgeBase)
|
||||
.where(
|
||||
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
|
||||
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
|
||||
persistence_rag.KnowledgeBase.embedding_dimension.is_(None),
|
||||
)
|
||||
.values(embedding_dimension=expected_dimension)
|
||||
)
|
||||
current_dimension = (await self.ap.persistence_mgr.execute_async(query)).scalar_one_or_none()
|
||||
|
||||
if expected_dimension is not None and current_dimension != expected_dimension:
|
||||
if current_dimension is None:
|
||||
raise ValueError('Knowledge base has no selected pgvector embedding dimension')
|
||||
raise ValueError(f'Knowledge base embedding dimension is {current_dimension}, not {expected_dimension}')
|
||||
|
||||
return PgVectorScope(
|
||||
workspace_uuid=execution_context.workspace_uuid,
|
||||
knowledge_base_uuid=knowledge_base_uuid,
|
||||
embedding_dimension=current_dimension,
|
||||
)
|
||||
|
||||
async def upsert(
|
||||
self,
|
||||
execution_context: ExecutionContext,
|
||||
@@ -219,6 +295,25 @@ class VectorDBManager:
|
||||
}
|
||||
for item in source_metadata
|
||||
]
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
if not vectors:
|
||||
return
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=len(vectors[0]),
|
||||
initialize_dimension=True,
|
||||
)
|
||||
await pgvector.add_embeddings(
|
||||
collection=collection_name,
|
||||
ids=ids,
|
||||
embeddings_list=vectors,
|
||||
metadatas=scoped_metadata,
|
||||
documents=documents,
|
||||
scope=scope,
|
||||
)
|
||||
return
|
||||
await self.vector_db.add_embeddings(
|
||||
collection=collection_name,
|
||||
ids=ids,
|
||||
@@ -248,15 +343,34 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
results = await self.vector_db.search(
|
||||
collection=collection_name,
|
||||
query_embedding=query_vector,
|
||||
k=limit,
|
||||
search_type=search_type,
|
||||
query_text=query_text,
|
||||
filter=filter,
|
||||
vector_weight=vector_weight,
|
||||
)
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=len(query_vector),
|
||||
initialize_dimension=False,
|
||||
)
|
||||
results = await pgvector.search(
|
||||
collection=collection_name,
|
||||
query_embedding=query_vector,
|
||||
k=limit,
|
||||
search_type=search_type,
|
||||
query_text=query_text,
|
||||
filter=filter,
|
||||
vector_weight=vector_weight,
|
||||
scope=scope,
|
||||
)
|
||||
else:
|
||||
results = await self.vector_db.search(
|
||||
collection=collection_name,
|
||||
query_embedding=query_vector,
|
||||
k=limit,
|
||||
search_type=search_type,
|
||||
query_text=query_text,
|
||||
filter=filter,
|
||||
vector_weight=vector_weight,
|
||||
)
|
||||
|
||||
if not results or 'ids' not in results or not results['ids']:
|
||||
return []
|
||||
@@ -297,8 +411,20 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
pgvector = self._pgvector_database()
|
||||
scope = None
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=None,
|
||||
initialize_dimension=False,
|
||||
)
|
||||
for file_id in file_ids:
|
||||
await self.vector_db.delete_by_file_id(collection_name, file_id)
|
||||
if pgvector is not None:
|
||||
await pgvector.delete_by_file_id(collection_name, file_id, scope=scope)
|
||||
else:
|
||||
await self.vector_db.delete_by_file_id(collection_name, file_id)
|
||||
|
||||
async def delete_collection(
|
||||
self,
|
||||
@@ -311,7 +437,17 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
await self.vector_db.delete_collection(collection_name)
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=None,
|
||||
initialize_dimension=False,
|
||||
)
|
||||
await pgvector.delete_collection(collection_name, scope=scope)
|
||||
else:
|
||||
await self.vector_db.delete_collection(collection_name)
|
||||
|
||||
async def delete_by_filter(
|
||||
self,
|
||||
@@ -328,6 +464,15 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=None,
|
||||
initialize_dimension=False,
|
||||
)
|
||||
return await pgvector.delete_by_filter(collection_name, filter, scope=scope)
|
||||
return await self.vector_db.delete_by_filter(collection_name, filter)
|
||||
|
||||
async def list_by_filter(
|
||||
@@ -347,4 +492,13 @@ class VectorDBManager:
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
)
|
||||
pgvector = self._pgvector_database()
|
||||
if pgvector is not None:
|
||||
scope = await self._resolve_pgvector_scope(
|
||||
execution_context,
|
||||
knowledge_base_uuid,
|
||||
expected_dimension=None,
|
||||
initialize_dimension=False,
|
||||
)
|
||||
return await pgvector.list_by_filter(collection_name, filter, limit, offset, scope=scope)
|
||||
return await self.vector_db.list_by_filter(collection_name, filter, limit, offset)
|
||||
|
||||
@@ -1,22 +1,31 @@
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict
|
||||
from sqlalchemy import create_engine, text, Column, String, Text
|
||||
from sqlalchemy.orm import declarative_base
|
||||
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
|
||||
|
||||
import contextlib
|
||||
import dataclasses
|
||||
from collections.abc import AsyncIterator
|
||||
from typing import Any
|
||||
|
||||
import sqlalchemy
|
||||
from pgvector.sqlalchemy import Vector
|
||||
from langbot.pkg.vector.vdb import VectorDatabase
|
||||
from langbot.pkg.vector.filter_utils import normalize_filter, strip_unsupported_fields
|
||||
from sqlalchemy.dialects.postgresql import insert as postgresql_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
|
||||
from sqlalchemy.orm import declarative_base
|
||||
|
||||
from langbot.pkg.core import app
|
||||
from langbot.pkg.vector.filter_utils import normalize_filter, strip_unsupported_fields
|
||||
from langbot.pkg.vector.vdb import VectorDatabase
|
||||
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
DEFAULT_ALLOWED_DIMENSIONS = (384, 512, 768, 1024, 1536)
|
||||
|
||||
# pgvector schema only stores these metadata fields.
|
||||
_PG_SUPPORTED_FIELDS = {'text', 'file_id', 'chunk_uuid'}
|
||||
|
||||
# Callers use canonical metadata key 'uuid' but pgvector stores it as 'chunk_uuid'.
|
||||
_PG_FIELD_ALIASES = {'uuid': 'chunk_uuid'}
|
||||
|
||||
# Map schema field names to SQLAlchemy columns (resolved lazily from PgVectorEntry).
|
||||
_PG_COLUMN_MAP = {
|
||||
'text': 'text',
|
||||
'file_id': 'file_id',
|
||||
@@ -24,21 +33,50 @@ _PG_COLUMN_MAP = {
|
||||
}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, slots=True)
|
||||
class PgVectorScope:
|
||||
"""Trusted relational tenant key for one knowledge-base operation."""
|
||||
|
||||
workspace_uuid: str
|
||||
knowledge_base_uuid: str
|
||||
embedding_dimension: int | None = None
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
for field_name in ('workspace_uuid', 'knowledge_base_uuid'):
|
||||
value = getattr(self, field_name)
|
||||
if not isinstance(value, str) or not value.strip():
|
||||
raise ValueError(f'{field_name} must not be empty')
|
||||
object.__setattr__(self, field_name, value.strip())
|
||||
dimension = self.embedding_dimension
|
||||
if dimension is not None and (isinstance(dimension, bool) or not isinstance(dimension, int) or dimension <= 0):
|
||||
raise ValueError('embedding_dimension must be a positive integer')
|
||||
|
||||
|
||||
class PgVectorEntry(Base):
|
||||
"""SQLAlchemy model for pgvector entries"""
|
||||
"""Tenant-scoped pgvector row created only by release/OSS migrations."""
|
||||
|
||||
__tablename__ = 'langbot_vectors'
|
||||
|
||||
id = Column(String, primary_key=True)
|
||||
collection = Column(String, index=True, nullable=False)
|
||||
embedding = Column(Vector(1536)) # Default dimension, will be created dynamically
|
||||
text = Column(Text)
|
||||
file_id = Column(String, index=True)
|
||||
chunk_uuid = Column(String)
|
||||
workspace_uuid = sqlalchemy.Column(sqlalchemy.String(36), primary_key=True)
|
||||
knowledge_base_uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
vector_id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
|
||||
embedding_dimension = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
|
||||
embedding = sqlalchemy.Column(Vector(), nullable=False)
|
||||
text = sqlalchemy.Column(sqlalchemy.Text)
|
||||
file_id = sqlalchemy.Column(sqlalchemy.String(255), index=True)
|
||||
chunk_uuid = sqlalchemy.Column(sqlalchemy.String(255))
|
||||
|
||||
__table_args__ = (
|
||||
sqlalchemy.CheckConstraint(
|
||||
'vector_dims(embedding) = embedding_dimension',
|
||||
name='ck_langbot_vectors_embedding_dimension',
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_pg_conditions(filter_dict: dict[str, Any]) -> list:
|
||||
"""Translate canonical filter dict into a list of SQLAlchemy conditions."""
|
||||
"""Translate canonical filter dict into SQLAlchemy conditions."""
|
||||
|
||||
triples = normalize_filter(filter_dict)
|
||||
triples = strip_unsupported_fields(triples, _PG_SUPPORTED_FIELDS, _PG_FIELD_ALIASES)
|
||||
|
||||
@@ -65,83 +103,139 @@ def _build_pg_conditions(filter_dict: dict[str, Any]) -> list:
|
||||
|
||||
|
||||
class PgVectorDatabase(VectorDatabase):
|
||||
"""PostgreSQL with pgvector extension database implementation"""
|
||||
"""PostgreSQL vector adapter with explicit Workspace/RLS scope.
|
||||
|
||||
Cloud reuses the business database engine and never performs DDL. OSS can
|
||||
still opt into a standalone pgvector database; that compatibility mode may
|
||||
create a fresh schema, but it uses the same explicit tenant keys.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ap: app.Application,
|
||||
connection_string: str = None,
|
||||
connection_string: str | None = None,
|
||||
host: str = 'localhost',
|
||||
port: int = 5432,
|
||||
database: str = 'langbot',
|
||||
user: str = 'postgres',
|
||||
password: str = 'postgres',
|
||||
):
|
||||
"""Initialize pgvector database
|
||||
|
||||
Args:
|
||||
ap: Application instance
|
||||
connection_string: Full PostgreSQL connection string (overrides other params)
|
||||
host: PostgreSQL host
|
||||
port: PostgreSQL port
|
||||
database: Database name
|
||||
user: Database user
|
||||
password: Database password
|
||||
"""
|
||||
*,
|
||||
use_business_database: bool = False,
|
||||
allowed_dimensions: list[int] | tuple[int, ...] = DEFAULT_ALLOWED_DIMENSIONS,
|
||||
) -> None:
|
||||
self.ap = ap
|
||||
self.use_business_database = use_business_database
|
||||
self.allowed_dimensions = self._normalize_allowed_dimensions(allowed_dimensions)
|
||||
self.engine = None
|
||||
self.async_engine = None
|
||||
self.AsyncSessionLocal: async_sessionmaker[AsyncSession] | None = None
|
||||
|
||||
if use_business_database:
|
||||
persistence_mgr = getattr(ap, 'persistence_mgr', None)
|
||||
if persistence_mgr is None:
|
||||
raise RuntimeError('Shared pgvector requires the initialized business persistence manager')
|
||||
business_engine = persistence_mgr.get_db_engine()
|
||||
if business_engine.dialect.name != 'postgresql':
|
||||
raise RuntimeError('Shared pgvector requires the PostgreSQL business database')
|
||||
self.async_engine = business_engine
|
||||
self.ap.logger.info('Connected pgvector adapter to the shared PostgreSQL business database')
|
||||
return
|
||||
|
||||
# Build connection string if not provided
|
||||
if connection_string:
|
||||
self.connection_string = connection_string
|
||||
else:
|
||||
self.connection_string = f'postgresql+psycopg://{user}:{password}@{host}:{port}/{database}'
|
||||
|
||||
self.async_connection_string = self.connection_string.replace('postgresql://', 'postgresql+asyncpg://').replace(
|
||||
'postgresql+psycopg://', 'postgresql+asyncpg://'
|
||||
)
|
||||
self._initialize_standalone_db()
|
||||
|
||||
self.engine = None
|
||||
self.async_engine = None
|
||||
self.SessionLocal = None
|
||||
self.AsyncSessionLocal = None
|
||||
self._collections = set()
|
||||
self._initialize_db()
|
||||
@staticmethod
|
||||
def _normalize_allowed_dimensions(dimensions: list[int] | tuple[int, ...]) -> frozenset[int]:
|
||||
if not isinstance(dimensions, (list, tuple)) or not dimensions:
|
||||
raise ValueError('pgvector allowed_dimensions must be a non-empty list')
|
||||
if any(isinstance(item, bool) or not isinstance(item, int) or item <= 0 for item in dimensions):
|
||||
raise ValueError('pgvector allowed_dimensions must contain positive integers')
|
||||
unsupported = set(dimensions) - set(DEFAULT_ALLOWED_DIMENSIONS)
|
||||
if unsupported:
|
||||
raise ValueError(f'pgvector dimensions do not have release-created ANN indexes: {sorted(unsupported)}')
|
||||
return frozenset(dimensions)
|
||||
|
||||
def _initialize_db(self):
|
||||
"""Initialize database connection and create tables"""
|
||||
try:
|
||||
# Create async engine for async operations
|
||||
self.async_engine = create_async_engine(self.async_connection_string, echo=False, pool_pre_ping=True)
|
||||
self.AsyncSessionLocal = async_sessionmaker(self.async_engine, class_=AsyncSession, expire_on_commit=False)
|
||||
def _initialize_standalone_db(self) -> None:
|
||||
"""Initialize the explicit OSS external database compatibility path."""
|
||||
|
||||
# Create sync engine for table creation
|
||||
sync_connection_string = self.connection_string.replace('postgresql+asyncpg://', 'postgresql+psycopg://')
|
||||
self.engine = create_engine(sync_connection_string, echo=False)
|
||||
from sqlalchemy import create_engine
|
||||
|
||||
# Create pgvector extension and tables
|
||||
with self.engine.connect() as conn:
|
||||
# Enable pgvector extension
|
||||
conn.execute(text('CREATE EXTENSION IF NOT EXISTS vector'))
|
||||
conn.commit()
|
||||
self.async_engine = create_async_engine(self.async_connection_string, echo=False, pool_pre_ping=True)
|
||||
self.AsyncSessionLocal = async_sessionmaker(self.async_engine, class_=AsyncSession, expire_on_commit=False)
|
||||
sync_connection_string = self.connection_string.replace('postgresql+asyncpg://', 'postgresql+psycopg://')
|
||||
self.engine = create_engine(sync_connection_string, echo=False)
|
||||
|
||||
# Create tables
|
||||
Base.metadata.create_all(self.engine)
|
||||
with self.engine.begin() as conn:
|
||||
conn.execute(sqlalchemy.text('CREATE EXTENSION IF NOT EXISTS vector'))
|
||||
existing_tables = set(sqlalchemy.inspect(conn).get_table_names())
|
||||
if PgVectorEntry.__tablename__ in existing_tables:
|
||||
columns = {
|
||||
column['name'] for column in sqlalchemy.inspect(conn).get_columns(PgVectorEntry.__tablename__)
|
||||
}
|
||||
required = {
|
||||
'workspace_uuid',
|
||||
'knowledge_base_uuid',
|
||||
'vector_id',
|
||||
'embedding_dimension',
|
||||
'embedding',
|
||||
}
|
||||
if not required.issubset(columns):
|
||||
raise RuntimeError(
|
||||
'The external pgvector database uses the legacy unscoped schema; '
|
||||
'migrate it before enabling multi-tenant vector access'
|
||||
)
|
||||
Base.metadata.create_all(conn)
|
||||
|
||||
self.ap.logger.info('Connected to PostgreSQL with pgvector')
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Failed to connect to PostgreSQL: {e}')
|
||||
raise
|
||||
self.ap.logger.info('Connected to standalone PostgreSQL pgvector database')
|
||||
|
||||
def _require_scope(self, scope: PgVectorScope | None, *, require_dimension: bool) -> PgVectorScope:
|
||||
if not isinstance(scope, PgVectorScope):
|
||||
raise ValueError('pgvector operations require a trusted PgVectorScope')
|
||||
dimension = scope.embedding_dimension
|
||||
if require_dimension and dimension is None:
|
||||
raise ValueError('pgvector operation requires an embedding dimension')
|
||||
if dimension is not None and dimension not in self.allowed_dimensions:
|
||||
raise ValueError(f'Embedding dimension {dimension} is not enabled for this pgvector deployment')
|
||||
return scope
|
||||
|
||||
@staticmethod
|
||||
def _scope_conditions(scope: PgVectorScope) -> tuple[Any, Any]:
|
||||
return (
|
||||
PgVectorEntry.workspace_uuid == scope.workspace_uuid,
|
||||
PgVectorEntry.knowledge_base_uuid == scope.knowledge_base_uuid,
|
||||
)
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def _session(self, scope: PgVectorScope) -> AsyncIterator[AsyncSession]:
|
||||
admission = getattr(self.ap, 'deployment_admission', None)
|
||||
if admission is not None:
|
||||
admission.require_active()
|
||||
|
||||
if self.use_business_database:
|
||||
async with self.ap.persistence_mgr.tenant_uow(scope.workspace_uuid) as uow:
|
||||
yield uow.session
|
||||
if admission is not None:
|
||||
admission.require_active()
|
||||
return
|
||||
|
||||
if self.AsyncSessionLocal is None: # pragma: no cover - constructor invariant
|
||||
raise RuntimeError('Standalone pgvector session factory is unavailable')
|
||||
async with self.AsyncSessionLocal() as session, session.begin():
|
||||
yield session
|
||||
if admission is not None:
|
||||
admission.require_active()
|
||||
|
||||
async def get_or_create_collection(self, collection: str):
|
||||
"""Get or create a collection (logical grouping in pgvector)
|
||||
"""Retain the common adapter API; relational rows need no collection DDL."""
|
||||
|
||||
Args:
|
||||
collection: Collection name (knowledge base UUID)
|
||||
"""
|
||||
# In pgvector, collections are logical - we just track them
|
||||
if collection not in self._collections:
|
||||
self._collections.add(collection)
|
||||
self.ap.logger.info(f"Registered pgvector collection '{collection}'")
|
||||
if not isinstance(collection, str) or not collection.strip():
|
||||
raise ValueError('collection must not be empty')
|
||||
return collection
|
||||
|
||||
async def add_embeddings(
|
||||
@@ -151,38 +245,59 @@ class PgVectorDatabase(VectorDatabase):
|
||||
embeddings_list: list[list[float]],
|
||||
metadatas: list[dict[str, Any]],
|
||||
documents: list[str] | None = None,
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> None:
|
||||
"""Add vector embeddings to pgvector
|
||||
|
||||
Args:
|
||||
collection: Collection name
|
||||
ids: List of unique IDs for each vector
|
||||
embeddings_list: List of embedding vectors
|
||||
metadatas: List of metadata dictionaries
|
||||
"""
|
||||
scope = self._require_scope(scope, require_dimension=True)
|
||||
await self.get_or_create_collection(collection)
|
||||
if not ids:
|
||||
return
|
||||
if len(ids) != len(embeddings_list) or len(metadatas) != len(ids):
|
||||
raise ValueError('pgvector ids, embeddings and metadata lengths must match')
|
||||
if documents is not None and len(documents) != len(ids):
|
||||
raise ValueError('pgvector documents length must match ids')
|
||||
if len(set(ids)) != len(ids) or any(not isinstance(item, str) or not item.strip() for item in ids):
|
||||
raise ValueError('pgvector vector IDs must be unique non-empty strings per upsert')
|
||||
expected_dimension = scope.embedding_dimension
|
||||
if any(len(embedding) != expected_dimension for embedding in embeddings_list):
|
||||
raise ValueError(f'All embeddings must have the selected dimension {expected_dimension}')
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
for i, vector_id in enumerate(ids):
|
||||
metadata = metadatas[i] if i < len(metadatas) else {}
|
||||
values = []
|
||||
for index, vector_id in enumerate(ids):
|
||||
metadata = metadatas[index]
|
||||
document = documents[index] if documents is not None else None
|
||||
values.append(
|
||||
{
|
||||
'workspace_uuid': scope.workspace_uuid,
|
||||
'knowledge_base_uuid': scope.knowledge_base_uuid,
|
||||
'vector_id': vector_id.strip(),
|
||||
'embedding_dimension': expected_dimension,
|
||||
'embedding': embeddings_list[index],
|
||||
'text': metadata.get('text', document or ''),
|
||||
'file_id': metadata.get('file_id', ''),
|
||||
'chunk_uuid': metadata.get('uuid', metadata.get('chunk_uuid', '')),
|
||||
}
|
||||
)
|
||||
|
||||
entry = PgVectorEntry(
|
||||
id=vector_id,
|
||||
collection=collection,
|
||||
embedding=embeddings_list[i],
|
||||
text=metadata.get('text', ''),
|
||||
file_id=metadata.get('file_id', ''),
|
||||
chunk_uuid=metadata.get('uuid', ''),
|
||||
)
|
||||
session.add(entry)
|
||||
|
||||
await session.commit()
|
||||
self.ap.logger.info(f"Added {len(ids)} embeddings to pgvector collection '{collection}'")
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
self.ap.logger.error(f'Error adding embeddings to pgvector: {e}')
|
||||
raise
|
||||
statement = postgresql_insert(PgVectorEntry).values(values)
|
||||
excluded = statement.excluded
|
||||
statement = statement.on_conflict_do_update(
|
||||
index_elements=[
|
||||
PgVectorEntry.workspace_uuid,
|
||||
PgVectorEntry.knowledge_base_uuid,
|
||||
PgVectorEntry.vector_id,
|
||||
],
|
||||
set_={
|
||||
'embedding_dimension': excluded.embedding_dimension,
|
||||
'embedding': excluded.embedding,
|
||||
'text': excluded.text,
|
||||
'file_id': excluded.file_id,
|
||||
'chunk_uuid': excluded.chunk_uuid,
|
||||
},
|
||||
)
|
||||
async with self._session(scope) as session:
|
||||
await session.execute(statement)
|
||||
self.ap.logger.info(f'Upserted {len(ids)} pgvector embeddings for knowledge base {scope.knowledge_base_uuid}')
|
||||
|
||||
async def search(
|
||||
self,
|
||||
@@ -193,125 +308,79 @@ class PgVectorDatabase(VectorDatabase):
|
||||
query_text: str = '',
|
||||
filter: dict[str, Any] | None = None,
|
||||
vector_weight: float | None = None,
|
||||
) -> Dict[str, Any]:
|
||||
"""Search for similar vectors using cosine distance
|
||||
|
||||
Args:
|
||||
collection: Collection name
|
||||
query_embedding: Query vector
|
||||
k: Number of top results to return
|
||||
|
||||
Returns:
|
||||
Dictionary with search results in Chroma-compatible format
|
||||
"""
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> dict[str, Any]:
|
||||
del query_text, vector_weight
|
||||
scope = self._require_scope(scope, require_dimension=True)
|
||||
await self.get_or_create_collection(collection)
|
||||
if search_type != 'vector':
|
||||
raise ValueError('pgvector currently supports vector search only')
|
||||
if k <= 0:
|
||||
raise ValueError('pgvector search limit must be positive')
|
||||
if len(query_embedding) != scope.embedding_dimension:
|
||||
raise ValueError(f'Query embedding must have the selected dimension {scope.embedding_dimension}')
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
# Use cosine distance for similarity search
|
||||
from sqlalchemy import select
|
||||
typed_embedding = sqlalchemy.cast(PgVectorEntry.embedding, Vector(scope.embedding_dimension))
|
||||
distance = typed_embedding.cosine_distance(query_embedding)
|
||||
statement = (
|
||||
sqlalchemy.select(
|
||||
PgVectorEntry.vector_id,
|
||||
PgVectorEntry.text,
|
||||
PgVectorEntry.file_id,
|
||||
PgVectorEntry.chunk_uuid,
|
||||
distance.label('distance'),
|
||||
)
|
||||
.where(*self._scope_conditions(scope), PgVectorEntry.embedding_dimension == scope.embedding_dimension)
|
||||
.order_by(distance)
|
||||
.limit(k)
|
||||
)
|
||||
for condition in _build_pg_conditions(filter or {}):
|
||||
statement = statement.where(condition)
|
||||
|
||||
# Query for similar vectors
|
||||
stmt = (
|
||||
select(
|
||||
PgVectorEntry.id,
|
||||
PgVectorEntry.text,
|
||||
PgVectorEntry.file_id,
|
||||
PgVectorEntry.chunk_uuid,
|
||||
PgVectorEntry.embedding.cosine_distance(query_embedding).label('distance'),
|
||||
)
|
||||
.filter(PgVectorEntry.collection == collection)
|
||||
.order_by(PgVectorEntry.embedding.cosine_distance(query_embedding))
|
||||
.limit(k)
|
||||
)
|
||||
async with self._session(scope) as session:
|
||||
rows = (await session.execute(statement)).all()
|
||||
|
||||
if filter:
|
||||
for cond in _build_pg_conditions(filter):
|
||||
stmt = stmt.filter(cond)
|
||||
ids = [row.vector_id for row in rows]
|
||||
distances = [float(row.distance) for row in rows]
|
||||
metadatas = [
|
||||
{'text': row.text or '', 'file_id': row.file_id or '', 'uuid': row.chunk_uuid or ''} for row in rows
|
||||
]
|
||||
return {'ids': [ids], 'distances': [distances], 'metadatas': [metadatas]}
|
||||
|
||||
result = await session.execute(stmt)
|
||||
rows = result.fetchall()
|
||||
|
||||
# Convert to Chroma-compatible format
|
||||
ids = []
|
||||
distances = []
|
||||
metadatas = []
|
||||
|
||||
for row in rows:
|
||||
ids.append(row.id)
|
||||
distances.append(float(row.distance))
|
||||
metadatas.append(
|
||||
{'text': row.text or '', 'file_id': row.file_id or '', 'uuid': row.chunk_uuid or ''}
|
||||
)
|
||||
|
||||
result_dict = {'ids': [ids], 'distances': [distances], 'metadatas': [metadatas]}
|
||||
|
||||
self.ap.logger.info(f"pgvector search in '{collection}' returned {len(ids)} results")
|
||||
return result_dict
|
||||
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Error searching pgvector: {e}')
|
||||
raise
|
||||
|
||||
async def delete_by_file_id(self, collection: str, file_id: str) -> None:
|
||||
"""Delete vectors by file_id
|
||||
|
||||
Args:
|
||||
collection: Collection name
|
||||
file_id: File ID to filter deletion
|
||||
"""
|
||||
async def delete_by_file_id(
|
||||
self,
|
||||
collection: str,
|
||||
file_id: str,
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> None:
|
||||
scope = self._require_scope(scope, require_dimension=False)
|
||||
await self.get_or_create_collection(collection)
|
||||
statement = sqlalchemy.delete(PgVectorEntry).where(
|
||||
*self._scope_conditions(scope),
|
||||
PgVectorEntry.file_id == file_id,
|
||||
)
|
||||
async with self._session(scope) as session:
|
||||
await session.execute(statement)
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
from sqlalchemy import delete
|
||||
|
||||
stmt = delete(PgVectorEntry).where(
|
||||
PgVectorEntry.collection == collection, PgVectorEntry.file_id == file_id
|
||||
)
|
||||
await session.execute(stmt)
|
||||
await session.commit()
|
||||
|
||||
self.ap.logger.info(
|
||||
f"Deleted embeddings from pgvector collection '{collection}' with file_id: {file_id}"
|
||||
)
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
self.ap.logger.error(f'Error deleting from pgvector: {e}')
|
||||
raise
|
||||
|
||||
async def delete_by_filter(self, collection: str, filter: dict[str, Any]) -> int:
|
||||
"""Delete vectors matching a metadata filter.
|
||||
|
||||
Args:
|
||||
collection: Collection name
|
||||
filter: Canonical metadata filter dict
|
||||
"""
|
||||
async def delete_by_filter(
|
||||
self,
|
||||
collection: str,
|
||||
filter: dict[str, Any],
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> int:
|
||||
scope = self._require_scope(scope, require_dimension=False)
|
||||
await self.get_or_create_collection(collection)
|
||||
conditions = _build_pg_conditions(filter)
|
||||
if not conditions:
|
||||
self.ap.logger.warning(
|
||||
f"pgvector delete_by_filter on '{collection}': filter produced no conditions, skipping"
|
||||
)
|
||||
self.ap.logger.warning('pgvector delete_by_filter produced no supported conditions; skipping')
|
||||
return 0
|
||||
|
||||
await self.get_or_create_collection(collection)
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
from sqlalchemy import delete
|
||||
|
||||
stmt = delete(PgVectorEntry).where(PgVectorEntry.collection == collection)
|
||||
for cond in conditions:
|
||||
stmt = stmt.where(cond)
|
||||
result = await session.execute(stmt)
|
||||
await session.commit()
|
||||
deleted = result.rowcount
|
||||
self.ap.logger.info(f"Deleted {deleted} embeddings from pgvector collection '{collection}' by filter")
|
||||
return deleted
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
self.ap.logger.error(f'Error deleting from pgvector by filter: {e}')
|
||||
raise
|
||||
statement = sqlalchemy.delete(PgVectorEntry).where(*self._scope_conditions(scope), *conditions)
|
||||
async with self._session(scope) as session:
|
||||
result = await session.execute(statement)
|
||||
return int(result.rowcount or 0)
|
||||
|
||||
async def list_by_filter(
|
||||
self,
|
||||
@@ -319,85 +388,62 @@ class PgVectorDatabase(VectorDatabase):
|
||||
filter: dict[str, Any] | None = None,
|
||||
limit: int = 20,
|
||||
offset: int = 0,
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> tuple[list[dict[str, Any]], int]:
|
||||
scope = self._require_scope(scope, require_dimension=False)
|
||||
await self.get_or_create_collection(collection)
|
||||
if limit <= 0 or offset < 0:
|
||||
raise ValueError('pgvector pagination requires limit > 0 and offset >= 0')
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
from sqlalchemy import select, func
|
||||
conditions = [*self._scope_conditions(scope), *_build_pg_conditions(filter or {})]
|
||||
statement = (
|
||||
sqlalchemy.select(
|
||||
PgVectorEntry.vector_id,
|
||||
PgVectorEntry.text,
|
||||
PgVectorEntry.file_id,
|
||||
PgVectorEntry.chunk_uuid,
|
||||
)
|
||||
.where(*conditions)
|
||||
.order_by(PgVectorEntry.vector_id)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
count_statement = sqlalchemy.select(sqlalchemy.func.count()).select_from(PgVectorEntry).where(*conditions)
|
||||
async with self._session(scope) as session:
|
||||
rows = (await session.execute(statement)).all()
|
||||
total = int((await session.execute(count_statement)).scalar_one())
|
||||
|
||||
stmt = (
|
||||
select(
|
||||
PgVectorEntry.id,
|
||||
PgVectorEntry.text,
|
||||
PgVectorEntry.file_id,
|
||||
PgVectorEntry.chunk_uuid,
|
||||
)
|
||||
.filter(PgVectorEntry.collection == collection)
|
||||
.offset(offset)
|
||||
.limit(limit)
|
||||
)
|
||||
return (
|
||||
[
|
||||
{
|
||||
'id': row.vector_id,
|
||||
'document': row.text or '',
|
||||
'metadata': {
|
||||
'text': row.text or '',
|
||||
'file_id': row.file_id or '',
|
||||
'uuid': row.chunk_uuid or '',
|
||||
},
|
||||
}
|
||||
for row in rows
|
||||
],
|
||||
total,
|
||||
)
|
||||
|
||||
count_stmt = (
|
||||
select(func.count()).select_from(PgVectorEntry).filter(PgVectorEntry.collection == collection)
|
||||
)
|
||||
async def delete_collection(
|
||||
self,
|
||||
collection: str,
|
||||
*,
|
||||
scope: PgVectorScope | None = None,
|
||||
) -> None:
|
||||
scope = self._require_scope(scope, require_dimension=False)
|
||||
await self.get_or_create_collection(collection)
|
||||
statement = sqlalchemy.delete(PgVectorEntry).where(*self._scope_conditions(scope))
|
||||
async with self._session(scope) as session:
|
||||
await session.execute(statement)
|
||||
|
||||
if filter:
|
||||
for cond in _build_pg_conditions(filter):
|
||||
stmt = stmt.filter(cond)
|
||||
count_stmt = count_stmt.filter(cond)
|
||||
|
||||
result = await session.execute(stmt)
|
||||
rows = result.fetchall()
|
||||
|
||||
count_result = await session.execute(count_stmt)
|
||||
total = count_result.scalar() or 0
|
||||
|
||||
items = []
|
||||
for row in rows:
|
||||
items.append(
|
||||
{
|
||||
'id': row.id,
|
||||
'document': row.text or '',
|
||||
'metadata': {
|
||||
'text': row.text or '',
|
||||
'file_id': row.file_id or '',
|
||||
'uuid': row.chunk_uuid or '',
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
return items, total
|
||||
except Exception as e:
|
||||
self.ap.logger.error(f'Error listing from pgvector: {e}')
|
||||
raise
|
||||
|
||||
async def delete_collection(self, collection: str):
|
||||
"""Delete all vectors in a collection
|
||||
|
||||
Args:
|
||||
collection: Collection name to delete
|
||||
"""
|
||||
if collection in self._collections:
|
||||
self._collections.remove(collection)
|
||||
|
||||
async with self.AsyncSessionLocal() as session:
|
||||
try:
|
||||
from sqlalchemy import delete
|
||||
|
||||
stmt = delete(PgVectorEntry).where(PgVectorEntry.collection == collection)
|
||||
await session.execute(stmt)
|
||||
await session.commit()
|
||||
|
||||
self.ap.logger.info(f"Deleted pgvector collection '{collection}'")
|
||||
except Exception as e:
|
||||
await session.rollback()
|
||||
self.ap.logger.error(f'Error deleting pgvector collection: {e}')
|
||||
raise
|
||||
|
||||
async def close(self):
|
||||
"""Close database connections"""
|
||||
if self.async_engine:
|
||||
async def close(self) -> None:
|
||||
if not self.use_business_database and self.async_engine is not None:
|
||||
await self.async_engine.dispose()
|
||||
if self.engine:
|
||||
if self.engine is not None:
|
||||
self.engine.dispose()
|
||||
|
||||
@@ -155,8 +155,20 @@ class WorkspaceCollaborationService:
|
||||
) -> ResolvedWorkspaceAccess:
|
||||
"""Resolve a selector against an active Account membership."""
|
||||
|
||||
normalized_workspace_uuid = requested_workspace_uuid.strip() if requested_workspace_uuid else None
|
||||
if normalized_workspace_uuid is None and self.policy.multi_workspace_enabled:
|
||||
raise WorkspaceNotFoundError('A Workspace selector is required')
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if session is None and normalized_workspace_uuid is not None and callable(tenant_uow):
|
||||
async with tenant_uow(normalized_workspace_uuid) as uow:
|
||||
return await self.resolve_account_workspace(
|
||||
account_uuid,
|
||||
normalized_workspace_uuid,
|
||||
session=uow.session,
|
||||
)
|
||||
|
||||
async def operation(active_session: AsyncSession) -> ResolvedWorkspaceAccess:
|
||||
workspace_uuid = requested_workspace_uuid.strip() if requested_workspace_uuid else None
|
||||
workspace_uuid = normalized_workspace_uuid
|
||||
if workspace_uuid is None:
|
||||
if self.policy.multi_workspace_enabled:
|
||||
raise WorkspaceNotFoundError('A Workspace selector is required')
|
||||
@@ -195,6 +207,40 @@ class WorkspaceCollaborationService:
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
) -> list[ResolvedWorkspaceAccess]:
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
account_uow = getattr(self.ap.persistence_mgr, 'account_discovery_uow', None)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if session is None and current_session() is None and callable(account_uow) and callable(tenant_uow):
|
||||
# Discovery exposes only this Account's active Membership rows.
|
||||
# Each resulting Workspace is then re-read under its own tenant
|
||||
# transaction; directory discovery never grants business access.
|
||||
async with account_uow(account_uuid) as discovery:
|
||||
workspace_uuids = list(
|
||||
(
|
||||
await discovery.session.scalars(
|
||||
sqlalchemy.select(WorkspaceMembership.workspace_uuid)
|
||||
.where(
|
||||
WorkspaceMembership.account_uuid == account_uuid,
|
||||
WorkspaceMembership.status == MembershipStatus.ACTIVE.value,
|
||||
)
|
||||
.order_by(WorkspaceMembership.workspace_uuid)
|
||||
)
|
||||
).all()
|
||||
)
|
||||
|
||||
accesses: list[ResolvedWorkspaceAccess] = []
|
||||
for workspace_uuid in workspace_uuids:
|
||||
async with tenant_uow(workspace_uuid) as workspace_uow:
|
||||
accesses.append(
|
||||
await self.resolve_account_workspace(
|
||||
account_uuid,
|
||||
workspace_uuid,
|
||||
session=workspace_uow.session,
|
||||
)
|
||||
)
|
||||
accesses.sort(key=lambda access: (access.workspace.created_at, access.workspace.uuid))
|
||||
return accesses
|
||||
|
||||
async def operation(active_session: AsyncSession) -> list[ResolvedWorkspaceAccess]:
|
||||
statement = (
|
||||
sqlalchemy.select(WorkspaceMembership, Workspace)
|
||||
@@ -346,6 +392,18 @@ class WorkspaceCollaborationService:
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
) -> tuple[WorkspaceInvitation, Workspace]:
|
||||
if session is None:
|
||||
scoped_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)()
|
||||
invitation_uow = getattr(self.ap.persistence_mgr, 'invitation_discovery_uow', None)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if scoped_session is None and callable(invitation_uow) and callable(tenant_uow):
|
||||
token_hash = hash_invitation_token(token)
|
||||
async with invitation_uow(token_hash) as discovery:
|
||||
invitation = await self._get_invitation_by_token(discovery.session, token, for_update=False)
|
||||
workspace_uuid = invitation.workspace_uuid
|
||||
async with tenant_uow(workspace_uuid) as workspace_uow:
|
||||
return await self.inspect_invitation(token, session=workspace_uow.session)
|
||||
|
||||
async def operation(active_session: AsyncSession) -> tuple[WorkspaceInvitation, Workspace]:
|
||||
invitation = await self._get_invitation_by_token(active_session, token, for_update=True)
|
||||
self._validate_invitation_state(invitation)
|
||||
@@ -365,6 +423,18 @@ class WorkspaceCollaborationService:
|
||||
*,
|
||||
session: AsyncSession | None = None,
|
||||
) -> WorkspaceMembership:
|
||||
if session is None:
|
||||
scoped_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)()
|
||||
invitation_uow = getattr(self.ap.persistence_mgr, 'invitation_discovery_uow', None)
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if scoped_session is None and callable(invitation_uow) and callable(tenant_uow):
|
||||
token_digest = hash_invitation_token(token)
|
||||
async with invitation_uow(token_digest) as discovery:
|
||||
invitation = await self._get_invitation_by_token(discovery.session, token, for_update=False)
|
||||
workspace_uuid = invitation.workspace_uuid
|
||||
async with tenant_uow(workspace_uuid) as workspace_uow:
|
||||
return await self.accept_invitation(token, account_uuid, session=workspace_uow.session)
|
||||
|
||||
async def operation(active_session: AsyncSession) -> WorkspaceMembership:
|
||||
invitation = await self._get_invitation_by_token(active_session, token, for_update=True)
|
||||
self._validate_invitation_state(invitation)
|
||||
@@ -677,6 +747,14 @@ class WorkspaceCollaborationService:
|
||||
) -> T:
|
||||
if session is not None:
|
||||
return await operation(session)
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)()
|
||||
if current_session is not None:
|
||||
return await operation(current_session)
|
||||
if getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime':
|
||||
require_current_session = getattr(self.ap.persistence_mgr, 'require_current_session', None)
|
||||
if callable(require_current_session):
|
||||
require_current_session()
|
||||
raise RuntimeError('Cloud collaboration services require an explicit persistence unit of work')
|
||||
async with self._session_factory()() as owned_session:
|
||||
if read_only:
|
||||
return await operation(owned_session)
|
||||
|
||||
@@ -6,6 +6,7 @@ import typing
|
||||
from collections.abc import Awaitable, Callable
|
||||
from typing import TypeVar
|
||||
|
||||
import sqlalchemy
|
||||
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
|
||||
|
||||
from ..entity.persistence.workspace import (
|
||||
@@ -68,6 +69,11 @@ class WorkspaceService:
|
||||
) -> Workspace:
|
||||
"""Load one Workspace projected onto this LangBot instance."""
|
||||
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if session is None and callable(tenant_uow):
|
||||
async with tenant_uow(workspace_uuid) as uow:
|
||||
return await self.get_workspace(workspace_uuid, session=uow.session)
|
||||
|
||||
async def operation(repository: WorkspaceRepository) -> Workspace:
|
||||
workspace = await repository.get_workspace(workspace_uuid)
|
||||
if workspace is None or workspace.instance_uuid != self.instance_uuid:
|
||||
@@ -97,6 +103,11 @@ class WorkspaceService:
|
||||
) -> WorkspaceExecutionState:
|
||||
"""Load a Workspace execution state and validate its instance binding."""
|
||||
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if session is None and callable(tenant_uow):
|
||||
async with tenant_uow(workspace_uuid) as uow:
|
||||
return await self.get_execution_state(workspace_uuid, session=uow.session)
|
||||
|
||||
async def operation(repository: WorkspaceRepository) -> WorkspaceExecutionState:
|
||||
execution_state = await repository.get_execution_state(workspace_uuid)
|
||||
if execution_state is None:
|
||||
@@ -109,6 +120,47 @@ class WorkspaceService:
|
||||
|
||||
return await self._run(operation, session=session)
|
||||
|
||||
async def list_active_execution_bindings(self) -> list[WorkspaceExecutionBinding]:
|
||||
"""Discover this instance's active Workspaces, then validate each tenant projection.
|
||||
|
||||
The instance discovery transaction may only reveal active, unfenced
|
||||
execution-state identifiers. It is closed before any Workspace data is
|
||||
read; every returned binding is revalidated inside its own tenant unit
|
||||
of work.
|
||||
"""
|
||||
|
||||
instance_discovery_uow = getattr(self.ap.persistence_mgr, 'instance_discovery_uow', None)
|
||||
statement = (
|
||||
sqlalchemy.select(WorkspaceExecutionState.workspace_uuid)
|
||||
.where(
|
||||
WorkspaceExecutionState.instance_uuid == self.instance_uuid,
|
||||
WorkspaceExecutionState.state == WorkspaceExecutionStatus.ACTIVE.value,
|
||||
WorkspaceExecutionState.write_fenced == sqlalchemy.false(),
|
||||
)
|
||||
.order_by(WorkspaceExecutionState.workspace_uuid)
|
||||
)
|
||||
if callable(instance_discovery_uow):
|
||||
async with instance_discovery_uow(self.instance_uuid) as uow:
|
||||
result = await uow.session.execute(statement)
|
||||
workspace_uuids = list(result.scalars().all())
|
||||
else:
|
||||
# Compatibility for lightweight test doubles. Production managers
|
||||
# always expose the explicit discovery scope.
|
||||
result = await self.ap.persistence_mgr.execute_async(statement)
|
||||
workspace_uuids = list(result.scalars().all())
|
||||
|
||||
bindings: list[WorkspaceExecutionBinding] = []
|
||||
for workspace_uuid in workspace_uuids:
|
||||
try:
|
||||
bindings.append(await self.get_execution_binding(workspace_uuid))
|
||||
except (
|
||||
WorkspaceExecutionUnavailableError,
|
||||
WorkspaceInvariantError,
|
||||
WorkspaceNotFoundError,
|
||||
) as exc:
|
||||
self.ap.logger.warning(f'Skipping invalid Workspace execution projection {workspace_uuid!r}: {exc}')
|
||||
return bindings
|
||||
|
||||
async def get_execution_binding(
|
||||
self,
|
||||
workspace_uuid: str | None = None,
|
||||
@@ -124,6 +176,18 @@ class WorkspaceService:
|
||||
a Workspace from source or recency.
|
||||
"""
|
||||
|
||||
self._require_deployment_admission()
|
||||
|
||||
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
|
||||
if session is None and workspace_uuid is not None and callable(tenant_uow):
|
||||
async with tenant_uow(workspace_uuid) as uow:
|
||||
return await self.get_execution_binding(
|
||||
workspace_uuid,
|
||||
expected_generation=expected_generation,
|
||||
session=uow.session,
|
||||
_require_local=_require_local,
|
||||
)
|
||||
|
||||
async def operation(repository: WorkspaceRepository) -> WorkspaceExecutionBinding:
|
||||
if workspace_uuid is None:
|
||||
workspaces = await repository.list_local_workspaces(self.instance_uuid)
|
||||
@@ -180,7 +244,11 @@ class WorkspaceService:
|
||||
state=execution_state.state,
|
||||
)
|
||||
|
||||
return await self._run(operation, session=session)
|
||||
binding = await self._run(operation, session=session)
|
||||
# The database lookup can cross the Manifest expiry boundary. Never
|
||||
# return a binding that is already invalid at a side-effect boundary.
|
||||
self._require_deployment_admission()
|
||||
return binding
|
||||
|
||||
async def get_local_execution_binding(
|
||||
self,
|
||||
@@ -408,6 +476,18 @@ class WorkspaceService:
|
||||
if session is not None:
|
||||
return await operation(WorkspaceRepository(session))
|
||||
|
||||
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
|
||||
active_session = current_session()
|
||||
if active_session is not None:
|
||||
return await operation(WorkspaceRepository(active_session))
|
||||
if getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime':
|
||||
# Do not let service-local session factories silently bypass the
|
||||
# request/task scope enforced by PersistenceManager.
|
||||
require_current_session = getattr(self.ap.persistence_mgr, 'require_current_session', None)
|
||||
if callable(require_current_session):
|
||||
require_current_session()
|
||||
raise RuntimeError('Cloud Workspace services require an explicit persistence unit of work')
|
||||
|
||||
session_factory = async_sessionmaker(
|
||||
self.ap.persistence_mgr.get_db_engine(),
|
||||
expire_on_commit=False,
|
||||
@@ -415,3 +495,8 @@ class WorkspaceService:
|
||||
async with session_factory() as owned_session:
|
||||
async with owned_session.begin():
|
||||
return await operation(WorkspaceRepository(owned_session))
|
||||
|
||||
def _require_deployment_admission(self) -> None:
|
||||
guard = getattr(self.ap, 'deployment_admission', None)
|
||||
if guard is not None:
|
||||
guard.require_active()
|
||||
|
||||
Reference in New Issue
Block a user