feat(tenancy): harden shared cloud runtime boundaries

This commit is contained in:
Junyan Qin
2026-07-20 01:47:42 +08:00
parent 41772920ef
commit a47bfe8167
121 changed files with 18152 additions and 5588 deletions
@@ -83,6 +83,7 @@ class RouterGroup(abc.ABC):
rule = self.path + rule
async def handler_error(*args, **kwargs):
request_context: RequestContext | None = None
if auth_type == AuthType.ACCOUNT_TOKEN:
authorization = quart.request.headers.get('Authorization', '')
if not authorization.startswith('Bearer '):
@@ -183,6 +184,17 @@ class RouterGroup(abc.ABC):
return self._auth_error_response(e)
try:
if request_context is not None:
persistence_mgr = getattr(self.ap, 'persistence_mgr', None)
tenant_scope_descriptor = getattr(type(persistence_mgr), 'tenant_scope', None)
if callable(tenant_scope_descriptor):
# Authorization discovery is complete. Carry the
# trusted Workspace identity across the handler, but
# do not reserve a database connection while it waits
# on providers, runtimes, uploads, or streamed clients.
# Services that need atomic writes open a short UoW.
async with persistence_mgr.tenant_scope(request_context.workspace_uuid):
return await f(*args, **kwargs)
return await f(*args, **kwargs)
except Exception as e: # 自动 500
@@ -1,7 +1,9 @@
from __future__ import annotations
from langbot.pkg.utils import constants
from langbot_plugin.box.errors import BoxAdmissionError
from langbot.pkg.cloud.entitlements import EntitlementUnavailableError
from ...authz import Permission
from ...context import RequestContext
from .. import group
@@ -18,7 +20,10 @@ class BoxRouterGroup(group.RouterGroup):
permission=Permission.RESOURCE_VIEW,
)
async def _(request_context: RequestContext) -> str:
status = await self.ap.box_service.get_status(request_context)
try:
status = await self.ap.box_service.get_status(request_context)
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
status['hidden'] = should_hide_box_runtime_status(constants.edition, status.get('enabled'))
return self.success(data=status)
@@ -29,7 +34,10 @@ class BoxRouterGroup(group.RouterGroup):
permission=Permission.AUDIT_VIEW,
)
async def _(request_context: RequestContext) -> str:
sessions = await self.ap.box_service.get_sessions(request_context)
try:
sessions = await self.ap.box_service.get_sessions(request_context)
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
return self.success(data=sessions)
@self.route(
@@ -39,5 +47,10 @@ class BoxRouterGroup(group.RouterGroup):
permission=Permission.AUDIT_VIEW,
)
async def _(request_context: RequestContext) -> str:
try:
if getattr(self.ap.box_service, 'managed_admission_required', False):
await self.ap.box_service.require_workspace_sandbox(request_context)
except (BoxAdmissionError, EntitlementUnavailableError) as exc:
return self.http_status(403, 'managed_sandbox_unavailable', str(exc))
errors = self.ap.box_service.get_recent_errors(request_context)
return self.success(data=errors)
@@ -13,6 +13,7 @@ import quart
from ....authz import Permission, permissions_for_role, require_permission
from ....context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
from ... import group
from ......core.task_boundary import run_in_workspace_uow
from ......platform.sources.websocket_manager import WebSocketScope, ws_connection_manager
logger = logging.getLogger(__name__)
@@ -113,7 +114,11 @@ class WebSocketChatRouterGroup(group.RouterGroup):
require_permission(current_context, Permission.RUNTIME_OPERATE)
async def _get_scoped_adapter(self, request_context: RequestContext, pipeline_uuid: str):
pipeline = await self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid)
pipeline = await run_in_workspace_uow(
self.ap,
request_context.workspace_uuid,
lambda: self.ap.pipeline_service.get_pipeline(request_context, pipeline_uuid),
)
if pipeline is None:
return None
proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(request_context)
@@ -16,11 +16,13 @@ import posixpath
import sqlalchemy
from .....core import taskmgr
from .....core.task_boundary import run_in_workspace_uow
from .....entity.persistence import plugin as persistence_plugin
from ...authz import Permission
from ...context import ExecutionContext, RequestContext
from .. import group
from .....workspace.errors import WorkspaceNotFoundError
from .....plugin.github import validate_github_plugin_install_info
from langbot_plugin.runtime.plugin.mgr import PluginInstallSource
@@ -308,7 +310,11 @@ class PluginsRouterGroup(group.RouterGroup):
):
"""Revalidate a captured task context immediately before Runtime I/O."""
await self.ap.plugin_connector.require_workspace_context(execution_context)
await run_in_workspace_uow(
self.ap,
execution_context.workspace_uuid,
lambda: self.ap.plugin_connector.require_workspace_context(execution_context),
)
return await operation()
async def _require_public_plugin_runtime_context(self) -> ExecutionContext:
@@ -757,27 +763,31 @@ class PluginsRouterGroup(group.RouterGroup):
if limit_error is not None:
return limit_error
data = await quart.request.json
asset_url = data.get('asset_url', '')
owner = data.get('owner', '')
repo = data.get('repo', '')
release_tag = data.get('release_tag', '')
data = await quart.request.json or {}
try:
install_info = validate_github_plugin_install_info(
{
'asset_url': data.get('asset_url'),
'asset_id': data.get('asset_id'),
'release_id': data.get('release_id'),
'owner': data.get('owner'),
'repo': data.get('repo'),
'release_tag': data.get('release_tag'),
'github_url': f'https://github.com/{data.get("owner", "")}/{data.get("repo", "")}',
}
)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
if not asset_url:
return self.http_status(400, -1, 'Missing asset_url parameter')
owner = install_info['owner']
repo = install_info['repo']
release_tag = install_info['release_tag']
execution_context = await self.ap.plugin_connector.require_workspace_context(request_context)
ctx = taskmgr.TaskContext.new()
ctx.metadata['plugin_name'] = f'{owner}/{repo}'
ctx.metadata['install_source'] = 'github'
install_info = {
'asset_url': asset_url,
'owner': owner,
'repo': repo,
'release_tag': release_tag,
'github_url': f'https://github.com/{owner}/{repo}',
}
wrapper = self.ap.task_mgr.create_user_task(
self._run_fenced_plugin_operation(
@@ -25,12 +25,33 @@ class SystemRouterGroup(group.RouterGroup):
account, _ = await self._authenticate_account(authorization.removeprefix('Bearer '))
request_context = await self._resolve_account_context(account, group.AuthType.USER_TOKEN)
if request_context is not None:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(WorkspaceMetadata).where(
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
WorkspaceMetadata.key.in_(['wizard_status', 'wizard_progress']),
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
async def load_workspace_metadata():
return await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
WorkspaceMetadata.key,
WorkspaceMetadata.value,
).where(
WorkspaceMetadata.workspace_uuid == request_context.workspace_uuid,
WorkspaceMetadata.key.in_(['wizard_status', 'wizard_progress']),
)
)
cloud_runtime = (
getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
)
if cloud_runtime:
if not callable(tenant_uow):
raise RuntimeError('Cloud system metadata requires an explicit tenant UoW')
async with tenant_uow(request_context.workspace_uuid):
result = await load_workspace_metadata()
else:
result = await load_workspace_metadata()
# ``execute_async`` deliberately preserves its historical
# AsyncConnection result shape. Selecting the two fields
# explicitly keeps this reader independent of ORM Session
# scalar semantics inside a tenant UoW.
for row in result:
if row.key == 'wizard_status':
wizard_status = row.value
@@ -44,11 +44,27 @@ class WebhookRouterGroup(group.RouterGroup):
if not hasattr(runtime_bot.adapter, 'handle_unified_webhook'):
return quart.jsonify({'error': 'Adapter does not support unified webhook'}), 501
response = await runtime_bot.adapter.handle_unified_webhook(
bot_uuid=bot_uuid,
path=path,
request=quart.request,
)
async def dispatch():
await self.ap.workspace_service.get_execution_binding(
runtime_bot.workspace_uuid,
expected_generation=runtime_bot.placement_generation,
)
return await runtime_bot.adapter.handle_unified_webhook(
bot_uuid=bot_uuid,
path=path,
request=quart.request,
)
persistence_mgr = self.ap.persistence_mgr
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if cloud_runtime:
tenant_scope = getattr(persistence_mgr, 'tenant_scope', None)
if not callable(tenant_scope):
raise RuntimeError('Cloud webhook dispatch requires an explicit tenant scope')
async with tenant_scope(runtime_bot.workspace_uuid):
response = await dispatch()
else:
response = await dispatch()
return response
+71 -19
View File
@@ -170,37 +170,89 @@ class ApiKeyService:
if not secret.startswith('lbk_'):
return None
secret_hash = self._hash_secret(secret)
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
)
key = result.first()
if key is None or key.status != apikey.ApiKeyStatus.ACTIVE.value:
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
discovery_uow = getattr(self.ap.persistence_mgr, 'api_key_discovery_uow', None)
if current_session() is None and callable(discovery_uow):
async with discovery_uow(secret_hash) as discovery:
key = await discovery.session.scalar(
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
)
else:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(apikey.ApiKey).where(apikey.ApiKey.key_hash == secret_hash)
)
key = result.first()
if key is None:
return None
discovered_workspace_uuid = key.workspace_uuid
discovered_key_id = key.id
now = self._utcnow()
if key.expires_at is not None and key.expires_at <= now:
return None
raw_scopes = list(key.scopes or [])
async def bind_and_record_use() -> tuple[typing.Any, typing.Any] | None:
# Re-read inside the tenant transaction. A revoke/expiry racing
# discovery must not result in an authenticated identity.
active_session = current_session()
if active_session is not None:
scoped_key = await active_session.scalar(
sqlalchemy.select(apikey.ApiKey).where(
apikey.ApiKey.id == discovered_key_id,
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
apikey.ApiKey.key_hash == secret_hash,
)
)
else: # compatibility for isolated service tests
scoped_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(apikey.ApiKey).where(
apikey.ApiKey.id == discovered_key_id,
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
apikey.ApiKey.key_hash == secret_hash,
)
)
scoped_key = scoped_result.first()
if scoped_key is None or scoped_key.status != apikey.ApiKeyStatus.ACTIVE.value:
return None
if scoped_key.expires_at is not None and scoped_key.expires_at <= now:
return None
binding = await self.ap.workspace_service.get_execution_binding(discovered_workspace_uuid)
updated = await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(apikey.ApiKey)
.where(
apikey.ApiKey.id == scoped_key.id,
apikey.ApiKey.workspace_uuid == discovered_workspace_uuid,
apikey.ApiKey.key_hash == secret_hash,
apikey.ApiKey.status == apikey.ApiKeyStatus.ACTIVE.value,
)
.values(last_used_at=now)
.returning(apikey.ApiKey.id)
)
# Authentication and revocation race on this atomic predicate. If
# revoke won, no active row is returned and the stale object read
# above must never become an authenticated identity.
if updated.scalar_one_or_none() is None:
return None
return binding, scoped_key
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if current_session() is None and callable(tenant_uow):
async with tenant_uow(discovered_workspace_uuid):
bound = await bind_and_record_use()
else:
bound = await bind_and_record_use()
if bound is None:
return None
binding, scoped_key = bound
raw_scopes = list(scoped_key.scopes or [])
permissions = (
frozenset(permission.value for permission in Permission)
if '*' in raw_scopes
else frozenset(self._normalize_scopes(raw_scopes))
)
binding = await self.ap.workspace_service.get_execution_binding(key.workspace_uuid)
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(apikey.ApiKey)
.where(
apikey.ApiKey.id == key.id,
apikey.ApiKey.workspace_uuid == key.workspace_uuid,
apikey.ApiKey.key_hash == secret_hash,
)
.values(last_used_at=now)
)
return ApiKeyIdentity(
instance_uuid=binding.instance_uuid,
workspace_uuid=binding.workspace_uuid,
placement_generation=binding.placement_generation,
api_key_uuid=key.uuid,
api_key_uuid=scoped_key.uuid,
permissions=permissions,
)
@@ -1,6 +1,7 @@
from __future__ import annotations
import datetime
import functools
import os
import re
from pathlib import Path
@@ -22,6 +23,25 @@ DEFAULT_LOG_RETENTION_DAYS = 3
UPLOAD_OWNER_TYPES = ('upload_image', 'upload_document', 'upload')
def _workspace_scope(method):
"""Bind maintenance work to a Workspace without spanning external I/O."""
@functools.wraps(method)
async def wrapped(self, context, *args, **kwargs):
workspace_uuid = require_workspace_uuid(context)
persistence_mgr = getattr(self.ap, 'persistence_mgr', None)
tenant_scope = getattr(persistence_mgr, 'tenant_scope', None)
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if cloud_runtime:
if not callable(tenant_scope):
raise RuntimeError('Cloud maintenance requires an explicit tenant scope')
async with tenant_scope(workspace_uuid):
return await method(self, context, *args, **kwargs)
return await method(self, context, *args, **kwargs)
return wrapped
class MaintenanceService:
"""Storage maintenance and diagnostics."""
@@ -30,6 +50,7 @@ class MaintenanceService:
def __init__(self, ap: app.Application) -> None:
self.ap = ap
@_workspace_scope
async def cleanup_expired_files(self, context: ExecutionContext) -> dict[str, int]:
if not isinstance(context, ExecutionContext):
raise WorkspaceRequiredError('Storage cleanup requires an ExecutionContext')
+9 -3
View File
@@ -1,6 +1,5 @@
from __future__ import annotations
import asyncio
import copy
import re
import uuid
@@ -8,6 +7,7 @@ import uuid
import sqlalchemy
from ....core import app, taskmgr
from ....core.task_boundary import create_detached_task
from ....entity.persistence import mcp as persistence_mcp
from ....entity.persistence import plugin as persistence_plugin
from ....provider.tools.loaders.mcp import MCPSessionStatus, RuntimeMCPSession
@@ -249,7 +249,10 @@ class MCPService:
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_mcp.MCPServer).values(payload))
created = await self._get_mcp_server_by_uuid_raw(execution_context, payload['uuid'])
if created and self.ap.tool_mgr.mcp_tool_loader:
task = asyncio.create_task(self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(execution_context, created))
task = create_detached_task(
self.ap.tool_mgr.mcp_tool_loader.host_mcp_server(execution_context, created),
after_commit_manager=self.ap.persistence_mgr,
)
self.ap.tool_mgr.mcp_tool_loader._hosted_mcp_tasks.append(task)
return payload['uuid']
@@ -351,7 +354,10 @@ class MCPService:
if old_enable and loader.has_session(execution_context, old_name):
await loader.remove_mcp_server(execution_context, old_name)
if new_enable:
task = asyncio.create_task(loader.host_mcp_server(execution_context, updated))
task = create_detached_task(
loader.host_mcp_server(execution_context, updated),
after_commit_manager=self.ap.persistence_mgr,
)
loader._hosted_mcp_tasks.append(task)
async def delete_mcp_server(self, context: TenantContext, server_uuid: str) -> None:
+138 -107
View File
@@ -2,8 +2,11 @@ from __future__ import annotations
import uuid
import datetime
import functools
import json
import sqlalchemy
from sqlalchemy.dialects import postgresql as postgresql_dialect
from sqlalchemy.dialects import sqlite as sqlite_dialect
from ....core import app
from ....entity.persistence import monitoring as persistence_monitoring
@@ -12,6 +15,21 @@ from ..context import ExecutionContext
from .tenant import TenantContext, require_workspace_uuid
def _workspace_transaction(method):
"""Run an explicit service entrypoint in one Workspace transaction."""
@functools.wraps(method)
async def wrapped(self, context, *args, **kwargs):
workspace_uuid = require_workspace_uuid(context)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if callable(tenant_uow):
async with tenant_uow(workspace_uuid):
return await method(self, context, *args, **kwargs)
return await method(self, context, *args, **kwargs)
return wrapped
class MonitoringService:
"""Monitoring service"""
@@ -49,7 +67,7 @@ class MonitoringService:
Returns:
A dict mapping table name to the number of deleted rows.
"""
self._require_write_context(context)
workspace_uuid = self._require_write_context(context)
if retention_days < 1:
raise ValueError('retention_days must be >= 1')
if batch_size < 1:
@@ -104,17 +122,27 @@ class MonitoringService:
),
]
deleted_counts: dict[str, int] = {}
async def delete_records() -> dict[str, int]:
deleted_counts: dict[str, int] = {}
for table_name, model_cls, ts_column, pk_column in tables_and_columns:
deleted_counts[table_name] = await self._delete_expired_in_batches(
context=context,
model_cls=model_cls,
ts_column=ts_column,
pk_column=pk_column,
cutoff=cutoff,
batch_size=batch_size,
)
return deleted_counts
for table_name, model_cls, ts_column, pk_column in tables_and_columns:
deleted_counts[table_name] = await self._delete_expired_in_batches(
context=context,
model_cls=model_cls,
ts_column=ts_column,
pk_column=pk_column,
cutoff=cutoff,
batch_size=batch_size,
)
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
if callable(tenant_scope):
# Carry the Workspace across the complete cleanup without holding a
# connection. Each select+delete batch opens and commits its own UoW.
async with tenant_scope(workspace_uuid):
deleted_counts = await delete_records()
else:
deleted_counts = await delete_records()
if sum(deleted_counts.values()) > 0:
await self._release_sqlite_space()
@@ -134,25 +162,36 @@ class MonitoringService:
deleted_total = 0
while True:
select_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(pk_column)
.where(model_cls.workspace_uuid == workspace_uuid, ts_column < cutoff)
.limit(batch_size)
)
pk_values = list(select_result.scalars().all())
if not pk_values:
break
delete_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.delete(model_cls).where(
model_cls.workspace_uuid == workspace_uuid,
pk_column.in_(pk_values),
async def delete_batch() -> tuple[int, int]:
select_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(pk_column)
.where(model_cls.workspace_uuid == workspace_uuid, ts_column < cutoff)
.limit(batch_size)
)
)
deleted = delete_result.rowcount or 0
deleted_total += deleted
pk_values = list(select_result.scalars().all())
if not pk_values:
return 0, 0
if len(pk_values) < batch_size:
delete_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.delete(model_cls).where(
model_cls.workspace_uuid == workspace_uuid,
pk_column.in_(pk_values),
)
)
return len(pk_values), int(delete_result.rowcount or 0)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if callable(tenant_uow):
async with tenant_uow(workspace_uuid):
selected, deleted = await delete_batch()
else:
selected, deleted = await delete_batch()
deleted_total += deleted
if selected == 0:
break
if selected < batch_size:
break
return deleted_total
@@ -192,22 +231,30 @@ class MonitoringService:
session_id: str | None = None,
):
workspace_uuid = self._require_write_context(context)
context_columns = (
persistence_monitoring.MonitoringMessage.id,
persistence_monitoring.MonitoringMessage.bot_id,
persistence_monitoring.MonitoringMessage.bot_name,
persistence_monitoring.MonitoringMessage.pipeline_id,
persistence_monitoring.MonitoringMessage.pipeline_name,
persistence_monitoring.MonitoringMessage.session_id,
)
if message_id:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_monitoring.MonitoringMessage).where(
sqlalchemy.select(*context_columns).where(
persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringMessage.id == message_id,
)
)
row = result.first()
if row:
return row[0]
return row
if not session_id:
return None
user_query = (
sqlalchemy.select(persistence_monitoring.MonitoringMessage)
sqlalchemy.select(*context_columns)
.where(
sqlalchemy.and_(
persistence_monitoring.MonitoringMessage.session_id == session_id,
@@ -221,10 +268,10 @@ class MonitoringService:
result = await self.ap.persistence_mgr.execute_async(user_query)
row = result.first()
if row:
return row[0]
return row
any_query = (
sqlalchemy.select(persistence_monitoring.MonitoringMessage)
sqlalchemy.select(*context_columns)
.where(
persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringMessage.session_id == session_id,
@@ -234,10 +281,11 @@ class MonitoringService:
)
result = await self.ap.persistence_mgr.execute_async(any_query)
row = result.first()
return row[0] if row else None
return row
# ========== Recording Methods ==========
@_workspace_transaction
async def record_message(
self,
context: ExecutionContext,
@@ -285,6 +333,7 @@ class MonitoringService:
return message_id
@_workspace_transaction
async def record_llm_call(
self,
context: ExecutionContext,
@@ -331,6 +380,7 @@ class MonitoringService:
return call_id
@_workspace_transaction
async def record_tool_call(
self,
context: ExecutionContext,
@@ -389,6 +439,7 @@ class MonitoringService:
return call_id
@_workspace_transaction
async def record_embedding_call(
self,
context: ExecutionContext,
@@ -432,6 +483,7 @@ class MonitoringService:
return call_id
@_workspace_transaction
async def record_session_start(
self,
context: ExecutionContext,
@@ -466,6 +518,7 @@ class MonitoringService:
sqlalchemy.insert(persistence_monitoring.MonitoringSession).values(session_data)
)
@_workspace_transaction
async def update_session_activity(
self,
context: ExecutionContext,
@@ -503,6 +556,7 @@ class MonitoringService:
# Check if any rows were updated
return result.rowcount > 0
@_workspace_transaction
async def record_error(
self,
context: ExecutionContext,
@@ -540,6 +594,7 @@ class MonitoringService:
return error_id
@_workspace_transaction
async def update_message_status(
self,
context: ExecutionContext,
@@ -1802,81 +1857,57 @@ class MonitoringService:
)
return None
# Check if record with this feedback_id already exists
existing_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(MonitoringFeedback).where(
MonitoringFeedback.workspace_uuid == workspace_uuid,
MonitoringFeedback.feedback_id == feedback_id,
)
)
existing_row = existing_result.first()
if existing_row:
# UPDATE existing record
existing = existing_row[0] if isinstance(existing_row, tuple) else existing_row
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(MonitoringFeedback)
.where(
MonitoringFeedback.workspace_uuid == workspace_uuid,
MonitoringFeedback.feedback_id == feedback_id,
)
.values(
timestamp=now,
feedback_type=feedback_type,
feedback_content=feedback_content,
inaccurate_reasons=reasons_json,
bot_id=bot_id or existing.bot_id,
bot_name=bot_name or existing.bot_name,
pipeline_id=pipeline_id or existing.pipeline_id,
pipeline_name=pipeline_name or existing.pipeline_name,
session_id=session_id or existing.session_id,
message_id=message_id or existing.message_id,
stream_id=stream_id or existing.stream_id,
user_id=user_id or existing.user_id,
platform=platform or existing.platform,
)
)
return existing.id
record_data = {
'id': str(uuid.uuid4()),
'workspace_uuid': workspace_uuid,
'timestamp': now,
'feedback_id': feedback_id,
'feedback_type': feedback_type,
'feedback_content': feedback_content,
'inaccurate_reasons': reasons_json,
'bot_id': bot_id,
'bot_name': bot_name,
'pipeline_id': pipeline_id,
'pipeline_name': pipeline_name,
'session_id': session_id,
'message_id': message_id,
'stream_id': stream_id,
'user_id': user_id,
'platform': platform,
}
dialect_name = self.ap.persistence_mgr.get_db_engine().dialect.name
if dialect_name == 'postgresql':
statement = postgresql_dialect.insert(MonitoringFeedback).values(record_data)
elif dialect_name == 'sqlite':
statement = sqlite_dialect.insert(MonitoringFeedback).values(record_data)
else:
# INSERT new record with IntegrityError defense
record_id = str(uuid.uuid4())
record_data = {
'id': record_id,
'workspace_uuid': workspace_uuid,
'timestamp': now,
'feedback_id': feedback_id,
'feedback_type': feedback_type,
'feedback_content': feedback_content,
'inaccurate_reasons': reasons_json,
'bot_id': bot_id,
'bot_name': bot_name,
'pipeline_id': pipeline_id,
'pipeline_name': pipeline_name,
'session_id': session_id,
'message_id': message_id,
'stream_id': stream_id,
'user_id': user_id,
'platform': platform,
}
try:
await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(MonitoringFeedback).values(record_data))
return record_id
except Exception:
# UNIQUE constraint conflict (concurrent feedback for same feedback_id)
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(MonitoringFeedback)
.where(
MonitoringFeedback.workspace_uuid == workspace_uuid,
MonitoringFeedback.feedback_id == feedback_id,
)
.values(
timestamp=now,
feedback_type=feedback_type,
feedback_content=feedback_content,
inaccurate_reasons=reasons_json,
)
)
return feedback_id
raise RuntimeError(f'Monitoring feedback upsert does not support {dialect_name!r}')
excluded = statement.excluded
def preserve_existing(column):
return sqlalchemy.func.coalesce(sqlalchemy.func.nullif(getattr(excluded, column.key), ''), column)
statement = statement.on_conflict_do_update(
index_elements=[MonitoringFeedback.workspace_uuid, MonitoringFeedback.feedback_id],
set_={
'timestamp': excluded.timestamp,
'feedback_type': excluded.feedback_type,
'feedback_content': excluded.feedback_content,
'inaccurate_reasons': excluded.inaccurate_reasons,
'bot_id': preserve_existing(MonitoringFeedback.bot_id),
'bot_name': preserve_existing(MonitoringFeedback.bot_name),
'pipeline_id': preserve_existing(MonitoringFeedback.pipeline_id),
'pipeline_name': preserve_existing(MonitoringFeedback.pipeline_name),
'session_id': preserve_existing(MonitoringFeedback.session_id),
'message_id': preserve_existing(MonitoringFeedback.message_id),
'stream_id': preserve_existing(MonitoringFeedback.stream_id),
'user_id': preserve_existing(MonitoringFeedback.user_id),
'platform': preserve_existing(MonitoringFeedback.platform),
},
).returning(MonitoringFeedback.id)
result = await self.ap.persistence_mgr.execute_async(statement)
return str(result.scalar_one())
async def get_feedback_stats(
self,
+53 -5
View File
@@ -4,6 +4,7 @@ import io
import inspect
import os
import posixpath
import stat
import zipfile
from typing import Optional
from urllib.parse import quote, unquote, urlparse
@@ -34,6 +35,12 @@ _GITHUB_ASSET_HOSTS = {
'raw.githubusercontent.com',
'codeload.github.com',
}
_MAX_GITHUB_ARCHIVE_BYTES = 10 * 1024 * 1024
_MAX_GITHUB_ARCHIVE_ENTRIES = 4096
_MAX_SKILL_ARCHIVE_FILES = 1024
_MAX_SKILL_FILE_BYTES = 10 * 1024 * 1024
_MAX_SKILL_UNCOMPRESSED_BYTES = 50 * 1024 * 1024
_MAX_SKILL_COMPRESSION_RATIO = 200
class SkillService:
@@ -322,9 +329,22 @@ class SkillService:
async def _download_github_asset(self, asset_url: str) -> bytes:
async with httpx.AsyncClient(follow_redirects=True, timeout=120) as client:
resp = await client.get(asset_url)
resp.raise_for_status()
return resp.content
async with client.stream('GET', asset_url) as resp:
resp.raise_for_status()
content_length = resp.headers.get('content-length')
if content_length is not None:
try:
if int(content_length) > _MAX_GITHUB_ARCHIVE_BYTES:
raise ValueError('GitHub skill archive exceeds the compressed size limit')
except ValueError as exc:
if 'exceeds' in str(exc):
raise
content = bytearray()
async for chunk in resp.aiter_bytes():
content.extend(chunk)
if len(content) > _MAX_GITHUB_ARCHIVE_BYTES:
raise ValueError('GitHub skill archive exceeds the compressed size limit')
return bytes(content)
async def _download_github_skill_directory_as_zip(
self, asset_url: str, *, owner: str, repo: str
@@ -339,7 +359,11 @@ class SkillService:
raise ValueError('GitHub repository archive must be a valid .zip archive') from exc
with source_archive as source_zip:
if len(source_zip.infolist()) > _MAX_GITHUB_ARCHIVE_ENTRIES:
raise ValueError('GitHub repository archive contains too many entries')
skill_entry = self._find_github_skill_archive_entry(source_zip, info['file_path'])
if skill_entry.file_size > _MAX_SKILL_FILE_BYTES:
raise ValueError('GitHub SKILL.md exceeds the file size limit')
try:
skill_md_content = source_zip.read(skill_entry).decode('utf-8')
except UnicodeDecodeError as exc:
@@ -377,6 +401,7 @@ class SkillService:
normalized_source_dir = posixpath.normpath(source_skill_dir)
source_prefix = f'{normalized_source_dir}/'
copied_files = 0
copied_bytes = 0
for member in source_zip.infolist():
normalized_member = posixpath.normpath(member.filename)
@@ -399,10 +424,33 @@ class SkillService:
if member.is_dir():
target_zip.writestr(target_info, b'')
continue
target_zip.writestr(target_info, source_zip.read(member))
if member.flag_bits & 0x1:
raise ValueError('Encrypted GitHub skill archive entries are not supported')
unix_mode = member.external_attr >> 16
if stat.S_IFMT(unix_mode) == stat.S_IFLNK:
raise ValueError(f'GitHub archive contains a symbolic link: {member.filename}')
if member.file_size > _MAX_SKILL_FILE_BYTES:
raise ValueError(f'GitHub skill file exceeds the size limit: {member.filename}')
if member.file_size and member.file_size > max(member.compress_size, 1) * _MAX_SKILL_COMPRESSION_RATIO:
raise ValueError(f'GitHub skill file exceeds the compression-ratio limit: {member.filename}')
copied_files += 1
copied_bytes += member.file_size
if copied_files > _MAX_SKILL_ARCHIVE_FILES:
raise ValueError('GitHub skill directory contains too many files')
if copied_bytes > _MAX_SKILL_UNCOMPRESSED_BYTES:
raise ValueError('GitHub skill directory exceeds the uncompressed size limit')
# Copy in bounded chunks instead of materialising a potentially
# large member in Core memory. The Box Runtime independently
# revalidates the resulting archive before installation.
with source_zip.open(member, 'r') as source_file, target_zip.open(target_info, 'w') as target_file:
remaining = member.file_size
while remaining:
chunk = source_file.read(min(64 * 1024, remaining))
if not chunk:
raise ValueError(f'GitHub skill file is truncated: {member.filename}')
target_file.write(chunk)
remaining -= len(chunk)
if copied_files == 0:
raise ValueError('GitHub skill directory is empty')
+69 -34
View File
@@ -151,10 +151,11 @@ class UserService:
)
async def is_initialized(self) -> bool:
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
result_list = result.all()
return result_list is not None and len(result_list) > 0
account = await self._identity_scalar(
sqlalchemy.select(user.User).limit(1),
f'instance:{self._jwt_identity()[1]}',
)
return account is not None
def _session_factory(self) -> async_sessionmaker[AsyncSession]:
return async_sessionmaker(self.ap.persistence_mgr.get_db_engine(), expire_on_commit=False)
@@ -247,28 +248,24 @@ class UserService:
async def get_user_by_email(self, user_email: str) -> user.User | None:
normalized_email = user_email.strip().casefold()
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email)
return await self._identity_scalar(
sqlalchemy.select(user.User).where(user.User.normalized_email == normalized_email),
f'email:{normalized_email}',
)
result_list = result.all()
return result_list[0] if result_list is not None and len(result_list) > 0 else None
async def get_user_by_uuid(self, account_uuid: str) -> user.User | None:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(user.User).where(user.User.uuid == account_uuid)
return await self._identity_scalar(
sqlalchemy.select(user.User).where(user.User.uuid == account_uuid),
f'uuid:{account_uuid}',
)
return result.first()
async def get_user_by_space_account_uuid(self, space_account_uuid: str) -> user.User | None:
"""Get user by Space account UUID"""
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid)
return await self._identity_scalar(
sqlalchemy.select(user.User).where(user.User.space_account_uuid == space_account_uuid),
f'space:{space_account_uuid}',
)
result_list = result.all()
return result_list[0] if result_list is not None and len(result_list) > 0 else None
async def authenticate(self, user_email: str, password: str) -> str | None:
user_obj = await self.get_user_by_email(user_email)
if user_obj is None:
@@ -389,11 +386,13 @@ class UserService:
async def reset_password(self, user_email: str, new_password: str) -> None:
hashed_password = await self._hash_password(new_password)
normalized_email = normalize_email(user_email)
await self.ap.persistence_mgr.execute_async(
await self._identity_execute(
sqlalchemy.update(user.User)
.where(user.User.normalized_email == normalize_email(user_email))
.values(password=hashed_password)
.where(user.User.normalized_email == normalized_email)
.values(password=hashed_password),
f'email:{normalized_email}',
)
async def change_password(self, user_email: str, current_password: str, new_password: str) -> None:
@@ -407,11 +406,13 @@ class UserService:
await self._verify_password(user_obj.password, current_password)
hashed_password = await self._hash_password(new_password)
normalized_email = normalize_email(user_email)
await self.ap.persistence_mgr.execute_async(
await self._identity_execute(
sqlalchemy.update(user.User)
.where(user.User.normalized_email == normalize_email(user_email))
.values(password=hashed_password)
.where(user.User.normalized_email == normalized_email)
.values(password=hashed_password),
f'email:{normalized_email}',
)
# Space user management
@@ -435,7 +436,7 @@ class UserService:
if existing_user:
# Update existing user's tokens
await self.ap.persistence_mgr.execute_async(
await self._identity_execute(
sqlalchemy.update(user.User)
.where(user.User.space_account_uuid == space_account_uuid)
.values(
@@ -443,7 +444,8 @@ class UserService:
space_refresh_token=refresh_token,
space_api_key=api_key,
space_access_token_expires_at=expires_at,
)
),
f'space:{space_account_uuid}',
)
await self._update_space_provider_for_account(existing_user, api_key)
return await self.get_user_by_space_account_uuid(space_account_uuid)
@@ -538,9 +540,38 @@ class UserService:
async def get_first_user(self) -> user.User | None:
"""Get the first user (for single-user mode)"""
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(user.User).limit(1))
result_list = result.all()
return result_list[0] if result_list else None
return await self._identity_scalar(
sqlalchemy.select(user.User).limit(1),
f'instance:{self._jwt_identity()[1]}',
)
async def _identity_scalar(
self,
statement: typing.Any,
identity: str,
) -> user.User | None:
"""Execute one exact Account lookup in an explicit discovery transaction."""
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
if current_session() is None and callable(identity_uow):
async with identity_uow(digest) as discovery:
return await discovery.session.scalar(statement)
result = await self.ap.persistence_mgr.execute_async(statement)
rows = result.all()
return rows[0] if rows else None
async def _identity_execute(self, statement: typing.Any, identity: str) -> typing.Any:
"""Execute one exact Account mutation in an explicit transaction."""
digest = hashlib.sha256(identity.encode('utf-8')).hexdigest()
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
identity_uow = getattr(self.ap.persistence_mgr, 'identity_discovery_uow', None)
if current_session() is None and callable(identity_uow):
async with identity_uow(digest) as discovery:
return await discovery.session.execute(statement)
return await self.ap.persistence_mgr.execute_async(statement)
async def set_password(self, user_email: str, new_password: str, current_password: str | None = None) -> None:
"""Set or change password for a user"""
@@ -557,10 +588,12 @@ class UserService:
await self._verify_password(user_obj.password, current_password)
hashed_password = await self._hash_password(new_password)
await self.ap.persistence_mgr.execute_async(
normalized_email = normalize_email(user_email)
await self._identity_execute(
sqlalchemy.update(user.User)
.where(user.User.normalized_email == normalize_email(user_email))
.values(password=hashed_password)
.where(user.User.normalized_email == normalized_email)
.values(password=hashed_password),
f'email:{normalized_email}',
)
async def bind_space_account(self, user_email: str, code: str) -> user.User:
@@ -596,9 +629,10 @@ class UserService:
raise ValueError('This Space account is already bound to another user')
# Update local account to Space account
await self.ap.persistence_mgr.execute_async(
normalized_email = normalize_email(user_email)
await self._identity_execute(
sqlalchemy.update(user.User)
.where(user.User.normalized_email == normalize_email(user_email))
.where(user.User.normalized_email == normalized_email)
.values(
user=normalize_email(space_email), # Update email to Space email
normalized_email=normalize_email(space_email),
@@ -608,7 +642,8 @@ class UserService:
space_refresh_token=refresh_token,
space_api_key=api_key,
space_access_token_expires_at=expires_at,
)
),
f'email:{normalized_email}',
)
# Update Space model provider API keys
+38 -5
View File
@@ -31,6 +31,9 @@ if typing.TYPE_CHECKING:
# JSON-RPC-ish 401 body returned before the MCP app is reached.
_UNAUTHORIZED_BODY = b'{"error":"unauthorized","message":"A valid LangBot API key is required for MCP access."}'
_ENTITLEMENT_UNAVAILABLE_BODY = (
b'{"error":"entitlement_unavailable","message":"Workspace entitlement is unavailable for MCP access."}'
)
def _extract_api_key(headers: list[tuple[bytes, bytes]]) -> str:
@@ -110,6 +113,29 @@ class MCPMount:
await send({'type': 'http.response.body', 'body': _UNAUTHORIZED_BODY})
return
deployment_admission = getattr(self.ap, 'deployment_admission', None)
try:
if deployment_admission is not None:
deployment_admission.require_active()
entitlement_revision = 0
deployment = getattr(self.ap, 'deployment', None)
if deployment is not None and getattr(deployment, 'multi_workspace_enabled', False):
resolver = getattr(self.ap, 'entitlement_resolver', None)
if resolver is None or identity.instance_uuid != resolver.instance_uuid:
raise RuntimeError('Workspace entitlement resolver is unavailable')
entitlement = await resolver.resolve(identity.workspace_uuid)
entitlement_revision = entitlement.entitlement_revision
except Exception:
await send(
{
'type': 'http.response.start',
'status': 403,
'headers': [(b'content-type', b'application/json')],
}
)
await send({'type': 'http.response.body', 'body': _ENTITLEMENT_UNAVAILABLE_BODY})
return
request_context = RequestContext(
instance_uuid=identity.instance_uuid,
placement_generation=identity.placement_generation,
@@ -125,11 +151,18 @@ class MCPMount:
role=None,
permissions=identity.permissions,
),
entitlement_revision=entitlement_revision,
)
token = bind_request_context(request_context)
try:
await mcp_asgi(scope, receive, send)
finally:
reset_request_context(token)
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
if not callable(tenant_scope):
raise RuntimeError('MCP request persistence scope is unavailable')
async with tenant_scope(identity.workspace_uuid):
token = bind_request_context(request_context)
try:
await mcp_asgi(scope, receive, send)
if deployment_admission is not None:
deployment_admission.require_active()
finally:
reset_request_context(token)
return dispatcher
+217
View File
@@ -0,0 +1,217 @@
from __future__ import annotations
import asyncio
import datetime as dt
import time
from collections.abc import Callable
from typing import TYPE_CHECKING
from langbot_plugin.box.errors import BoxAdmissionError, BoxRuntimeUnavailableError
from langbot_plugin.box.models import (
SandboxAdmissionGrant,
SandboxAdmissionPolicy,
SandboxAdmissionRevocation,
)
from ..api.http.context import ExecutionContext
from ..cloud.entitlements import EntitlementSnapshot, EntitlementUnavailableError
if TYPE_CHECKING:
from langbot_plugin.box.client import BoxRuntimeClient
from ..core.app import Application
_UTC = dt.timezone.utc
_MANAGED_SANDBOX_FEATURE = 'managed_sandbox'
_MANAGED_SANDBOX_SESSION_LIMIT = 'managed_sandbox_sessions'
_MAX_GRANT_TTL_SEC = 300
class SandboxAdmissionController:
"""Project Cloud entitlements into short-lived Box Runtime grants.
Product and plan names intentionally never cross this boundary. The
closed Control Plane supplies a versioned generic entitlement, while Core
installs only the numeric authority understood by the shared Box Runtime.
No state is allocated for a Workspace until it attempts to use the
managed sandbox. Per-Workspace locks serialize renewal/revocation so a
concurrent first use cannot install conflicting grants.
"""
def __init__(
self,
ap: Application,
client: BoxRuntimeClient,
*,
policy: SandboxAdmissionPolicy,
wall_time: Callable[[], float] = time.time,
) -> None:
self.ap = ap
self.client = client
self.policy = policy
self._wall_time = wall_time
self._locks: dict[str, asyncio.Lock] = {}
self._highest_revisions: dict[str, int] = {}
def _workspace_lock(self, workspace_uuid: str) -> asyncio.Lock:
lock = self._locks.get(workspace_uuid)
if lock is None:
lock = asyncio.Lock()
self._locks[workspace_uuid] = lock
return lock
@staticmethod
def _context_revision(context: ExecutionContext) -> int:
revision = getattr(context, 'entitlement_revision', 0)
if isinstance(revision, bool) or not isinstance(revision, int):
return 0
return max(revision, 0)
def _revocation_revision(self, context: ExecutionContext, candidate_revision: int = 0) -> int:
return max(
1,
self._highest_revisions.get(context.workspace_uuid, 0),
self._context_revision(context),
candidate_revision,
)
async def _revoke_locked(
self,
context: ExecutionContext,
*,
candidate_revision: int = 0,
) -> None:
revision = self._revocation_revision(context, candidate_revision)
revocation = SandboxAdmissionRevocation(
instance_uuid=context.instance_uuid,
workspace_uuid=context.workspace_uuid,
entitlement_revision=revision,
)
try:
result = await self.client.revoke_sandbox_admission_grant(revocation)
if (
not isinstance(result, dict)
or result.get('revoked') is not True
or result.get('workspace_uuid') != context.workspace_uuid
or result.get('entitlement_revision') != revision
):
raise BoxRuntimeUnavailableError('Box Runtime returned an invalid sandbox revocation receipt')
except Exception as exc:
# The caller still fails closed even if the control connection is
# unavailable. A previously installed grant expires independently
# in at most five minutes inside the Runtime.
self.ap.logger.warning(
'Failed to install Box sandbox admission revocation: '
f'workspace_uuid={context.workspace_uuid} revision={revision} error={exc}'
)
self._highest_revisions[context.workspace_uuid] = revision
@staticmethod
def _require_managed_sandbox(snapshot: EntitlementSnapshot) -> None:
snapshot.require_feature(_MANAGED_SANDBOX_FEATURE)
sessions = snapshot.limit(_MANAGED_SANDBOX_SESSION_LIMIT)
if sessions != 1:
raise EntitlementUnavailableError('Workspace entitlement must grant exactly one managed sandbox session')
def _grant_expiry(self, snapshot: EntitlementSnapshot) -> dt.datetime:
now_epoch = int(self._wall_time())
ttl_sec = min(self.policy.max_grant_ttl_sec, _MAX_GRANT_TTL_SEC)
expires_epoch = min(snapshot.expires_at, now_epoch + ttl_sec)
if expires_epoch <= now_epoch:
raise EntitlementUnavailableError('Workspace entitlement expired before sandbox admission')
return dt.datetime.fromtimestamp(expires_epoch, tz=_UTC)
async def require(self, context: ExecutionContext) -> SandboxAdmissionGrant:
"""Validate entitlement freshness and install/renew one Runtime grant."""
resolver = getattr(self.ap, 'entitlement_resolver', None)
if resolver is None:
raise EntitlementUnavailableError('Workspace entitlement resolver is unavailable')
if context.instance_uuid != resolver.instance_uuid:
raise EntitlementUnavailableError('Workspace entitlement targets another LangBot instance')
lock = self._workspace_lock(context.workspace_uuid)
async with lock:
try:
snapshot = await resolver.resolve(
context.workspace_uuid,
minimum_revision=self._context_revision(context),
now=int(self._wall_time()),
)
except EntitlementUnavailableError as exc:
# Only a verified, scoped snapshot can authoritatively revoke
# a revision. Provider timeouts, malformed responses, and
# rollback/equivocation errors fail this request closed but do
# not tombstone a still-valid revision forever.
authoritative_revision = exc.entitlement_revision
if authoritative_revision is not None:
await self._revoke_locked(
context,
candidate_revision=authoritative_revision,
)
raise
try:
self._require_managed_sandbox(snapshot)
except EntitlementUnavailableError:
await self._revoke_locked(
context,
candidate_revision=snapshot.entitlement_revision,
)
raise
grant = SandboxAdmissionGrant(
instance_uuid=context.instance_uuid,
workspace_uuid=context.workspace_uuid,
execution_generation=context.placement_generation,
entitlement_revision=snapshot.entitlement_revision,
expires_at=self._grant_expiry(snapshot),
max_sessions=1,
max_managed_processes=0,
)
result = await self.client.upsert_sandbox_admission_grant(grant)
if (
not isinstance(result, dict)
or result.get('installed') is not True
or result.get('workspace_uuid') != context.workspace_uuid
or result.get('execution_generation') != context.placement_generation
or result.get('entitlement_revision') != snapshot.entitlement_revision
or result.get('max_sessions') != 1
or result.get('max_managed_processes') != 0
):
raise BoxRuntimeUnavailableError('Box Runtime returned an invalid sandbox admission receipt')
self._highest_revisions[context.workspace_uuid] = max(
self._highest_revisions.get(context.workspace_uuid, 0),
snapshot.entitlement_revision,
)
return grant
async def revoke(self, context: ExecutionContext, *, entitlement_revision: int = 0) -> None:
"""Explicitly revoke a Workspace grant using a monotonic tombstone."""
async with self._workspace_lock(context.workspace_uuid):
await self._revoke_locked(context, candidate_revision=entitlement_revision)
def require_cloud_admission_policy(raw_policy: object) -> SandboxAdmissionPolicy:
"""Parse the Cloud Box policy without permitting an OSS downgrade."""
try:
policy = SandboxAdmissionPolicy.model_validate(raw_policy)
except Exception as exc:
raise BoxAdmissionError('Cloud Box sandbox admission policy is invalid') from exc
if not policy.required:
raise BoxAdmissionError('Cloud Box sandbox admission must be required')
if policy.logical_session_id != 'global':
raise BoxAdmissionError('Cloud Box sandbox session ID must be global')
if policy.required_backend != 'nsjail':
raise BoxAdmissionError('Cloud Box sandbox backend must be nsjail')
if policy.max_sessions != 1 or policy.max_managed_processes != 0:
raise BoxAdmissionError('Cloud Box sandbox policy must allow one session and zero managed processes')
if policy.max_grant_ttl_sec > _MAX_GRANT_TTL_SEC:
raise BoxAdmissionError('Cloud Box sandbox admission grant TTL must not exceed 300 seconds')
if policy.workspace_quota_mb <= 0:
raise BoxAdmissionError('Cloud Box sandbox workspace quota must be a positive integer')
return policy
+285
View File
@@ -0,0 +1,285 @@
from __future__ import annotations
import contextlib
import errno
import os
import stat
from collections.abc import Iterable
class UnsafeWorkspacePathError(OSError):
"""A tenant-controlled path could not be opened without following links."""
_DIRECTORY_FLAGS = (
os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
)
_FILE_READ_FLAGS = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
_FILE_WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
_MAX_REMOVAL_ENTRIES = 4096
_MAX_REMOVAL_DEPTH = 16
def _component(value: str) -> str:
normalized = str(value or '').strip()
if (
not normalized
or normalized in {'.', '..'}
or '/' in normalized
or '\\' in normalized
or '\x00' in normalized
or len(os.fsencode(normalized)) > 240
):
raise UnsafeWorkspacePathError('Unsafe Workspace path component')
return normalized
def _unsafe(path: str, exc: BaseException | None = None) -> UnsafeWorkspacePathError:
error = UnsafeWorkspacePathError(f'Workspace path is not a link-free directory: {path}')
if exc is not None:
error.__cause__ = exc
return error
@contextlib.contextmanager
def _root_fd(root: str):
try:
fd = os.open(root, _DIRECTORY_FLAGS)
except OSError as exc:
raise _unsafe(root, exc)
try:
if not stat.S_ISDIR(os.fstat(fd).st_mode):
raise _unsafe(root)
yield fd
finally:
os.close(fd)
def _open_dir_at(parent_fd: int, name: str, *, create: bool) -> int:
name = _component(name)
if create:
try:
os.mkdir(name, mode=0o700, dir_fd=parent_fd)
except FileExistsError:
pass
try:
fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
except OSError as exc:
raise _unsafe(name, exc)
if not stat.S_ISDIR(os.fstat(fd).st_mode):
os.close(fd)
raise _unsafe(name)
return fd
def _remove_entry(
parent_fd: int,
name: str,
*,
budget: list[int] | None = None,
depth: int = 0,
) -> None:
"""Remove an entry recursively without following a symlink at any depth."""
name = _component(name)
budget = budget if budget is not None else [_MAX_REMOVAL_ENTRIES]
if depth > _MAX_REMOVAL_DEPTH or budget[0] <= 0:
raise UnsafeWorkspacePathError('Workspace cleanup exceeded its inode budget')
budget[0] -= 1
for _ in range(4):
try:
child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
except FileNotFoundError:
return
except OSError as exc:
if exc.errno not in {errno.ELOOP, errno.ENOTDIR, errno.EACCES}:
raise
try:
os.unlink(name, dir_fd=parent_fd)
return
except FileNotFoundError:
return
except IsADirectoryError:
continue
else:
try:
_clear_dir(child_fd, budget=budget, depth=depth + 1)
finally:
os.close(child_fd)
try:
os.rmdir(name, dir_fd=parent_fd)
return
except FileNotFoundError:
return
except NotADirectoryError:
continue
raise UnsafeWorkspacePathError('Workspace entry changed while it was being removed')
def _clear_dir(directory_fd: int, *, budget: list[int], depth: int) -> None:
# ``scandir(fd)`` enumerates the already-open directory. Names are then
# resolved relative to the same fd, so a tenant cannot redirect the walk by
# swapping an ancestor symlink between validation and use.
# Do not materialize the whole directory: an attacker-controlled outbox
# may contain an inode bomb even when its byte size is tiny. Removal is
# deliberately budgeted and fails closed once the per-operation cap is
# reached; hard filesystem/inode quota remains a Cloud readiness gate.
with os.scandir(directory_fd) as iterator:
for entry in iterator:
_remove_entry(directory_fd, entry.name, budget=budget, depth=depth)
@contextlib.contextmanager
def _query_fd(root: str, subdir: str, query_key: str, *, create: bool, reset: bool = False):
subdir = _component(subdir)
query_key = _component(query_key)
with _root_fd(root) as root_fd:
subdir_fd = _open_dir_at(root_fd, subdir, create=create)
try:
if reset:
_remove_entry(subdir_fd, query_key)
query_fd = _open_dir_at(subdir_fd, query_key, create=create)
try:
yield query_fd
finally:
os.close(query_fd)
finally:
os.close(subdir_fd)
def write_files(
root: str,
subdir: str,
query_key: str,
files: Iterable[tuple[str, bytes]],
) -> None:
"""Atomically recreate one query directory and write regular files only."""
with _query_fd(root, subdir, query_key, create=True, reset=True) as query_fd:
for raw_name, data in files:
name = _component(raw_name)
try:
file_fd = os.open(name, _FILE_WRITE_FLAGS, 0o600, dir_fd=query_fd)
except OSError as exc:
raise UnsafeWorkspacePathError(f'Could not create a link-free Workspace file: {name}') from exc
with os.fdopen(file_fd, 'wb') as file_obj:
file_obj.write(data)
def _read_directory(
directory_fd: int,
*,
prefix: str,
max_file_bytes: int,
max_files: int,
max_total_bytes: int,
output: list[tuple[str, bytes]],
total: list[int],
remaining_entries: list[int],
remaining_directories: list[int],
depth: int,
) -> None:
if depth > 8:
return
with os.scandir(directory_fd) as iterator:
for entry in iterator:
if len(output) >= max_files or total[0] >= max_total_bytes:
return
if remaining_entries[0] <= 0:
raise UnsafeWorkspacePathError('Sandbox outbox exceeds the directory-entry limit')
remaining_entries[0] -= 1
name = _component(entry.name)
relative = f'{prefix}/{name}' if prefix else name
if entry.is_symlink():
continue
if entry.is_dir(follow_symlinks=False):
if remaining_directories[0] <= 0:
raise UnsafeWorkspacePathError('Sandbox outbox exceeds the directory limit')
remaining_directories[0] -= 1
try:
child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=directory_fd)
except OSError:
continue
try:
_read_directory(
child_fd,
prefix=relative,
max_file_bytes=max_file_bytes,
max_files=max_files,
max_total_bytes=max_total_bytes,
output=output,
total=total,
remaining_entries=remaining_entries,
remaining_directories=remaining_directories,
depth=depth + 1,
)
finally:
os.close(child_fd)
continue
try:
file_fd = os.open(name, _FILE_READ_FLAGS, dir_fd=directory_fd)
except OSError:
continue
try:
metadata = os.fstat(file_fd)
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > max_file_bytes:
continue
remaining = max_total_bytes - total[0]
if metadata.st_size > remaining:
continue
with os.fdopen(file_fd, 'rb', closefd=False) as file_obj:
data = file_obj.read(max_file_bytes + 1)
if len(data) > max_file_bytes or len(data) > remaining:
continue
output.append((relative, data))
total[0] += len(data)
finally:
os.close(file_fd)
def read_regular_files(
root: str,
subdir: str,
query_key: str,
*,
max_file_bytes: int,
max_files: int,
max_total_bytes: int,
max_entries: int = 512,
max_directories: int = 64,
) -> list[tuple[str, bytes]]:
"""Read bounded regular files without following tenant-created links."""
output: list[tuple[str, bytes]] = []
try:
with _query_fd(root, subdir, query_key, create=False) as query_fd:
_read_directory(
query_fd,
prefix='',
max_file_bytes=max_file_bytes,
max_files=max_files,
max_total_bytes=max_total_bytes,
output=output,
total=[0],
remaining_entries=[max_entries],
remaining_directories=[max_directories],
depth=0,
)
except (FileNotFoundError, UnsafeWorkspacePathError):
# A missing directory is an empty outbox. An unsafe existing path is
# deliberately surfaced to the caller rather than followed.
if os.path.lexists(os.path.join(root, subdir, query_key)):
raise
return output
def reset_directory(root: str, subdir: str, query_key: str) -> None:
with _query_fd(root, subdir, query_key, create=True, reset=True):
return
def purge_subdirectory(root: str, subdir: str) -> None:
"""Remove one known subtree without following a hostile replacement link."""
with _root_fd(root) as root_fd:
_remove_entry(root_fd, _component(subdir))
+377 -69
View File
@@ -4,8 +4,10 @@ import asyncio
import collections
import datetime as _dt
import enum
import hashlib
import json
import os
import secrets
from typing import TYPE_CHECKING
import pydantic
@@ -13,11 +15,14 @@ import pydantic
from langbot_plugin.box.client import BoxRuntimeClient
from langbot_plugin.entities.io.context import ActionContext
from langbot_plugin.box.tenancy import box_namespace
from langbot_plugin.box.security import BOX_SHARED_WORKSPACE_PROBE_PREFIX
from .admission import SandboxAdmissionController, require_cloud_admission_policy
from .connector import BoxRuntimeConnector, _get_box_config
from . import secure_fs
from ..telemetry import features as telemetry_features
from ..api.http.context import ExecutionContext
from ..api.http.service.tenant import TenantContext, require_workspace_uuid
from langbot_plugin.box.errors import BoxError, BoxValidationError
from langbot_plugin.box.errors import BoxAdmissionError, BoxError, BoxValidationError
from langbot_plugin.box.models import (
BUILTIN_PROFILES,
BoxExecutionResult,
@@ -51,6 +56,7 @@ class BoxService:
output_limit_chars: int = 4000,
):
self.ap = ap
self._cloud_managed = bool(getattr(getattr(ap, 'deployment', None), 'multi_workspace_enabled', False))
self._enabled = self._load_enabled()
self._runtime_connector: BoxRuntimeConnector | None = None
if client is None:
@@ -67,6 +73,18 @@ class BoxService:
self.profile = self._load_profile()
self.custom_image = self._load_custom_image()
self.workspace_quota_mb = self._load_workspace_quota_mb()
self._admission_policy = (
require_cloud_admission_policy(_get_box_config(ap).get('admission')) if self._cloud_managed else None
)
self._admission = (
SandboxAdmissionController(
ap,
self.client,
policy=self._admission_policy,
)
if self._cloud_managed and self._admission_policy is not None
else None
)
self._recent_errors: collections.deque[dict] = collections.deque(maxlen=_MAX_RECENT_ERRORS)
self._shutdown_task = None
self._available = False
@@ -85,6 +103,10 @@ class BoxService:
``available = False`` to consumers, but distinguished in get_status."""
return self._enabled
@property
def managed_admission_required(self) -> bool:
return self._cloud_managed
async def initialize(self):
if not self._enabled:
# Disabled by config: do NOT connect to a remote runtime, do NOT
@@ -103,17 +125,24 @@ class BoxService:
else:
await self.client.initialize()
self._ensure_default_workspace()
await self._verify_cloud_runtime()
self._available = True
self._connector_error = ''
self.ap.logger.info(
f'LangBot Box runtime initialized: profile={self.profile.name} '
f'default_workspace={self.default_workspace or "(none)"}'
)
await self._purge_attachment_dirs()
# Cloud query directories use globally opaque query UUIDs. Never
# sweep all tenants when a future replica joins the same logical
# instance; that could delete another replica's in-flight files.
if not self._cloud_managed:
await self._purge_attachment_dirs()
except Exception as exc:
self.ap.logger.warning(f'LangBot Box runtime unavailable, sandbox features disabled: {exc}')
self._available = False
self._connector_error = str(exc)
if self._cloud_managed:
raise
async def _on_runtime_disconnect(self, connector: BoxRuntimeConnector) -> None:
"""Called by the connector when the Box runtime connection drops.
@@ -143,6 +172,7 @@ class BoxService:
try:
connector.dispose()
await connector.initialize()
await self._verify_cloud_runtime()
self._available = True
self._connector_error = ''
self.ap.logger.info('Box runtime reconnected, sandbox features restored.')
@@ -154,6 +184,83 @@ class BoxService:
finally:
self._reconnecting = False
async def _verify_cloud_runtime(self) -> None:
if not self._cloud_managed:
return
self._ensure_cloud_shared_workspace()
await self._challenge_cloud_shared_workspace()
backend_info = await self.client.get_backend_info()
if (
not isinstance(backend_info, dict)
or backend_info.get('name') != 'nsjail'
or backend_info.get('available') is not True
):
raise BoxValidationError('Cloud Box nsjail isolation readiness failed')
async def _challenge_cloud_shared_workspace(self) -> None:
"""Prove Core and Box Runtime see the same durable filesystem.
Equal configured path strings are not evidence of a shared container
volume. Core creates one high-entropy, no-follow marker under its
canonical root and the authenticated Runtime host-control action reads
that basename only. Any mismatch fails Cloud startup/reconnect closed.
"""
if self.default_workspace is None:
raise BoxValidationError('Cloud Box shared default_workspace is unavailable')
marker_name = f'{BOX_SHARED_WORKSPACE_PROBE_PREFIX}{secrets.token_hex(16)}'
marker_payload = secrets.token_bytes(64)
expected_digest = hashlib.sha256(marker_payload).hexdigest()
directory_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0)
nofollow = getattr(os, 'O_NOFOLLOW', 0)
root_fd: int | None = None
marker_fd: int | None = None
marker_created = False
try:
root_fd = os.open(self.default_workspace, directory_flags | nofollow)
marker_fd = os.open(
marker_name,
os.O_WRONLY | os.O_CREAT | os.O_EXCL | nofollow,
0o600,
dir_fd=root_fd,
)
marker_created = True
remaining = memoryview(marker_payload)
while remaining:
written = os.write(marker_fd, remaining)
if written <= 0:
raise BoxValidationError('Failed to write Cloud Box shared-volume probe')
remaining = remaining[written:]
os.fsync(marker_fd)
os.close(marker_fd)
marker_fd = None
result = await self.client.verify_shared_workspace(marker_name)
if (
not isinstance(result, dict)
or result.get('marker_name') != marker_name
or result.get('size') != len(marker_payload)
or not secrets.compare_digest(str(result.get('sha256') or ''), expected_digest)
):
raise BoxValidationError(
'Cloud Box Core and Runtime do not share the configured durable Workspace volume'
)
except BoxValidationError:
raise
except Exception as exc:
raise BoxValidationError('Cloud Box shared durable Workspace volume verification failed') from exc
finally:
if marker_fd is not None:
os.close(marker_fd)
if root_fd is not None:
if marker_created:
try:
os.unlink(marker_name, dir_fd=root_fd)
except FileNotFoundError:
pass
os.close(root_fd)
@property
def available(self) -> bool:
return self._available
@@ -200,10 +307,14 @@ class BoxService:
bot_uuid=getattr(context, 'bot_uuid', None),
pipeline_uuid=getattr(context, 'pipeline_uuid', None),
query_uuid=getattr(context, 'query_uuid', None),
entitlement_revision=getattr(context, 'entitlement_revision', 0),
)
@classmethod
def _query_execution_context(cls, query: pipeline_query.Query) -> ExecutionContext:
attached_context = getattr(query, '_execution_context', None)
if isinstance(attached_context, ExecutionContext):
return cls._execution_context(attached_context)
return cls._execution_context(
ExecutionContext(
instance_uuid=str(getattr(query, 'instance_uuid', '') or ''),
@@ -212,6 +323,7 @@ class BoxService:
bot_uuid=getattr(query, 'bot_uuid', None),
pipeline_uuid=getattr(query, 'pipeline_uuid', None),
query_uuid=getattr(query, 'query_uuid', None),
entitlement_revision=getattr(query, 'entitlement_revision', 0),
)
)
@@ -241,6 +353,116 @@ class BoxService:
raise BoxValidationError('Box execution context belongs to a stale Workspace placement')
return execution_context
async def require_workspace_sandbox(self, context: TenantContext) -> ExecutionContext:
"""Fence a Workspace and install its short-lived Cloud admission grant.
OSS keeps the existing singleton behavior and does not require a
Control Plane entitlement. Cloud always resolves a fresh generic
entitlement before a sandbox-visible operation.
"""
execution_context = await self._validated_execution_context(context)
await self._require_validated_workspace_sandbox(execution_context)
return execution_context
async def _require_validated_workspace_sandbox(self, execution_context: ExecutionContext) -> None:
if not self._available:
raise BoxError('Box runtime is not available. Install and start Docker to use sandbox features.')
if self._cloud_managed:
if self._admission is None:
raise BoxAdmissionError('Cloud Box sandbox admission is unavailable')
await self._admission.require(execution_context)
async def is_workspace_sandbox_available(self, context: TenantContext) -> bool:
"""Return tenant-specific availability for UI and tool discovery.
This method deliberately catches entitlement failures so callers can
hide tools without leaking plan details. Direct execution APIs use
:meth:`require_workspace_sandbox` and retain an explicit failure.
"""
if not self._available:
return False
try:
await self.require_workspace_sandbox(context)
return True
except Exception:
return False
def _managed_policy_payload(
self,
context: TenantContext,
spec_payload: dict,
) -> dict:
"""Reject tenant-owned policy fields and apply the Cloud hard policy."""
payload = dict(spec_payload)
if not self._cloud_managed:
return payload
policy = self._admission_policy
if policy is None:
raise BoxAdmissionError('Cloud Box sandbox admission policy is unavailable')
forged_fields = {
'plan',
'subscription',
'managed_sandbox',
'entitlement',
'entitlement_revision',
'max_sessions',
'max_managed_processes',
'backend',
}
submitted_forged_fields = sorted(forged_fields.intersection(payload))
if submitted_forged_fields:
raise BoxAdmissionError(
'Managed sandbox policy fields are host-controlled: ' + ', '.join(submitted_forged_fields)
)
submitted_session_id = str(payload.get('session_id', '') or '').strip()
if submitted_session_id and submitted_session_id != policy.logical_session_id:
raise BoxAdmissionError('Managed sandbox session_id is runtime-owned')
submitted_network = str(getattr(payload.get('network'), 'value', payload.get('network', 'off')) or 'off')
if submitted_network != 'off':
raise BoxAdmissionError('Managed sandbox network access is disabled')
if payload.get('extra_mounts'):
raise BoxAdmissionError('Managed sandbox additional host mounts are disabled')
submitted_mount_path = str(payload.get('mount_path', '/workspace') or '/workspace')
if submitted_mount_path != '/workspace':
raise BoxAdmissionError('Managed sandbox mount_path is runtime-owned')
canonical_host_path = self._tenant_workspace(context)
if canonical_host_path is None:
raise BoxAdmissionError('Managed sandbox Workspace path is unavailable')
submitted_host_path = str(payload.get('host_path', '') or '').strip()
if submitted_host_path and os.path.realpath(submitted_host_path) != os.path.realpath(canonical_host_path):
raise BoxAdmissionError('Managed sandbox host_path is runtime-owned')
timeout = payload.get('timeout_sec', policy.max_timeout_sec)
if isinstance(timeout, bool) or not isinstance(timeout, int):
raise BoxValidationError('timeout_sec must be an integer')
payload.update(
{
'session_id': policy.logical_session_id,
'network': 'off',
'host_path': canonical_host_path,
'mount_path': '/workspace',
'extra_mounts': [],
'persistent': True,
'timeout_sec': min(timeout, policy.max_timeout_sec),
'cpus': policy.cpus,
'memory_mb': policy.memory_mb,
'pids_limit': policy.pids_limit,
'read_only_rootfs': policy.read_only_rootfs,
'workspace_quota_mb': policy.workspace_quota_mb,
}
)
return payload
def _reject_cloud_managed_process(self) -> None:
if self._cloud_managed:
raise BoxAdmissionError('Managed processes are disabled for Cloud sandboxes')
def _tenant_workspace(self, context: TenantContext) -> str | None:
if self.default_workspace is None:
return None
@@ -257,7 +479,8 @@ class BoxService:
if not self._available:
raise BoxError('Box runtime is not available. Install and start Docker to use sandbox features.')
execution_context = await self._validated_execution_context(self._query_execution_context(query))
spec_payload = dict(spec_payload)
spec_payload = self._managed_policy_payload(execution_context, spec_payload)
await self._require_validated_workspace_sandbox(execution_context)
if spec_payload.get('host_path') in (None, ''):
tenant_workspace = self._tenant_workspace(execution_context)
if tenant_workspace is not None:
@@ -284,6 +507,11 @@ class BoxService:
spec,
action_context=self._action_context(execution_context),
)
# A placement may be cut over while a long-running sandbox call is
# in flight. Never accept a result produced by the superseded
# generation. Runtime-side generation fencing prevents new work;
# this second Core check closes the response race.
await self._validated_execution_context(execution_context)
except BoxError as exc:
self._record_error(exc, query)
raise
@@ -311,6 +539,8 @@ class BoxService:
by editing the pipeline config directly through the API (which only
gates the web UI).
"""
if self._cloud_managed:
return 'global'
forced_template = self._forced_box_session_id_template()
if forced_template:
template = forced_template
@@ -362,6 +592,8 @@ class BoxService:
skills it discovered on its own filesystem, so the path is valid there
by construction.
"""
if self._cloud_managed:
return []
skill_mgr = getattr(self.ap, 'skill_mgr', None)
if skill_mgr is None:
return []
@@ -392,13 +624,21 @@ class BoxService:
)
return mounts
async def execute_tool(self, parameters: dict, query: pipeline_query.Query) -> dict:
async def execute_tool(
self,
parameters: dict,
query: pipeline_query.Query,
*,
skill_name: str | None = None,
) -> dict:
"""Execute an agent-facing ``exec`` tool call.
Translates the agent-facing ``command`` field to the internal
``BoxSpec.cmd`` field and injects the session id from the query.
"""
spec_payload: dict = {'cmd': parameters['command']}
if skill_name is not None:
spec_payload['skill_name'] = skill_name
# Pass through allowed agent-facing fields
for key in ('workdir', 'timeout_sec', 'env'):
@@ -424,7 +664,8 @@ class BoxService:
"""Execute trusted internal Box work inside one Workspace namespace."""
execution_context = await self._validated_execution_context(context)
payload = dict(spec_payload)
payload = self._managed_policy_payload(execution_context, spec_payload)
await self._require_validated_workspace_sandbox(execution_context)
if payload.get('host_path') in (None, ''):
tenant_workspace = self._tenant_workspace(execution_context)
if tenant_workspace is not None:
@@ -432,7 +673,9 @@ class BoxService:
if self.shares_filesystem_with_box:
os.makedirs(tenant_workspace, exist_ok=True)
spec = self.build_spec(payload, skip_host_mount_validation=skip_host_mount_validation)
return await self.client.execute(spec, action_context=self._action_context(execution_context))
result = await self.client.execute(spec, action_context=self._action_context(execution_context))
await self._validated_execution_context(execution_context)
return result
# ── Attachment passthrough (inbound / outbound) ──────────────────
#
@@ -462,10 +705,22 @@ class BoxService:
# Hard cap on a single attachment. The HTTP upload endpoints already cap
# uploads at 10MiB; keep parity.
_ATTACHMENT_MAX_BYTES = 10 * _MIB
_ATTACHMENT_MAX_FILES = 20
_ATTACHMENT_MAX_TOTAL_BYTES = 50 * _MIB
# Conservative cap for the exec FALLBACK path only (ARG_MAX / stdout
# truncation). The host-filesystem path has no such limit.
_EXEC_FALLBACK_MAX_BYTES = 256 * 1024
def _attachment_query_key(self, query: pipeline_query.Query) -> str:
query_uuid = str(getattr(query, 'query_uuid', '') or '').strip()
if query_uuid:
if query_uuid in {'.', '..'} or '/' in query_uuid or '\\' in query_uuid or '\x00' in query_uuid:
raise BoxValidationError('Query attachment identity is invalid')
return query_uuid
if self._cloud_managed:
raise BoxValidationError('Cloud attachment transfer requires query_uuid')
return str(query.query_id)
def _host_query_dir(self, subdir: str, query: pipeline_query.Query) -> str | None:
"""Host path for ``/workspace/<subdir>/<query_id>`` when LangBot can
access the bind-mounted workspace directly, else ``None``.
@@ -477,9 +732,9 @@ class BoxService:
E2B and remote runtimes, where we must fall back to the exec channel.
"""
root = self._tenant_workspace(self._query_execution_context(query))
if not root or not os.path.isdir(root):
if not root or not os.path.isdir(root) or os.path.islink(root):
return None
return os.path.join(root, subdir, str(query.query_id))
return os.path.join(root, subdir, self._attachment_query_key(query))
async def _purge_attachment_dirs(self) -> None:
"""Remove leftover inbox/outbox directories on startup.
@@ -498,14 +753,12 @@ class BoxService:
if not root or not os.path.isdir(root):
return
import shutil
host_survivors: list[str] = []
def _host_purge() -> list[str]:
candidates = [
os.path.join(root, self.INBOX_SUBDIR),
os.path.join(root, self.OUTBOX_SUBDIR),
candidates: list[tuple[str, str]] = [
(root, self.INBOX_SUBDIR),
(root, self.OUTBOX_SUBDIR),
]
tenants_root = os.path.join(root, 'tenants')
if os.path.isdir(tenants_root):
@@ -515,17 +768,18 @@ class BoxService:
continue
candidates.extend(
[
os.path.join(tenant_entry.path, self.INBOX_SUBDIR),
os.path.join(tenant_entry.path, self.OUTBOX_SUBDIR),
(tenant_entry.path, self.INBOX_SUBDIR),
(tenant_entry.path, self.OUTBOX_SUBDIR),
]
)
survivors: list[str] = []
for path in candidates:
if not os.path.isdir(path):
continue
shutil.rmtree(path, ignore_errors=True)
if os.path.exists(path):
survivors.append(path)
for candidate_root, subdir in candidates:
path = os.path.join(candidate_root, subdir)
try:
secure_fs.purge_subdirectory(candidate_root, subdir)
except OSError:
if os.path.lexists(path):
survivors.append(path)
return survivors
try:
@@ -640,14 +894,15 @@ class BoxService:
(the webchat session uses small sequential ids) never inherits stale
files from an earlier turn.
"""
import shutil
shutil.rmtree(host_dir, ignore_errors=True)
os.makedirs(host_dir, exist_ok=True)
query_key = os.path.basename(host_dir)
subdir = os.path.basename(os.path.dirname(host_dir))
tenant_root = os.path.dirname(os.path.dirname(host_dir))
try:
secure_fs.write_files(tenant_root, subdir, query_key, files)
except secure_fs.UnsafeWorkspacePathError as exc:
raise BoxValidationError('Sandbox attachment path contains an unsafe symbolic link') from exc
written: list[str] = []
for name, data in files:
with open(os.path.join(host_dir, name), 'wb') as fh:
fh.write(data)
for name, _data in files:
written.append(f'{target_mount_dir}/{name}')
return written
@@ -718,6 +973,8 @@ class BoxService:
"""
if not self._available:
return []
if self._cloud_managed:
await self.require_workspace_sandbox(self._query_execution_context(query))
import langbot_plugin.api.entities.builtin.platform.message as platform_message
@@ -766,7 +1023,8 @@ class BoxService:
if not pending:
return []
target_dir = f'{self.INBOX_MOUNT_DIR}/{query.query_id}'
query_key = self._attachment_query_key(query)
target_dir = f'{self.INBOX_MOUNT_DIR}/{query_key}'
written = await self._write_files_into_sandbox(query, self.INBOX_SUBDIR, target_dir, pending)
written_basenames = {os.path.basename(p) for p in written}
@@ -794,6 +1052,8 @@ class BoxService:
"""
if not self._available:
return []
if self._cloud_managed:
await self.require_workspace_sandbox(self._query_execution_context(query))
host_dir = self._host_query_dir(self.OUTBOX_SUBDIR, query)
if host_dir is not None:
@@ -817,22 +1077,21 @@ class BoxService:
"""Read outbox files straight off the bind-mounted host directory."""
import base64 as _b64
entries: list[dict] = []
if not os.path.isdir(host_dir):
return entries
for root, _dirs, names in os.walk(host_dir):
for name in sorted(names):
path = os.path.join(root, name)
try:
if os.path.getsize(path) > self._ATTACHMENT_MAX_BYTES:
continue
with open(path, 'rb') as fh:
data = fh.read()
except OSError:
continue
rel = os.path.relpath(path, host_dir)
entries.append({'name': rel, 'b64': _b64.b64encode(data).decode('ascii')})
return entries
query_key = os.path.basename(host_dir)
subdir = os.path.basename(os.path.dirname(host_dir))
tenant_root = os.path.dirname(os.path.dirname(host_dir))
try:
files = secure_fs.read_regular_files(
tenant_root,
subdir,
query_key,
max_file_bytes=self._ATTACHMENT_MAX_BYTES,
max_files=self._ATTACHMENT_MAX_FILES,
max_total_bytes=self._ATTACHMENT_MAX_TOTAL_BYTES,
)
except secure_fs.UnsafeWorkspacePathError as exc:
raise BoxValidationError('Sandbox outbox contains an unsafe symbolic link') from exc
return [{'name': name, 'b64': _b64.b64encode(data).decode('ascii')} for name, data in files]
async def _read_outbox_via_exec(self, query: pipeline_query.Query) -> list[dict]:
"""Fallback: read the outbox over the exec channel (E2B / remote).
@@ -842,7 +1101,7 @@ class BoxService:
"""
import json as _json
target_dir = f'{self.OUTBOX_MOUNT_DIR}/{query.query_id}'
target_dir = f'{self.OUTBOX_MOUNT_DIR}/{self._attachment_query_key(query)}'
max_bytes = self._EXEC_FALLBACK_MAX_BYTES
script = (
'import base64, json, os\n'
@@ -887,16 +1146,19 @@ class BoxService:
container's root can remove its own files. Best-effort: never raise
into the pipeline.
"""
target_dir = f'{self.OUTBOX_MOUNT_DIR}/{query.query_id}'
target_dir = f'{self.OUTBOX_MOUNT_DIR}/{self._attachment_query_key(query)}'
if host_dir is not None:
import shutil
def _clear() -> bool:
shutil.rmtree(host_dir, ignore_errors=True)
survived = os.path.exists(host_dir) and bool(os.listdir(host_dir))
os.makedirs(host_dir, exist_ok=True)
return survived
query_key = os.path.basename(host_dir)
subdir = os.path.basename(os.path.dirname(host_dir))
tenant_root = os.path.dirname(os.path.dirname(host_dir))
try:
secure_fs.reset_directory(tenant_root, subdir, query_key)
return False
except OSError:
return True
survived = await asyncio.to_thread(_clear)
if not survived:
@@ -952,9 +1214,9 @@ class BoxService:
self._shutdown_task = loop.create_task(self.shutdown())
async def get_sessions(self, context: TenantContext) -> list[dict]:
execution_context = await self._validated_execution_context(context)
if not self._available:
return []
execution_context = await self.require_workspace_sandbox(context)
try:
return await self.client.get_sessions(action_context=self._action_context(execution_context))
except Exception:
@@ -992,7 +1254,8 @@ class BoxService:
skip_host_mount_validation: bool = False,
) -> dict:
execution_context = await self._validated_execution_context(context)
spec_payload = dict(spec_payload)
spec_payload = self._managed_policy_payload(execution_context, spec_payload)
await self._require_validated_workspace_sandbox(execution_context)
if spec_payload.get('host_path') in (None, ''):
tenant_workspace = self._tenant_workspace(execution_context)
if tenant_workspace is not None:
@@ -1008,6 +1271,7 @@ class BoxService:
session_id: str,
process_payload: dict,
) -> BoxManagedProcessInfo:
self._reject_cloud_managed_process()
execution_context = await self._validated_execution_context(context)
process_spec = BoxManagedProcessSpec.model_validate(process_payload)
return await self.client.start_managed_process(
@@ -1022,6 +1286,7 @@ class BoxService:
session_id: str,
process_id: str = 'default',
) -> BoxManagedProcessInfo:
self._reject_cloud_managed_process()
execution_context = await self._validated_execution_context(context)
return await self.client.get_managed_process(
session_id,
@@ -1035,6 +1300,7 @@ class BoxService:
session_id: str,
process_id: str = 'default',
) -> None:
self._reject_cloud_managed_process()
execution_context = await self._validated_execution_context(context)
return await self.client.stop_managed_process(
session_id,
@@ -1076,6 +1342,7 @@ class BoxService:
execution context used by the action RPC that created the process.
"""
self._reject_cloud_managed_process()
execution_context = await self._validated_execution_context(context)
if self._runtime_connector is None:
raise BoxValidationError(
@@ -1092,23 +1359,32 @@ class BoxService:
)
async def list_skills(self, context: TenantContext) -> list[dict]:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
return await self.client.list_skills(action_context=self._action_context(execution_context))
async def get_skill(self, context: TenantContext, name: str) -> dict | None:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
return await self.client.get_skill(name, action_context=self._action_context(execution_context))
async def create_skill(self, context: TenantContext, skill: dict) -> dict:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
payload = dict(skill)
payload.pop('workspace_uuid', None)
if self._cloud_managed and str(payload.get('package_root', '') or '').strip():
raise BoxAdmissionError('Cloud skill package_root is runtime-owned')
if self._cloud_managed:
payload.pop('package_root', None)
return await self.client.create_skill(payload, action_context=self._action_context(execution_context))
async def update_skill(self, context: TenantContext, name: str, skill: dict) -> dict:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
payload = dict(skill)
payload.pop('workspace_uuid', None)
if self._cloud_managed:
# The runtime already owns the package path for an existing skill.
# A serialized read response may contain it, but it is never an
# authority-bearing update field in shared Cloud mode.
payload.pop('package_root', None)
return await self.client.update_skill(
name,
payload,
@@ -1116,13 +1392,20 @@ class BoxService:
)
async def delete_skill(self, context: TenantContext, name: str) -> None:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
await self.client.delete_skill(name, action_context=self._action_context(execution_context))
async def scan_skill_directory(self, context: TenantContext, path: str) -> dict:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
if self._cloud_managed:
raise BoxAdmissionError('Scanning arbitrary host skill directories is disabled in Cloud')
return await self.client.scan_skill_directory(path, action_context=self._action_context(execution_context))
async def _validated_skill_execution_context(self, context: TenantContext) -> ExecutionContext:
execution_context = await self._validated_execution_context(context)
await self._require_validated_workspace_sandbox(execution_context)
return execution_context
async def list_skill_files(
self,
context: TenantContext,
@@ -1131,7 +1414,7 @@ class BoxService:
include_hidden: bool = False,
max_entries: int = 200,
) -> dict:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
return await self.client.list_skill_files(
name,
path,
@@ -1141,7 +1424,7 @@ class BoxService:
)
async def read_skill_file(self, context: TenantContext, name: str, path: str) -> dict:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
return await self.client.read_skill_file(
name,
path,
@@ -1149,7 +1432,7 @@ class BoxService:
)
async def write_skill_file(self, context: TenantContext, name: str, path: str, content: str) -> dict:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
return await self.client.write_skill_file(
name,
path,
@@ -1165,7 +1448,7 @@ class BoxService:
source_subdir: str = '',
target_suffix: str = 'upload',
) -> list[dict]:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
return await self.client.preview_skill_zip(
file_bytes,
filename,
@@ -1184,7 +1467,7 @@ class BoxService:
source_subdir: str = '',
target_suffix: str = 'upload',
) -> list[dict]:
execution_context = await self._validated_execution_context(context)
execution_context = await self._validated_skill_execution_context(context)
return await self.client.install_skill_zip(
file_bytes,
filename,
@@ -1405,6 +1688,20 @@ class BoxService:
allowed_roots = ', '.join(self.allowed_mount_roots)
raise BoxValidationError(f'box.local.default_workspace is outside allowed_mount_roots: {allowed_roots}')
def _ensure_cloud_shared_workspace(self) -> None:
"""Require the Core-side view of the Cloud Box durable volume."""
if self.default_workspace is None:
raise BoxValidationError('Cloud Box requires box.local.default_workspace')
if not os.path.isabs(self.default_workspace) or not os.path.isdir(self.default_workspace):
raise BoxValidationError('Cloud Box shared default_workspace must be an existing absolute directory')
if not os.access(self.default_workspace, os.R_OK | os.W_OK | os.X_OK):
raise BoxValidationError('Cloud Box shared default_workspace must be writable by LangBot Core')
if not self.allowed_mount_roots or not any(
_is_path_under(self.default_workspace, allowed_root) for allowed_root in self.allowed_mount_roots
):
raise BoxValidationError('Cloud Box shared default_workspace is outside allowed_mount_roots')
def _validate_host_mount(self, spec: BoxSpec):
if spec.host_path is None:
return
@@ -1547,13 +1844,13 @@ class BoxService:
and error.get('workspace_uuid') == execution_context.workspace_uuid
]
def get_system_guidance(self, query_id=None) -> str:
def get_system_guidance(self, query: pipeline_query.Query | int | str | None = None) -> str:
"""Return LLM system-prompt guidance for the exec tool.
All execution-specific prompt text is kept here so that callers
(e.g. LocalAgentRunner) stay free of box domain knowledge.
``query_id`` is the current turn's pipeline query id. When provided,
``query`` is the current turn's pipeline query. When provided,
the guidance ALWAYS advertises the per-query outbox path so the agent
knows how to deliver generated files back to the user even on turns
where the user sent no inbound attachment (e.g. "generate a QR code"),
@@ -1574,8 +1871,17 @@ class BoxService:
'modify local files in the working directory, use exec with /workspace paths directly; do not ask the '
'user for directory parameters unless they explicitly need a different directory.'
)
if query_id is not None:
outbox_dir = f'{self.OUTBOX_MOUNT_DIR}/{query_id}'
if query is not None:
if not isinstance(query, (int, str)):
query_key = self._attachment_query_key(query)
else:
# Backwards compatibility for OSS callers/tests that passed
# the old process-local integer identity. Cloud callers must
# pass the full Query so an opaque UUID is always advertised.
if self._cloud_managed:
raise BoxValidationError('Cloud outbox guidance requires a pipeline Query')
query_key = str(query)
outbox_dir = f'{self.OUTBOX_MOUNT_DIR}/{query_key}'
guidance += (
f' If you produce any file (image, audio, document, etc.) that should be sent back to the user, '
f'write it into {outbox_dir}/ (create the directory if needed). Every file placed there will be '
@@ -1593,6 +1899,8 @@ class BoxService:
async def get_status(self, context: TenantContext) -> dict:
execution_context = await self._validated_execution_context(context)
if self._cloud_managed and self._available:
await self._require_validated_workspace_sandbox(execution_context)
action_context = self._action_context(execution_context)
recent_error_count = len(self.get_recent_errors(execution_context))
if not self._available:
+15 -9
View File
@@ -126,24 +126,30 @@ def should_prepare_python_env(host_path: str | None) -> bool:
return bool(list_python_manifest_files(normalized_root))
def wrap_python_command_with_env(command: str, *, mount_path: str = '/workspace') -> str:
def wrap_python_command_with_env(
command: str,
*,
mount_path: str = '/workspace',
state_path: str | None = None,
) -> str:
"""Wrap a command with a reusable sandbox-local Python env bootstrap.
This is the generic "workspace is a Python project" path used by mutable
workspaces such as skills. Read-only installation strategies stay in the
higher-level caller because they are application policy, not workspace
semantics.
``mount_path`` is always the source tree used for manifest hashing and
installation. ``state_path`` may point at a separate writable directory
for read-only source mounts; when omitted, legacy mutable-workspace behavior
stores the environment beside the source.
"""
writable_state_path = state_path or mount_path
bootstrap = textwrap.dedent(
f"""
set -e
_LB_VENV_DIR="{mount_path}/.venv"
_LB_META_DIR="{mount_path}/.langbot"
_LB_VENV_DIR="{writable_state_path}/.venv"
_LB_META_DIR="{writable_state_path}/.langbot"
_LB_META_FILE="$_LB_META_DIR/python-env.json"
_LB_LOCK_DIR="$_LB_META_DIR/python-env.lock"
_LB_TMP_DIR="{mount_path}/.tmp"
_LB_PIP_CACHE_DIR="{mount_path}/.cache/pip"
_LB_TMP_DIR="{writable_state_path}/.tmp"
_LB_PIP_CACHE_DIR="{writable_state_path}/.cache/pip"
mkdir -p "$_LB_META_DIR" "$_LB_TMP_DIR" "$_LB_PIP_CACHE_DIR"
_LB_SYSTEM_PYTHON="$(command -v python3 || command -v python || true)"
+161
View File
@@ -3,6 +3,8 @@ from __future__ import annotations
import dataclasses
import importlib.metadata
import inspect
import os
import threading
import time
from collections.abc import Awaitable, Callable
from typing import Any, Protocol
@@ -13,12 +15,17 @@ from .entitlements import EntitlementProvider, OpenSourceEntitlementProvider
CLOUD_BOOTSTRAP_ENTRY_POINT = 'langbot.cloud_bootstrap'
REQUIRED_TENANT_ISOLATION_VERSION = 2
SUPPORTED_PGVECTOR_DIMENSIONS = frozenset({384, 512, 768, 1024, 1536})
class CloudBootstrapError(RuntimeError):
"""Fail-closed Cloud bootstrap validation error."""
class CloudRuntimeUnavailableError(CloudBootstrapError):
"""The verified Cloud receipt no longer admits runtime work."""
@dataclasses.dataclass(frozen=True, slots=True)
class OpenSourceDeployment:
"""Default deployment selected when no closed bootstrap is installed."""
@@ -88,8 +95,70 @@ class VerifiedCloudDeployment:
raise CloudBootstrapError('Cloud runtime requires database.use=postgresql')
if config.get('vdb', {}).get('use') != self.required_vector_backend:
raise CloudBootstrapError('Cloud runtime requires vdb.use=pgvector')
pgvector_config = config.get('vdb', {}).get('pgvector', {})
if pgvector_config.get('use_business_database') is not True:
raise CloudBootstrapError('Cloud runtime requires vdb.pgvector.use_business_database=true')
dimensions = pgvector_config.get('allowed_dimensions')
if (
not isinstance(dimensions, list)
or not dimensions
or any(isinstance(item, bool) or not isinstance(item, int) for item in dimensions)
or not set(dimensions).issubset(SUPPORTED_PGVECTOR_DIMENSIONS)
):
supported = ', '.join(str(item) for item in sorted(SUPPORTED_PGVECTOR_DIMENSIONS))
raise CloudBootstrapError(f'Cloud pgvector allowed_dimensions must be a non-empty subset of: {supported}')
if config.get('mcp', {}).get('stdio', {}).get('enabled', True) is not False:
raise CloudBootstrapError('Cloud runtime requires mcp.stdio.enabled=false')
plugin_worker = config.get('plugin', {}).get('worker', {})
if plugin_worker.get('require_hard_limits') is not True:
raise CloudBootstrapError('Cloud Runtime requires plugin.worker.require_hard_limits=true')
box_config = config.get('box', {})
if box_config.get('enabled') is not True:
raise CloudBootstrapError('Cloud runtime requires box.enabled=true')
if box_config.get('backend') != 'nsjail':
raise CloudBootstrapError('Cloud runtime requires box.backend=nsjail')
runtime_endpoint = str(box_config.get('runtime', {}).get('endpoint', '') or '').strip()
if not runtime_endpoint:
raise CloudBootstrapError('Cloud runtime requires a shared external box.runtime.endpoint')
admission = box_config.get('admission', {})
required_admission = {
'required': True,
'logical_session_id': 'global',
'required_backend': 'nsjail',
'max_sessions': 1,
'max_managed_processes': 0,
}
if any(admission.get(name) != value for name, value in required_admission.items()):
raise CloudBootstrapError(
'Cloud runtime requires grant-enforced Box admission with one global session and zero managed processes'
)
grant_ttl = admission.get('max_grant_ttl_sec')
if isinstance(grant_ttl, bool) or not isinstance(grant_ttl, int) or not 1 <= grant_ttl <= 300:
raise CloudBootstrapError('Cloud Box admission max_grant_ttl_sec must be between 1 and 300')
workspace_quota_mb = admission.get('workspace_quota_mb')
if isinstance(workspace_quota_mb, bool) or not isinstance(workspace_quota_mb, int) or workspace_quota_mb <= 0:
raise CloudBootstrapError('Cloud Box admission workspace_quota_mb must be a positive integer')
local_config = box_config.get('local', {})
host_root = str(local_config.get('host_root', '') or '').strip()
default_workspace = str(local_config.get('default_workspace', '') or '').strip()
allowed_mount_roots = local_config.get('allowed_mount_roots')
if not host_root or not os.path.isabs(host_root):
raise CloudBootstrapError('Cloud Box local.host_root must be an absolute shared-volume path')
if not default_workspace or not os.path.isabs(default_workspace):
raise CloudBootstrapError('Cloud Box local.default_workspace must be an absolute shared-volume path')
if (
not isinstance(allowed_mount_roots, list)
or not allowed_mount_roots
or any(not isinstance(root, str) or not os.path.isabs(root) for root in allowed_mount_roots)
):
raise CloudBootstrapError('Cloud Box local.allowed_mount_roots must contain absolute shared-volume paths')
resolved_workspace = os.path.realpath(default_workspace)
if not any(
resolved_workspace == os.path.realpath(root)
or resolved_workspace.startswith(f'{os.path.realpath(root)}{os.sep}')
for root in allowed_mount_roots
):
raise CloudBootstrapError('Cloud Box local.default_workspace must be under allowed_mount_roots')
class CloudBootstrapProvider(Protocol):
@@ -101,6 +170,98 @@ class CloudBootstrapProvider(Protocol):
) -> VerifiedCloudDeployment | Awaitable[VerifiedCloudDeployment]: ...
class DeploymentAdmissionGuard:
"""Continuously enforce one verified deployment receipt.
Startup verification alone is insufficient because a long-running process
could otherwise keep serving after the signed Manifest expires. The guard
tracks both wall-clock expiry and a monotonic deadline so moving the system
clock backwards cannot extend an already admitted receipt.
A closed bootstrap may atomically replace the receipt with a strictly newer
Manifest generation after performing its own signature verification. The
logical instance and deployment mode cannot change during the process.
"""
def __init__(
self,
instance_uuid: str,
deployment: OpenSourceDeployment | VerifiedCloudDeployment,
*,
wall_time: Callable[[], float] = time.time,
monotonic_time: Callable[[], float] = time.monotonic,
) -> None:
self.instance_uuid = instance_uuid
self._wall_time = wall_time
self._monotonic_time = monotonic_time
self._lock = threading.Lock()
self._deployment = deployment
self._deadline: float | None = None
self._install_initial(deployment)
@property
def deployment(self) -> OpenSourceDeployment | VerifiedCloudDeployment:
with self._lock:
return self._deployment
def _install_initial(self, deployment: OpenSourceDeployment | VerifiedCloudDeployment) -> None:
now = int(self._wall_time())
if isinstance(deployment, VerifiedCloudDeployment):
deployment.validate(self.instance_uuid, now=now)
self._deadline = self._monotonic_time() + (deployment.expires_at - now)
elif not isinstance(deployment, OpenSourceDeployment):
raise TypeError('Deployment admission requires a verified deployment object')
@staticmethod
def _receipt_identity(deployment: VerifiedCloudDeployment) -> tuple[Any, ...]:
return (
deployment.instance_uuid,
deployment.manifest_jti,
deployment.manifest_generation,
deployment.expires_at,
deployment.release,
tuple(sorted(deployment.capabilities)),
deployment.tenant_isolation_version,
deployment.verification_key_id,
)
def replace(self, deployment: VerifiedCloudDeployment) -> None:
"""Atomically install a verified, non-rollback Cloud receipt."""
now = int(self._wall_time())
deployment.validate(self.instance_uuid, now=now)
with self._lock:
current = self._deployment
if not isinstance(current, VerifiedCloudDeployment):
raise CloudRuntimeUnavailableError('Deployment mode cannot change while LangBot is running')
if deployment.manifest_generation < current.manifest_generation:
raise CloudRuntimeUnavailableError('Cloud Manifest generation rolled back')
if deployment.manifest_generation == current.manifest_generation and self._receipt_identity(
deployment
) != self._receipt_identity(current):
raise CloudRuntimeUnavailableError('Cloud Manifest generation has conflicting contents')
self._deployment = deployment
self._deadline = self._monotonic_time() + (deployment.expires_at - now)
def require_active(self) -> OpenSourceDeployment | VerifiedCloudDeployment:
"""Return the active deployment or fail closed after Manifest expiry."""
now = int(self._wall_time())
monotonic_now = self._monotonic_time()
with self._lock:
deployment = self._deployment
deadline = self._deadline
if isinstance(deployment, OpenSourceDeployment):
return deployment
try:
deployment.validate(self.instance_uuid, now=now)
except CloudBootstrapError as exc:
raise CloudRuntimeUnavailableError(str(exc)) from exc
if deadline is None or monotonic_now >= deadline:
raise CloudRuntimeUnavailableError('Verified Cloud Manifest is expired')
return deployment
async def _invoke_provider(
loaded: Any,
*,
+26 -3
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import json
import time
from collections.abc import Callable
from typing import Protocol, runtime_checkable
import pydantic
@@ -11,6 +12,10 @@ import pydantic
class EntitlementUnavailableError(RuntimeError):
"""Raised when a trusted, currently-active entitlement is unavailable."""
def __init__(self, message: str, *, entitlement_revision: int | None = None) -> None:
super().__init__(message)
self.entitlement_revision = entitlement_revision
class EntitlementSnapshot(pydantic.BaseModel):
"""Plan-agnostic capability projection consumed by open-source Core.
@@ -60,9 +65,15 @@ class EntitlementSnapshot(pydantic.BaseModel):
if self.instance_uuid != instance_uuid or self.workspace_uuid != workspace_uuid:
raise EntitlementUnavailableError('Entitlement scope does not match the Workspace execution context')
if self.status != 'active':
raise EntitlementUnavailableError('Workspace entitlement is not active')
raise EntitlementUnavailableError(
'Workspace entitlement is not active',
entitlement_revision=self.entitlement_revision,
)
if current_time < self.not_before or current_time >= self.expires_at:
raise EntitlementUnavailableError('Workspace entitlement is not currently valid')
raise EntitlementUnavailableError(
'Workspace entitlement is not currently valid',
entitlement_revision=self.entitlement_revision,
)
return self
def require_feature(self, feature: str) -> None:
@@ -95,9 +106,16 @@ class OpenSourceEntitlementProvider:
class EntitlementResolver:
"""Validate scope/freshness and reject revision rollback or equivocation."""
def __init__(self, instance_uuid: str, provider: EntitlementProvider) -> None:
def __init__(
self,
instance_uuid: str,
provider: EntitlementProvider,
*,
deployment_admission: Callable[[], object] | None = None,
) -> None:
self.instance_uuid = instance_uuid
self.provider = provider
self._deployment_admission = deployment_admission
self._lock = asyncio.Lock()
self._snapshots: dict[str, tuple[int, str, EntitlementSnapshot]] = {}
@@ -112,7 +130,12 @@ class EntitlementResolver:
minimum_revision: int = 0,
now: int | None = None,
) -> EntitlementSnapshot:
if self._deployment_admission is not None:
self._deployment_admission()
candidate = await self.provider.get_workspace_entitlement(workspace_uuid)
if self._deployment_admission is not None:
# A provider call may cross the Manifest expiry boundary.
self._deployment_admission()
if not isinstance(candidate, EntitlementSnapshot):
raise EntitlementUnavailableError('Entitlement provider returned an invalid snapshot')
# Deep-copy untrusted provider-owned containers before caching them.
+7
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import typing
import inspect
from ..core import app
from . import operator
@@ -63,6 +64,12 @@ class CommandManager:
) -> typing.AsyncGenerator[command_context.CommandReturn, None]:
"""执行命令"""
require_context = getattr(self.ap.plugin_connector, 'require_workspace_context', None)
if require_context is not None:
result = require_context(context)
if inspect.isawaitable(result):
await result
command_list = await self.ap.plugin_connector.list_commands(bound_plugins)
for command in command_list:
+12 -24
View File
@@ -4,7 +4,6 @@ import logging
import asyncio
import traceback
import os
import sqlalchemy
from ..platform import botmgr as im_mgr
from ..platform.webhook_pusher import WebhookPusher
@@ -51,7 +50,6 @@ from ..workspace import collaboration as workspace_collaboration_module
from ..cloud import bootstrap as cloud_bootstrap_module
from ..cloud import entitlements as cloud_entitlements_module
from ..api.http.context import ExecutionContext, PrincipalContext, PrincipalType
from ..entity.persistence.workspace import WorkspaceExecutionState, WorkspaceExecutionStatus
class Application:
@@ -132,6 +130,8 @@ class Application:
deployment: cloud_bootstrap_module.OpenSourceDeployment | cloud_bootstrap_module.VerifiedCloudDeployment = None
deployment_admission: cloud_bootstrap_module.DeploymentAdmissionGuard = None
entitlement_resolver: cloud_entitlements_module.EntitlementResolver | None = None
vector_db_mgr: vectordb_mgr.VectorDBManager = None
@@ -250,18 +250,12 @@ class Application:
check_interval_seconds = check_interval_hours * 3600
while True:
try:
execution_states = await self.persistence_mgr.execute_async(
sqlalchemy.select(WorkspaceExecutionState).where(
WorkspaceExecutionState.instance_uuid == self.workspace_service.instance_uuid,
WorkspaceExecutionState.state == WorkspaceExecutionStatus.ACTIVE.value,
WorkspaceExecutionState.write_fenced == sqlalchemy.false(),
)
)
for execution_state in execution_states.all():
bindings = await self.workspace_service.list_active_execution_bindings()
for binding in bindings:
context = ExecutionContext(
instance_uuid=execution_state.instance_uuid,
workspace_uuid=execution_state.workspace_uuid,
placement_generation=execution_state.active_generation,
instance_uuid=binding.instance_uuid,
workspace_uuid=binding.workspace_uuid,
placement_generation=binding.placement_generation,
trigger_principal=PrincipalContext(PrincipalType.SYSTEM),
)
deleted = await self.monitoring_service.cleanup_expired_records(
@@ -298,18 +292,12 @@ class Application:
check_interval_seconds = check_interval_hours * 3600
while True:
try:
execution_states = await self.persistence_mgr.execute_async(
sqlalchemy.select(WorkspaceExecutionState).where(
WorkspaceExecutionState.instance_uuid == self.workspace_service.instance_uuid,
WorkspaceExecutionState.state == WorkspaceExecutionStatus.ACTIVE.value,
WorkspaceExecutionState.write_fenced == sqlalchemy.false(),
)
)
for execution_state in execution_states.all():
bindings = await self.workspace_service.list_active_execution_bindings()
for binding in bindings:
context = ExecutionContext(
instance_uuid=execution_state.instance_uuid,
workspace_uuid=execution_state.workspace_uuid,
placement_generation=execution_state.active_generation,
instance_uuid=binding.instance_uuid,
workspace_uuid=binding.workspace_uuid,
placement_generation=binding.placement_generation,
trigger_principal=PrincipalContext(PrincipalType.SYSTEM),
)
deleted = await self.maintenance_service.cleanup_expired_files(context)
+9 -1
View File
@@ -61,8 +61,16 @@ class BuildAppStage(stage.BootingStage):
instance_config=ap.instance_config.data,
)
ap.deployment = deployment
ap.deployment_admission = cloud_bootstrap.DeploymentAdmissionGuard(
constants.instance_id,
deployment,
)
ap.entitlement_resolver = (
EntitlementResolver(constants.instance_id, deployment.entitlement_provider)
EntitlementResolver(
constants.instance_id,
deployment.entitlement_provider,
deployment_admission=ap.deployment_admission.require_active,
)
if deployment.multi_workspace_enabled
else None
)
+11 -2
View File
@@ -21,6 +21,7 @@ _RUNTIME_POLICY_DEFAULTS = {
'max_pids': 128,
'max_open_files': 256,
'max_file_size_mb': 512,
'require_hard_limits': False,
}
},
'mcp': {'stdio': {'enabled': True}},
@@ -103,11 +104,19 @@ def _apply_env_overrides_to_config(cfg: dict) -> dict:
if '__' not in env_key:
continue
print(f'apply env overrides to config: env_key: {env_key}, env_value: {env_value}')
# Convert environment variable name to config path
# e.g., CONCURRENCY__PIPELINE -> ['concurrency', 'pipeline']
keys = [key.lower() for key in env_key.split('__')]
# macOS and some launchers expose variables such as
# ``__CF_USER_TEXT_ENCODING``. They are not LangBot config paths and
# must not create an empty top-level YAML key when config is dumped.
if any(not key for key in keys):
continue
# Values may contain database passwords, runtime control tokens, or
# provider credentials. Keep the useful audit breadcrumb without ever
# copying the secret into startup logs.
print(f'apply env override to config: env_key: {env_key}')
# Navigate to the target value and validate the path
current = cfg
+74
View File
@@ -0,0 +1,74 @@
from __future__ import annotations
import asyncio
import contextvars
import typing
T = typing.TypeVar('T')
def create_detached_task(
coro: typing.Coroutine[typing.Any, typing.Any, T],
*,
loop: asyncio.AbstractEventLoop | None = None,
name: str | None = None,
after_commit_manager: typing.Any | None = None,
) -> asyncio.Task[T]:
"""Create a task that inherits no request-local ContextVars.
A normal ``asyncio.create_task`` copies the caller's context. That is
unsafe for work which outlives an HTTP request because it can copy the
request's active database transaction or trusted tenant scope into a
different asyncio task. Detached work must receive durable identity such
as ``ExecutionContext`` through explicit arguments and establish its own
tenant scope or unit of work whenever it accesses persistence.
"""
task_loop = loop or asyncio.get_running_loop()
gate: asyncio.Future[None] | None = None
# Inspect the type so dynamic Mock/AsyncMock attributes do not turn into a
# fake gate in lightweight tests or embedders.
gate_factory = getattr(type(after_commit_manager), 'create_after_commit_gate', None)
if callable(gate_factory):
gate = gate_factory(after_commit_manager)
task_coro = _wait_for_commit(coro, gate) if gate is not None else coro
return task_loop.create_task(task_coro, name=name, context=contextvars.Context())
async def _wait_for_commit(
coro: typing.Coroutine[typing.Any, typing.Any, T],
gate: asyncio.Future[None],
) -> T:
try:
await gate
except BaseException:
coro.close()
raise
return await coro
async def run_in_workspace_uow(
ap: typing.Any,
workspace_uuid: str,
operation: typing.Callable[[], typing.Awaitable[T]],
) -> T:
"""Run one short persistence section in a detached Cloud task scope.
This helper deliberately scopes only the supplied operation. Callers
should not wrap long-running network or runtime work in a database
transaction.
"""
persistence_mgr = getattr(ap, 'persistence_mgr', None)
if persistence_mgr is None:
return await operation()
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if not cloud_runtime:
return await operation()
tenant_uow = getattr(persistence_mgr, 'tenant_uow', None)
if not callable(tenant_uow):
raise RuntimeError('Detached Cloud tasks require an explicit tenant unit of work')
async with tenant_uow(workspace_uuid):
return await operation()
+7 -1
View File
@@ -7,6 +7,7 @@ import time
from . import app
from . import entities as core_entities
from .task_boundary import create_detached_task
class TaskContext:
@@ -125,7 +126,12 @@ class TaskWrapper:
TaskWrapper._id_index += 1
self.ap = ap
self.task_context = context or TaskContext()
self.task = self.ap.event_loop.create_task(coro)
self.task = create_detached_task(
coro,
loop=self.ap.event_loop,
name=name or None,
after_commit_manager=getattr(self.ap, 'persistence_mgr', None),
)
self.task_type = task_type
self.kind = kind
self.name = name
+26 -1
View File
@@ -1,3 +1,6 @@
import hashlib
import uuid
import sqlalchemy
from .base import Base
@@ -15,6 +18,17 @@ class PluginSetting(Base):
)
plugin_author = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
plugin_name = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
installation_uuid = sqlalchemy.Column(
sqlalchemy.String(36),
nullable=False,
default=lambda: str(uuid.uuid4()),
)
artifact_digest = sqlalchemy.Column(
sqlalchemy.String(64),
nullable=False,
default=lambda: hashlib.sha256(f'pending:{uuid.uuid4()}'.encode()).hexdigest(),
)
runtime_revision = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=1)
enabled = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=True)
priority = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0)
config = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default=dict)
@@ -28,4 +42,15 @@ class PluginSetting(Base):
onupdate=sqlalchemy.func.now(),
)
__table_args__ = (sqlalchemy.Index('ix_plugin_settings_workspace_enabled', 'workspace_uuid', 'enabled'),)
__table_args__ = (
sqlalchemy.UniqueConstraint('installation_uuid', name='uq_plugin_settings_installation_uuid'),
sqlalchemy.CheckConstraint('runtime_revision >= 1', name='ck_plugin_settings_runtime_revision_positive'),
sqlalchemy.CheckConstraint('length(artifact_digest) = 64', name='ck_plugin_settings_artifact_digest_length'),
sqlalchemy.Index('ix_plugin_settings_workspace_enabled', 'workspace_uuid', 'enabled'),
sqlalchemy.Index(
'ix_plugin_settings_workspace_installation',
'workspace_uuid',
'installation_uuid',
unique=True,
),
)
@@ -29,6 +29,9 @@ class KnowledgeBase(Base):
)
creation_settings = sqlalchemy.Column(sqlalchemy.JSON, nullable=True, default=None)
retrieval_settings = sqlalchemy.Column(sqlalchemy.JSON, nullable=True, default=None)
# Server-selected pgvector dimension. ``None`` means no embedding has been
# written yet; the first pgvector upsert binds it atomically.
embedding_dimension = sqlalchemy.Column(sqlalchemy.Integer, nullable=True)
# Field sets for different operations
MUTABLE_FIELDS = {'name', 'description', 'retrieval_settings'}
@@ -40,6 +43,7 @@ class KnowledgeBase(Base):
ALL_DB_FIELDS = CREATE_FIELDS | {
'workspace_uuid',
'legacy_vector_collection',
'embedding_dimension',
'emoji',
'created_at',
'updated_at',
@@ -57,6 +61,10 @@ class KnowledgeBase(Base):
sqlite_where=sqlalchemy.text('collection_id IS NOT NULL'),
postgresql_where=sqlalchemy.text('collection_id IS NOT NULL'),
),
sqlalchemy.CheckConstraint(
'embedding_dimension IS NULL OR embedding_dimension > 0',
name='ck_knowledge_bases_embedding_dimension_positive',
),
)
@@ -1,11 +1,13 @@
"""enforce PostgreSQL tenant isolation with row-level security
"""enforce PostgreSQL tenant isolation with exact discovery contracts
Revision ID: 0011_postgres_tenant_rls
Revises: 0010_scope_resources
Create Date: 2026-07-19
The table list is deliberately duplicated from the runtime contract. Alembic
revisions must remain self-contained after application code evolves.
The table and policy lists are deliberately duplicated from the runtime
contract. Alembic revisions must remain self-contained after application code
evolves. Discovery policies are SELECT-only and reveal the minimum rows needed
to turn an authenticated credential into one Workspace transaction.
"""
from __future__ import annotations
@@ -20,8 +22,18 @@ depends_on = None
_POLICY_NAME = 'langbot_workspace_isolation'
_ACCOUNT_POLICY_NAME = 'langbot_account_discovery'
_API_KEY_POLICY_NAME = 'langbot_api_key_discovery'
_INVITATION_POLICY_NAME = 'langbot_invitation_discovery'
_INSTANCE_POLICY_NAME = 'langbot_instance_discovery'
_TENANT_SETTING = 'langbot.workspace_uuid'
_ACCOUNT_SETTING = 'langbot.account_uuid'
_API_KEY_HASH_SETTING = 'langbot.api_key_hash'
_INVITATION_HASH_SETTING = 'langbot.invitation_hash'
_INSTANCE_SETTING = 'langbot.instance_uuid'
_OSS_WORKSPACE_METADATA_KEY = 'oss_workspace_uuid'
_TENANT_TABLE_COLUMNS: dict[str, str] = {
'workspaces': 'uuid',
'workspace_memberships': 'workspace_uuid',
@@ -54,17 +66,42 @@ _TENANT_TABLE_COLUMNS: dict[str, str] = {
}
def _setting(name: str) -> str:
return f"NULLIF(current_setting('{name}', true), '')"
def _tenant_expression(column: str) -> str:
return f'{column}::text = {_setting(_TENANT_SETTING)}'
_DISCOVERY_POLICIES: dict[str, dict[str, str]] = {
'workspace_memberships': {
_ACCOUNT_POLICY_NAME: (f"account_uuid::text = {_setting(_ACCOUNT_SETTING)} AND status = 'active'"),
},
'workspace_execution_states': {
_INSTANCE_POLICY_NAME: (
f"instance_uuid = {_setting(_INSTANCE_SETTING)} AND state = 'active' AND write_fenced = false"
),
},
'api_keys': {
_API_KEY_POLICY_NAME: (
f"key_hash = {_setting(_API_KEY_HASH_SETTING)} AND status = 'active' "
'AND (expires_at IS NULL OR expires_at > CURRENT_TIMESTAMP)'
),
},
'workspace_invitations': {
_INVITATION_POLICY_NAME: f'token_hash = {_setting(_INVITATION_HASH_SETTING)}',
},
}
def _quote_identifier(conn: sa.Connection, identifier: str) -> str:
return conn.dialect.identifier_preparer.quote(identifier)
def _record_oss_workspace_scope(conn: sa.Connection) -> None:
"""Keep PostgreSQL OSS usable after FORCE RLS is enabled.
"""Keep PostgreSQL OSS usable after FORCE RLS is enabled."""
The runtime reads this instance-level metadata and installs the singleton
Workspace as a transaction-local default. Cloud databases have no local
Workspace and therefore do not receive this compatibility value.
"""
local_workspaces = (
conn.execute(sa.text("SELECT uuid FROM workspaces WHERE source = 'local' ORDER BY uuid")).scalars().all()
)
@@ -84,6 +121,43 @@ def _record_oss_workspace_scope(conn: sa.Connection) -> None:
raise RuntimeError('Stored OSS Workspace scope does not match the local Workspace')
def _drop_all_policies(conn: sa.Connection, table_name: str) -> None:
policy_names = conn.execute(
sa.text(
"""
SELECT p.polname
FROM pg_policy p
JOIN pg_class c ON c.oid = p.polrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = current_schema() AND c.relname = :table_name
"""
),
{'table_name': table_name},
).scalars()
table = _quote_identifier(conn, table_name)
for policy in policy_names:
op.execute(sa.text(f'DROP POLICY {_quote_identifier(conn, policy)} ON {table}'))
def _create_policy(
conn: sa.Connection,
table_name: str,
policy_name: str,
expression: str,
*,
command: str,
) -> None:
table = _quote_identifier(conn, table_name)
policy = _quote_identifier(conn, policy_name)
if command == 'ALL':
sql = f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR ALL TO PUBLIC USING ({expression}) WITH CHECK ({expression})'
elif command == 'SELECT':
sql = f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR SELECT TO PUBLIC USING ({expression})'
else: # pragma: no cover - migration-local invariant
raise AssertionError(f'Unsupported RLS command: {command}')
op.execute(sa.text(sql))
def upgrade() -> None:
conn = op.get_bind()
if conn.dialect.name != 'postgresql':
@@ -96,22 +170,20 @@ def upgrade() -> None:
_record_oss_workspace_scope(conn)
policy_name = _quote_identifier(conn, _POLICY_NAME)
for table_name, tenant_column in _TENANT_TABLE_COLUMNS.items():
table = _quote_identifier(conn, table_name)
column = _quote_identifier(conn, tenant_column)
tenant_value = f"CAST(NULLIF(current_setting('{_TENANT_SETTING}', true), '') AS VARCHAR(36))"
_drop_all_policies(conn, table_name)
op.execute(sa.text(f'ALTER TABLE {table} ENABLE ROW LEVEL SECURITY'))
op.execute(sa.text(f'ALTER TABLE {table} FORCE ROW LEVEL SECURITY'))
op.execute(sa.text(f'DROP POLICY IF EXISTS {policy_name} ON {table}'))
op.execute(
sa.text(
f'CREATE POLICY {policy_name} ON {table} '
f'USING ({column} = {tenant_value}) '
f'WITH CHECK ({column} = {tenant_value})'
)
_create_policy(
conn,
table_name,
_POLICY_NAME,
_tenant_expression(_quote_identifier(conn, tenant_column)),
command='ALL',
)
for policy_name, expression in _DISCOVERY_POLICIES.get(table_name, {}).items():
_create_policy(conn, table_name, policy_name, expression, command='SELECT')
def downgrade() -> None:
@@ -120,11 +192,10 @@ def downgrade() -> None:
return
existing_tables = set(sa.inspect(conn).get_table_names())
policy_name = _quote_identifier(conn, _POLICY_NAME)
for table_name in _TENANT_TABLE_COLUMNS:
if table_name not in existing_tables:
continue
table = _quote_identifier(conn, table_name)
op.execute(sa.text(f'DROP POLICY IF EXISTS {policy_name} ON {table}'))
_drop_all_policies(conn, table_name)
op.execute(sa.text(f'ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY'))
op.execute(sa.text(f'ALTER TABLE {table} DISABLE ROW LEVEL SECURITY'))
@@ -0,0 +1,179 @@
"""add immutable plugin installation identity
Revision ID: 0012_plugin_identity
Revises: 0011_postgres_tenant_rls
Create Date: 2026-07-19
The migration gives every legacy row a random, stable installation UUID. A
legacy artifact digest is only a valid SHA-256-shaped recovery marker; Core
replaces it with the package digest and increments ``runtime_revision`` before
the next package apply.
"""
from __future__ import annotations
import hashlib
import uuid
import sqlalchemy as sa
from alembic import op
revision = '0012_plugin_identity'
down_revision = '0011_postgres_tenant_rls'
branch_labels = None
depends_on = None
_TABLE = 'plugin_settings'
_INSTALLATION_INDEX = 'ix_plugin_settings_workspace_installation'
_INSTALLATION_UNIQUE = 'uq_plugin_settings_installation_uuid'
_REVISION_CHECK = 'ck_plugin_settings_runtime_revision_positive'
_DIGEST_CHECK = 'ck_plugin_settings_artifact_digest_length'
def _column_names(conn: sa.Connection) -> set[str]:
inspector = sa.inspect(conn)
if _TABLE not in inspector.get_table_names():
return set()
return {column['name'] for column in inspector.get_columns(_TABLE)}
def _legacy_digest(installation_uuid: str) -> str:
return hashlib.sha256(f'legacy-installation:{installation_uuid}'.encode()).hexdigest()
def _suspend_postgres_rls(conn: sa.Connection) -> tuple[bool, bool]:
"""Let the release migration backfill every tenant row after revision 0011."""
if conn.dialect.name != 'postgresql':
return False, False
row = conn.execute(
sa.text('SELECT relrowsecurity, relforcerowsecurity FROM pg_class WHERE oid = to_regclass(:table_name)'),
{'table_name': _TABLE},
).one()
rls_enabled, rls_forced = bool(row.relrowsecurity), bool(row.relforcerowsecurity)
if rls_forced:
op.execute(sa.text(f'ALTER TABLE {_TABLE} NO FORCE ROW LEVEL SECURITY'))
if rls_enabled:
op.execute(sa.text(f'ALTER TABLE {_TABLE} DISABLE ROW LEVEL SECURITY'))
return rls_enabled, rls_forced
def _restore_postgres_rls(conn: sa.Connection, state: tuple[bool, bool]) -> None:
if conn.dialect.name != 'postgresql':
return
rls_enabled, rls_forced = state
if rls_enabled:
op.execute(sa.text(f'ALTER TABLE {_TABLE} ENABLE ROW LEVEL SECURITY'))
if rls_forced:
op.execute(sa.text(f'ALTER TABLE {_TABLE} FORCE ROW LEVEL SECURITY'))
def _backfill(conn: sa.Connection) -> None:
table = sa.table(
_TABLE,
sa.column('workspace_uuid', sa.String(36)),
sa.column('plugin_author', sa.String(255)),
sa.column('plugin_name', sa.String(255)),
sa.column('installation_uuid', sa.String(36)),
sa.column('artifact_digest', sa.String(64)),
sa.column('runtime_revision', sa.Integer()),
)
rows = conn.execute(
sa.select(
table.c.workspace_uuid,
table.c.plugin_author,
table.c.plugin_name,
table.c.installation_uuid,
table.c.artifact_digest,
table.c.runtime_revision,
)
).all()
for row in rows:
installation_uuid = str(row.installation_uuid or uuid.uuid4())
values: dict[str, object] = {}
if not row.installation_uuid:
values['installation_uuid'] = installation_uuid
if not row.artifact_digest or len(str(row.artifact_digest)) != 64:
values['artifact_digest'] = _legacy_digest(installation_uuid)
if row.runtime_revision is None or row.runtime_revision < 1:
values['runtime_revision'] = 1
if values:
conn.execute(
table.update()
.where(table.c.workspace_uuid == row.workspace_uuid)
.where(table.c.plugin_author == row.plugin_author)
.where(table.c.plugin_name == row.plugin_name)
.values(**values)
)
def _constraint_names(conn: sa.Connection, kind: str) -> set[str]:
inspector = sa.inspect(conn)
getter = inspector.get_unique_constraints if kind == 'unique' else inspector.get_check_constraints
return {str(item.get('name')) for item in getter(_TABLE) if item.get('name')}
def upgrade() -> None:
conn = op.get_bind()
columns = _column_names(conn)
if not columns:
return
if 'installation_uuid' not in columns:
op.add_column(_TABLE, sa.Column('installation_uuid', sa.String(36), nullable=True))
if 'artifact_digest' not in columns:
op.add_column(_TABLE, sa.Column('artifact_digest', sa.String(64), nullable=True))
if 'runtime_revision' not in columns:
op.add_column(_TABLE, sa.Column('runtime_revision', sa.Integer(), nullable=True))
rls_state = _suspend_postgres_rls(conn)
try:
_backfill(conn)
finally:
_restore_postgres_rls(conn, rls_state)
# Fresh databases are created from current metadata before Alembic runs;
# guards make the revision safe for that path and for interrupted upgrades.
indexes = {index['name'] for index in sa.inspect(conn).get_indexes(_TABLE)}
unique_constraints = _constraint_names(conn, 'unique')
check_constraints = _constraint_names(conn, 'check')
with op.batch_alter_table(_TABLE) as batch:
batch.alter_column('installation_uuid', existing_type=sa.String(36), nullable=False)
batch.alter_column('artifact_digest', existing_type=sa.String(64), nullable=False)
batch.alter_column('runtime_revision', existing_type=sa.Integer(), nullable=False)
if _REVISION_CHECK not in check_constraints:
batch.create_check_constraint(_REVISION_CHECK, 'runtime_revision >= 1')
if _DIGEST_CHECK not in check_constraints:
batch.create_check_constraint(_DIGEST_CHECK, 'length(artifact_digest) = 64')
if _INSTALLATION_UNIQUE not in unique_constraints:
batch.create_unique_constraint(_INSTALLATION_UNIQUE, ['installation_uuid'])
if _INSTALLATION_INDEX not in indexes and _INSTALLATION_INDEX not in unique_constraints:
batch.create_index(
_INSTALLATION_INDEX,
['workspace_uuid', 'installation_uuid'],
unique=True,
)
def downgrade() -> None:
conn = op.get_bind()
columns = _column_names(conn)
if not columns:
return
indexes = {index['name'] for index in sa.inspect(conn).get_indexes(_TABLE)}
checks = _constraint_names(conn, 'check')
uniques = _constraint_names(conn, 'unique')
with op.batch_alter_table(_TABLE) as batch:
if _INSTALLATION_INDEX in indexes:
batch.drop_index(_INSTALLATION_INDEX)
if _DIGEST_CHECK in checks:
batch.drop_constraint(_DIGEST_CHECK, type_='check')
if _REVISION_CHECK in checks:
batch.drop_constraint(_REVISION_CHECK, type_='check')
if _INSTALLATION_UNIQUE in uniques:
batch.drop_constraint(_INSTALLATION_UNIQUE, type_='unique')
for column_name in ('runtime_revision', 'artifact_digest', 'installation_uuid'):
if column_name in columns:
batch.drop_column(column_name)
@@ -0,0 +1,382 @@
"""create tenant-scoped pgvector storage in the business database
Revision ID: 0013_tenant_pgvector
Revises: 0012_plugin_identity
Create Date: 2026-07-19
The Cloud application role never executes this DDL. A release migration role
installs pgvector once, creates an untyped vector column, and builds a bounded
set of expression/partial ANN indexes. Existing legacy rows are migrated only
when each row maps unambiguously to one knowledge base.
"""
from __future__ import annotations
import contextlib
import typing
import sqlalchemy as sa
from alembic import op
from pgvector.sqlalchemy import Vector
revision = '0013_tenant_pgvector'
down_revision = '0012_plugin_identity'
branch_labels = None
depends_on = None
_VECTOR_TABLE = 'langbot_vectors'
_LEGACY_TABLE = 'langbot_vectors_legacy_0013'
_TENANT_POLICY = 'langbot_workspace_isolation'
_TENANT_SETTING = 'langbot.workspace_uuid'
_KB_DIMENSION_CHECK = 'ck_knowledge_bases_embedding_dimension_positive'
_VECTOR_DIMENSION_CHECK = 'ck_langbot_vectors_embedding_dimension'
_VECTOR_ALLOWED_CHECK = 'ck_langbot_vectors_embedding_dimension_enabled'
_ALLOWED_DIMENSIONS = (384, 512, 768, 1024, 1536)
_LEGACY_SOURCE_TABLES = (
'knowledge_bases',
'knowledge_base_files',
'knowledge_base_chunks',
)
def _quote(conn: sa.Connection, identifier: str) -> str:
return conn.dialect.identifier_preparer.quote(identifier)
def _columns(conn: sa.Connection, table_name: str) -> set[str]:
inspector = sa.inspect(conn)
if table_name not in inspector.get_table_names():
return set()
return {column['name'] for column in inspector.get_columns(table_name)}
def _checks(conn: sa.Connection, table_name: str) -> set[str]:
return {str(item['name']) for item in sa.inspect(conn).get_check_constraints(table_name) if item.get('name')}
def _ensure_knowledge_base_dimension(conn: sa.Connection) -> None:
columns = _columns(conn, 'knowledge_bases')
if 'embedding_dimension' not in columns:
op.add_column('knowledge_bases', sa.Column('embedding_dimension', sa.Integer(), nullable=True))
if _KB_DIMENSION_CHECK not in _checks(conn, 'knowledge_bases'):
with op.batch_alter_table('knowledge_bases') as batch:
batch.create_check_constraint(
_KB_DIMENSION_CHECK,
'embedding_dimension IS NULL OR embedding_dimension > 0',
)
def _create_vector_table() -> None:
enabled = ', '.join(str(item) for item in _ALLOWED_DIMENSIONS)
op.create_table(
_VECTOR_TABLE,
sa.Column('workspace_uuid', sa.String(36), nullable=False),
sa.Column('knowledge_base_uuid', sa.String(255), nullable=False),
sa.Column('vector_id', sa.String(255), nullable=False),
sa.Column('embedding_dimension', sa.Integer(), nullable=False),
sa.Column('embedding', Vector(), nullable=False),
sa.Column('text', sa.Text(), nullable=True),
sa.Column('file_id', sa.String(255), nullable=True),
sa.Column('chunk_uuid', sa.String(255), nullable=True),
sa.PrimaryKeyConstraint(
'workspace_uuid',
'knowledge_base_uuid',
'vector_id',
name='pk_langbot_vectors',
),
sa.ForeignKeyConstraint(
['workspace_uuid', 'knowledge_base_uuid'],
['knowledge_bases.workspace_uuid', 'knowledge_bases.uuid'],
name='fk_langbot_vectors_workspace_kb',
ondelete='CASCADE',
),
sa.CheckConstraint(
'vector_dims(embedding) = embedding_dimension',
name=_VECTOR_DIMENSION_CHECK,
),
sa.CheckConstraint(
f'embedding_dimension IN ({enabled})',
name=_VECTOR_ALLOWED_CHECK,
),
)
op.create_index(
'ix_langbot_vectors_workspace_kb_file',
_VECTOR_TABLE,
['workspace_uuid', 'knowledge_base_uuid', 'file_id'],
)
op.create_index(
'ix_langbot_vectors_workspace_kb_chunk',
_VECTOR_TABLE,
['workspace_uuid', 'knowledge_base_uuid', 'chunk_uuid'],
)
def _legacy_mapping_predicate() -> str:
return """
kb.collection_id = legacy.collection
OR EXISTS (
SELECT 1
FROM knowledge_base_files AS files
LEFT JOIN knowledge_base_chunks AS chunks
ON chunks.workspace_uuid = files.workspace_uuid
AND chunks.file_id = files.uuid
WHERE files.workspace_uuid = kb.workspace_uuid
AND files.kb_id = kb.uuid
AND (files.uuid = legacy.file_id OR chunks.uuid = legacy.chunk_uuid)
)
"""
def _legacy_source_rls_states(conn: sa.Connection) -> dict[str, tuple[bool, bool]]:
rows = (
conn.execute(
sa.text(
"""
SELECT c.relname, c.relrowsecurity, c.relforcerowsecurity
FROM pg_class AS c
JOIN pg_namespace AS n ON n.oid = c.relnamespace
WHERE n.nspname = current_schema()
AND c.relname IN :table_names
AND c.relkind IN ('r', 'p')
"""
).bindparams(sa.bindparam('table_names', expanding=True)),
{'table_names': _LEGACY_SOURCE_TABLES},
)
.mappings()
.all()
)
states = {str(row['relname']): (bool(row['relrowsecurity']), bool(row['relforcerowsecurity'])) for row in rows}
missing = set(_LEGACY_SOURCE_TABLES) - set(states)
if missing:
raise RuntimeError(f'Legacy pgvector source tables are missing: {sorted(missing)!r}')
return states
@contextlib.contextmanager
def _suspend_legacy_source_rls(conn: sa.Connection) -> typing.Iterator[None]:
"""Temporarily let the table-owning migrator map all legacy tenant rows.
Revision 0011 enables and forces RLS on each source table. The release
migrator intentionally has neither superuser nor BYPASSRLS, so even a table
owner cannot read those rows until FORCE RLS is paused. Preserve both flags
independently and restore them in ``finally`` so mixed pre-existing states
survive successful, rejected, and interrupted legacy migrations.
"""
states = _legacy_source_rls_states(conn)
try:
for table_name in _LEGACY_SOURCE_TABLES:
table = _quote(conn, table_name)
conn.execute(sa.text(f'ALTER TABLE {table} NO FORCE ROW LEVEL SECURITY'))
conn.execute(sa.text(f'ALTER TABLE {table} DISABLE ROW LEVEL SECURITY'))
yield
finally:
for table_name in _LEGACY_SOURCE_TABLES:
table = _quote(conn, table_name)
rls_enabled, rls_forced = states[table_name]
enabled_clause = 'ENABLE' if rls_enabled else 'DISABLE'
forced_clause = 'FORCE' if rls_forced else 'NO FORCE'
conn.execute(sa.text(f'ALTER TABLE {table} {enabled_clause} ROW LEVEL SECURITY'))
conn.execute(sa.text(f'ALTER TABLE {table} {forced_clause} ROW LEVEL SECURITY'))
def _migrate_legacy_rows(conn: sa.Connection) -> None:
predicate = _legacy_mapping_predicate()
ambiguous = conn.execute(
sa.text(
f"""
WITH candidates AS (
SELECT legacy.id, kb.workspace_uuid, kb.uuid AS knowledge_base_uuid
FROM {_LEGACY_TABLE} AS legacy
JOIN knowledge_bases AS kb ON ({predicate})
), candidate_counts AS (
SELECT id, COUNT(*) AS count
FROM candidates
GROUP BY id
)
SELECT legacy.id, COALESCE(candidate_counts.count, 0) AS candidate_count
FROM {_LEGACY_TABLE} AS legacy
LEFT JOIN candidate_counts ON candidate_counts.id = legacy.id
WHERE COALESCE(candidate_counts.count, 0) <> 1
LIMIT 1
"""
)
).first()
if ambiguous is not None:
raise RuntimeError(
'Legacy pgvector row cannot be mapped to exactly one Workspace/knowledge base: '
f'{ambiguous.id!r} has {ambiguous.candidate_count} candidates'
)
conn.execute(
sa.text(
f"""
INSERT INTO {_VECTOR_TABLE} (
workspace_uuid,
knowledge_base_uuid,
vector_id,
embedding_dimension,
embedding,
text,
file_id,
chunk_uuid
)
SELECT
kb.workspace_uuid,
kb.uuid,
legacy.id,
vector_dims(legacy.embedding),
legacy.embedding,
legacy.text,
legacy.file_id,
legacy.chunk_uuid
FROM {_LEGACY_TABLE} AS legacy
JOIN knowledge_bases AS kb ON ({predicate})
"""
)
)
mixed_dimension = conn.execute(
sa.text(
f"""
SELECT workspace_uuid, knowledge_base_uuid
FROM {_VECTOR_TABLE}
GROUP BY workspace_uuid, knowledge_base_uuid
HAVING MIN(embedding_dimension) <> MAX(embedding_dimension)
LIMIT 1
"""
)
).first()
if mixed_dimension is not None:
raise RuntimeError('Legacy knowledge base contains mixed embedding dimensions')
conn.execute(
sa.text(
f"""
UPDATE knowledge_bases AS kb
SET embedding_dimension = dimensions.embedding_dimension
FROM (
SELECT workspace_uuid, knowledge_base_uuid, MIN(embedding_dimension) AS embedding_dimension
FROM {_VECTOR_TABLE}
GROUP BY workspace_uuid, knowledge_base_uuid
) AS dimensions
WHERE kb.workspace_uuid = dimensions.workspace_uuid
AND kb.uuid = dimensions.knowledge_base_uuid
AND kb.embedding_dimension IS NULL
"""
)
)
def _drop_all_policies(conn: sa.Connection) -> None:
table = _quote(conn, _VECTOR_TABLE)
policies = conn.execute(
sa.text(
"""
SELECT p.polname
FROM pg_policy p
JOIN pg_class c ON c.oid = p.polrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
WHERE n.nspname = current_schema() AND c.relname = :table_name
"""
),
{'table_name': _VECTOR_TABLE},
).scalars()
for policy_name in policies:
op.execute(sa.text(f'DROP POLICY {_quote(conn, policy_name)} ON {table}'))
def _enable_rls(conn: sa.Connection) -> None:
table = _quote(conn, _VECTOR_TABLE)
policy = _quote(conn, _TENANT_POLICY)
expression = f"workspace_uuid::text = NULLIF(current_setting('{_TENANT_SETTING}', true), '')"
_drop_all_policies(conn)
op.execute(sa.text(f'ALTER TABLE {table} ENABLE ROW LEVEL SECURITY'))
op.execute(sa.text(f'ALTER TABLE {table} FORCE ROW LEVEL SECURITY'))
op.execute(
sa.text(
f'CREATE POLICY {policy} ON {table} AS PERMISSIVE FOR ALL TO PUBLIC '
f'USING ({expression}) WITH CHECK ({expression})'
)
)
def _create_ann_indexes(conn: sa.Connection) -> None:
table = _quote(conn, _VECTOR_TABLE)
for dimension in _ALLOWED_DIMENSIONS:
index = _quote(conn, f'ix_langbot_vectors_hnsw_cosine_{dimension}')
op.execute(
sa.text(
f'CREATE INDEX {index} ON {table} USING hnsw '
f'((embedding::vector({dimension})) vector_cosine_ops) '
f'WHERE embedding_dimension = {dimension}'
)
)
def upgrade() -> None:
conn = op.get_bind()
if 'knowledge_bases' not in sa.inspect(conn).get_table_names():
if conn.dialect.name == 'postgresql':
# The supported PostgreSQL release path creates the business
# tables before stamping 0010 and reaching this migration. Missing
# knowledge_bases therefore means the operator bypassed the
# release bootstrap or the schema is incomplete; stamping head in
# that state would make Cloud runtime validation unrecoverable.
raise RuntimeError('PostgreSQL release migration requires the knowledge_bases table')
# A direct empty SQLite Alembic walk is still used by migration tooling;
# Core creates the complete ORM schema on its following compatibility
# pass, including the portable embedding_dimension field.
return
_ensure_knowledge_base_dimension(conn)
# pgvector storage is PostgreSQL-only, but ``embedding_dimension`` is an
# ORM field used by both deployment modes. Existing OSS SQLite databases
# must receive the column before this revision becomes a no-op.
if conn.dialect.name != 'postgresql':
return
op.execute(sa.text('CREATE EXTENSION IF NOT EXISTS vector'))
columns = _columns(conn, _VECTOR_TABLE)
if columns:
scoped_columns = {
'workspace_uuid',
'knowledge_base_uuid',
'vector_id',
'embedding_dimension',
'embedding',
}
if scoped_columns.issubset(columns):
raise RuntimeError('Tenant pgvector table exists before its owning release migration')
legacy_columns = {'id', 'collection', 'embedding'}
if not legacy_columns.issubset(columns):
raise RuntimeError('Existing pgvector table has an unsupported schema')
if _LEGACY_TABLE in sa.inspect(conn).get_table_names():
raise RuntimeError(f'Interrupted pgvector migration left {_LEGACY_TABLE!r} behind')
op.rename_table(_VECTOR_TABLE, _LEGACY_TABLE)
_create_vector_table()
if _LEGACY_TABLE in sa.inspect(conn).get_table_names():
with _suspend_legacy_source_rls(conn):
_migrate_legacy_rows(conn)
op.drop_table(_LEGACY_TABLE)
_create_ann_indexes(conn)
_enable_rls(conn)
def downgrade() -> None:
conn = op.get_bind()
if conn.dialect.name == 'postgresql' and _VECTOR_TABLE in sa.inspect(conn).get_table_names():
_drop_all_policies(conn)
op.drop_table(_VECTOR_TABLE)
columns = _columns(conn, 'knowledge_bases')
if 'embedding_dimension' in columns:
checks = _checks(conn, 'knowledge_bases')
with op.batch_alter_table('knowledge_bases') as batch:
if _KB_DIMENSION_CHECK in checks:
batch.drop_constraint(_KB_DIMENSION_CHECK, type_='check')
batch.drop_column('embedding_dimension')
+8 -1
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import abc
import sqlalchemy
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
from ..core import app
@@ -30,8 +31,14 @@ class BaseDatabaseManager(abc.ABC):
engine: sqlalchemy_asyncio.AsyncEngine
def __init__(self, ap: app.Application) -> None:
def __init__(
self,
ap: app.Application,
*,
url_override: sqlalchemy.engine.URL | None = None,
) -> None:
self.ap = ap
self.url_override = url_override
@abc.abstractmethod
async def initialize(self) -> None:
@@ -1,8 +1,10 @@
from __future__ import annotations
import sqlalchemy
import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
from .. import database
from ..postgresql_url import normalize_asyncpg_url
@database.manager_class('postgresql')
@@ -10,12 +12,29 @@ class PostgreSQLDatabaseManager(database.BaseDatabaseManager):
"""PostgreSQL database manager"""
async def initialize(self) -> None:
postgresql_config = self.ap.instance_config.data.get('database', {}).get('postgresql', {})
host = postgresql_config.get('host', '127.0.0.1')
port = postgresql_config.get('port', 5432)
user = postgresql_config.get('user', 'postgres')
password = postgresql_config.get('password', 'postgres')
database = postgresql_config.get('database', 'postgres')
engine_url = f'postgresql+asyncpg://{user}:{password}@{host}:{port}/{database}'
if self.url_override is not None:
engine_url = self.url_override
else:
postgresql_config = self.ap.instance_config.data.get('database', {}).get('postgresql', {})
explicit_url = postgresql_config.get('url')
if explicit_url:
if not isinstance(explicit_url, str):
raise ValueError('database.postgresql.url must be a string')
try:
engine_url = sqlalchemy.engine.make_url(explicit_url)
except Exception:
raise ValueError('database.postgresql.url is invalid') from None
try:
engine_url = normalize_asyncpg_url(engine_url)
except ValueError:
raise ValueError('database.postgresql.url must use valid PostgreSQL asyncpg options') from None
else:
engine_url = sqlalchemy.URL.create(
'postgresql+asyncpg',
username=postgresql_config.get('user', 'postgres'),
password=postgresql_config.get('password', 'postgres'),
host=postgresql_config.get('host', '127.0.0.1'),
port=postgresql_config.get('port', 5432),
database=postgresql_config.get('database', 'postgres'),
)
self.engine = sqlalchemy_asyncio.create_async_engine(engine_url)
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,25 @@
"""Safe PostgreSQL URL normalization shared by runtime and migration jobs."""
from __future__ import annotations
import sqlalchemy
def normalize_asyncpg_url(url: sqlalchemy.engine.URL) -> sqlalchemy.engine.URL:
"""Select asyncpg and translate the common libpq TLS query spelling."""
if url.drivername == 'postgresql':
url = url.set(drivername='postgresql+asyncpg')
elif url.drivername != 'postgresql+asyncpg':
raise ValueError('PostgreSQL URL must use PostgreSQL with the asyncpg driver')
query = dict(url.query)
sslmode = query.pop('sslmode', None)
if sslmode is not None:
if 'ssl' in query and query['ssl'] != sslmode:
raise ValueError('PostgreSQL URL cannot specify conflicting ssl and sslmode options')
# SQLAlchemy expands URL query keys into asyncpg keyword arguments.
# asyncpg calls this keyword ``ssl`` even though PostgreSQL DSNs
# conventionally spell the same mode ``sslmode``.
query['ssl'] = sslmode
return url.set(query=query)
@@ -0,0 +1,193 @@
"""One-shot, operator-only Cloud PostgreSQL release migration entrypoint."""
from __future__ import annotations
import asyncio
import os
import re
from collections.abc import Mapping
import sqlalchemy
from ..cloud.bootstrap import SUPPORTED_PGVECTOR_DIMENSIONS
from ..core import app as core_app
from ..core.stages.load_config import LoadConfigStage
from ..core.stages.setup_logger import SetupLoggerStage
from .mgr import PersistenceManager, PersistenceMode
from .postgresql_url import normalize_asyncpg_url
DEFAULT_OPERATOR_DSN_ENV = 'LANGBOT_CLOUD_MIGRATION_DSN'
_ENV_NAME = re.compile(r'^[A-Z_][A-Z0-9_]*$')
class CloudReleaseMigrationConfigurationError(RuntimeError):
"""Raised before any database operation when migration input is unsafe."""
def _url_endpoint(url: sqlalchemy.engine.URL, *, label: str) -> tuple[str, int]:
"""Return a comparison-safe PostgreSQL endpoint without leaking its DSN."""
try:
host = (url.host or '').strip().casefold()
port = url.port or 5432
except (TypeError, ValueError):
raise CloudReleaseMigrationConfigurationError(f'{label} PostgreSQL host or port is invalid') from None
if not host or isinstance(port, bool) or not isinstance(port, int) or not 1 <= port <= 65535:
raise CloudReleaseMigrationConfigurationError(f'{label} PostgreSQL host or port is invalid')
return host, port
def _operator_database_url(
instance_config: dict,
*,
environ: Mapping[str, str],
) -> sqlalchemy.engine.URL:
database_config = instance_config.get('database')
if not isinstance(database_config, dict) or database_config.get('use') != 'postgresql':
raise CloudReleaseMigrationConfigurationError(
'Cloud release migration requires explicit database.use=postgresql; SQLite fallback is forbidden'
)
runtime_config = database_config.get('postgresql')
if not isinstance(runtime_config, dict):
raise CloudReleaseMigrationConfigurationError('Cloud runtime PostgreSQL configuration is missing')
migration_config = database_config.get('cloud_migration', {})
if not isinstance(migration_config, dict):
raise CloudReleaseMigrationConfigurationError('database.cloud_migration must be a mapping')
dsn_env = migration_config.get('operator_dsn_env', DEFAULT_OPERATOR_DSN_ENV)
if not isinstance(dsn_env, str) or not _ENV_NAME.fullmatch(dsn_env):
raise CloudReleaseMigrationConfigurationError(
'database.cloud_migration.operator_dsn_env must name an uppercase environment variable'
)
raw_dsn = environ.get(dsn_env, '').strip()
if not raw_dsn:
raise CloudReleaseMigrationConfigurationError(
f'Cloud release migration requires the operator DSN in environment variable {dsn_env}'
)
try:
operator_url = sqlalchemy.engine.make_url(raw_dsn)
except Exception:
# Never echo a malformed DSN because it may contain an unescaped secret.
raise CloudReleaseMigrationConfigurationError('Cloud release migration operator DSN is invalid') from None
try:
operator_url = normalize_asyncpg_url(operator_url)
except ValueError:
raise CloudReleaseMigrationConfigurationError(
'Cloud release migration operator DSN must use valid PostgreSQL asyncpg options'
) from None
operator_user = (operator_url.username or '').strip()
operator_database = (operator_url.database or '').strip()
operator_host, operator_port = _url_endpoint(operator_url, label='Cloud release migration operator')
runtime_url_value = runtime_config.get('url')
if runtime_url_value:
if not isinstance(runtime_url_value, str):
raise CloudReleaseMigrationConfigurationError('Cloud runtime PostgreSQL URL must be a string')
try:
runtime_url = sqlalchemy.engine.make_url(runtime_url_value)
except Exception:
raise CloudReleaseMigrationConfigurationError('Cloud runtime PostgreSQL URL is invalid') from None
if runtime_url.drivername not in {'postgresql', 'postgresql+asyncpg'}:
raise CloudReleaseMigrationConfigurationError('Cloud runtime database URL must use PostgreSQL')
runtime_user = (runtime_url.username or '').strip()
runtime_database = (runtime_url.database or '').strip()
runtime_host, runtime_port = _url_endpoint(runtime_url, label='Cloud runtime')
else:
runtime_user = str(runtime_config.get('user', 'postgres') or '').strip()
runtime_database = str(runtime_config.get('database', 'postgres') or '').strip()
runtime_host = str(runtime_config.get('host', '') or '').strip().casefold()
runtime_port = runtime_config.get('port', 5432)
if not operator_user or not operator_database:
raise CloudReleaseMigrationConfigurationError(
'Cloud release migration operator DSN must include a user, host, and database'
)
if (
not runtime_user
or not runtime_database
or not runtime_host
or isinstance(runtime_port, bool)
or not isinstance(runtime_port, int)
or not 1 <= runtime_port <= 65535
):
raise CloudReleaseMigrationConfigurationError(
'Cloud runtime PostgreSQL user, host, port, and database are required'
)
if operator_user == runtime_user:
raise CloudReleaseMigrationConfigurationError(
'Cloud release migration requires a distinct operator role from the runtime PostgreSQL role'
)
if operator_database != runtime_database:
raise CloudReleaseMigrationConfigurationError(
'Cloud release migration operator DSN must target the configured runtime database'
)
if operator_host != runtime_host or operator_port != runtime_port:
# The first Cloud release intentionally requires the migrator and
# runtime to use the same PostgreSQL endpoint. Supporting a direct
# operator endpoint plus a runtime pooler requires a database-backed
# immutable cluster identity check; accepting aliases here would turn a
# same-named database on another cluster into a silent migration target.
raise CloudReleaseMigrationConfigurationError(
'Cloud release migration operator DSN must target the configured runtime PostgreSQL endpoint'
)
vdb_config = instance_config.get('vdb')
if not isinstance(vdb_config, dict) or vdb_config.get('use') != 'pgvector':
raise CloudReleaseMigrationConfigurationError('Cloud release migration requires vdb.use=pgvector')
pgvector_config = vdb_config.get('pgvector')
if not isinstance(pgvector_config, dict) or pgvector_config.get('use_business_database') is not True:
raise CloudReleaseMigrationConfigurationError(
'Cloud release migration requires vdb.pgvector.use_business_database=true'
)
dimensions = pgvector_config.get('allowed_dimensions')
if (
not isinstance(dimensions, list)
or not dimensions
or any(isinstance(item, bool) or not isinstance(item, int) for item in dimensions)
or not set(dimensions).issubset(SUPPORTED_PGVECTOR_DIMENSIONS)
):
raise CloudReleaseMigrationConfigurationError(
'Cloud release migration pgvector dimensions are outside the release-created index set'
)
return operator_url
async def run_cloud_release_migration(
ap: core_app.Application,
*,
environ: Mapping[str, str] | None = None,
) -> None:
"""Run and validate one release migration with an isolated operator DSN."""
operator_url = _operator_database_url(
ap.instance_config.data,
environ=os.environ if environ is None else environ,
)
manager = PersistenceManager(
ap,
mode=PersistenceMode.RELEASE_MIGRATION,
database_url=operator_url,
)
ap.persistence_mgr = manager
try:
await manager.initialize()
ap.logger.info('Cloud PostgreSQL release migration reached and validated the exact release head.')
finally:
db = getattr(manager, 'db', None)
engine = getattr(db, 'engine', None)
if engine is not None:
await engine.dispose()
async def run_cloud_release_migration_from_config(loop: asyncio.AbstractEventLoop) -> None:
"""Load only process configuration/logging, then run the one-shot job."""
ap = core_app.Application()
ap.event_loop = loop
await LoadConfigStage().run(ap)
await SetupLoggerStage().run(ap)
await run_cloud_release_migration(ap)
+396 -19
View File
@@ -1,5 +1,9 @@
from __future__ import annotations
import asyncio
import contextvars
import dataclasses
import enum
import types
import typing
@@ -8,7 +12,17 @@ import sqlalchemy.ext.asyncio as sqlalchemy_asyncio
TENANT_SETTING = 'langbot.workspace_uuid'
ACCOUNT_SETTING = 'langbot.account_uuid'
API_KEY_HASH_SETTING = 'langbot.api_key_hash'
INVITATION_HASH_SETTING = 'langbot.invitation_hash'
INSTANCE_SETTING = 'langbot.instance_uuid'
IDENTITY_DIGEST_SETTING = 'langbot.identity_digest'
TENANT_POLICY_NAME = 'langbot_workspace_isolation'
ACCOUNT_DISCOVERY_POLICY_NAME = 'langbot_account_discovery'
API_KEY_DISCOVERY_POLICY_NAME = 'langbot_api_key_discovery'
INVITATION_DISCOVERY_POLICY_NAME = 'langbot_invitation_discovery'
INSTANCE_DISCOVERY_POLICY_NAME = 'langbot_instance_discovery'
# Keep this contract explicit. A new tenant-owned table must be added to both
# this runtime list and the corresponding Alembic migration before release.
@@ -41,26 +55,269 @@ TENANT_TABLE_COLUMNS: dict[str, str] = {
'monitoring_errors': 'workspace_uuid',
'monitoring_embedding_calls': 'workspace_uuid',
'monitoring_feedback': 'workspace_uuid',
# Created by 0013 rather than ORM metadata; it is still part of the same
# business-database RLS contract and permits no discovery policies.
'langbot_vectors': 'workspace_uuid',
}
class TenantUnitOfWork:
"""Bind one Workspace scope to one database transaction.
class PersistenceScopeKind(enum.StrEnum):
WORKSPACE = 'workspace'
ACCOUNT_DISCOVERY = 'account_discovery'
API_KEY_DISCOVERY = 'api_key_discovery'
INVITATION_DISCOVERY = 'invitation_discovery'
INSTANCE_DISCOVERY = 'instance_discovery'
IDENTITY_DISCOVERY = 'identity_discovery'
PostgreSQL receives a transaction-local setting used by RLS policies.
SQLite keeps the same transaction boundary without a database setting so
OSS callers can adopt this API without changing database engines.
@dataclasses.dataclass(frozen=True, slots=True)
class PersistenceScope:
"""A trusted, transaction-local database visibility scope."""
kind: PersistenceScopeKind
settings: tuple[tuple[str, str], ...]
@classmethod
def workspace(cls, workspace_uuid: str) -> PersistenceScope:
return cls._one(PersistenceScopeKind.WORKSPACE, TENANT_SETTING, workspace_uuid)
@classmethod
def account(cls, account_uuid: str) -> PersistenceScope:
return cls._one(PersistenceScopeKind.ACCOUNT_DISCOVERY, ACCOUNT_SETTING, account_uuid)
@classmethod
def api_key(cls, key_hash: str) -> PersistenceScope:
return cls._one(PersistenceScopeKind.API_KEY_DISCOVERY, API_KEY_HASH_SETTING, key_hash)
@classmethod
def invitation(cls, token_hash: str) -> PersistenceScope:
return cls._one(PersistenceScopeKind.INVITATION_DISCOVERY, INVITATION_HASH_SETTING, token_hash)
@classmethod
def instance(cls, instance_uuid: str) -> PersistenceScope:
return cls._one(PersistenceScopeKind.INSTANCE_DISCOVERY, INSTANCE_SETTING, instance_uuid)
@classmethod
def identity(cls, identity_digest: str) -> PersistenceScope:
return cls._one(PersistenceScopeKind.IDENTITY_DISCOVERY, IDENTITY_DIGEST_SETTING, identity_digest)
@classmethod
def _one(cls, kind: PersistenceScopeKind, setting: str, value: str) -> PersistenceScope:
return cls(kind, ((setting, cls._normalize(value, kind.value)),))
@staticmethod
def _normalize(value: str, label: str) -> str:
if not isinstance(value, str):
raise TypeError(f'{label} must be a string')
normalized = value.strip()
if not normalized:
raise ValueError(f'{label} must not be empty')
if len(normalized) > 512:
raise ValueError(f'{label} exceeds the database scope limit')
return normalized
class TenantScopeRequiredError(RuntimeError):
"""Raised when Cloud business data is accessed without a trusted scope."""
class CrossScopeTransactionError(RuntimeError):
"""Raised when code tries to change scope inside an active transaction."""
class TransactionRollbackOnlyError(RuntimeError):
"""Raised when a caught failure made the outer transaction unsafe to commit."""
@dataclasses.dataclass(slots=True)
class ActiveScopedTransaction:
scope: PersistenceScope
session: sqlalchemy_asyncio.AsyncSession
engine: sqlalchemy_asyncio.AsyncEngine
owner_task: asyncio.Task[typing.Any] | None
depth: int = 1
rollback_only: bool = False
rollback_only_cause: BaseException | None = None
after_commit_waiters: list[asyncio.Future[None]] = dataclasses.field(default_factory=list)
def mark_rollback_only(self, cause: BaseException | None = None) -> None:
"""Remember that this transaction must never release commit-gated work."""
self.rollback_only = True
if self.rollback_only_cause is None:
self.rollback_only_cause = cause
ActiveTransactionVar = contextvars.ContextVar[ActiveScopedTransaction | None]
# ``AsyncSession`` delegates database I/O to a greenlet-backed synchronous
# Engine, but Python context variables are preserved across that boundary. A
# per-engine ``handle_error`` listener therefore covers DBAPI failures from all
# direct Session APIs (execute/scalar/get/flush) as well as callers which obtain
# the transaction-bound Connection. The owner-task and engine checks prevent
# copied child-task contexts or unrelated database engines from poisoning this
# transaction.
_DATABASE_OPERATION_TRANSACTION: contextvars.ContextVar[ActiveScopedTransaction | None] = contextvars.ContextVar(
'langbot_database_operation_transaction',
default=None,
)
def _mark_current_transaction_rollback_only(exception_context: typing.Any) -> None:
state = _DATABASE_OPERATION_TRANSACTION.get()
if state is None:
return
try:
current_task = asyncio.current_task()
except RuntimeError: # pragma: no cover - async engines always run in an event loop
return
if state.owner_task is not current_task or exception_context.engine is not state.engine.sync_engine:
return
cause = exception_context.sqlalchemy_exception or exception_context.original_exception
state.mark_rollback_only(cause)
def _install_rollback_only_error_listener(engine: sqlalchemy_asyncio.AsyncEngine) -> None:
sync_engine = engine.sync_engine
if not sqlalchemy.event.contains(sync_engine, 'handle_error', _mark_current_transaction_rollback_only):
sqlalchemy.event.listen(sync_engine, 'handle_error', _mark_current_transaction_rollback_only)
@dataclasses.dataclass(slots=True)
class ActivePersistenceScope:
"""A trusted visibility scope without an open database transaction."""
scope: PersistenceScope
owner_task: asyncio.Task[typing.Any] | None
depth: int = 1
ActivePersistenceScopeVar = contextvars.ContextVar[ActivePersistenceScope | None]
class PersistenceScopeBoundary:
"""Carry a trusted persistence scope across long-running async work.
Unlike :class:`TenantUnitOfWork`, this boundary never opens a database
session. ``PersistenceManager.execute_async`` materializes the scope as a
short transaction for each database call. This makes the boundary suitable
for request and pipeline lifetimes which can spend substantial time waiting
on model providers, tools, or streaming clients.
Context variables are copied into child tasks, so implicit use from a child
task is rejected by the manager. A child may establish its own explicit
boundary because the scope value still comes from trusted application code.
"""
def __init__(self, engine: sqlalchemy_asyncio.AsyncEngine, workspace_uuid: str) -> None:
workspace_uuid = workspace_uuid.strip()
if not workspace_uuid:
raise ValueError('TenantUnitOfWork requires a non-empty workspace_uuid')
def __init__(
self,
scope: PersistenceScope,
*,
active_scope: ActivePersistenceScopeVar,
active_transaction: ActiveTransactionVar,
) -> None:
self.scope = scope
self._active_scope = active_scope
self._active_transaction = active_transaction
self._active_state: ActivePersistenceScope | None = None
self._context_token: contextvars.Token[ActivePersistenceScope | None] | None = None
self._used = False
self._owns_scope = False
async def __aenter__(self) -> PersistenceScopeBoundary:
if self._used:
raise RuntimeError('PersistenceScopeBoundary instances cannot be reused')
self._used = True
transaction = self._active_transaction.get()
if transaction is not None:
if transaction.owner_task is not asyncio.current_task():
raise CrossScopeTransactionError(
'Scoped database transactions cannot be inherited by child tasks; open an explicit task scope'
)
if transaction.scope != self.scope:
raise CrossScopeTransactionError(
f'Cannot enter {self.scope.kind.value} scope while {transaction.scope.kind.value} scope is active'
)
active = self._active_scope.get()
if active is not None and active.owner_task is asyncio.current_task():
if active.scope != self.scope:
raise CrossScopeTransactionError(
f'Cannot enter {self.scope.kind.value} scope while {active.scope.kind.value} scope is active'
)
active.depth += 1
self._active_state = active
return self
state = ActivePersistenceScope(
scope=self.scope,
owner_task=asyncio.current_task(),
)
self._active_state = state
self._context_token = self._active_scope.set(state)
self._owns_scope = True
return self
async def __aexit__(
self,
exc_type: type[BaseException] | None,
exc_value: BaseException | None,
traceback: types.TracebackType | None,
) -> bool:
del exc_type, exc_value, traceback
state = self._active_state
if state is None:
raise RuntimeError('PersistenceScopeBoundary has no active state')
if state.owner_task is not asyncio.current_task():
raise CrossScopeTransactionError('Scoped persistence boundaries cannot be exited by a different task')
if not self._owns_scope:
state.depth -= 1
self._active_state = None
return False
if self._context_token is None:
raise RuntimeError('PersistenceScopeBoundary has no context token')
self._active_scope.reset(self._context_token)
state.depth = 0
self._active_state = None
return False
class TenantUnitOfWork:
"""Bind one trusted visibility scope to one database transaction.
PostgreSQL receives transaction-local settings consumed by RLS policies.
SQLite keeps the same transaction boundary so OSS code exercises the same
request/task ownership and rollback semantics. A manager-owned ContextVar
lets all legacy ``execute_async`` helpers reuse this exact session.
"""
def __init__(
self,
engine: sqlalchemy_asyncio.AsyncEngine,
workspace_uuid: str | None = None,
*,
scope: PersistenceScope | None = None,
active_transaction: ActiveTransactionVar | None = None,
active_scope: ActivePersistenceScopeVar | None = None,
) -> None:
if (workspace_uuid is None) == (scope is None):
raise ValueError('TenantUnitOfWork requires exactly one Workspace or persistence scope')
self._engine = engine
self.workspace_uuid = workspace_uuid
self.scope = scope or PersistenceScope.workspace(typing.cast(str, workspace_uuid))
self.workspace_uuid = self.scope.settings[0][1] if self.scope.kind == PersistenceScopeKind.WORKSPACE else None
self._active_transaction = active_transaction
self._active_scope = active_scope
self._session: sqlalchemy_asyncio.AsyncSession | None = None
self._active_state: ActiveScopedTransaction | None = None
self._context_token: contextvars.Token[ActiveScopedTransaction | None] | None = None
self._database_operation_token: contextvars.Token[ActiveScopedTransaction | None] | None = None
self._used = False
self._owns_transaction = False
_install_rollback_only_error_listener(engine)
@property
def session(self) -> sqlalchemy_asyncio.AsyncSession:
@@ -73,16 +330,67 @@ class TenantUnitOfWork:
raise RuntimeError('TenantUnitOfWork instances cannot be reused')
self._used = True
active_scope = self._active_scope.get() if self._active_scope is not None else None
if active_scope is not None and active_scope.owner_task is asyncio.current_task():
if active_scope.scope != self.scope:
# A request/runtime Workspace boundary is transaction-free and
# may temporarily enter a narrower trusted discovery UoW (for
# example, resolving an account record by its verified UUID).
# It must still never switch directly to another Workspace.
workspace_to_discovery = (
active_scope.scope.kind == PersistenceScopeKind.WORKSPACE
and self.scope.kind != PersistenceScopeKind.WORKSPACE
)
if not workspace_to_discovery:
raise CrossScopeTransactionError(
f'Cannot enter {self.scope.kind.value} scope '
f'while {active_scope.scope.kind.value} scope is active'
)
active = self._active_transaction.get() if self._active_transaction is not None else None
if active is not None:
if active.owner_task is asyncio.current_task():
if active.scope != self.scope:
raise CrossScopeTransactionError(
f'Cannot enter {self.scope.kind.value} scope while {active.scope.kind.value} scope is active'
)
active.depth += 1
self._active_state = active
self._session = active.session
return self
# ContextVars are copied into child tasks. Merely calling a helper
# there must fail (PersistenceManager enforces that), but an
# explicit UoW is allowed to replace the inherited pointer with an
# independent transaction owned by the child task.
session = sqlalchemy_asyncio.AsyncSession(self._engine, expire_on_commit=False)
self._session = session
try:
await session.begin()
if self._engine.dialect.name == 'postgresql':
await session.execute(
sqlalchemy.text(f"SELECT set_config('{TENANT_SETTING}', :workspace_uuid, true)"),
{'workspace_uuid': self.workspace_uuid},
)
for setting_name, setting_value in self.scope.settings:
await session.execute(
sqlalchemy.text('SELECT set_config(:setting_name, :setting_value, true)'),
{'setting_name': setting_name, 'setting_value': setting_value},
)
state = ActiveScopedTransaction(
scope=self.scope,
session=session,
engine=self._engine,
owner_task=asyncio.current_task(),
)
self._active_state = state
if self._active_transaction is not None:
self._context_token = self._active_transaction.set(state)
self._database_operation_token = _DATABASE_OPERATION_TRANSACTION.set(state)
self._owns_transaction = True
except BaseException:
if self._database_operation_token is not None:
_DATABASE_OPERATION_TRANSACTION.reset(self._database_operation_token)
self._database_operation_token = None
if self._active_transaction is not None and self._context_token is not None:
self._active_transaction.reset(self._context_token)
self._context_token = None
await session.rollback()
await session.close()
self._session = None
@@ -95,19 +403,88 @@ class TenantUnitOfWork:
exc_value: BaseException | None,
traceback: types.TracebackType | None,
) -> bool:
state = self._active_state
if state is None:
raise RuntimeError('TenantUnitOfWork has no active transaction state')
self._require_owner_task(state)
if not self._owns_transaction:
if exc_type is not None:
state.mark_rollback_only(exc_value)
state.depth -= 1
self._session = None
self._active_state = None
return False
session = self.session
if exc_type is not None:
state.mark_rollback_only(exc_value)
rollback_only = state.rollback_only
committed = False
try:
if exc_type is None:
if exc_type is None and not rollback_only:
await session.commit()
committed = True
else:
await session.rollback()
finally:
await session.close()
self._session = None
try:
if self._active_transaction is not None and self._context_token is not None:
self._active_transaction.reset(self._context_token)
await session.close()
finally:
if self._database_operation_token is not None:
_DATABASE_OPERATION_TRANSACTION.reset(self._database_operation_token)
self._database_operation_token = None
self._session = None
self._active_state = None
state.depth = 0
self._complete_after_commit_waiters(state, committed=committed)
if exc_type is None and rollback_only:
raise TransactionRollbackOnlyError(
'A scoped database operation failed or rolled back; '
'the transaction was rolled back and after-commit work was cancelled'
) from state.rollback_only_cause
return False
async def execute(self, *args: typing.Any, **kwargs: typing.Any) -> sqlalchemy.engine.Result[typing.Any]:
return await self.session.execute(*args, **kwargs)
try:
return await self.session.execute(*args, **kwargs)
except BaseException as exc:
self._mark_rollback_only(exc)
raise
async def flush(self) -> None:
await self.session.flush()
try:
await self.session.flush()
except BaseException as exc:
self._mark_rollback_only(exc)
raise
def _mark_rollback_only(self, cause: BaseException) -> None:
state = self._active_state
if state is not None:
state.mark_rollback_only(cause)
@staticmethod
def _require_owner_task(state: ActiveScopedTransaction) -> None:
if state.owner_task is not asyncio.current_task():
raise CrossScopeTransactionError(
'Scoped database transactions cannot be inherited by child tasks; open an explicit task scope'
)
@staticmethod
def _complete_after_commit_waiters(
state: ActiveScopedTransaction,
*,
committed: bool,
) -> None:
for waiter in state.after_commit_waiters:
if waiter.done():
continue
if committed:
waiter.set_result(None)
else:
waiter.cancel()
state.after_commit_waiters.clear()
+10 -2
View File
@@ -13,6 +13,7 @@ import langbot_plugin.api.entities.builtin.platform.message as platform_message
import langbot_plugin.api.entities.builtin.provider.session as provider_session
from ..api.http.context import ExecutionContext
from ..core.task_boundary import create_detached_task, run_in_workspace_uow
from .pool import ExecutionContextMismatchError
from ..workspace.errors import WorkspaceError, WorkspaceInvariantError
@@ -203,7 +204,10 @@ class MessageAggregator:
if len(buffer.messages) >= MAX_BUFFER_MESSAGES:
force_flush = True
else:
buffer.timer_task = asyncio.create_task(self._delayed_flush(aggregation_key, delay, execution_context))
buffer.timer_task = create_detached_task(
self._delayed_flush(aggregation_key, delay, execution_context),
after_commit_manager=getattr(self.ap, 'persistence_mgr', None),
)
if force_flush:
await self._flush_buffer(aggregation_key, execution_context)
@@ -218,7 +222,11 @@ class MessageAggregator:
try:
await asyncio.sleep(delay)
await self._flush_buffer(aggregation_key, execution_context)
await run_in_workspace_uow(
self.ap,
execution_context.workspace_uuid,
lambda: self._flush_buffer(aggregation_key, execution_context),
)
except asyncio.CancelledError:
pass
except WorkspaceError as exc:
+28 -13
View File
@@ -43,22 +43,37 @@ class Controller:
try:
async with self.semaphore:
execution_context = await self._assert_query_execution_active(selected_query)
pipeline_uuid = selected_query.pipeline_uuid
queued_context = get_query_execution_context(selected_query)
if pipeline_uuid:
pipeline = await self.ap.pipeline_mgr.get_pipeline_by_uuid(
execution_context,
pipeline_uuid,
)
if pipeline:
await pipeline.run(selected_query)
else:
self.ap.logger.warning(
f'Pipeline {pipeline_uuid} not found for query {selected_query.query_id}, query dropped'
async def run_scoped_query() -> None:
execution_context = await self._assert_query_execution_active(selected_query)
pipeline_uuid = selected_query.pipeline_uuid
if pipeline_uuid:
pipeline = await self.ap.pipeline_mgr.get_pipeline_by_uuid(
execution_context,
pipeline_uuid,
)
if pipeline:
await pipeline.run(selected_query)
else:
self.ap.logger.warning(
f'Pipeline {pipeline_uuid} not found for query {selected_query.query_id}, query dropped'
)
else:
self.ap.logger.warning(f'No pipeline_uuid for query {selected_query.query_id}, query dropped')
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
cloud_runtime = (
getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
)
if cloud_runtime:
if not callable(tenant_scope):
raise RuntimeError('Cloud query processing requires an explicit tenant scope')
async with tenant_scope(queued_context.workspace_uuid):
await run_scoped_query()
else:
self.ap.logger.warning(f'No pipeline_uuid for query {selected_query.query_id}, query dropped')
await run_scoped_query()
except WorkspaceError as exc:
self.ap.logger.info(
f'Dropped query {selected_query.query_id} because its Workspace execution binding is stale: {exc}'
+28
View File
@@ -506,6 +506,34 @@ class PipelineManager:
async def load_pipelines_from_db(self):
self.ap.logger.info('Loading pipelines from db...')
self.pipelines = []
list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if cloud_runtime:
if not callable(list_bindings) or not callable(tenant_uow):
raise RuntimeError('Cloud pipeline loading requires explicit instance discovery and tenant UoWs')
for binding in await list_bindings():
async with tenant_uow(binding.workspace_uuid):
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_pipeline.LegacyPipeline)
.where(persistence_pipeline.LegacyPipeline.workspace_uuid == binding.workspace_uuid)
.order_by(persistence_pipeline.LegacyPipeline.uuid)
)
for pipeline in result.all():
await self.load_pipeline(
ExecutionContext(
instance_uuid=binding.instance_uuid,
workspace_uuid=binding.workspace_uuid,
placement_generation=binding.placement_generation,
pipeline_uuid=pipeline.uuid,
trigger_principal=PrincipalContext(PrincipalType.SYSTEM),
),
pipeline,
)
return
# Compatibility path for isolated manager tests and older embedders.
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_pipeline.LegacyPipeline))
pipelines = result.all()
+66 -9
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import asyncio
import dataclasses
import functools
import json
import re
import traceback
@@ -14,6 +15,7 @@ from ..discover import engine
from ..entity.persistence import bot as persistence_bot
from ..entity.persistence import pipeline as persistence_pipeline
from ..entity.persistence import workspace as persistence_workspace
from ..entity.errors import platform as platform_errors
from ..api.http.context import ExecutionContext, PrincipalContext, PrincipalType, RequestContext
@@ -256,6 +258,24 @@ class RuntimeBot:
await self.logger.error(f'Failed to record discarded message: {e}')
async def initialize(self):
def tenant_scoped_listener(listener):
"""Bind adapter callbacks to a Workspace without holding a DB transaction."""
@functools.wraps(listener)
async def wrapped(*args, **kwargs):
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
cloud_runtime = (
getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
)
if cloud_runtime:
if not callable(tenant_scope):
raise RuntimeError('Cloud platform callbacks require an explicit tenant scope')
async with tenant_scope(self.workspace_uuid):
return await listener(*args, **kwargs)
return await listener(*args, **kwargs)
return wrapped
async def on_friend_message(
event: platform_events.FriendMessage,
adapter: abstract_platform_adapter.AbstractMessagePlatformAdapter,
@@ -396,8 +416,8 @@ class RuntimeBot:
else:
await self.logger.info('Pipeline skipped for group message due to webhook response')
self.adapter.register_listener(platform_events.FriendMessage, on_friend_message)
self.adapter.register_listener(platform_events.GroupMessage, on_group_message)
self.adapter.register_listener(platform_events.FriendMessage, tenant_scoped_listener(on_friend_message))
self.adapter.register_listener(platform_events.GroupMessage, tenant_scoped_listener(on_group_message))
# Register feedback listener (only effective on adapters that support it)
async def on_feedback(
@@ -444,7 +464,7 @@ class RuntimeBot:
except Exception:
await self.logger.error(f'Failed to record feedback: {traceback.format_exc()}')
self.adapter.register_listener(platform_events.FeedbackEvent, on_feedback)
self.adapter.register_listener(platform_events.FeedbackEvent, tenant_scoped_listener(on_feedback))
async def run(self):
async def exception_wrapper():
@@ -642,14 +662,51 @@ class PlatformManager:
self.bots = []
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_bot.Bot))
instance_uow = getattr(self.ap.persistence_mgr, 'instance_discovery_uow', None)
tenant_scope = getattr(self.ap.persistence_mgr, 'tenant_scope', None)
if callable(instance_uow) and callable(tenant_scope):
async with instance_uow(self.ap.workspace_service.instance_uuid) as discovery:
workspace_uuids = list(
(
await discovery.session.scalars(
sqlalchemy.select(persistence_workspace.WorkspaceExecutionState.workspace_uuid)
.where(
persistence_workspace.WorkspaceExecutionState.instance_uuid
== self.ap.workspace_service.instance_uuid,
persistence_workspace.WorkspaceExecutionState.state
== persistence_workspace.WorkspaceExecutionStatus.ACTIVE.value,
persistence_workspace.WorkspaceExecutionState.write_fenced.is_(False),
)
.order_by(persistence_workspace.WorkspaceExecutionState.workspace_uuid)
)
).all()
)
else:
# Compatibility for lightweight tests and pre-tenancy managers.
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_bot.Bot))
workspace_uuids = sorted({bot.workspace_uuid for bot in result.all()})
bots = result.all()
for bot in bots:
# load all bots here, enable or disable will be handled in runtime
for workspace_uuid in workspace_uuids:
try:
binding = await self.ap.workspace_service.get_execution_binding(bot.workspace_uuid)
if callable(tenant_scope):
async with tenant_scope(workspace_uuid):
await self._load_workspace_bots(workspace_uuid)
else:
await self._load_workspace_bots(workspace_uuid)
except Exception as e:
self.ap.logger.error(
f'Failed to load Workspace bots for {workspace_uuid}: {e}\n{traceback.format_exc()}'
)
async def _load_workspace_bots(self, workspace_uuid: str) -> None:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_bot.Bot)
.where(persistence_bot.Bot.workspace_uuid == workspace_uuid)
.order_by(persistence_bot.Bot.uuid)
)
for bot in result.all():
try:
binding = await self.ap.workspace_service.get_execution_binding(workspace_uuid)
execution_context = ExecutionContext(
instance_uuid=binding.instance_uuid,
workspace_uuid=binding.workspace_uuid,
@@ -286,19 +286,30 @@ class OpenClawWeixinAdapter(abstract_platform_adapter.AbstractMessagePlatformAda
if execution_context.bot_uuid != self._bot_uuid:
raise RuntimeError('Weixin Bot UUID does not match its ExecutionContext')
binding = await ap.workspace_service.get_execution_binding(
execution_context.workspace_uuid,
expected_generation=execution_context.placement_generation,
)
if binding.instance_uuid != execution_context.instance_uuid:
raise RuntimeError('Weixin Bot Workspace belongs to another LangBot instance')
async def persist() -> None:
binding = await ap.workspace_service.get_execution_binding(
execution_context.workspace_uuid,
expected_generation=execution_context.placement_generation,
)
if binding.instance_uuid != execution_context.instance_uuid:
raise RuntimeError('Weixin Bot Workspace belongs to another LangBot instance')
await ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_bot.Bot)
.where(persistence_bot.Bot.workspace_uuid == execution_context.workspace_uuid)
.where(persistence_bot.Bot.uuid == self._bot_uuid)
.values(adapter_config=self.config)
)
await ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_bot.Bot)
.where(persistence_bot.Bot.workspace_uuid == execution_context.workspace_uuid)
.where(persistence_bot.Bot.uuid == self._bot_uuid)
.values(adapter_config=self.config)
)
cloud_runtime = getattr(getattr(ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if cloud_runtime:
tenant_uow = getattr(ap.persistence_mgr, 'tenant_uow', None)
if not callable(tenant_uow):
raise RuntimeError('Cloud adapter persistence requires an explicit tenant UoW')
async with tenant_uow(execution_context.workspace_uuid):
await persist()
else:
await persist()
except Exception as e:
await self.logger.warning(f'Failed to persist adapter config: {e}')
File diff suppressed because it is too large Load Diff
+97
View File
@@ -0,0 +1,97 @@
from __future__ import annotations
import re
from typing import Any
from urllib.parse import unquote, urlparse
_GITHUB_OWNER_PATTERN = re.compile(r'^[A-Za-z0-9](?:[A-Za-z0-9-]{0,38})$')
_GITHUB_REPO_PATTERN = re.compile(r'^[A-Za-z0-9._-]{1,100}$')
def _positive_github_id(value: object, field_name: str) -> int:
if isinstance(value, bool):
raise ValueError(f'{field_name} must be a positive GitHub identifier')
try:
identifier = int(value)
except (TypeError, ValueError) as exc:
raise ValueError(f'{field_name} must be a positive GitHub identifier') from exc
if identifier <= 0 or str(value).strip() != str(identifier):
raise ValueError(f'{field_name} must be a positive GitHub identifier')
return identifier
def validate_github_release_asset_url(
asset_url: object,
*,
owner: str,
repo: str,
release_tag: str,
) -> str:
"""Accept only a GitHub browser release URL tied to the requested release."""
normalized_url = str(asset_url or '').strip()
parsed = urlparse(normalized_url)
try:
port = parsed.port
except ValueError as exc:
raise ValueError('asset_url has an invalid port') from exc
if (
parsed.scheme != 'https'
or (parsed.hostname or '').lower() != 'github.com'
or parsed.username is not None
or parsed.password is not None
or port not in {None, 443}
or parsed.fragment
):
raise ValueError('asset_url must be an HTTPS GitHub release asset URL')
decoded_path = unquote(parsed.path)
expected_prefix = f'/{owner}/{repo}/releases/download/{release_tag}/'
if not decoded_path.casefold().startswith(
f'/{owner}/{repo}/releases/download/'.casefold()
) or not decoded_path.startswith(expected_prefix):
raise ValueError('asset_url does not match the requested GitHub release')
if decoded_path == expected_prefix or decoded_path.endswith('/'):
raise ValueError('asset_url must identify a GitHub release asset')
return normalized_url
def validate_github_plugin_install_info(install_info: dict[str, Any]) -> dict[str, Any]:
"""Normalize a GitHub install request without trusting a tenant-provided URL."""
owner = str(install_info.get('owner') or '').strip()
repo = str(install_info.get('repo') or '').strip()
release_tag = str(install_info.get('release_tag') or '').strip()
if _GITHUB_OWNER_PATTERN.fullmatch(owner) is None:
raise ValueError('owner must be a valid GitHub repository owner')
if _GITHUB_REPO_PATTERN.fullmatch(repo) is None:
raise ValueError('repo must be a valid GitHub repository name')
if not release_tag or '\x00' in release_tag or len(release_tag) > 255:
raise ValueError('release_tag must identify a GitHub release')
release_id_value = install_info.get('release_id')
asset_id_value = install_info.get('asset_id')
normalized = dict(install_info)
normalized.update(
{
'owner': owner,
'repo': repo,
'release_tag': release_tag,
'github_url': f'https://github.com/{owner}/{repo}',
}
)
if release_id_value is not None or asset_id_value is not None:
if release_id_value is None or asset_id_value is None:
raise ValueError('release_id and asset_id must be provided together')
normalized['release_id'] = _positive_github_id(release_id_value, 'release_id')
normalized['asset_id'] = _positive_github_id(asset_id_value, 'asset_id')
normalized.pop('asset_url', None)
return normalized
normalized['asset_url'] = validate_github_release_asset_url(
install_info.get('asset_url'),
owner=owner,
repo=repo,
release_tag=release_tag,
)
return normalized
+362 -162
View File
@@ -4,15 +4,26 @@ import inspect
import typing
from typing import Any
import base64
import contextlib
import contextvars
import traceback
import uuid
from dataclasses import dataclass
import sqlalchemy
from langbot_plugin.runtime.io import handler
from langbot_plugin.runtime.io.connection import Connection
from langbot_plugin.entities.io.context import ActionContext
from langbot_plugin.entities.io.context import (
ActionContext,
ApplyPluginInstallationRequest,
InstallationBinding,
PluginInstallationDesiredState,
PluginWorkerPolicy,
ReconcilePluginInstallationsRequest,
RemovePluginInstallationRequest,
RuntimeConfig,
RuntimeIdentity,
)
from langbot_plugin.entities.io.actions.enums import (
CommonAction,
RuntimeToLangBotAction,
@@ -61,6 +72,9 @@ class _PluginInstallationIdentity:
workspace_uuid: str
plugin_author: str
plugin_name: str
installation_uuid: str
runtime_revision: int
artifact_digest: str
_UNTRUSTED_SCOPE_FIELDS = frozenset(
@@ -71,12 +85,13 @@ _UNTRUSTED_SCOPE_FIELDS = frozenset(
'workspace_uuid',
'placement_generation',
'installation_uuid',
'runtime_revision',
'artifact_digest',
}
)
_RUNTIME_SCOPED_ACTIONS = frozenset(
{
CommonAction.PING.value,
CommonAction.FILE_CHUNK.value,
RuntimeToLangBotAction.INITIALIZE_PLUGIN_SETTINGS.value,
RuntimeToLangBotAction.GET_PLUGIN_SETTINGS.value,
@@ -89,113 +104,129 @@ class RuntimeConnectionHandler(handler.Handler):
ap: app.Application
@staticmethod
def derive_installation_uuid(
action_context: ActionContext,
plugin_author: str,
plugin_name: str,
) -> str:
"""Derive the current stable installation capability without trusting plugin data."""
return str(
uuid.uuid5(
uuid.NAMESPACE_URL,
'langbot:plugin-installation:'
f'{action_context.instance_uuid}:{action_context.workspace_uuid}:'
f'{plugin_author}/{plugin_name}',
)
)
def validate_inbound_action_context(
self,
action: str,
action_context: ActionContext | None,
) -> ActionContext | None:
"""Keep a validated child installation capability from the request envelope."""
"""Require a complete installation tuple on every tenant action."""
validated = super().validate_inbound_action_context(action, action_context)
if action_context is not None and action_context.installation_uuid is not None:
# The base handler already checked instance, Workspace and placement
# generation against this connector's immutable binding. The
# installation is resolved against trusted Host state asynchronously
# before dispatch.
if action == CommonAction.PING.value:
if action_context is not None:
raise ValueError('PING does not accept an installation binding')
return None
if isinstance(action_context, InstallationBinding):
return action_context
return validated
if self._allow_legacy_oss_context(action, action_context):
return action_context
raise ValueError(f'{action} requires a complete InstallationBinding context')
def _allow_legacy_oss_context(self, action: str, action_context: ActionContext | None) -> bool:
"""Keep pre-v4 local plugins usable without weakening shared Runtime."""
if not (
getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') == 'oss'
and isinstance(action_context, ActionContext)
and not isinstance(action_context, InstallationBinding)
):
return False
if action in {
RuntimeToLangBotAction.INITIALIZE_PLUGIN_SETTINGS.value,
RuntimeToLangBotAction.GET_PLUGIN_SETTINGS.value,
CommonAction.FILE_CHUNK.value,
}:
return True
# Pre-v4 OSS workers receive this capability from Core's settings
# response, but their pinned SDK cannot carry revision/digest fields.
# Shared Runtime never enters this compatibility branch.
return bool(action_context.installation_uuid)
def _require_runtime_action_context(self) -> ActionContext:
action_context = self.current_action_context or self.bound_action_context
action_context = self.current_action_context
if action_context is None:
raise ValueError('Plugin Runtime action is missing a trusted Workspace context')
bound = self.require_bound_action_context()
if not bound.same_workspace(action_context):
raise ValueError('Plugin Runtime action context does not match its connector binding')
return action_context
def _remember_installation(
self,
action_context: ActionContext,
plugin_author: str,
plugin_name: str,
) -> str:
installation_uuid = self.derive_installation_uuid(
action_context,
plugin_author,
plugin_name,
)
self._installation_bindings[installation_uuid] = _PluginInstallationIdentity(
workspace_uuid=action_context.workspace_uuid,
plugin_author=plugin_author,
plugin_name=plugin_name,
)
return installation_uuid
async def _resolve_installation_identity(
self,
action_context: ActionContext,
action_context: InstallationBinding,
) -> _PluginInstallationIdentity:
installation_uuid = action_context.installation_uuid
if installation_uuid is None:
raise ValueError('Plugin action is missing installation_uuid')
identity = self._installation_bindings.get(installation_uuid)
if identity is not None:
if identity.workspace_uuid != action_context.workspace_uuid:
raise ValueError('Plugin installation belongs to another Workspace')
cached = self._installation_bindings.get(action_context.installation_uuid)
if cached is not None:
cached_binding, identity = cached
if cached_binding != action_context:
raise ValueError('Plugin installation revision or artifact is stale')
return identity
# A control connection may reconnect while the Runtime keeps plugin
# processes alive. Rebuild the capability map from Workspace-scoped
# settings instead of accepting an installation asserted by the peer.
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
persistence_plugin.PluginSetting.plugin_author,
persistence_plugin.PluginSetting.plugin_name,
).where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
)
for setting in result.all():
candidate_uuid = self.derive_installation_uuid(
action_context,
setting.plugin_author,
setting.plugin_name,
persistence_plugin.PluginSetting.installation_uuid,
persistence_plugin.PluginSetting.runtime_revision,
persistence_plugin.PluginSetting.artifact_digest,
)
if candidate_uuid == installation_uuid:
return self._installation_bindings.setdefault(
installation_uuid,
_PluginInstallationIdentity(
workspace_uuid=action_context.workspace_uuid,
plugin_author=setting.plugin_author,
plugin_name=setting.plugin_name,
),
)
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
.where(persistence_plugin.PluginSetting.installation_uuid == action_context.installation_uuid)
)
setting = result.first()
if setting is None:
raise ValueError('Plugin installation is not registered in this Workspace')
if (
setting.runtime_revision != action_context.runtime_revision
or setting.artifact_digest != action_context.artifact_digest
):
raise ValueError('Plugin installation revision or artifact is stale')
identity = _PluginInstallationIdentity(
workspace_uuid=action_context.workspace_uuid,
plugin_author=setting.plugin_author,
plugin_name=setting.plugin_name,
installation_uuid=setting.installation_uuid,
runtime_revision=setting.runtime_revision,
artifact_digest=setting.artifact_digest,
)
self._installation_bindings[action_context.installation_uuid] = (action_context, identity)
return identity
raise ValueError('Plugin installation is not registered in this Workspace')
async def _resolve_legacy_oss_installation_identity(
self,
action_context: ActionContext,
) -> _PluginInstallationIdentity:
if not action_context.installation_uuid:
raise ValueError('Legacy OSS plugin action is missing its installation capability')
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
persistence_plugin.PluginSetting.plugin_author,
persistence_plugin.PluginSetting.plugin_name,
persistence_plugin.PluginSetting.installation_uuid,
persistence_plugin.PluginSetting.runtime_revision,
persistence_plugin.PluginSetting.artifact_digest,
)
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
.where(persistence_plugin.PluginSetting.installation_uuid == action_context.installation_uuid)
)
setting = result.first()
if setting is None:
raise ValueError('Plugin installation is not registered in this Workspace')
return _PluginInstallationIdentity(
workspace_uuid=action_context.workspace_uuid,
plugin_author=setting.plugin_author,
plugin_name=setting.plugin_name,
installation_uuid=setting.installation_uuid,
runtime_revision=setting.runtime_revision,
artifact_digest=setting.artifact_digest,
)
async def _require_plugin_action_context(
self,
) -> tuple[ActionContext, _PluginInstallationIdentity]:
action_context = self._require_runtime_action_context()
identity = await self._resolve_installation_identity(action_context)
if isinstance(action_context, InstallationBinding):
identity = await self._resolve_installation_identity(action_context)
elif self._allow_legacy_oss_context('plugin_action', action_context):
identity = await self._resolve_legacy_oss_installation_identity(action_context)
else:
raise ValueError('Plugin action requires a complete InstallationBinding context')
return action_context, identity
async def _require_active_action_context(self, action_context: ActionContext) -> None:
@@ -215,25 +246,53 @@ class RuntimeConnectionHandler(handler.Handler):
):
raise ValueError('Plugin Runtime action uses a stale Workspace execution binding')
@contextlib.asynccontextmanager
async def _tenant_action_scope(self, action_context: ActionContext):
"""Bind Runtime-origin work to the trusted Workspace database scope.
Cloud persistence deliberately rejects unscoped access and PostgreSQL
RLS reads the Workspace id from each short database transaction. The
wire envelope is validated before this helper is entered, so plugin
payload fields can never select the scope. A transaction-free boundary
avoids reserving one pooled connection across provider and network waits.
"""
persistence_mgr = getattr(self.ap, 'persistence_mgr', None)
if persistence_mgr is None:
yield
return
tenant_scope_descriptor = getattr(type(persistence_mgr), 'tenant_scope', None)
if not callable(tenant_scope_descriptor):
# Lightweight test doubles and older OSS persistence managers do
# not expose the transaction-free scope API.
yield
return
async with persistence_mgr.tenant_scope(action_context.workspace_uuid):
yield
def _secure_plugin_actions(self) -> None:
"""Wrap plugin-origin actions with installation validation and payload scrubbing."""
for action_name, action_handler in list(self.actions.items()):
if action_name in _RUNTIME_SCOPED_ACTIONS:
if action_name == CommonAction.PING.value:
continue
async def secured_action(
data: dict[str, Any],
*,
_action_handler=action_handler,
_runtime_scoped=action_name in _RUNTIME_SCOPED_ACTIONS,
) -> handler.ActionResponse:
action_context, _ = await self._require_plugin_action_context()
await self._require_active_action_context(action_context)
safe_data = {key: value for key, value in data.items() if key not in _UNTRUSTED_SCOPE_FIELDS}
response = _action_handler(safe_data)
if inspect.isawaitable(response):
response = await response
return response
action_context = self._require_runtime_action_context()
async with self._tenant_action_scope(action_context):
if not _runtime_scoped:
action_context, _ = await self._require_plugin_action_context()
await self._require_active_action_context(action_context)
safe_data = {key: value for key, value in data.items() if key not in _UNTRUSTED_SCOPE_FIELDS}
response = _action_handler(safe_data)
if inspect.isawaitable(response):
response = await response
return response
self.actions[action_name] = secured_action
@@ -253,6 +312,31 @@ class RuntimeConnectionHandler(handler.Handler):
raise ValueError('Plugin installation setting was not found')
return setting
async def _get_plugin_setting_by_name(
self,
action_context: ActionContext,
plugin_author: str,
plugin_name: str,
):
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_plugin.PluginSetting)
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
.where(persistence_plugin.PluginSetting.plugin_author == plugin_author)
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
)
return result.first()
@staticmethod
def _require_setting_binding(setting, action_context: InstallationBinding) -> None:
if setting is None:
raise ValueError('Plugin installation setting was not found')
if (
setting.installation_uuid != action_context.installation_uuid
or setting.runtime_revision != action_context.runtime_revision
or setting.artifact_digest != action_context.artifact_digest
):
raise ValueError('Plugin installation binding does not match the active setting')
async def _resolve_query(
self,
data: dict[str, Any],
@@ -355,18 +439,24 @@ class RuntimeConnectionHandler(handler.Handler):
connection: Connection,
disconnect_callback: typing.Callable[[], typing.Coroutine[typing.Any, typing.Any, bool]],
ap: app.Application,
action_context: ActionContext,
):
super().__init__(connection, disconnect_callback)
self.ap = ap
self.bind_action_context(action_context.without_installation())
self._installation_bindings: dict[str, _PluginInstallationIdentity] = {}
self._outbound_installation_context: contextvars.ContextVar[InstallationBinding | None] = (
contextvars.ContextVar(
f'{self.__class__.__name__}_{id(self)}_outbound_installation',
default=None,
)
)
self._installation_bindings: dict[
str,
tuple[InstallationBinding, _PluginInstallationIdentity],
] = {}
@self.action(RuntimeToLangBotAction.INITIALIZE_PLUGIN_SETTINGS)
async def initialize_plugin_settings(data: dict[str, Any]) -> handler.ActionResponse:
"""Initialize plugin settings"""
action_context = self._require_runtime_action_context()
await self._require_active_action_context(action_context)
# check if exists plugin setting
plugin_author = data['plugin_author']
plugin_name = data['plugin_name']
@@ -374,39 +464,37 @@ class RuntimeConnectionHandler(handler.Handler):
install_info = data['install_info']
try:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_plugin.PluginSetting)
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
.where(persistence_plugin.PluginSetting.plugin_author == plugin_author)
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
setting = await self._get_plugin_setting_by_name(
action_context,
plugin_author,
plugin_name,
)
setting = result.first()
if setting is not None:
# delete plugin setting
if isinstance(action_context, InstallationBinding):
self._require_setting_binding(setting, action_context)
elif setting is None:
# OSS debug and pre-v4 data/plugins are the only callers
# allowed to create settings without a desired-state row.
await self.ap.persistence_mgr.execute_async(
sqlalchemy.delete(persistence_plugin.PluginSetting)
sqlalchemy.insert(persistence_plugin.PluginSetting).values(
workspace_uuid=action_context.workspace_uuid,
plugin_author=plugin_author,
plugin_name=plugin_name,
install_source=install_source,
install_info=install_info,
enabled=True,
priority=0,
config={},
)
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_plugin.PluginSetting)
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
.where(persistence_plugin.PluginSetting.plugin_author == plugin_author)
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
.values(install_source=install_source, install_info=install_info)
)
# create plugin setting
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(persistence_plugin.PluginSetting).values(
workspace_uuid=action_context.workspace_uuid,
plugin_author=plugin_author,
plugin_name=plugin_name,
install_source=install_source,
install_info=install_info,
# inherit from existing setting
enabled=setting.enabled if setting is not None else True,
priority=setting.priority if setting is not None else 0,
config=setting.config if setting is not None else {}, # noqa: F821
)
)
return handler.ActionResponse.success(
data={},
)
@@ -421,16 +509,16 @@ class RuntimeConnectionHandler(handler.Handler):
"""Get plugin settings"""
action_context = self._require_runtime_action_context()
await self._require_active_action_context(action_context)
plugin_author = data['plugin_author']
plugin_name = data['plugin_name']
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_plugin.PluginSetting)
.where(persistence_plugin.PluginSetting.workspace_uuid == action_context.workspace_uuid)
.where(persistence_plugin.PluginSetting.plugin_author == plugin_author)
.where(persistence_plugin.PluginSetting.plugin_name == plugin_name)
setting = await self._get_plugin_setting_by_name(
action_context,
plugin_author,
plugin_name,
)
if isinstance(action_context, InstallationBinding):
self._require_setting_binding(setting, action_context)
data = {
'enabled': True,
@@ -438,21 +526,20 @@ class RuntimeConnectionHandler(handler.Handler):
'plugin_config': {},
'install_source': 'local',
'install_info': {},
'installation_uuid': self._remember_installation(
action_context,
plugin_author,
plugin_name,
),
'installation_uuid': None,
'runtime_revision': None,
'artifact_digest': None,
}
setting = result.first()
if setting is not None:
data['enabled'] = setting.enabled
data['priority'] = setting.priority
data['plugin_config'] = setting.config
data['install_source'] = setting.install_source
data['install_info'] = setting.install_info
data['installation_uuid'] = setting.installation_uuid
data['runtime_revision'] = setting.runtime_revision
data['artifact_digest'] = setting.artifact_digest
return handler.ActionResponse.success(
data=data,
@@ -1369,24 +1456,144 @@ class RuntimeConnectionHandler(handler.Handler):
self._secure_plugin_actions()
async def ping(self) -> dict[str, Any]:
"""Ping the runtime"""
return await self.call_action(
CommonAction.PING,
{},
timeout=10,
@contextlib.contextmanager
def installation_scope(self, binding: InstallationBinding | None):
"""Attach one immutable installation tuple to nested wire actions."""
token = self._outbound_installation_context.set(binding)
try:
yield
finally:
self._outbound_installation_context.reset(token)
def register_installation_binding(
self,
binding: InstallationBinding,
*,
plugin_author: str,
plugin_name: str,
) -> None:
"""Install Core's current desired-state fence for inbound actions."""
existing = self._installation_bindings.get(binding.installation_uuid)
if existing is not None:
existing_binding = existing[0]
if existing_binding.workspace_uuid != binding.workspace_uuid:
raise ValueError('Plugin installation cannot move between Workspaces')
if binding.placement_generation < existing_binding.placement_generation or (
binding.placement_generation == existing_binding.placement_generation
and binding.runtime_revision < existing_binding.runtime_revision
):
raise ValueError('Cannot register a stale plugin installation binding')
self._installation_bindings[binding.installation_uuid] = (
binding,
_PluginInstallationIdentity(
workspace_uuid=binding.workspace_uuid,
plugin_author=plugin_author,
plugin_name=plugin_name,
installation_uuid=binding.installation_uuid,
runtime_revision=binding.runtime_revision,
artifact_digest=binding.artifact_digest,
),
)
async def set_runtime_config(self, cloud_service_url: str | None) -> dict[str, Any]:
"""Push runtime configuration (e.g. marketplace URL) to the runtime."""
data = {}
if cloud_service_url:
data['cloud_service_url'] = cloud_service_url
return await self.call_action(
LangBotToRuntimeAction.SET_RUNTIME_CONFIG,
data,
timeout=10,
def unregister_installation_binding(self, binding: InstallationBinding) -> None:
existing = self._installation_bindings.get(binding.installation_uuid)
if existing is not None and existing[0] == binding:
self._installation_bindings.pop(binding.installation_uuid, None)
def resolve_outbound_action_context(
self,
action_context: InstallationBinding | ActionContext | dict[str, Any] | None,
) -> InstallationBinding | ActionContext | None:
if action_context is not None:
return super().resolve_outbound_action_context(action_context)
inbound_context = self.current_action_context
if inbound_context is not None:
return inbound_context
return self._outbound_installation_context.get()
def require_outbound_installation_context(self) -> InstallationBinding:
binding = self._outbound_installation_context.get()
if not isinstance(binding, InstallationBinding):
raise ValueError('Host plugin action requires an InstallationBinding scope')
return binding
async def ping(self) -> dict[str, Any]:
"""Ping the runtime"""
with self.installation_scope(None):
return await self.call_action(
CommonAction.PING,
{},
timeout=10,
)
async def set_runtime_config(
self,
*,
runtime_identity: RuntimeIdentity,
worker_policy: PluginWorkerPolicy,
runtime_profile: typing.Literal['oss_dev', 'shared'],
cloud_service_url: str | None,
) -> dict[str, Any]:
"""Push the instance-scoped, immutable Runtime handshake."""
runtime_config = RuntimeConfig(
runtime_identity=runtime_identity,
worker_policy=worker_policy,
runtime_profile=runtime_profile,
cloud_service_url=cloud_service_url,
)
with self.installation_scope(None):
return await self.call_action(
LangBotToRuntimeAction.SET_RUNTIME_CONFIG,
runtime_config.model_dump(exclude_none=True),
timeout=10,
)
async def reconcile_plugin_installations(
self,
installations: tuple[PluginInstallationDesiredState, ...],
) -> dict[str, Any]:
request = ReconcilePluginInstallationsRequest(installations=installations)
with self.installation_scope(None):
return await self.call_action(
LangBotToRuntimeAction.RECONCILE_PLUGIN_INSTALLATIONS,
request.model_dump(),
timeout=120,
)
async def apply_plugin_installation(
self,
binding: InstallationBinding,
*,
artifact_package: bytes | None,
enabled: bool,
) -> dict[str, Any]:
with self.installation_scope(binding):
artifact_file_key = None
if artifact_package is not None:
artifact_file_key = await self.send_file(artifact_package, 'lbpkg')
request = ApplyPluginInstallationRequest(
artifact_file_key=artifact_file_key,
enabled=enabled,
)
return await self.call_action(
LangBotToRuntimeAction.APPLY_PLUGIN_INSTALLATION,
request.model_dump(exclude_none=True),
timeout=120,
)
async def remove_plugin_installation(
self,
binding: InstallationBinding,
) -> dict[str, Any]:
with self.installation_scope(binding):
return await self.call_action(
LangBotToRuntimeAction.REMOVE_PLUGIN_INSTALLATION,
RemovePluginInstallationRequest().model_dump(),
timeout=120,
)
async def install_plugin(
self, install_source: str, install_info: dict[str, Any]
@@ -1455,7 +1662,7 @@ class RuntimeConnectionHandler(handler.Handler):
async def set_plugin_config(self, plugin_author: str, plugin_name: str, config: dict[str, Any]) -> dict[str, Any]:
"""Set plugin config"""
action_context = self.require_bound_action_context()
action_context = self.require_outbound_installation_context()
# update plugin setting
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_plugin.PluginSetting)
@@ -1642,12 +1849,7 @@ class RuntimeConnectionHandler(handler.Handler):
async def cleanup_plugin_data(self, plugin_author: str, plugin_name: str) -> None:
"""Cleanup plugin settings and binary storage"""
action_context = self.require_bound_action_context()
installation_uuid = self.derive_installation_uuid(
action_context,
plugin_author,
plugin_name,
)
action_context = self.require_outbound_installation_context()
# Delete plugin settings
await self.ap.persistence_mgr.execute_async(
sqlalchemy.delete(persistence_plugin.PluginSetting)
@@ -1664,9 +1866,6 @@ class RuntimeConnectionHandler(handler.Handler):
.where(persistence_bstorage.BinaryStorage.owner_type == 'plugin')
.where(persistence_bstorage.BinaryStorage.owner == owner)
)
installation_bindings = getattr(self, '_installation_bindings', None)
if installation_bindings is not None:
installation_bindings.pop(installation_uuid, None)
async def call_tool(
self,
@@ -1744,11 +1943,12 @@ class RuntimeConnectionHandler(handler.Handler):
async def get_debug_info(self) -> dict[str, Any]:
"""Get debug information including debug key and WS URL"""
result = await self.call_action(
LangBotToRuntimeAction.GET_DEBUG_INFO,
{},
timeout=10,
)
with self.installation_scope(None):
result = await self.call_action(
LangBotToRuntimeAction.GET_DEBUG_INFO,
{},
timeout=10,
)
return result
# ================= RAG Capability Callers (LangBot -> Runtime) =================
@@ -197,6 +197,23 @@ class ModelManager:
self.llm_model_dict = {}
self.embedding_model_dict = {}
self.rerank_model_dict = {}
list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if cloud_runtime:
if not callable(list_bindings) or not callable(tenant_uow):
raise RuntimeError('Cloud model loading requires explicit instance discovery and tenant UoWs')
for binding in await list_bindings():
context = self._context_from_binding(
binding,
trigger_principal=PrincipalContext(principal_type=PrincipalType.SYSTEM),
)
async with tenant_uow(binding.workspace_uuid):
await self._load_workspace_models(context)
return
# Compatibility path for isolated manager tests and older embedders.
contexts: dict[str, ExecutionContext] = {}
async def context_for(workspace_uuid: str | None) -> ExecutionContext:
@@ -263,6 +280,61 @@ class ModelManager:
except Exception as exc:
self.ap.logger.error(f'Failed to load model {model_entity.uuid}: {exc}\n{traceback.format_exc()}')
async def _load_workspace_models(self, context: ExecutionContext) -> None:
"""Load one Workspace while its tenant transaction is active."""
providers_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_model.ModelProvider).where(
persistence_model.ModelProvider.workspace_uuid == context.workspace_uuid
)
)
for provider_entity in providers_result.all():
try:
runtime_provider = await self._build_provider(context, provider_entity)
self.provider_dict[self._cache_key(context, provider_entity.uuid)] = runtime_provider
except provider_errors.RequesterNotFoundError as exc:
self.ap.logger.warning(
f'Requester {exc.requester_name} not found, skipping provider {provider_entity.uuid}'
)
except Exception as exc:
self.ap.logger.error(f'Failed to load provider {provider_entity.uuid}: {exc}\n{traceback.format_exc()}')
await self._load_workspace_model_kind(
context,
persistence_model.LLMModel,
self.llm_model_dict,
self._build_llm_model,
)
await self._load_workspace_model_kind(
context,
persistence_model.EmbeddingModel,
self.embedding_model_dict,
self._build_embedding_model,
)
await self._load_workspace_model_kind(
context,
persistence_model.RerankModel,
self.rerank_model_dict,
self._build_rerank_model,
)
async def _load_workspace_model_kind(self, context, entity_type, cache: dict, builder) -> None:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(entity_type).where(entity_type.workspace_uuid == context.workspace_uuid)
)
for model_entity in result.all():
try:
provider = self.provider_dict.get(self._cache_key(context, model_entity.provider_uuid))
if provider is None:
self.ap.logger.warning(
f'Provider {model_entity.provider_uuid} not found for model {model_entity.uuid}'
)
continue
runtime_model = builder(context, model_entity, provider)
cache[self._cache_key(context, model_entity.uuid)] = runtime_model
except Exception as exc:
self.ap.logger.error(f'Failed to load model {model_entity.uuid}: {exc}\n{traceback.format_exc()}')
async def sync_new_models_from_space(self, context: ExecutionContext) -> None:
"""Sync legacy Space models for the explicitly selected OSS Workspace."""
@@ -211,7 +211,7 @@ class LocalAgentRunner(runner.RequestRunner):
req_messages.append(
provider_message.Message(
role='system',
content=self.ap.box_service.get_system_guidance(query.query_id),
content=self.ap.box_service.get_system_guidance(query),
)
)
+47 -7
View File
@@ -23,6 +23,7 @@ from pydantic import AnyUrl
from .. import loader
from ....core import app
from ....core.task_boundary import create_detached_task, run_in_workspace_uow
from ....api.http.context import ExecutionContext
from ....api.http.service.tenant import TenantContext, require_workspace_uuid
from ....workspace.errors import WorkspaceError, WorkspaceInvariantError
@@ -1437,11 +1438,41 @@ class MCPLoader(loader.ToolLoader):
self.sessions = {}
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_mcp.MCPServer))
servers = result.all()
server_configs: list[tuple[typing.Any, typing.Any, dict]] = []
list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if cloud_runtime:
if not callable(list_bindings) or not callable(tenant_uow):
raise RuntimeError('Cloud MCP loading requires explicit instance discovery and tenant UoWs')
for binding in await list_bindings():
async with tenant_uow(binding.workspace_uuid):
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_mcp.MCPServer)
.where(persistence_mcp.MCPServer.workspace_uuid == binding.workspace_uuid)
.order_by(persistence_mcp.MCPServer.uuid)
)
for server in result.all():
server_configs.append(
(
binding,
server,
self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server),
)
)
else:
# Compatibility path for isolated loader tests and older embedders.
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_mcp.MCPServer))
for server in result.all():
server_configs.append(
(
None,
server,
self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server),
)
)
for server in servers:
config = self.ap.persistence_mgr.serialize_model(persistence_mcp.MCPServer, server)
for binding, server, config in server_configs:
if config.get('mode') == 'stdio' and not stdio_mcp_enabled(self.ap):
self.ap.logger.info(
f'Skipping disabled stdio MCP server {server.uuid}; '
@@ -1449,7 +1480,8 @@ class MCPLoader(loader.ToolLoader):
)
continue
try:
binding = await self.ap.workspace_service.get_execution_binding(server.workspace_uuid)
if binding is None:
binding = await self.ap.workspace_service.get_execution_binding(server.workspace_uuid)
execution_context = ExecutionContext(
instance_uuid=binding.instance_uuid,
workspace_uuid=binding.workspace_uuid,
@@ -1461,7 +1493,10 @@ class MCPLoader(loader.ToolLoader):
)
continue
task = asyncio.create_task(self.host_mcp_server(execution_context, config))
task = create_detached_task(
self.host_mcp_server(execution_context, config),
after_commit_manager=getattr(self.ap, 'persistence_mgr', None),
)
self._hosted_mcp_tasks.append(task)
@staticmethod
@@ -1482,7 +1517,12 @@ class MCPLoader(loader.ToolLoader):
return [session for key, session in self.sessions.items() if key[:3] == scope_key]
async def host_mcp_server(self, context: TenantContext, server_config: dict):
execution_context = await self._assert_execution_active(context)
requested_context = _execution_context_from_tenant(context)
execution_context = await run_in_workspace_uow(
self.ap,
requested_context.workspace_uuid,
lambda: self._assert_execution_active(requested_context),
)
configured_workspace = str(server_config.get('workspace_uuid') or '').strip()
if configured_workspace and configured_workspace != execution_context.workspace_uuid:
raise ValueError('MCP server configuration belongs to another Workspace')
+504 -192
View File
@@ -1,8 +1,15 @@
from __future__ import annotations
import base64
import contextlib
import errno
import json
import os
import posixpath
import stat
from collections.abc import Iterator
from dataclasses import dataclass
from pathlib import PurePosixPath
import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
from langbot_plugin.api.entities.events import pipeline_query
@@ -34,6 +41,158 @@ _GREP_MAX_MATCHES = 200
_GREP_MAX_FILES = 5000
_GREP_MAX_LINE_CHARS = 500
_DIRECTORY_OPEN_FLAGS = (
os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
)
_FILE_OPEN_FLAGS = getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0) | getattr(os, 'O_NONBLOCK', 0)
_SECURE_HOST_FILE_OPS_AVAILABLE = bool(
getattr(os, 'O_NOFOLLOW', 0)
and os.open in os.supports_dir_fd
and os.stat in os.supports_dir_fd
and os.mkdir in os.supports_dir_fd
and os.listdir in os.supports_fd
and os.scandir in os.supports_fd
)
@dataclass(frozen=True)
class _HostLocation:
root: str
relative_parts: tuple[str, ...]
selected_skill: dict | None
workspace_anchor: str | None = None
def _unsafe_host_path(path: str, exc: BaseException | None = None) -> ValueError:
error = ValueError(f'Path escapes the workspace boundary or contains a symbolic link: {path}')
if exc is not None:
error.__cause__ = exc
return error
def _relative_workspace_parts(path: str) -> tuple[str, ...]:
normalized = posixpath.normpath(str(path or '/workspace').strip() or '/workspace')
if normalized == '/workspace':
return ()
if not normalized.startswith('/workspace/'):
raise ValueError('Path escapes the workspace boundary.')
parts = tuple(part for part in normalized.removeprefix('/workspace/').split('/') if part)
if any(part in {'.', '..'} or '\x00' in part for part in parts):
raise ValueError('Path escapes the workspace boundary.')
return parts
def _is_symlink_at(parent_fd: int, name: str) -> bool:
try:
return stat.S_ISLNK(os.stat(name, dir_fd=parent_fd, follow_symlinks=False).st_mode)
except FileNotFoundError:
return False
def _open_directory_at(parent_fd: int, name: str, *, create: bool) -> int:
if create:
try:
os.mkdir(name, mode=0o777, dir_fd=parent_fd)
except FileExistsError:
pass
try:
directory_fd = os.open(name, _DIRECTORY_OPEN_FLAGS, dir_fd=parent_fd)
except OSError as exc:
if exc.errno == errno.ELOOP or _is_symlink_at(parent_fd, name):
raise _unsafe_host_path(name, exc)
raise
if not stat.S_ISDIR(os.fstat(directory_fd).st_mode):
os.close(directory_fd)
raise NotADirectoryError(name)
return directory_fd
def _open_directory_parts(root_fd: int, parts: tuple[str, ...], *, create: bool) -> int:
current_fd = os.dup(root_fd)
try:
for part in parts:
next_fd = _open_directory_at(current_fd, part, create=create)
os.close(current_fd)
current_fd = next_fd
return current_fd
except BaseException:
os.close(current_fd)
raise
@contextlib.contextmanager
def _open_host_root(location: _HostLocation, *, create: bool) -> Iterator[int]:
"""Open and pin the tenant root before resolving tenant-controlled names."""
if location.workspace_anchor is None:
root_path = os.path.realpath(location.root)
try:
root_fd = os.open(root_path, _DIRECTORY_OPEN_FLAGS)
except OSError as exc:
if exc.errno == errno.ELOOP:
raise _unsafe_host_path(location.root, exc)
raise
else:
anchor_path = os.path.abspath(location.workspace_anchor)
root_path = os.path.abspath(location.root)
try:
if os.path.commonpath((anchor_path, root_path)) != anchor_path:
raise _unsafe_host_path(location.root)
except ValueError as exc:
raise _unsafe_host_path(location.root, exc)
anchor_real_path = os.path.realpath(anchor_path)
try:
anchor_fd = os.open(anchor_real_path, _DIRECTORY_OPEN_FLAGS)
except OSError as exc:
if exc.errno == errno.ELOOP:
raise _unsafe_host_path(location.workspace_anchor, exc)
raise
try:
root_relative = os.path.relpath(root_path, anchor_path)
root_parts = () if root_relative == '.' else tuple(root_relative.split(os.sep))
root_fd = _open_directory_parts(anchor_fd, root_parts, create=create)
finally:
os.close(anchor_fd)
try:
if not stat.S_ISDIR(os.fstat(root_fd).st_mode):
raise _unsafe_host_path(location.root)
yield root_fd
finally:
os.close(root_fd)
@contextlib.contextmanager
def _open_location_fd(
root_fd: int,
relative_parts: tuple[str, ...],
flags: int,
*,
create_parents: bool = False,
mode: int = 0o666,
) -> Iterator[int]:
if not relative_parts:
target_fd = os.dup(root_fd)
else:
parent_fd = _open_directory_parts(root_fd, relative_parts[:-1], create=create_parents)
try:
try:
target_fd = os.open(relative_parts[-1], flags | _FILE_OPEN_FLAGS, mode, dir_fd=parent_fd)
except OSError as exc:
if exc.errno == errno.ELOOP or _is_symlink_at(parent_fd, relative_parts[-1]):
raise _unsafe_host_path(relative_parts[-1], exc)
raise
finally:
os.close(parent_fd)
try:
yield target_fd
finally:
os.close(target_fd)
class NativeToolLoader(loader.ToolLoader):
def __init__(self, ap):
@@ -59,6 +218,9 @@ class NativeToolLoader(loader.ToolLoader):
@staticmethod
def _execution_context(query: pipeline_query.Query) -> ExecutionContext:
attached_context = getattr(query, '_execution_context', None)
if isinstance(attached_context, ExecutionContext):
return attached_context
return ExecutionContext(
instance_uuid=str(getattr(query, 'instance_uuid', '') or ''),
workspace_uuid=str(getattr(query, 'workspace_uuid', '') or ''),
@@ -66,6 +228,7 @@ class NativeToolLoader(loader.ToolLoader):
bot_uuid=getattr(query, 'bot_uuid', None),
pipeline_uuid=getattr(query, 'pipeline_uuid', None),
query_uuid=getattr(query, 'query_uuid', None),
entitlement_revision=getattr(query, 'entitlement_revision', 0),
)
async def get_tools(self, bound_plugins: list[str] | None = None) -> list[resource_tool.LLMTool]:
@@ -86,6 +249,13 @@ class NativeToolLoader(loader.ToolLoader):
return name in _ALL_TOOL_NAMES and self._is_sandbox_available()
async def invoke_tool(self, name: str, parameters: dict, query: pipeline_query.Query):
require_sandbox = getattr(
getattr(self.ap, 'box_service', None),
'require_workspace_sandbox',
None,
)
if callable(require_sandbox):
await require_sandbox(self._execution_context(query))
if name == EXEC_TOOL_NAME:
self.ap.logger.info(
'exec tool invoked: '
@@ -111,6 +281,7 @@ class NativeToolLoader(loader.ToolLoader):
async def _invoke_exec(self, parameters: dict, query: pipeline_query.Query) -> dict:
command = str(parameters['command'])
workdir = str(parameters.get('workdir', '/workspace') or '/workspace')
selected_skill_name: str | None = None
# Validate that skill references target activated skills.
selected_skill, _ = skill_loader.resolve_virtual_skill_path(
@@ -140,31 +311,51 @@ class NativeToolLoader(loader.ToolLoader):
if not package_root:
raise ValueError(f'Activated skill "{selected_skill_name}" has no package_root.')
# Pass only the logical name across the authenticated Core→Runtime
# boundary. In Cloud mode the shared Box Runtime resolves the
# Workspace-scoped package root and constructs the read-only mount;
# Core host paths are never accepted as mount authority.
# Wrap command with Python venv bootstrap if the skill has a Python project.
# The venv is created inside the skill's mount path.
skill_mount = f'/workspace/.skills/{selected_skill_name}'
if skill_loader.should_prepare_skill_python_env(package_root):
python_project = selected_skill.get('python_project') is True
if 'python_project' not in selected_skill and bool(
getattr(self.ap.box_service, 'shares_filesystem_with_box', False)
):
# Backward compatibility for a same-process OSS Runtime that
# predates trusted Box metadata. Never probe a path reported by
# an external Runtime from the Core filesystem.
python_project = skill_loader.should_prepare_skill_python_env(package_root)
if python_project:
parameters = dict(parameters)
parameters['command'] = skill_loader.wrap_skill_command_with_python_env(command, mount_path=skill_mount)
parameters['command'] = skill_loader.wrap_skill_command_with_python_env(
command,
mount_path=skill_mount,
state_path=f'/workspace/.skill-envs/{selected_skill_name}',
)
# All exec calls (with or without skills) go through the same container
# via execute_tool. Skills are mounted at /workspace/.skills/{name}/
# via extra_mounts built by BoxService.
result = await self.ap.box_service.execute_tool(parameters, query)
result = await self.ap.box_service.execute_tool(
parameters,
query,
skill_name=selected_skill_name,
)
result = self._normalize_exec_result(result)
if selected_skill is not None:
self._refresh_skill_from_disk(query, selected_skill)
return result
def _resolve_host_path(
def _resolve_host_location(
self,
query: pipeline_query.Query,
sandbox_path: str,
*,
include_visible: bool,
include_activated: bool,
) -> tuple[str, dict | None]:
) -> _HostLocation:
selected_skill, rewritten_path = skill_loader.resolve_virtual_skill_path(
self.ap,
query,
@@ -174,26 +365,26 @@ class NativeToolLoader(loader.ToolLoader):
)
box_service = self.ap.box_service
host_root = (
selected_skill.get('package_root')
if selected_skill is not None
else box_service._tenant_workspace(self._execution_context(query))
)
if selected_skill is not None:
if not self._can_interpret_skill_host_paths():
raise ValueError(
'Skill package paths are owned by the Box Runtime; '
'this operation requires a Runtime skill-file API.'
)
host_root = selected_skill.get('package_root')
workspace_anchor = None
else:
host_root = box_service._tenant_workspace(self._execution_context(query))
workspace_anchor = getattr(box_service, 'default_workspace', None)
if not host_root:
raise ValueError('No host workspace configured for file operations.')
mount_path = '/workspace'
if not rewritten_path.startswith(mount_path):
raise ValueError(f'Path must be under {mount_path}.')
relative = rewritten_path[len(mount_path) :].lstrip('/')
host_path = os.path.realpath(os.path.join(host_root, relative))
host_root = os.path.realpath(host_root)
if not (host_path == host_root or host_path.startswith(host_root + os.sep)):
raise ValueError('Path escapes the workspace boundary.')
return host_path, selected_skill
return _HostLocation(
root=str(host_root),
relative_parts=_relative_workspace_parts(rewritten_path),
selected_skill=selected_skill,
workspace_anchor=str(workspace_anchor) if workspace_anchor else None,
)
def _resolve_skill_relative_path(
self,
@@ -213,21 +404,259 @@ class NativeToolLoader(loader.ToolLoader):
if selected_skill is None:
return None
mount_path = '/workspace'
if not rewritten_path.startswith(mount_path):
raise ValueError(f'Path must be under {mount_path}.')
relative = rewritten_path[len(mount_path) :].lstrip('/') or '.'
relative = '/'.join(_relative_workspace_parts(rewritten_path)) or '.'
return selected_skill, relative
def _can_interpret_skill_host_paths(self) -> bool:
"""Require an explicitly proven shared Core/Runtime filesystem view."""
return _SECURE_HOST_FILE_OPS_AVAILABLE and bool(
getattr(self.ap.box_service, 'shares_filesystem_with_box', False)
)
def _should_use_box_workspace_files(self, selected_skill: dict | None) -> bool:
if selected_skill is not None:
return False
box_service = getattr(self.ap, 'box_service', None)
if box_service is None or not hasattr(box_service, 'execute_tool'):
return False
if not _SECURE_HOST_FILE_OPS_AVAILABLE:
# Preserve the OSS API on platforms without openat/O_NOFOLLOW by
# running inside the tenant-scoped Box mount, never via a racy
# host-path fallback.
return True
default_workspace = getattr(box_service, 'default_workspace', None)
return bool(default_workspace and not os.path.isdir(os.path.realpath(default_workspace)))
def _read_host_location(self, location: _HostLocation, parameters: dict) -> dict:
with _open_host_root(location, create=False) as root_fd:
with _open_location_fd(root_fd, location.relative_parts, os.O_RDONLY) as target_fd:
metadata = os.fstat(target_fd)
if stat.S_ISDIR(metadata.st_mode):
return self._build_directory_result(os.listdir(target_fd))
if not stat.S_ISREG(metadata.st_mode):
raise ValueError('Path must reference a regular file or directory.')
return self._read_text_file_preview(target_fd, parameters, metadata=metadata)
def _write_host_location(self, location: _HostLocation, content: str, parameters: dict) -> None:
if not location.relative_parts:
raise ValueError('Path must reference a file under /workspace.')
encoding, mode = self._write_options(parameters)
if encoding == 'base64':
try:
payload = base64.b64decode(content, validate=True)
except Exception as exc:
raise ValueError(f'invalid base64 content: {exc}') from exc
else:
payload = content.encode('utf-8')
flags = os.O_WRONLY | os.O_CREAT
if mode == 'append':
flags |= os.O_APPEND
with _open_host_root(location, create=True) as root_fd:
with _open_location_fd(
root_fd,
location.relative_parts,
flags,
create_parents=True,
) as target_fd:
if not stat.S_ISREG(os.fstat(target_fd).st_mode):
raise ValueError('Path must reference a regular file.')
if mode != 'append':
os.ftruncate(target_fd, 0)
os.lseek(target_fd, 0, os.SEEK_SET)
self._write_all(target_fd, payload)
def _edit_host_location(
self,
location: _HostLocation,
old_string: str,
new_string: str,
) -> tuple[bool, str | None]:
if not location.relative_parts:
raise ValueError('Path must reference a file under /workspace.')
with _open_host_root(location, create=False) as root_fd:
with _open_location_fd(root_fd, location.relative_parts, os.O_RDWR) as target_fd:
if not stat.S_ISREG(os.fstat(target_fd).st_mode):
return False, 'File not found.'
with os.fdopen(os.dup(target_fd), 'r', encoding='utf-8', errors='replace') as file_obj:
content = file_obj.read()
count = content.count(old_string)
if count == 0:
return False, 'old_string not found in file.'
if count > 1:
return False, f'old_string matches {count} locations; provide a more unique string.'
payload = content.replace(old_string, new_string, 1).encode('utf-8')
os.ftruncate(target_fd, 0)
os.lseek(target_fd, 0, os.SEEK_SET)
self._write_all(target_fd, payload)
return True, None
@staticmethod
def _write_all(file_fd: int, payload: bytes) -> None:
view = memoryview(payload)
while view:
written = os.write(file_fd, view)
if written <= 0:
raise OSError('Could not write the complete workspace file.')
view = view[written:]
@staticmethod
def _rglob_matches(relative_path: str, pattern: str) -> bool:
candidates = {pattern}
pending = [pattern]
while pending:
candidate = pending.pop()
marker = candidate.find('**/')
while marker >= 0:
without_recursive_segment = candidate[:marker] + candidate[marker + 3 :]
if without_recursive_segment not in candidates:
candidates.add(without_recursive_segment)
pending.append(without_recursive_segment)
marker = candidate.find('**/', marker + 3)
return any(candidate and PurePosixPath(relative_path).match(candidate) for candidate in candidates)
def _glob_host_location(self, location: _HostLocation, pattern: str, sandbox_base: str) -> dict:
hits: list[tuple[str, float]] = []
def walk(directory_fd: int, prefix: str) -> None:
with os.scandir(directory_fd) as entries:
for entry in entries:
name = entry.name
if name in _SKIP_DIRS:
continue
try:
child_fd = os.open(name, os.O_RDONLY | _FILE_OPEN_FLAGS, dir_fd=directory_fd)
except OSError:
continue
try:
metadata = os.fstat(child_fd)
relative = f'{prefix}/{name}' if prefix else name
if self._rglob_matches(relative, pattern):
hits.append((relative, metadata.st_mtime))
if stat.S_ISDIR(metadata.st_mode):
walk(child_fd, relative)
finally:
os.close(child_fd)
with _open_host_root(location, create=False) as root_fd:
with _open_location_fd(root_fd, location.relative_parts, os.O_RDONLY) as target_fd:
if not stat.S_ISDIR(os.fstat(target_fd).st_mode):
return {'ok': False, 'error': f'Path is not a directory: {sandbox_base}'}
walk(target_fd, '')
hits.sort(key=lambda item: item[1], reverse=True)
total = len(hits)
sandbox_paths: list[str] = []
output_bytes = 0
truncated_by_bytes = False
for relative, _mtime in hits[:_GLOB_MAX_MATCHES]:
sandbox_path = self._sandbox_child_path(sandbox_base, relative)
entry_bytes = len(sandbox_path.encode('utf-8')) + (1 if sandbox_paths else 0)
if output_bytes + entry_bytes > _DEFAULT_TOOL_RESULT_MAX_BYTES:
truncated_by_bytes = True
break
sandbox_paths.append(sandbox_path)
output_bytes += entry_bytes
return {
'ok': True,
'matches': sandbox_paths,
'preview': '\n'.join(sandbox_paths),
'total': total,
'truncated': total > len(sandbox_paths) or truncated_by_bytes,
'truncated_by': 'bytes' if truncated_by_bytes else ('matches' if total > len(sandbox_paths) else None),
}
def _grep_host_location(
self,
location: _HostLocation,
regex,
include: str | None,
sandbox_base: str,
) -> dict:
matches: list[dict] = []
output_bytes = 0
truncated_by: str | None = None
files_seen = 0
def grep_file(file_fd: int, sandbox_path: str) -> bool:
nonlocal output_bytes, truncated_by
with os.fdopen(os.dup(file_fd), 'r', encoding='utf-8', errors='ignore') as handle:
for lineno, line in enumerate(handle, 1):
if not regex.search(line):
continue
content, line_truncated = self._truncate_grep_line(line.rstrip())
entry = {'file': sandbox_path, 'line': lineno, 'content': content}
entry_bytes = len(json.dumps(entry, ensure_ascii=False).encode('utf-8')) + 1
if output_bytes + entry_bytes > _DEFAULT_TOOL_RESULT_MAX_BYTES:
truncated_by = 'bytes'
return True
if line_truncated and truncated_by is None:
truncated_by = 'line'
matches.append(entry)
output_bytes += entry_bytes
if len(matches) >= _GREP_MAX_MATCHES:
truncated_by = truncated_by or 'matches'
return True
return False
def walk(directory_fd: int, prefix: str) -> bool:
nonlocal files_seen
with os.scandir(directory_fd) as entries:
for entry in entries:
name = entry.name
if name in _SKIP_DIRS:
continue
try:
child_fd = os.open(name, os.O_RDONLY | _FILE_OPEN_FLAGS, dir_fd=directory_fd)
except OSError:
continue
try:
metadata = os.fstat(child_fd)
relative = f'{prefix}/{name}' if prefix else name
if stat.S_ISDIR(metadata.st_mode):
if walk(child_fd, relative):
return True
continue
if not stat.S_ISREG(metadata.st_mode):
continue
if include and not self._rglob_matches(relative, include):
continue
files_seen += 1
if grep_file(child_fd, self._sandbox_child_path(sandbox_base, relative)):
return True
if files_seen >= _GREP_MAX_FILES:
return True
finally:
os.close(child_fd)
return False
with _open_host_root(location, create=False) as root_fd:
with _open_location_fd(root_fd, location.relative_parts, os.O_RDONLY) as target_fd:
metadata = os.fstat(target_fd)
if stat.S_ISREG(metadata.st_mode):
grep_file(target_fd, sandbox_base)
elif stat.S_ISDIR(metadata.st_mode):
walk(target_fd, '')
else:
return {'ok': False, 'error': f'Path not found: {sandbox_base}'}
return {
'ok': True,
'matches': matches,
'total': len(matches),
'truncated': truncated_by is not None,
'truncated_by': truncated_by,
}
@staticmethod
def _sandbox_child_path(base: str, relative: str) -> str:
return f'{str(base).rstrip("/")}/{relative}'
async def _run_workspace_file_script(self, script: str, query: pipeline_query.Query) -> dict:
result = await self.ap.box_service.execute_tool(
{
@@ -531,11 +960,15 @@ else:
)
if skill_request is not None and hasattr(self.ap.box_service, 'read_skill_file'):
selected_skill, relative = skill_request
host_path = self._resolve_skill_host_path(selected_skill, relative)
if host_path and os.path.exists(host_path):
if os.path.isdir(host_path):
return self._build_directory_result(os.listdir(host_path))
return self._read_text_file_preview(host_path, parameters)
if self._can_interpret_skill_host_paths():
host_location = self._resolve_skill_host_location(selected_skill, relative)
else:
host_location = None
if host_location is not None:
try:
return self._read_host_location(host_location, parameters)
except FileNotFoundError:
pass
try:
result = await self.ap.box_service.read_skill_file(
@@ -556,20 +989,18 @@ else:
except Exception as exc:
return {'ok': False, 'error': str(exc)}
host_path, selected_skill = self._resolve_host_path(
host_location = self._resolve_host_location(
query,
path,
include_visible=True,
include_activated=True,
)
if self._should_use_box_workspace_files(selected_skill):
if self._should_use_box_workspace_files(host_location.selected_skill):
return await self._read_workspace_via_box(path, parameters, query)
if not os.path.exists(host_path):
try:
return self._read_host_location(host_location, parameters)
except (FileNotFoundError, NotADirectoryError):
return {'ok': False, 'error': f'File not found: {path}'}
if os.path.isdir(host_path):
entries = os.listdir(host_path)
return self._build_directory_result(entries)
return self._read_text_file_preview(host_path, parameters)
async def _invoke_write(self, parameters: dict, query: pipeline_query.Query) -> dict:
path = parameters['path']
@@ -591,20 +1022,19 @@ else:
await self.ap.skill_mgr.reload_skills(execution_context)
return {'ok': True, 'path': path}
host_path, selected_skill = self._resolve_host_path(
host_location = self._resolve_host_location(
query,
path,
include_visible=False,
include_activated=True,
)
if self._should_use_box_workspace_files(selected_skill):
if self._should_use_box_workspace_files(host_location.selected_skill):
return await self._write_workspace_via_box(path, content, parameters, query)
os.makedirs(os.path.dirname(host_path), exist_ok=True)
try:
self._write_host_file(host_path, content, parameters)
self._write_host_location(host_location, content, parameters)
except ValueError as exc:
return {'ok': False, 'error': str(exc)}
self._refresh_skill_from_disk(query, selected_skill)
self._refresh_skill_from_disk(query, host_location.selected_skill)
return {'ok': True, 'path': path}
async def _invoke_edit(self, parameters: dict, query: pipeline_query.Query) -> dict:
@@ -652,27 +1082,21 @@ else:
await self.ap.skill_mgr.reload_skills(execution_context)
return {'ok': True, 'path': path}
host_path, selected_skill = self._resolve_host_path(
host_location = self._resolve_host_location(
query,
path,
include_visible=False,
include_activated=True,
)
if self._should_use_box_workspace_files(selected_skill):
if self._should_use_box_workspace_files(host_location.selected_skill):
return await self._edit_workspace_via_box(path, old_string, new_string, query)
if not os.path.isfile(host_path):
try:
changed, error = self._edit_host_location(host_location, old_string, new_string)
except (FileNotFoundError, NotADirectoryError):
return {'ok': False, 'error': f'File not found: {path}'}
with open(host_path, 'r', encoding='utf-8', errors='replace') as f:
content = f.read()
count = content.count(old_string)
if count == 0:
return {'ok': False, 'error': 'old_string not found in file.'}
if count > 1:
return {'ok': False, 'error': f'old_string matches {count} locations; provide a more unique string.'}
new_content = content.replace(old_string, new_string, 1)
with open(host_path, 'w', encoding='utf-8') as f:
f.write(new_content)
self._refresh_skill_from_disk(query, selected_skill)
if not changed:
return {'ok': False, 'error': error or f'File not found: {path}'}
self._refresh_skill_from_disk(query, host_location.selected_skill)
return {'ok': True, 'path': path}
def _refresh_skill_from_disk(self, query: pipeline_query.Query, selected_skill: dict | None) -> None:
@@ -934,55 +1358,19 @@ else:
path = str(parameters.get('path', '/workspace') or '/workspace')
self.ap.logger.info(f'glob tool invoked: query_id={query.query_id} pattern={pattern} path={path}')
host_path, selected_skill = self._resolve_host_path(
host_location = self._resolve_host_location(
query,
path,
include_visible=True,
include_activated=True,
)
if self._should_use_box_workspace_files(selected_skill):
if self._should_use_box_workspace_files(host_location.selected_skill):
return await self._glob_workspace_via_box(path, pattern, query)
if not os.path.isdir(host_path):
try:
return self._glob_host_location(host_location, pattern, path)
except (FileNotFoundError, NotADirectoryError):
return {'ok': False, 'error': f'Path is not a directory: {path}'}
from pathlib import Path
base = Path(host_path)
hits = list(base.rglob(pattern))
# Filter out skipped directories
hits = [h for h in hits if not any(skip in h.parts for skip in _SKIP_DIRS)]
# Sort by mtime, newest first
hits.sort(key=lambda p: p.stat().st_mtime if p.exists() else 0, reverse=True)
total = len(hits)
shown = hits[:_GLOB_MAX_MATCHES]
# Convert back to sandbox paths
sandbox_paths = []
output_bytes = 0
truncated_by_bytes = False
for h in shown:
rel = os.path.relpath(str(h), host_path)
sandbox_path = os.path.join(path, rel)
entry_bytes = len(sandbox_path.encode('utf-8')) + (1 if sandbox_paths else 0)
if output_bytes + entry_bytes > _DEFAULT_TOOL_RESULT_MAX_BYTES:
truncated_by_bytes = True
break
sandbox_paths.append(sandbox_path)
output_bytes += entry_bytes
return {
'ok': True,
'matches': sandbox_paths,
'preview': '\n'.join(sandbox_paths),
'total': total,
'truncated': total > len(sandbox_paths) or truncated_by_bytes,
'truncated_by': 'bytes' if truncated_by_bytes else ('matches' if total > len(sandbox_paths) else None),
}
async def _invoke_grep(self, parameters: dict, query: pipeline_query.Query) -> dict:
pattern = parameters['pattern']
path = str(parameters.get('path', '/workspace') or '/workspace')
@@ -990,99 +1378,36 @@ else:
self.ap.logger.info(f'grep tool invoked: query_id={query.query_id} pattern={pattern} path={path}')
import re
from pathlib import Path
try:
regex = re.compile(pattern)
except re.error as e:
return {'ok': False, 'error': f'Invalid regex: {e}'}
host_path, selected_skill = self._resolve_host_path(
host_location = self._resolve_host_location(
query,
path,
include_visible=True,
include_activated=True,
)
if self._should_use_box_workspace_files(selected_skill):
if self._should_use_box_workspace_files(host_location.selected_skill):
return await self._grep_workspace_via_box(path, pattern, include, query)
if not os.path.exists(host_path):
try:
return self._grep_host_location(host_location, regex, include, path)
except (FileNotFoundError, NotADirectoryError):
return {'ok': False, 'error': f'Path not found: {path}'}
base = Path(host_path)
if base.is_file():
files = [base]
else:
files = self._grep_walk(base, include)
matches = []
output_bytes = 0
truncated_by = None
for fp in files:
try:
handle = fp.open('r', encoding='utf-8', errors='ignore')
except OSError:
continue
with handle:
for lineno, line in enumerate(handle, 1):
if regex.search(line):
rel = os.path.relpath(str(fp), host_path)
sandbox_path = os.path.join(path, rel)
content, line_truncated = self._truncate_grep_line(line.rstrip())
entry = {
'file': sandbox_path,
'line': lineno,
'content': content,
}
entry_bytes = len(json.dumps(entry, ensure_ascii=False).encode('utf-8')) + 1
if output_bytes + entry_bytes > _DEFAULT_TOOL_RESULT_MAX_BYTES:
truncated_by = 'bytes'
break
if line_truncated and truncated_by is None:
truncated_by = 'line'
matches.append(entry)
output_bytes += entry_bytes
if len(matches) >= _GREP_MAX_MATCHES:
truncated_by = truncated_by or 'matches'
break
if truncated_by == 'bytes' or len(matches) >= _GREP_MAX_MATCHES:
break
if truncated_by == 'bytes' or len(matches) >= _GREP_MAX_MATCHES:
break
return {
'ok': True,
'matches': matches,
'total': len(matches),
'truncated': truncated_by is not None,
'truncated_by': truncated_by,
}
@staticmethod
def _grep_walk(root, include: str | None) -> list:
"""Walk dir tree for grep, skipping junk dirs."""
results = []
for item in root.rglob(include or '*'):
if any(skip in item.parts for skip in _SKIP_DIRS):
continue
if item.is_file():
results.append(item)
if len(results) >= _GREP_MAX_FILES:
break
return results
@staticmethod
def _resolve_skill_host_path(selected_skill: dict, relative: str) -> str | None:
def _resolve_skill_host_location(selected_skill: dict, relative: str) -> _HostLocation | None:
package_root = str(selected_skill.get('package_root', '') or '').strip()
if not package_root:
return None
host_root = os.path.realpath(package_root)
host_path = os.path.realpath(os.path.join(host_root, relative))
if not (host_path == host_root or host_path.startswith(host_root + os.sep)):
raise ValueError('Path escapes the skill package boundary.')
return host_path
relative_path = '/workspace' if relative in {'', '.'} else f'/workspace/{relative}'
return _HostLocation(
root=package_root,
relative_parts=_relative_workspace_parts(relative_path),
selected_skill=selected_skill,
)
def _normalize_exec_result(self, result: dict) -> dict:
normalized = dict(result)
@@ -1122,9 +1447,9 @@ else:
'truncated_by': 'bytes' if truncated else None,
}
def _read_text_file_preview(self, host_path: str, parameters: dict) -> dict:
def _read_text_file_preview(self, file_fd: int, parameters: dict, *, metadata: os.stat_result) -> dict:
if self._read_encoding(parameters) == 'base64':
return self._read_binary_file_chunk(host_path, parameters)
return self._read_binary_file_chunk(file_fd, parameters, metadata=metadata)
offset = self._positive_int(parameters.get('offset'), default=1)
max_lines = self._positive_int(
@@ -1144,7 +1469,7 @@ else:
truncated_by: str | None = None
next_offset: int | None = None
with open(host_path, 'r', encoding='utf-8', errors='replace') as f:
with os.fdopen(os.dup(file_fd), 'r', encoding='utf-8', errors='replace') as f:
for line_number, line in enumerate(f, 1):
if line_number < offset:
continue
@@ -1185,15 +1510,15 @@ else:
'max_bytes': max_bytes,
}
def _read_binary_file_chunk(self, host_path: str, parameters: dict) -> dict:
def _read_binary_file_chunk(self, file_fd: int, parameters: dict, *, metadata: os.stat_result) -> dict:
byte_offset = self._non_negative_int(parameters.get('byte_offset'), default=0)
max_bytes = self._positive_int(
parameters.get('max_bytes'),
default=_DEFAULT_TOOL_RESULT_MAX_BYTES,
max_value=_DEFAULT_TOOL_RESULT_MAX_BYTES,
)
size_bytes = os.path.getsize(host_path)
with open(host_path, 'rb') as f:
size_bytes = metadata.st_size
with os.fdopen(os.dup(file_fd), 'rb') as f:
f.seek(byte_offset)
data = f.read(max_bytes + 1)
chunk = data[:max_bytes]
@@ -1210,19 +1535,6 @@ else:
'max_bytes': max_bytes,
}
def _write_host_file(self, host_path: str, content: str, parameters: dict) -> None:
encoding, mode = self._write_options(parameters)
if encoding == 'base64':
try:
data = base64.b64decode(content, validate=True)
except Exception as exc:
raise ValueError(f'invalid base64 content: {exc}') from exc
with open(host_path, 'ab' if mode == 'append' else 'wb') as f:
f.write(data)
return
with open(host_path, 'a' if mode == 'append' else 'w', encoding='utf-8') as f:
f.write(content)
@staticmethod
def _read_encoding(parameters: dict) -> str:
return 'base64' if parameters.get('encoding') == 'base64' else 'text'
@@ -201,5 +201,14 @@ def should_prepare_skill_python_env(package_root: str | None) -> bool:
return box_workspace.should_prepare_python_env(package_root)
def wrap_skill_command_with_python_env(command: str, *, mount_path: str = '/workspace') -> str:
return box_workspace.wrap_python_command_with_env(command, mount_path=mount_path).rstrip()
def wrap_skill_command_with_python_env(
command: str,
*,
mount_path: str = '/workspace',
state_path: str | None = None,
) -> str:
return box_workspace.wrap_python_command_with_env(
command,
mount_path=mount_path,
state_path=state_path,
).rstrip()
@@ -65,12 +65,34 @@ class SkillToolLoader(loader.ToolLoader):
return self._has_skill_manager() and self._sandbox_available
async def invoke_tool(self, name: str, parameters: dict, query) -> typing.Any:
require_sandbox = getattr(
getattr(self.ap, 'box_service', None),
'require_workspace_sandbox',
None,
)
if callable(require_sandbox):
await require_sandbox(self._execution_context(query))
if name == ACTIVATE_SKILL_TOOL_NAME:
return await self._invoke_activate_skill(parameters, query)
if name == REGISTER_SKILL_TOOL_NAME:
return await self._invoke_register_skill(parameters, query)
raise ValueError(f'Unknown skill tool: {name}')
@staticmethod
def _execution_context(query) -> ExecutionContext:
attached_context = getattr(query, '_execution_context', None)
if isinstance(attached_context, ExecutionContext):
return attached_context
return ExecutionContext(
instance_uuid=str(getattr(query, 'instance_uuid', '') or ''),
workspace_uuid=str(getattr(query, 'workspace_uuid', '') or ''),
placement_generation=getattr(query, 'placement_generation', 0) or 0,
bot_uuid=getattr(query, 'bot_uuid', None),
pipeline_uuid=getattr(query, 'pipeline_uuid', None),
query_uuid=getattr(query, 'query_uuid', None),
entitlement_revision=getattr(query, 'entitlement_revision', 0),
)
async def shutdown(self):
pass
@@ -128,7 +150,8 @@ class SkillToolLoader(loader.ToolLoader):
raise ValueError('path is required')
# Resolve sandbox path to host path
host_path = self._resolve_workspace_directory(sandbox_path)
execution_context = self._execution_context(query)
host_path = self._resolve_workspace_directory(sandbox_path, execution_context)
# Get or create skill service
skill_service = getattr(self.ap, 'skill_service', None)
@@ -136,14 +159,6 @@ class SkillToolLoader(loader.ToolLoader):
raise ValueError('Skill service not available')
# Scan and register the skill
execution_context = ExecutionContext(
instance_uuid=str(getattr(query, 'instance_uuid', '') or ''),
workspace_uuid=str(getattr(query, 'workspace_uuid', '') or ''),
placement_generation=getattr(query, 'placement_generation', 0) or 0,
bot_uuid=getattr(query, 'bot_uuid', None),
pipeline_uuid=getattr(query, 'pipeline_uuid', None),
query_uuid=getattr(query, 'query_uuid', None),
)
scanned = await skill_service.scan_directory_async(execution_context, host_path)
# Override name if provided
@@ -170,10 +185,19 @@ class SkillToolLoader(loader.ToolLoader):
'skill': created,
}
def _resolve_workspace_directory(self, sandbox_path: str) -> str:
def _resolve_workspace_directory(
self,
sandbox_path: str,
execution_context: ExecutionContext,
) -> str:
"""Resolve sandbox path to host filesystem path."""
box_service = getattr(self.ap, 'box_service', None)
workspace_root = getattr(box_service, 'default_workspace', None)
tenant_workspace = getattr(box_service, '_tenant_workspace', None)
workspace_root = (
tenant_workspace(execution_context)
if callable(tenant_workspace)
else getattr(box_service, 'default_workspace', None)
)
if not workspace_root:
raise ValueError('No default workspace configured')
+57 -12
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import typing
import time
import inspect
from typing import TYPE_CHECKING
import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
@@ -35,6 +36,36 @@ class ToolManager:
def __init__(self, ap: app.Application):
self.ap = ap
async def _bind_plugin_workspace(self, context: TenantContext) -> None:
"""Select the tenant before any plugin catalog lookup.
Tool discovery happens before invocation, so relying on ``call_tool``
to bind the Workspace is too late and can expose another task's
catalog in a shared Runtime.
"""
connector = getattr(self.ap, 'plugin_connector', None)
require_context = getattr(connector, 'require_workspace_context', None)
if require_context is None:
return
result = require_context(context)
if inspect.isawaitable(result):
await result
async def _workspace_sandbox_available(self, context: TenantContext) -> bool:
"""Resolve the Workspace capability before exposing sandbox tools."""
box_service = getattr(self.ap, 'box_service', None)
checker = getattr(box_service, 'is_workspace_sandbox_available', None)
if not callable(checker):
# Compatibility for OSS embedders and isolated manager tests. The
# BoxService execution path remains the final authority.
return True
try:
return bool(await checker(context))
except Exception:
return False
async def initialize(self):
from langbot.pkg.utils import importutil
from langbot.pkg.provider.tools import loaders
@@ -65,10 +96,13 @@ class ToolManager:
include_skill_authoring: bool = False,
include_mcp_resource_tools: bool = True,
) -> list[resource_tool.LLMTool]:
await self._bind_plugin_workspace(context)
all_functions: list[resource_tool.LLMTool] = []
all_functions.extend(await self.native_tool_loader.get_tools())
if include_skill_authoring:
sandbox_available = await self._workspace_sandbox_available(context)
if sandbox_available:
all_functions.extend(await self.native_tool_loader.get_tools())
if include_skill_authoring and sandbox_available:
all_functions.extend(await self.skill_tool_loader.get_tools())
all_functions.extend(await self.plugin_tool_loader.get_tools(bound_plugins))
all_functions.extend(
@@ -89,6 +123,7 @@ class ToolManager:
include_skill_authoring: bool = False,
include_mcp_resource_tools: bool = False,
) -> list[dict[str, typing.Any]]:
await self._bind_plugin_workspace(context)
catalog: list[dict[str, typing.Any]] = []
def append_tools(source: str, source_name: str, tools: list[resource_tool.LLMTool]) -> None:
@@ -104,8 +139,10 @@ class ToolManager:
}
)
append_tools('builtin', 'LangBot', await self.native_tool_loader.get_tools())
if include_skill_authoring:
sandbox_available = await self._workspace_sandbox_available(context)
if sandbox_available:
append_tools('builtin', 'LangBot', await self.native_tool_loader.get_tools())
if include_skill_authoring and sandbox_available:
append_tools('skill', 'LangBot', await self.skill_tool_loader.get_tools())
catalog.extend(await self.plugin_tool_loader.get_tool_catalog(bound_plugins))
@@ -121,14 +158,20 @@ class ToolManager:
async def get_tool_by_name(self, context: TenantContext, name: str) -> tool_loader.ToolLookupResult | None:
"""Get tool by name from any active loader."""
for active_loader in (
self.native_tool_loader,
self.plugin_tool_loader,
self.skill_tool_loader,
):
await self._bind_plugin_workspace(context)
sandbox_available = await self._workspace_sandbox_available(context)
if sandbox_available:
tool = await self.native_tool_loader.get_tool(name)
if tool:
return tool
for active_loader in (self.plugin_tool_loader,):
tool = await active_loader.get_tool(name)
if tool:
return tool
if sandbox_available:
tool = await self.skill_tool_loader.get_tool(name)
if tool:
return tool
return await self.mcp_tool_loader.get_tool(context, name)
@@ -237,7 +280,10 @@ class ToolManager:
async def execute_func_call(self, name: str, parameters: dict, query: pipeline_query.Query) -> typing.Any:
from langbot.pkg.telemetry import features as telemetry_features
if await self.native_tool_loader.has_tool(name):
execution_context = get_query_execution_context(query)
await self._bind_plugin_workspace(execution_context)
sandbox_available = await self._workspace_sandbox_available(execution_context)
if sandbox_available and await self.native_tool_loader.has_tool(name):
telemetry_features.increment(query, 'tool_calls', 'native')
return await self._invoke_tool_with_monitoring(
source='native',
@@ -255,7 +301,6 @@ class ToolManager:
query=query,
invoke=lambda: self.plugin_tool_loader.invoke_tool(name, parameters, query),
)
execution_context = get_query_execution_context(query)
if await self.mcp_tool_loader.has_tool(execution_context, name):
telemetry_features.increment(query, 'tool_calls', 'mcp')
return await self._invoke_tool_with_monitoring(
@@ -265,7 +310,7 @@ class ToolManager:
query=query,
invoke=lambda: self.mcp_tool_loader.invoke_tool(name, parameters, query),
)
if await self.skill_tool_loader.has_tool(name):
if sandbox_available and await self.skill_tool_loader.has_tool(name):
telemetry_features.increment(query, 'tool_calls', 'skill')
return await self._invoke_tool_with_monitoring(
source='skill',
+81 -23
View File
@@ -1,4 +1,5 @@
from __future__ import annotations
import asyncio
import io
import mimetypes
import os.path
@@ -14,6 +15,7 @@ from langbot.pkg.api.http.authz import WorkspaceRequiredError
from langbot.pkg.api.http.context import ExecutionContext, RequestContext
from langbot.pkg.api.http.service.tenant import TenantContext, require_workspace_uuid
from langbot.pkg.core import app, taskmgr
from langbot.pkg.core.task_boundary import run_in_workspace_uow
from langbot.pkg.entity.persistence import rag as persistence_rag
from langbot.pkg.workspace.errors import WorkspaceNotFoundError
@@ -90,16 +92,23 @@ class RuntimeKnowledgeBase(KnowledgeBaseInterface):
task_context: taskmgr.TaskContext,
parser_plugin_id: str | None = None,
):
await self._assert_execution_context(execution_context)
await run_in_workspace_uow(
self.ap,
execution_context.workspace_uuid,
lambda: self._assert_execution_context(execution_context),
)
self._require_upload_object_key(execution_context, file.file_name)
try:
# set file status to processing
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_rag.File)
.where(persistence_rag.File.workspace_uuid == execution_context.workspace_uuid)
.where(persistence_rag.File.uuid == file.uuid)
.values(status='processing')
)
status_visible = False
for retry_delay in (0.0, 0.01, 0.05, 0.1):
if retry_delay:
await asyncio.sleep(retry_delay)
if await self._set_file_status(execution_context, file.uuid, 'processing'):
status_visible = True
break
if not status_visible:
raise WorkspaceNotFoundError('Knowledge file was not committed before its background task started')
task_context.set_current_action('Processing file')
@@ -152,13 +161,8 @@ class RuntimeKnowledgeBase(KnowledgeBaseInterface):
raise Exception(error_msg)
# set file status to completed
await self._assert_execution_context(execution_context)
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_rag.File)
.where(persistence_rag.File.workspace_uuid == execution_context.workspace_uuid)
.where(persistence_rag.File.uuid == file.uuid)
.values(status='completed')
)
if not await self._set_file_status(execution_context, file.uuid, 'completed'):
raise WorkspaceNotFoundError('Knowledge file not found')
except Exception as e:
self.ap.logger.error(f'Error storing file {file.uuid}: {e}')
@@ -166,16 +170,10 @@ class RuntimeKnowledgeBase(KnowledgeBaseInterface):
# A stale placement is fenced from all writes, including failure
# status updates from an old background task.
try:
await self._assert_execution_context(execution_context)
if not await self._set_file_status(execution_context, file.uuid, 'failed'):
raise WorkspaceNotFoundError('Knowledge file not found')
except Exception:
self.ap.logger.warning(f'Skipping stale RAG task status update for file {file.uuid}')
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_rag.File)
.where(persistence_rag.File.workspace_uuid == execution_context.workspace_uuid)
.where(persistence_rag.File.uuid == file.uuid)
.values(status='failed')
)
raise
finally:
@@ -191,6 +189,37 @@ class RuntimeKnowledgeBase(KnowledgeBaseInterface):
except (WorkspaceRequiredError, WorkspaceNotFoundError):
self.ap.logger.warning(f'Skipping stale RAG upload cleanup for file {file.uuid}')
async def _set_file_status(
self,
execution_context: ExecutionContext,
file_uuid: str,
status: str,
) -> bool:
"""Commit one detached-task status transition in its own tenant UoW."""
async def update() -> bool:
await self._assert_execution_context(execution_context)
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_rag.File)
.where(persistence_rag.File.workspace_uuid == execution_context.workspace_uuid)
.where(persistence_rag.File.uuid == file_uuid)
.values(status=status)
)
return getattr(result, 'rowcount', 0) > 0
persistence_mgr = self.ap.persistence_mgr
managed_mode = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) in {
'cloud_runtime',
'oss_compat',
}
tenant_uow = getattr(persistence_mgr, 'tenant_uow', None)
if managed_mode:
if not callable(tenant_uow):
raise RuntimeError('Knowledge tasks require an explicit tenant UoW')
async with tenant_uow(execution_context.workspace_uuid):
return await update()
return await update()
async def store_file(
self,
execution_context: ExecutionContext,
@@ -735,7 +764,36 @@ class RAGManager:
self.knowledge_bases = {}
# Load knowledge bases
list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
cloud_runtime = getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if cloud_runtime:
if not callable(list_bindings) or not callable(tenant_uow):
raise RuntimeError('Cloud knowledge loading requires explicit instance discovery and tenant UoWs')
for binding in await list_bindings():
async with tenant_uow(binding.workspace_uuid):
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_rag.KnowledgeBase)
.where(persistence_rag.KnowledgeBase.workspace_uuid == binding.workspace_uuid)
.order_by(persistence_rag.KnowledgeBase.uuid)
)
for knowledge_base in result.all():
try:
await self.load_knowledge_base(
ExecutionContext(
instance_uuid=binding.instance_uuid,
workspace_uuid=binding.workspace_uuid,
placement_generation=binding.placement_generation,
),
knowledge_base,
)
except Exception as e:
self.ap.logger.error(
f'Error loading knowledge base {knowledge_base.uuid}: {e}\n{traceback.format_exc()}'
)
return
# Compatibility path for isolated manager tests and older embedders.
result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(persistence_rag.KnowledgeBase))
knowledge_bases = result.all()
+63 -32
View File
@@ -3,6 +3,7 @@
from __future__ import annotations
import asyncio
import contextlib
import json
import typing
import httpx
@@ -10,6 +11,7 @@ import sqlalchemy
from ..core import app as core_app
from ..entity.persistence.metadata import Metadata
from ..persistence.tenant_uow import CrossScopeTransactionError
from ..utils import constants
SURVEY_TRIGGERED_KEY = 'survey_triggered_events'
@@ -36,15 +38,41 @@ class SurveyManager:
await self._load_triggered_events()
await self._load_bot_response_count()
@contextlib.asynccontextmanager
async def _instance_transaction(self):
"""Bind instance-global survey metadata to an explicit Cloud transaction."""
persistence_mgr = self.ap.persistence_mgr
try:
active_session = getattr(persistence_mgr, 'current_session', lambda: None)()
except CrossScopeTransactionError:
# A newly-created child task inherited its parent's ContextVar;
# opening an explicit UoW below gives it an independent session.
active_session = None
if active_session is not None:
yield
return
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if cloud_runtime:
instance_uow = getattr(persistence_mgr, 'instance_discovery_uow', None)
if not callable(instance_uow):
raise RuntimeError('Cloud survey metadata requires an explicit instance UoW')
async with instance_uow(self.ap.workspace_service.instance_uuid):
yield
return
yield
async def _load_triggered_events(self):
"""Load previously triggered events from metadata table."""
try:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(Metadata).where(Metadata.key == SURVEY_TRIGGERED_KEY)
)
row = result.first()
if row:
self._triggered_events = set(json.loads(row[0].value))
async with self._instance_transaction():
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(Metadata.value).where(Metadata.key == SURVEY_TRIGGERED_KEY)
)
value = result.scalar_one_or_none()
if value is not None:
self._triggered_events = set(json.loads(value))
except Exception:
self._triggered_events = set()
@@ -52,17 +80,18 @@ class SurveyManager:
"""Persist triggered events to metadata table."""
try:
value = json.dumps(list(self._triggered_events))
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(Metadata).where(Metadata.key == SURVEY_TRIGGERED_KEY)
)
if result.first():
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(Metadata).where(Metadata.key == SURVEY_TRIGGERED_KEY).values(value=value)
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(Metadata).values(key=SURVEY_TRIGGERED_KEY, value=value)
async with self._instance_transaction():
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(Metadata.value).where(Metadata.key == SURVEY_TRIGGERED_KEY)
)
if result.scalar_one_or_none() is not None:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(Metadata).where(Metadata.key == SURVEY_TRIGGERED_KEY).values(value=value)
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(Metadata).values(key=SURVEY_TRIGGERED_KEY, value=value)
)
except Exception as e:
self.ap.logger.debug(f'Failed to save survey triggered events: {e}')
@@ -75,12 +104,13 @@ class SurveyManager:
async def _load_bot_response_count(self):
"""Load the persisted successful bot response count from metadata table."""
try:
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY)
)
row = result.first()
if row:
self._bot_response_count = int(row[0].value)
async with self._instance_transaction():
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(Metadata.value).where(Metadata.key == BOT_RESPONSE_COUNT_KEY)
)
value = result.scalar_one_or_none()
if value is not None:
self._bot_response_count = int(value)
except Exception:
self._bot_response_count = 0
@@ -88,17 +118,18 @@ class SurveyManager:
"""Persist the successful bot response count to metadata table."""
try:
value = str(self._bot_response_count)
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY)
)
if result.first():
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY).values(value=value)
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(Metadata).values(key=BOT_RESPONSE_COUNT_KEY, value=value)
async with self._instance_transaction():
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(Metadata.value).where(Metadata.key == BOT_RESPONSE_COUNT_KEY)
)
if result.scalar_one_or_none() is not None:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(Metadata).where(Metadata.key == BOT_RESPONSE_COUNT_KEY).values(value=value)
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(Metadata).values(key=BOT_RESPONSE_COUNT_KEY, value=value)
)
except Exception as e:
self.ap.logger.debug(f'Failed to save survey bot response count: {e}')
+14
View File
@@ -30,6 +30,20 @@ HEARTBEAT_INTERVAL_SECONDS = 24 * 3600
async def _count(ap: core_app.Application, table) -> int:
"""Count rows in a persistence table; -1 when unavailable."""
try:
persistence_mgr = ap.persistence_mgr
cloud_runtime = getattr(getattr(persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime'
if cloud_runtime:
tenant_uow = getattr(persistence_mgr, 'tenant_uow', None)
if not callable(tenant_uow):
return -1
total = 0
for binding in await ap.workspace_service.list_active_execution_bindings():
async with tenant_uow(binding.workspace_uuid):
result = await persistence_mgr.execute_async(
sqlalchemy.select(sqlalchemy.func.count()).select_from(table)
)
total += int(result.scalar() or 0)
return total
result = await ap.persistence_mgr.execute_async(sqlalchemy.select(sqlalchemy.func.count()).select_from(table))
return int(result.scalar() or 0)
except Exception:
+183 -29
View File
@@ -64,9 +64,26 @@ class VectorDBManager:
# Get pgvector configuration
pgvector_config = kb_config.get('pgvector', {})
use_business_database = pgvector_config.get('use_business_database', False)
allowed_dimensions = pgvector_config.get(
'allowed_dimensions',
[384, 512, 768, 1024, 1536],
)
common_options = {
'use_business_database': use_business_database,
'allowed_dimensions': allowed_dimensions,
}
if use_business_database:
self.vector_db = PgVectorDatabase(self.ap, **common_options)
self.ap.logger.info('Initialized pgvector on the shared business PostgreSQL database.')
return
connection_string = pgvector_config.get('connection_string')
if connection_string:
self.vector_db = PgVectorDatabase(self.ap, connection_string=connection_string)
self.vector_db = PgVectorDatabase(
self.ap,
connection_string=connection_string,
**common_options,
)
else:
# Use individual parameters
host = pgvector_config.get('host', 'localhost')
@@ -75,7 +92,13 @@ class VectorDBManager:
user = pgvector_config.get('user', 'postgres')
password = pgvector_config.get('password', 'postgres')
self.vector_db = PgVectorDatabase(
self.ap, host=host, port=port, database=database, user=user, password=password
self.ap,
host=host,
port=port,
database=database,
user=user,
password=password,
**common_options,
)
self.ap.logger.info('Initialized pgvector database backend.')
@@ -156,23 +179,24 @@ class VectorDBManager:
"""
await self._validate_execution_context(execution_context)
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
persistence_rag.KnowledgeBase.collection_id,
persistence_rag.KnowledgeBase.legacy_vector_collection,
persistence_workspace.Workspace.source,
async with self.ap.persistence_mgr.tenant_uow(execution_context.workspace_uuid):
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
persistence_rag.KnowledgeBase.collection_id,
persistence_rag.KnowledgeBase.legacy_vector_collection,
persistence_workspace.Workspace.source,
)
.join(
persistence_workspace.Workspace,
persistence_workspace.Workspace.uuid == persistence_rag.KnowledgeBase.workspace_uuid,
)
.where(
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
persistence_workspace.Workspace.instance_uuid == execution_context.instance_uuid,
)
.limit(1)
)
.join(
persistence_workspace.Workspace,
persistence_workspace.Workspace.uuid == persistence_rag.KnowledgeBase.workspace_uuid,
)
.where(
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
persistence_workspace.Workspace.instance_uuid == execution_context.instance_uuid,
)
.limit(1)
)
row = result.first()
if row is None:
raise WorkspaceNotFoundError('Knowledge base not found')
@@ -194,6 +218,58 @@ class VectorDBManager:
return self.physical_collection_name(execution_context, knowledge_base_uuid)
def _pgvector_database(self):
from .vdbs.pgvector_db import PgVectorDatabase
return self.vector_db if isinstance(self.vector_db, PgVectorDatabase) else None
async def _resolve_pgvector_scope(
self,
execution_context: ExecutionContext,
knowledge_base_uuid: str,
*,
expected_dimension: int | None,
initialize_dimension: bool,
):
"""Bind and verify the server-owned knowledge-base vector dimension."""
from .vdbs.pgvector_db import PgVectorScope
pgvector = self._pgvector_database()
if pgvector is None: # pragma: no cover - private call invariant
raise RuntimeError('pgvector scope requested for another vector backend')
if expected_dimension is not None and expected_dimension not in pgvector.allowed_dimensions:
raise ValueError(f'Embedding dimension {expected_dimension} is not enabled for this deployment')
async with self.ap.persistence_mgr.tenant_uow(execution_context.workspace_uuid):
query = sqlalchemy.select(persistence_rag.KnowledgeBase.embedding_dimension).where(
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
)
current_dimension = (await self.ap.persistence_mgr.execute_async(query)).scalar_one_or_none()
if current_dimension is None and expected_dimension is not None and initialize_dimension:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_rag.KnowledgeBase)
.where(
persistence_rag.KnowledgeBase.workspace_uuid == execution_context.workspace_uuid,
persistence_rag.KnowledgeBase.uuid == knowledge_base_uuid,
persistence_rag.KnowledgeBase.embedding_dimension.is_(None),
)
.values(embedding_dimension=expected_dimension)
)
current_dimension = (await self.ap.persistence_mgr.execute_async(query)).scalar_one_or_none()
if expected_dimension is not None and current_dimension != expected_dimension:
if current_dimension is None:
raise ValueError('Knowledge base has no selected pgvector embedding dimension')
raise ValueError(f'Knowledge base embedding dimension is {current_dimension}, not {expected_dimension}')
return PgVectorScope(
workspace_uuid=execution_context.workspace_uuid,
knowledge_base_uuid=knowledge_base_uuid,
embedding_dimension=current_dimension,
)
async def upsert(
self,
execution_context: ExecutionContext,
@@ -219,6 +295,25 @@ class VectorDBManager:
}
for item in source_metadata
]
pgvector = self._pgvector_database()
if pgvector is not None:
if not vectors:
return
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=len(vectors[0]),
initialize_dimension=True,
)
await pgvector.add_embeddings(
collection=collection_name,
ids=ids,
embeddings_list=vectors,
metadatas=scoped_metadata,
documents=documents,
scope=scope,
)
return
await self.vector_db.add_embeddings(
collection=collection_name,
ids=ids,
@@ -248,15 +343,34 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
results = await self.vector_db.search(
collection=collection_name,
query_embedding=query_vector,
k=limit,
search_type=search_type,
query_text=query_text,
filter=filter,
vector_weight=vector_weight,
)
pgvector = self._pgvector_database()
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=len(query_vector),
initialize_dimension=False,
)
results = await pgvector.search(
collection=collection_name,
query_embedding=query_vector,
k=limit,
search_type=search_type,
query_text=query_text,
filter=filter,
vector_weight=vector_weight,
scope=scope,
)
else:
results = await self.vector_db.search(
collection=collection_name,
query_embedding=query_vector,
k=limit,
search_type=search_type,
query_text=query_text,
filter=filter,
vector_weight=vector_weight,
)
if not results or 'ids' not in results or not results['ids']:
return []
@@ -297,8 +411,20 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
pgvector = self._pgvector_database()
scope = None
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=None,
initialize_dimension=False,
)
for file_id in file_ids:
await self.vector_db.delete_by_file_id(collection_name, file_id)
if pgvector is not None:
await pgvector.delete_by_file_id(collection_name, file_id, scope=scope)
else:
await self.vector_db.delete_by_file_id(collection_name, file_id)
async def delete_collection(
self,
@@ -311,7 +437,17 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
await self.vector_db.delete_collection(collection_name)
pgvector = self._pgvector_database()
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=None,
initialize_dimension=False,
)
await pgvector.delete_collection(collection_name, scope=scope)
else:
await self.vector_db.delete_collection(collection_name)
async def delete_by_filter(
self,
@@ -328,6 +464,15 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
pgvector = self._pgvector_database()
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=None,
initialize_dimension=False,
)
return await pgvector.delete_by_filter(collection_name, filter, scope=scope)
return await self.vector_db.delete_by_filter(collection_name, filter)
async def list_by_filter(
@@ -347,4 +492,13 @@ class VectorDBManager:
execution_context,
knowledge_base_uuid,
)
pgvector = self._pgvector_database()
if pgvector is not None:
scope = await self._resolve_pgvector_scope(
execution_context,
knowledge_base_uuid,
expected_dimension=None,
initialize_dimension=False,
)
return await pgvector.list_by_filter(collection_name, filter, limit, offset, scope=scope)
return await self.vector_db.list_by_filter(collection_name, filter, limit, offset)
+322 -276
View File
@@ -1,22 +1,31 @@
from __future__ import annotations
from typing import Any, Dict
from sqlalchemy import create_engine, text, Column, String, Text
from sqlalchemy.orm import declarative_base
from sqlalchemy.ext.asyncio import create_async_engine, AsyncSession, async_sessionmaker
import contextlib
import dataclasses
from collections.abc import AsyncIterator
from typing import Any
import sqlalchemy
from pgvector.sqlalchemy import Vector
from langbot.pkg.vector.vdb import VectorDatabase
from langbot.pkg.vector.filter_utils import normalize_filter, strip_unsupported_fields
from sqlalchemy.dialects.postgresql import insert as postgresql_insert
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
from sqlalchemy.orm import declarative_base
from langbot.pkg.core import app
from langbot.pkg.vector.filter_utils import normalize_filter, strip_unsupported_fields
from langbot.pkg.vector.vdb import VectorDatabase
Base = declarative_base()
DEFAULT_ALLOWED_DIMENSIONS = (384, 512, 768, 1024, 1536)
# pgvector schema only stores these metadata fields.
_PG_SUPPORTED_FIELDS = {'text', 'file_id', 'chunk_uuid'}
# Callers use canonical metadata key 'uuid' but pgvector stores it as 'chunk_uuid'.
_PG_FIELD_ALIASES = {'uuid': 'chunk_uuid'}
# Map schema field names to SQLAlchemy columns (resolved lazily from PgVectorEntry).
_PG_COLUMN_MAP = {
'text': 'text',
'file_id': 'file_id',
@@ -24,21 +33,50 @@ _PG_COLUMN_MAP = {
}
@dataclasses.dataclass(frozen=True, slots=True)
class PgVectorScope:
"""Trusted relational tenant key for one knowledge-base operation."""
workspace_uuid: str
knowledge_base_uuid: str
embedding_dimension: int | None = None
def __post_init__(self) -> None:
for field_name in ('workspace_uuid', 'knowledge_base_uuid'):
value = getattr(self, field_name)
if not isinstance(value, str) or not value.strip():
raise ValueError(f'{field_name} must not be empty')
object.__setattr__(self, field_name, value.strip())
dimension = self.embedding_dimension
if dimension is not None and (isinstance(dimension, bool) or not isinstance(dimension, int) or dimension <= 0):
raise ValueError('embedding_dimension must be a positive integer')
class PgVectorEntry(Base):
"""SQLAlchemy model for pgvector entries"""
"""Tenant-scoped pgvector row created only by release/OSS migrations."""
__tablename__ = 'langbot_vectors'
id = Column(String, primary_key=True)
collection = Column(String, index=True, nullable=False)
embedding = Column(Vector(1536)) # Default dimension, will be created dynamically
text = Column(Text)
file_id = Column(String, index=True)
chunk_uuid = Column(String)
workspace_uuid = sqlalchemy.Column(sqlalchemy.String(36), primary_key=True)
knowledge_base_uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
vector_id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
embedding_dimension = sqlalchemy.Column(sqlalchemy.Integer, nullable=False)
embedding = sqlalchemy.Column(Vector(), nullable=False)
text = sqlalchemy.Column(sqlalchemy.Text)
file_id = sqlalchemy.Column(sqlalchemy.String(255), index=True)
chunk_uuid = sqlalchemy.Column(sqlalchemy.String(255))
__table_args__ = (
sqlalchemy.CheckConstraint(
'vector_dims(embedding) = embedding_dimension',
name='ck_langbot_vectors_embedding_dimension',
),
)
def _build_pg_conditions(filter_dict: dict[str, Any]) -> list:
"""Translate canonical filter dict into a list of SQLAlchemy conditions."""
"""Translate canonical filter dict into SQLAlchemy conditions."""
triples = normalize_filter(filter_dict)
triples = strip_unsupported_fields(triples, _PG_SUPPORTED_FIELDS, _PG_FIELD_ALIASES)
@@ -65,83 +103,139 @@ def _build_pg_conditions(filter_dict: dict[str, Any]) -> list:
class PgVectorDatabase(VectorDatabase):
"""PostgreSQL with pgvector extension database implementation"""
"""PostgreSQL vector adapter with explicit Workspace/RLS scope.
Cloud reuses the business database engine and never performs DDL. OSS can
still opt into a standalone pgvector database; that compatibility mode may
create a fresh schema, but it uses the same explicit tenant keys.
"""
def __init__(
self,
ap: app.Application,
connection_string: str = None,
connection_string: str | None = None,
host: str = 'localhost',
port: int = 5432,
database: str = 'langbot',
user: str = 'postgres',
password: str = 'postgres',
):
"""Initialize pgvector database
Args:
ap: Application instance
connection_string: Full PostgreSQL connection string (overrides other params)
host: PostgreSQL host
port: PostgreSQL port
database: Database name
user: Database user
password: Database password
"""
*,
use_business_database: bool = False,
allowed_dimensions: list[int] | tuple[int, ...] = DEFAULT_ALLOWED_DIMENSIONS,
) -> None:
self.ap = ap
self.use_business_database = use_business_database
self.allowed_dimensions = self._normalize_allowed_dimensions(allowed_dimensions)
self.engine = None
self.async_engine = None
self.AsyncSessionLocal: async_sessionmaker[AsyncSession] | None = None
if use_business_database:
persistence_mgr = getattr(ap, 'persistence_mgr', None)
if persistence_mgr is None:
raise RuntimeError('Shared pgvector requires the initialized business persistence manager')
business_engine = persistence_mgr.get_db_engine()
if business_engine.dialect.name != 'postgresql':
raise RuntimeError('Shared pgvector requires the PostgreSQL business database')
self.async_engine = business_engine
self.ap.logger.info('Connected pgvector adapter to the shared PostgreSQL business database')
return
# Build connection string if not provided
if connection_string:
self.connection_string = connection_string
else:
self.connection_string = f'postgresql+psycopg://{user}:{password}@{host}:{port}/{database}'
self.async_connection_string = self.connection_string.replace('postgresql://', 'postgresql+asyncpg://').replace(
'postgresql+psycopg://', 'postgresql+asyncpg://'
)
self._initialize_standalone_db()
self.engine = None
self.async_engine = None
self.SessionLocal = None
self.AsyncSessionLocal = None
self._collections = set()
self._initialize_db()
@staticmethod
def _normalize_allowed_dimensions(dimensions: list[int] | tuple[int, ...]) -> frozenset[int]:
if not isinstance(dimensions, (list, tuple)) or not dimensions:
raise ValueError('pgvector allowed_dimensions must be a non-empty list')
if any(isinstance(item, bool) or not isinstance(item, int) or item <= 0 for item in dimensions):
raise ValueError('pgvector allowed_dimensions must contain positive integers')
unsupported = set(dimensions) - set(DEFAULT_ALLOWED_DIMENSIONS)
if unsupported:
raise ValueError(f'pgvector dimensions do not have release-created ANN indexes: {sorted(unsupported)}')
return frozenset(dimensions)
def _initialize_db(self):
"""Initialize database connection and create tables"""
try:
# Create async engine for async operations
self.async_engine = create_async_engine(self.async_connection_string, echo=False, pool_pre_ping=True)
self.AsyncSessionLocal = async_sessionmaker(self.async_engine, class_=AsyncSession, expire_on_commit=False)
def _initialize_standalone_db(self) -> None:
"""Initialize the explicit OSS external database compatibility path."""
# Create sync engine for table creation
sync_connection_string = self.connection_string.replace('postgresql+asyncpg://', 'postgresql+psycopg://')
self.engine = create_engine(sync_connection_string, echo=False)
from sqlalchemy import create_engine
# Create pgvector extension and tables
with self.engine.connect() as conn:
# Enable pgvector extension
conn.execute(text('CREATE EXTENSION IF NOT EXISTS vector'))
conn.commit()
self.async_engine = create_async_engine(self.async_connection_string, echo=False, pool_pre_ping=True)
self.AsyncSessionLocal = async_sessionmaker(self.async_engine, class_=AsyncSession, expire_on_commit=False)
sync_connection_string = self.connection_string.replace('postgresql+asyncpg://', 'postgresql+psycopg://')
self.engine = create_engine(sync_connection_string, echo=False)
# Create tables
Base.metadata.create_all(self.engine)
with self.engine.begin() as conn:
conn.execute(sqlalchemy.text('CREATE EXTENSION IF NOT EXISTS vector'))
existing_tables = set(sqlalchemy.inspect(conn).get_table_names())
if PgVectorEntry.__tablename__ in existing_tables:
columns = {
column['name'] for column in sqlalchemy.inspect(conn).get_columns(PgVectorEntry.__tablename__)
}
required = {
'workspace_uuid',
'knowledge_base_uuid',
'vector_id',
'embedding_dimension',
'embedding',
}
if not required.issubset(columns):
raise RuntimeError(
'The external pgvector database uses the legacy unscoped schema; '
'migrate it before enabling multi-tenant vector access'
)
Base.metadata.create_all(conn)
self.ap.logger.info('Connected to PostgreSQL with pgvector')
except Exception as e:
self.ap.logger.error(f'Failed to connect to PostgreSQL: {e}')
raise
self.ap.logger.info('Connected to standalone PostgreSQL pgvector database')
def _require_scope(self, scope: PgVectorScope | None, *, require_dimension: bool) -> PgVectorScope:
if not isinstance(scope, PgVectorScope):
raise ValueError('pgvector operations require a trusted PgVectorScope')
dimension = scope.embedding_dimension
if require_dimension and dimension is None:
raise ValueError('pgvector operation requires an embedding dimension')
if dimension is not None and dimension not in self.allowed_dimensions:
raise ValueError(f'Embedding dimension {dimension} is not enabled for this pgvector deployment')
return scope
@staticmethod
def _scope_conditions(scope: PgVectorScope) -> tuple[Any, Any]:
return (
PgVectorEntry.workspace_uuid == scope.workspace_uuid,
PgVectorEntry.knowledge_base_uuid == scope.knowledge_base_uuid,
)
@contextlib.asynccontextmanager
async def _session(self, scope: PgVectorScope) -> AsyncIterator[AsyncSession]:
admission = getattr(self.ap, 'deployment_admission', None)
if admission is not None:
admission.require_active()
if self.use_business_database:
async with self.ap.persistence_mgr.tenant_uow(scope.workspace_uuid) as uow:
yield uow.session
if admission is not None:
admission.require_active()
return
if self.AsyncSessionLocal is None: # pragma: no cover - constructor invariant
raise RuntimeError('Standalone pgvector session factory is unavailable')
async with self.AsyncSessionLocal() as session, session.begin():
yield session
if admission is not None:
admission.require_active()
async def get_or_create_collection(self, collection: str):
"""Get or create a collection (logical grouping in pgvector)
"""Retain the common adapter API; relational rows need no collection DDL."""
Args:
collection: Collection name (knowledge base UUID)
"""
# In pgvector, collections are logical - we just track them
if collection not in self._collections:
self._collections.add(collection)
self.ap.logger.info(f"Registered pgvector collection '{collection}'")
if not isinstance(collection, str) or not collection.strip():
raise ValueError('collection must not be empty')
return collection
async def add_embeddings(
@@ -151,38 +245,59 @@ class PgVectorDatabase(VectorDatabase):
embeddings_list: list[list[float]],
metadatas: list[dict[str, Any]],
documents: list[str] | None = None,
*,
scope: PgVectorScope | None = None,
) -> None:
"""Add vector embeddings to pgvector
Args:
collection: Collection name
ids: List of unique IDs for each vector
embeddings_list: List of embedding vectors
metadatas: List of metadata dictionaries
"""
scope = self._require_scope(scope, require_dimension=True)
await self.get_or_create_collection(collection)
if not ids:
return
if len(ids) != len(embeddings_list) or len(metadatas) != len(ids):
raise ValueError('pgvector ids, embeddings and metadata lengths must match')
if documents is not None and len(documents) != len(ids):
raise ValueError('pgvector documents length must match ids')
if len(set(ids)) != len(ids) or any(not isinstance(item, str) or not item.strip() for item in ids):
raise ValueError('pgvector vector IDs must be unique non-empty strings per upsert')
expected_dimension = scope.embedding_dimension
if any(len(embedding) != expected_dimension for embedding in embeddings_list):
raise ValueError(f'All embeddings must have the selected dimension {expected_dimension}')
async with self.AsyncSessionLocal() as session:
try:
for i, vector_id in enumerate(ids):
metadata = metadatas[i] if i < len(metadatas) else {}
values = []
for index, vector_id in enumerate(ids):
metadata = metadatas[index]
document = documents[index] if documents is not None else None
values.append(
{
'workspace_uuid': scope.workspace_uuid,
'knowledge_base_uuid': scope.knowledge_base_uuid,
'vector_id': vector_id.strip(),
'embedding_dimension': expected_dimension,
'embedding': embeddings_list[index],
'text': metadata.get('text', document or ''),
'file_id': metadata.get('file_id', ''),
'chunk_uuid': metadata.get('uuid', metadata.get('chunk_uuid', '')),
}
)
entry = PgVectorEntry(
id=vector_id,
collection=collection,
embedding=embeddings_list[i],
text=metadata.get('text', ''),
file_id=metadata.get('file_id', ''),
chunk_uuid=metadata.get('uuid', ''),
)
session.add(entry)
await session.commit()
self.ap.logger.info(f"Added {len(ids)} embeddings to pgvector collection '{collection}'")
except Exception as e:
await session.rollback()
self.ap.logger.error(f'Error adding embeddings to pgvector: {e}')
raise
statement = postgresql_insert(PgVectorEntry).values(values)
excluded = statement.excluded
statement = statement.on_conflict_do_update(
index_elements=[
PgVectorEntry.workspace_uuid,
PgVectorEntry.knowledge_base_uuid,
PgVectorEntry.vector_id,
],
set_={
'embedding_dimension': excluded.embedding_dimension,
'embedding': excluded.embedding,
'text': excluded.text,
'file_id': excluded.file_id,
'chunk_uuid': excluded.chunk_uuid,
},
)
async with self._session(scope) as session:
await session.execute(statement)
self.ap.logger.info(f'Upserted {len(ids)} pgvector embeddings for knowledge base {scope.knowledge_base_uuid}')
async def search(
self,
@@ -193,125 +308,79 @@ class PgVectorDatabase(VectorDatabase):
query_text: str = '',
filter: dict[str, Any] | None = None,
vector_weight: float | None = None,
) -> Dict[str, Any]:
"""Search for similar vectors using cosine distance
Args:
collection: Collection name
query_embedding: Query vector
k: Number of top results to return
Returns:
Dictionary with search results in Chroma-compatible format
"""
*,
scope: PgVectorScope | None = None,
) -> dict[str, Any]:
del query_text, vector_weight
scope = self._require_scope(scope, require_dimension=True)
await self.get_or_create_collection(collection)
if search_type != 'vector':
raise ValueError('pgvector currently supports vector search only')
if k <= 0:
raise ValueError('pgvector search limit must be positive')
if len(query_embedding) != scope.embedding_dimension:
raise ValueError(f'Query embedding must have the selected dimension {scope.embedding_dimension}')
async with self.AsyncSessionLocal() as session:
try:
# Use cosine distance for similarity search
from sqlalchemy import select
typed_embedding = sqlalchemy.cast(PgVectorEntry.embedding, Vector(scope.embedding_dimension))
distance = typed_embedding.cosine_distance(query_embedding)
statement = (
sqlalchemy.select(
PgVectorEntry.vector_id,
PgVectorEntry.text,
PgVectorEntry.file_id,
PgVectorEntry.chunk_uuid,
distance.label('distance'),
)
.where(*self._scope_conditions(scope), PgVectorEntry.embedding_dimension == scope.embedding_dimension)
.order_by(distance)
.limit(k)
)
for condition in _build_pg_conditions(filter or {}):
statement = statement.where(condition)
# Query for similar vectors
stmt = (
select(
PgVectorEntry.id,
PgVectorEntry.text,
PgVectorEntry.file_id,
PgVectorEntry.chunk_uuid,
PgVectorEntry.embedding.cosine_distance(query_embedding).label('distance'),
)
.filter(PgVectorEntry.collection == collection)
.order_by(PgVectorEntry.embedding.cosine_distance(query_embedding))
.limit(k)
)
async with self._session(scope) as session:
rows = (await session.execute(statement)).all()
if filter:
for cond in _build_pg_conditions(filter):
stmt = stmt.filter(cond)
ids = [row.vector_id for row in rows]
distances = [float(row.distance) for row in rows]
metadatas = [
{'text': row.text or '', 'file_id': row.file_id or '', 'uuid': row.chunk_uuid or ''} for row in rows
]
return {'ids': [ids], 'distances': [distances], 'metadatas': [metadatas]}
result = await session.execute(stmt)
rows = result.fetchall()
# Convert to Chroma-compatible format
ids = []
distances = []
metadatas = []
for row in rows:
ids.append(row.id)
distances.append(float(row.distance))
metadatas.append(
{'text': row.text or '', 'file_id': row.file_id or '', 'uuid': row.chunk_uuid or ''}
)
result_dict = {'ids': [ids], 'distances': [distances], 'metadatas': [metadatas]}
self.ap.logger.info(f"pgvector search in '{collection}' returned {len(ids)} results")
return result_dict
except Exception as e:
self.ap.logger.error(f'Error searching pgvector: {e}')
raise
async def delete_by_file_id(self, collection: str, file_id: str) -> None:
"""Delete vectors by file_id
Args:
collection: Collection name
file_id: File ID to filter deletion
"""
async def delete_by_file_id(
self,
collection: str,
file_id: str,
*,
scope: PgVectorScope | None = None,
) -> None:
scope = self._require_scope(scope, require_dimension=False)
await self.get_or_create_collection(collection)
statement = sqlalchemy.delete(PgVectorEntry).where(
*self._scope_conditions(scope),
PgVectorEntry.file_id == file_id,
)
async with self._session(scope) as session:
await session.execute(statement)
async with self.AsyncSessionLocal() as session:
try:
from sqlalchemy import delete
stmt = delete(PgVectorEntry).where(
PgVectorEntry.collection == collection, PgVectorEntry.file_id == file_id
)
await session.execute(stmt)
await session.commit()
self.ap.logger.info(
f"Deleted embeddings from pgvector collection '{collection}' with file_id: {file_id}"
)
except Exception as e:
await session.rollback()
self.ap.logger.error(f'Error deleting from pgvector: {e}')
raise
async def delete_by_filter(self, collection: str, filter: dict[str, Any]) -> int:
"""Delete vectors matching a metadata filter.
Args:
collection: Collection name
filter: Canonical metadata filter dict
"""
async def delete_by_filter(
self,
collection: str,
filter: dict[str, Any],
*,
scope: PgVectorScope | None = None,
) -> int:
scope = self._require_scope(scope, require_dimension=False)
await self.get_or_create_collection(collection)
conditions = _build_pg_conditions(filter)
if not conditions:
self.ap.logger.warning(
f"pgvector delete_by_filter on '{collection}': filter produced no conditions, skipping"
)
self.ap.logger.warning('pgvector delete_by_filter produced no supported conditions; skipping')
return 0
await self.get_or_create_collection(collection)
async with self.AsyncSessionLocal() as session:
try:
from sqlalchemy import delete
stmt = delete(PgVectorEntry).where(PgVectorEntry.collection == collection)
for cond in conditions:
stmt = stmt.where(cond)
result = await session.execute(stmt)
await session.commit()
deleted = result.rowcount
self.ap.logger.info(f"Deleted {deleted} embeddings from pgvector collection '{collection}' by filter")
return deleted
except Exception as e:
await session.rollback()
self.ap.logger.error(f'Error deleting from pgvector by filter: {e}')
raise
statement = sqlalchemy.delete(PgVectorEntry).where(*self._scope_conditions(scope), *conditions)
async with self._session(scope) as session:
result = await session.execute(statement)
return int(result.rowcount or 0)
async def list_by_filter(
self,
@@ -319,85 +388,62 @@ class PgVectorDatabase(VectorDatabase):
filter: dict[str, Any] | None = None,
limit: int = 20,
offset: int = 0,
*,
scope: PgVectorScope | None = None,
) -> tuple[list[dict[str, Any]], int]:
scope = self._require_scope(scope, require_dimension=False)
await self.get_or_create_collection(collection)
if limit <= 0 or offset < 0:
raise ValueError('pgvector pagination requires limit > 0 and offset >= 0')
async with self.AsyncSessionLocal() as session:
try:
from sqlalchemy import select, func
conditions = [*self._scope_conditions(scope), *_build_pg_conditions(filter or {})]
statement = (
sqlalchemy.select(
PgVectorEntry.vector_id,
PgVectorEntry.text,
PgVectorEntry.file_id,
PgVectorEntry.chunk_uuid,
)
.where(*conditions)
.order_by(PgVectorEntry.vector_id)
.offset(offset)
.limit(limit)
)
count_statement = sqlalchemy.select(sqlalchemy.func.count()).select_from(PgVectorEntry).where(*conditions)
async with self._session(scope) as session:
rows = (await session.execute(statement)).all()
total = int((await session.execute(count_statement)).scalar_one())
stmt = (
select(
PgVectorEntry.id,
PgVectorEntry.text,
PgVectorEntry.file_id,
PgVectorEntry.chunk_uuid,
)
.filter(PgVectorEntry.collection == collection)
.offset(offset)
.limit(limit)
)
return (
[
{
'id': row.vector_id,
'document': row.text or '',
'metadata': {
'text': row.text or '',
'file_id': row.file_id or '',
'uuid': row.chunk_uuid or '',
},
}
for row in rows
],
total,
)
count_stmt = (
select(func.count()).select_from(PgVectorEntry).filter(PgVectorEntry.collection == collection)
)
async def delete_collection(
self,
collection: str,
*,
scope: PgVectorScope | None = None,
) -> None:
scope = self._require_scope(scope, require_dimension=False)
await self.get_or_create_collection(collection)
statement = sqlalchemy.delete(PgVectorEntry).where(*self._scope_conditions(scope))
async with self._session(scope) as session:
await session.execute(statement)
if filter:
for cond in _build_pg_conditions(filter):
stmt = stmt.filter(cond)
count_stmt = count_stmt.filter(cond)
result = await session.execute(stmt)
rows = result.fetchall()
count_result = await session.execute(count_stmt)
total = count_result.scalar() or 0
items = []
for row in rows:
items.append(
{
'id': row.id,
'document': row.text or '',
'metadata': {
'text': row.text or '',
'file_id': row.file_id or '',
'uuid': row.chunk_uuid or '',
},
}
)
return items, total
except Exception as e:
self.ap.logger.error(f'Error listing from pgvector: {e}')
raise
async def delete_collection(self, collection: str):
"""Delete all vectors in a collection
Args:
collection: Collection name to delete
"""
if collection in self._collections:
self._collections.remove(collection)
async with self.AsyncSessionLocal() as session:
try:
from sqlalchemy import delete
stmt = delete(PgVectorEntry).where(PgVectorEntry.collection == collection)
await session.execute(stmt)
await session.commit()
self.ap.logger.info(f"Deleted pgvector collection '{collection}'")
except Exception as e:
await session.rollback()
self.ap.logger.error(f'Error deleting pgvector collection: {e}')
raise
async def close(self):
"""Close database connections"""
if self.async_engine:
async def close(self) -> None:
if not self.use_business_database and self.async_engine is not None:
await self.async_engine.dispose()
if self.engine:
if self.engine is not None:
self.engine.dispose()
+79 -1
View File
@@ -155,8 +155,20 @@ class WorkspaceCollaborationService:
) -> ResolvedWorkspaceAccess:
"""Resolve a selector against an active Account membership."""
normalized_workspace_uuid = requested_workspace_uuid.strip() if requested_workspace_uuid else None
if normalized_workspace_uuid is None and self.policy.multi_workspace_enabled:
raise WorkspaceNotFoundError('A Workspace selector is required')
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if session is None and normalized_workspace_uuid is not None and callable(tenant_uow):
async with tenant_uow(normalized_workspace_uuid) as uow:
return await self.resolve_account_workspace(
account_uuid,
normalized_workspace_uuid,
session=uow.session,
)
async def operation(active_session: AsyncSession) -> ResolvedWorkspaceAccess:
workspace_uuid = requested_workspace_uuid.strip() if requested_workspace_uuid else None
workspace_uuid = normalized_workspace_uuid
if workspace_uuid is None:
if self.policy.multi_workspace_enabled:
raise WorkspaceNotFoundError('A Workspace selector is required')
@@ -195,6 +207,40 @@ class WorkspaceCollaborationService:
*,
session: AsyncSession | None = None,
) -> list[ResolvedWorkspaceAccess]:
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
account_uow = getattr(self.ap.persistence_mgr, 'account_discovery_uow', None)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if session is None and current_session() is None and callable(account_uow) and callable(tenant_uow):
# Discovery exposes only this Account's active Membership rows.
# Each resulting Workspace is then re-read under its own tenant
# transaction; directory discovery never grants business access.
async with account_uow(account_uuid) as discovery:
workspace_uuids = list(
(
await discovery.session.scalars(
sqlalchemy.select(WorkspaceMembership.workspace_uuid)
.where(
WorkspaceMembership.account_uuid == account_uuid,
WorkspaceMembership.status == MembershipStatus.ACTIVE.value,
)
.order_by(WorkspaceMembership.workspace_uuid)
)
).all()
)
accesses: list[ResolvedWorkspaceAccess] = []
for workspace_uuid in workspace_uuids:
async with tenant_uow(workspace_uuid) as workspace_uow:
accesses.append(
await self.resolve_account_workspace(
account_uuid,
workspace_uuid,
session=workspace_uow.session,
)
)
accesses.sort(key=lambda access: (access.workspace.created_at, access.workspace.uuid))
return accesses
async def operation(active_session: AsyncSession) -> list[ResolvedWorkspaceAccess]:
statement = (
sqlalchemy.select(WorkspaceMembership, Workspace)
@@ -346,6 +392,18 @@ class WorkspaceCollaborationService:
*,
session: AsyncSession | None = None,
) -> tuple[WorkspaceInvitation, Workspace]:
if session is None:
scoped_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)()
invitation_uow = getattr(self.ap.persistence_mgr, 'invitation_discovery_uow', None)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if scoped_session is None and callable(invitation_uow) and callable(tenant_uow):
token_hash = hash_invitation_token(token)
async with invitation_uow(token_hash) as discovery:
invitation = await self._get_invitation_by_token(discovery.session, token, for_update=False)
workspace_uuid = invitation.workspace_uuid
async with tenant_uow(workspace_uuid) as workspace_uow:
return await self.inspect_invitation(token, session=workspace_uow.session)
async def operation(active_session: AsyncSession) -> tuple[WorkspaceInvitation, Workspace]:
invitation = await self._get_invitation_by_token(active_session, token, for_update=True)
self._validate_invitation_state(invitation)
@@ -365,6 +423,18 @@ class WorkspaceCollaborationService:
*,
session: AsyncSession | None = None,
) -> WorkspaceMembership:
if session is None:
scoped_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)()
invitation_uow = getattr(self.ap.persistence_mgr, 'invitation_discovery_uow', None)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if scoped_session is None and callable(invitation_uow) and callable(tenant_uow):
token_digest = hash_invitation_token(token)
async with invitation_uow(token_digest) as discovery:
invitation = await self._get_invitation_by_token(discovery.session, token, for_update=False)
workspace_uuid = invitation.workspace_uuid
async with tenant_uow(workspace_uuid) as workspace_uow:
return await self.accept_invitation(token, account_uuid, session=workspace_uow.session)
async def operation(active_session: AsyncSession) -> WorkspaceMembership:
invitation = await self._get_invitation_by_token(active_session, token, for_update=True)
self._validate_invitation_state(invitation)
@@ -677,6 +747,14 @@ class WorkspaceCollaborationService:
) -> T:
if session is not None:
return await operation(session)
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)()
if current_session is not None:
return await operation(current_session)
if getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime':
require_current_session = getattr(self.ap.persistence_mgr, 'require_current_session', None)
if callable(require_current_session):
require_current_session()
raise RuntimeError('Cloud collaboration services require an explicit persistence unit of work')
async with self._session_factory()() as owned_session:
if read_only:
return await operation(owned_session)
+86 -1
View File
@@ -6,6 +6,7 @@ import typing
from collections.abc import Awaitable, Callable
from typing import TypeVar
import sqlalchemy
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ..entity.persistence.workspace import (
@@ -68,6 +69,11 @@ class WorkspaceService:
) -> Workspace:
"""Load one Workspace projected onto this LangBot instance."""
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if session is None and callable(tenant_uow):
async with tenant_uow(workspace_uuid) as uow:
return await self.get_workspace(workspace_uuid, session=uow.session)
async def operation(repository: WorkspaceRepository) -> Workspace:
workspace = await repository.get_workspace(workspace_uuid)
if workspace is None or workspace.instance_uuid != self.instance_uuid:
@@ -97,6 +103,11 @@ class WorkspaceService:
) -> WorkspaceExecutionState:
"""Load a Workspace execution state and validate its instance binding."""
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if session is None and callable(tenant_uow):
async with tenant_uow(workspace_uuid) as uow:
return await self.get_execution_state(workspace_uuid, session=uow.session)
async def operation(repository: WorkspaceRepository) -> WorkspaceExecutionState:
execution_state = await repository.get_execution_state(workspace_uuid)
if execution_state is None:
@@ -109,6 +120,47 @@ class WorkspaceService:
return await self._run(operation, session=session)
async def list_active_execution_bindings(self) -> list[WorkspaceExecutionBinding]:
"""Discover this instance's active Workspaces, then validate each tenant projection.
The instance discovery transaction may only reveal active, unfenced
execution-state identifiers. It is closed before any Workspace data is
read; every returned binding is revalidated inside its own tenant unit
of work.
"""
instance_discovery_uow = getattr(self.ap.persistence_mgr, 'instance_discovery_uow', None)
statement = (
sqlalchemy.select(WorkspaceExecutionState.workspace_uuid)
.where(
WorkspaceExecutionState.instance_uuid == self.instance_uuid,
WorkspaceExecutionState.state == WorkspaceExecutionStatus.ACTIVE.value,
WorkspaceExecutionState.write_fenced == sqlalchemy.false(),
)
.order_by(WorkspaceExecutionState.workspace_uuid)
)
if callable(instance_discovery_uow):
async with instance_discovery_uow(self.instance_uuid) as uow:
result = await uow.session.execute(statement)
workspace_uuids = list(result.scalars().all())
else:
# Compatibility for lightweight test doubles. Production managers
# always expose the explicit discovery scope.
result = await self.ap.persistence_mgr.execute_async(statement)
workspace_uuids = list(result.scalars().all())
bindings: list[WorkspaceExecutionBinding] = []
for workspace_uuid in workspace_uuids:
try:
bindings.append(await self.get_execution_binding(workspace_uuid))
except (
WorkspaceExecutionUnavailableError,
WorkspaceInvariantError,
WorkspaceNotFoundError,
) as exc:
self.ap.logger.warning(f'Skipping invalid Workspace execution projection {workspace_uuid!r}: {exc}')
return bindings
async def get_execution_binding(
self,
workspace_uuid: str | None = None,
@@ -124,6 +176,18 @@ class WorkspaceService:
a Workspace from source or recency.
"""
self._require_deployment_admission()
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if session is None and workspace_uuid is not None and callable(tenant_uow):
async with tenant_uow(workspace_uuid) as uow:
return await self.get_execution_binding(
workspace_uuid,
expected_generation=expected_generation,
session=uow.session,
_require_local=_require_local,
)
async def operation(repository: WorkspaceRepository) -> WorkspaceExecutionBinding:
if workspace_uuid is None:
workspaces = await repository.list_local_workspaces(self.instance_uuid)
@@ -180,7 +244,11 @@ class WorkspaceService:
state=execution_state.state,
)
return await self._run(operation, session=session)
binding = await self._run(operation, session=session)
# The database lookup can cross the Manifest expiry boundary. Never
# return a binding that is already invalid at a side-effect boundary.
self._require_deployment_admission()
return binding
async def get_local_execution_binding(
self,
@@ -408,6 +476,18 @@ class WorkspaceService:
if session is not None:
return await operation(WorkspaceRepository(session))
current_session = getattr(self.ap.persistence_mgr, 'current_session', lambda: None)
active_session = current_session()
if active_session is not None:
return await operation(WorkspaceRepository(active_session))
if getattr(getattr(self.ap.persistence_mgr, 'mode', None), 'value', None) == 'cloud_runtime':
# Do not let service-local session factories silently bypass the
# request/task scope enforced by PersistenceManager.
require_current_session = getattr(self.ap.persistence_mgr, 'require_current_session', None)
if callable(require_current_session):
require_current_session()
raise RuntimeError('Cloud Workspace services require an explicit persistence unit of work')
session_factory = async_sessionmaker(
self.ap.persistence_mgr.get_db_engine(),
expire_on_commit=False,
@@ -415,3 +495,8 @@ class WorkspaceService:
async with session_factory() as owned_session:
async with owned_session.begin():
return await operation(WorkspaceRepository(owned_session))
def _require_deployment_admission(self) -> None:
guard = getattr(self.ap, 'deployment_admission', None)
if guard is not None:
guard.require_active()