Merge master into dev/4.11.x and preserve plugin runner architecture

Reconcile migration branches without rewriting published revisions; retain additive Codex, monitoring, provider and platform fixes. Keep dynamic runner schemas and Host ownership, restore compatibility regressions, and preserve safe model-test error handling.
This commit is contained in:
RockChinQ
2026-09-16 08:13:07 +00:00
228 changed files with 22138 additions and 5057 deletions
+1 -1
View File
@@ -16,7 +16,7 @@ asciiart = r"""
|___/
⭐️ Open Source 开源地址: https://github.com/langbot-app/LangBot
📖 Documentation 文档地址: https://docs.langbot.app
📖 Documentation 文档地址: https://langbot.app/docs
"""
@@ -15,6 +15,7 @@ from ....workspace.collaboration import MembershipPermissionError, WorkspaceColl
from ....workspace.errors import WorkspaceNotFoundError
from ....cloud.entitlements import EntitlementUnavailableError
from ....core.errors import TaskCapacityError
from ....provider.modelmgr.codex_errors import CodexProviderError
from ..authz import (
AuthenticationDeniedError,
AuthorizationError,
@@ -249,6 +250,8 @@ class RouterGroup(abc.ABC):
return await f(*args, **kwargs)
except Exception as e: # 自动 500
if isinstance(e, CodexProviderError):
return self.http_status(e.status_code, e.error_code, str(e))
if isinstance(e, AuthorizationError):
return self.http_status(e.status_code, e.error_code, str(e))
if isinstance(e, WorkspaceNotFoundError):
@@ -47,6 +47,21 @@ class FilesRouterGroup(group.RouterGroup):
return quart.Response(image_bytes, mimetype=mime_type)
@self.route(
'/media/<filename>',
methods=['GET'],
auth_type=group.AuthType.NONE,
)
async def get_media_file(filename: str) -> quart.Response:
media = await self.ap.storage_mgr.media_cache.get_media(filename)
if media is None:
return quart.Response('Media not found or expired', status=404)
media_bytes, mime_type = media
headers = {
'Cache-Control': 'public, max-age=2592000, immutable',
}
return quart.Response(media_bytes, mimetype=mime_type, headers=headers)
@self.route(
'/images',
methods=['POST'],
@@ -5,6 +5,7 @@ import quart
from ...authz import Permission
from ...context import RequestContext
from ...service.monitoring_traffic import get_traffic_series
from .. import group
@@ -218,6 +219,7 @@ class MonitoringRouterGroup(group.RouterGroup):
pipeline_ids = quart.request.args.getlist('pipelineId')
start_time_str = quart.request.args.get('startTime')
end_time_str = quart.request.args.get('endTime')
user_query = quart.request.args.get('userQuery')
is_active_str = quart.request.args.get('isActive')
limit = int(quart.request.args.get('limit', 100))
offset = int(quart.request.args.get('offset', 0))
@@ -237,6 +239,7 @@ class MonitoringRouterGroup(group.RouterGroup):
pipeline_ids=pipeline_ids if pipeline_ids else None,
start_time=start_time,
end_time=end_time,
user_query=user_query,
is_active=is_active,
limit=limit,
offset=offset,
@@ -375,6 +378,14 @@ class MonitoringRouterGroup(group.RouterGroup):
return self.success(
data={
'traffic': await get_traffic_series(
self.ap,
request_context,
bot_ids=bot_ids or None,
pipeline_ids=pipeline_ids or None,
start_time=start_time,
end_time=end_time,
),
'overview': overview,
'messages': messages,
'llmCalls': llm_calls,
@@ -396,7 +407,15 @@ class MonitoringRouterGroup(group.RouterGroup):
@self.route('/sessions/<session_id>/analysis', methods=['GET'], permission=Permission.RESOURCE_VIEW)
async def get_session_analysis(session_id: str, request_context: RequestContext) -> str:
"""Get detailed analysis for a specific session"""
analysis = await self.ap.monitoring_service.get_session_analysis(request_context, session_id)
start_time = parse_iso_datetime(quart.request.args.get('startTime'))
end_time = parse_iso_datetime(quart.request.args.get('endTime'))
analysis = await self.ap.monitoring_service.get_session_analysis(
request_context,
session_id,
start_time=start_time,
end_time=end_time,
bot_id=quart.request.args.get('botId'),
)
# Always return success with the analysis data
# The frontend will handle the 'found: false' case
@@ -4,6 +4,7 @@ from langbot.pkg.provider.modelmgr import errors as provider_errors
from ....authz import Permission, has_permission
from ....context import RequestContext
from ... import group
from .query import resolve_include_secret
MODEL_TEST_ERROR_CODE = 'model_test_failed'
@@ -20,7 +21,12 @@ class LLMModelsRouterGroup(group.RouterGroup):
)
async def _(request_context: RequestContext) -> str:
provider_uuid = quart.request.args.get('provider_uuid')
include_secret = has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE)
include_secret, error = resolve_include_secret(
quart.request.args.get('include_secret'),
permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
)
if error:
return self.http_status(400, -1, error)
if provider_uuid:
models = await self.ap.llm_model_service.get_llm_models_by_provider(
request_context,
@@ -57,10 +63,16 @@ class LLMModelsRouterGroup(group.RouterGroup):
permission=Permission.RESOURCE_VIEW,
)
async def _(model_uuid: str, request_context: RequestContext) -> str:
include_secret, error = resolve_include_secret(
quart.request.args.get('include_secret'),
permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
)
if error:
return self.http_status(400, -1, error)
model = await self.ap.llm_model_service.get_llm_model(
request_context,
model_uuid,
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
include_secret=include_secret,
)
if model is None:
return self.http_status(404, -1, 'model not found')
@@ -106,7 +118,7 @@ class LLMModelsRouterGroup(group.RouterGroup):
model_uuid,
await quart.request.json,
)
except (provider_errors.RequesterError, ValueError) as exc:
except provider_errors.RequesterError as exc:
return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc))
return self.success()
@@ -122,7 +134,12 @@ class EmbeddingModelsRouterGroup(group.RouterGroup):
)
async def _(request_context: RequestContext) -> str:
provider_uuid = quart.request.args.get('provider_uuid')
include_secret = has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE)
include_secret, error = resolve_include_secret(
quart.request.args.get('include_secret'),
permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
)
if error:
return self.http_status(400, -1, error)
if provider_uuid:
models = await self.ap.embedding_models_service.get_embedding_models_by_provider(
request_context,
@@ -159,10 +176,16 @@ class EmbeddingModelsRouterGroup(group.RouterGroup):
permission=Permission.RESOURCE_VIEW,
)
async def _(model_uuid: str, request_context: RequestContext) -> str:
include_secret, error = resolve_include_secret(
quart.request.args.get('include_secret'),
permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
)
if error:
return self.http_status(400, -1, error)
model = await self.ap.embedding_models_service.get_embedding_model(
request_context,
model_uuid,
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
include_secret=include_secret,
)
if model is None:
return self.http_status(404, -1, 'model not found')
@@ -208,7 +231,7 @@ class EmbeddingModelsRouterGroup(group.RouterGroup):
model_uuid,
await quart.request.json,
)
except (provider_errors.RequesterError, ValueError) as exc:
except provider_errors.RequesterError as exc:
return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc))
return self.success()
@@ -224,7 +247,12 @@ class RerankModelsRouterGroup(group.RouterGroup):
)
async def _(request_context: RequestContext) -> str:
provider_uuid = quart.request.args.get('provider_uuid')
include_secret = has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE)
include_secret, error = resolve_include_secret(
quart.request.args.get('include_secret'),
permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
)
if error:
return self.http_status(400, -1, error)
if provider_uuid:
models = await self.ap.rerank_models_service.get_rerank_models_by_provider(
request_context,
@@ -261,10 +289,16 @@ class RerankModelsRouterGroup(group.RouterGroup):
permission=Permission.RESOURCE_VIEW,
)
async def _(model_uuid: str, request_context: RequestContext) -> str:
include_secret, error = resolve_include_secret(
quart.request.args.get('include_secret'),
permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
)
if error:
return self.http_status(400, -1, error)
model = await self.ap.rerank_models_service.get_rerank_model(
request_context,
model_uuid,
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
include_secret=include_secret,
)
if model is None:
return self.http_status(404, -1, 'model not found')
@@ -310,6 +344,6 @@ class RerankModelsRouterGroup(group.RouterGroup):
model_uuid,
await quart.request.json,
)
except (provider_errors.RequesterError, ValueError) as exc:
except provider_errors.RequesterError as exc:
return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc))
return self.success()
@@ -3,11 +3,86 @@ import quart
from ....authz import Permission, has_permission
from ....context import RequestContext
from ... import group
from .query import resolve_include_secret
@group.group_class('models/providers', '/api/v1/provider/providers')
class ModelProvidersRouterGroup(group.RouterGroup):
async def initialize(self) -> None:
# Subscription authorization is an interactive, browser-user-only surface.
@self.route(
'/<provider_uuid>/codex/status',
methods=['GET'],
auth_type=group.AuthType.USER_TOKEN,
permission=Permission.PROVIDER_SECRET_MANAGE,
)
async def codex_status(provider_uuid: str, request_context: RequestContext):
try:
return self.success(
data=await self.ap.provider_service.codex_auth.status(request_context, provider_uuid)
)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
@self.route(
'/<provider_uuid>/codex/device',
methods=['POST'],
auth_type=group.AuthType.USER_TOKEN,
permission=Permission.PROVIDER_SECRET_MANAGE,
)
async def codex_device(provider_uuid: str, request_context: RequestContext):
try:
return self.success(
data=await self.ap.provider_service.codex_auth.start(request_context, provider_uuid)
)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
@self.route(
'/<provider_uuid>/codex/device/poll',
methods=['POST'],
auth_type=group.AuthType.USER_TOKEN,
permission=Permission.PROVIDER_SECRET_MANAGE,
)
async def codex_poll(provider_uuid: str, request_context: RequestContext):
body = await quart.request.get_json()
if not isinstance(body, dict):
return self.http_status(400, -1, 'JSON object required')
try:
return self.success(
data=await self.ap.provider_service.codex_auth.poll(
request_context, provider_uuid, body.get('authorization_id')
)
)
except ValueError as exc:
return self.http_status(400, -1, str(exc))
@self.route(
'/<provider_uuid>/codex/auth',
methods=['DELETE'],
auth_type=group.AuthType.USER_TOKEN,
permission=Permission.PROVIDER_SECRET_MANAGE,
)
async def codex_disconnect(provider_uuid: str, request_context: RequestContext):
try:
await self.ap.provider_service.codex_auth.disconnect(request_context, provider_uuid)
return self.success()
except ValueError as exc:
return self.http_status(400, -1, str(exc))
@self.route(
'/<provider_uuid>/codex/device/<authorization_id>',
methods=['DELETE'],
auth_type=group.AuthType.USER_TOKEN,
permission=Permission.PROVIDER_SECRET_MANAGE,
)
async def codex_cancel(provider_uuid: str, authorization_id: str, request_context: RequestContext):
try:
await self.ap.provider_service.codex_auth.cancel(request_context, provider_uuid, authorization_id)
return self.success()
except ValueError as exc:
return self.http_status(400, -1, str(exc))
@self.route(
'',
methods=['GET'],
@@ -15,9 +90,15 @@ class ModelProvidersRouterGroup(group.RouterGroup):
permission=Permission.RESOURCE_VIEW,
)
async def _(request_context: RequestContext) -> str:
include_secret, error = resolve_include_secret(
quart.request.args.get('include_secret'),
permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
)
if error:
return self.http_status(400, -1, error)
providers = await self.ap.provider_service.get_providers(
request_context,
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
include_secret=include_secret,
)
for provider in providers:
counts = await self.ap.provider_service.get_provider_model_counts(request_context, provider['uuid'])
@@ -47,10 +128,16 @@ class ModelProvidersRouterGroup(group.RouterGroup):
permission=Permission.RESOURCE_VIEW,
)
async def _(provider_uuid: str, request_context: RequestContext) -> str:
include_secret, error = resolve_include_secret(
quart.request.args.get('include_secret'),
permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
)
if error:
return self.http_status(400, -1, error)
provider = await self.ap.provider_service.get_provider(
request_context,
provider_uuid,
include_secret=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE),
include_secret=include_secret,
)
if provider is None:
return self.http_status(404, -1, 'provider not found')
@@ -82,7 +169,15 @@ class ModelProvidersRouterGroup(group.RouterGroup):
)
async def _(provider_uuid: str, request_context: RequestContext) -> str:
try:
await self.ap.provider_service.delete_provider(request_context, provider_uuid)
cascade_values = quart.request.args.getlist('cascade')
if cascade_values:
if len(cascade_values) != 1 or cascade_values[0] not in ('true', 'false'):
return self.http_status(400, -1, 'cascade must be a single true or false value')
await self.ap.provider_service.delete_provider(
request_context, provider_uuid, cascade=cascade_values[0] == 'true'
)
else:
await self.ap.provider_service.delete_provider(request_context, provider_uuid)
return self.success()
except ValueError as e:
return self.http_status(400, -1, str(e))
@@ -0,0 +1,15 @@
from __future__ import annotations
def resolve_include_secret(raw_value: str | None, *, permitted: bool) -> tuple[bool, str | None]:
"""Resolve the optional secret projection query parameter."""
if raw_value is None:
return permitted, None
value = raw_value.strip().lower()
if value == 'false':
return False, None
if value == 'true':
return permitted, None
return False, 'include_secret must be either true or false'
@@ -7,14 +7,116 @@ from .. import group
from .....utils import constants
from .....entity.persistence.metadata import WorkspaceMetadata
from ...authz import Permission
from ...context import RequestContext
from ...context import PrincipalType, RequestContext
from .....provider.tools.loaders.mcp_policy import stdio_mcp_enabled
from .....workspace.invitation_delivery import InvitationDeliveryService
SYSTEM_CAPABILITY_OPERATIONS = (
'bot.list',
'bot.get',
'bot.create',
'bot.update',
'bot.delete',
'pipeline.list',
'pipeline.get',
'pipeline.create',
'pipeline.update',
'pipeline.delete',
'pipeline.copy',
'task.list',
'task.get',
'knowledge_base.list',
'knowledge_base.get',
'knowledge_base.create',
'knowledge_base.update',
'knowledge_base.delete',
'knowledge_base.file.list',
'knowledge_base.file.store',
'knowledge_base.file.delete',
'knowledge_base.retrieve',
'file.document.upload',
'plugin.install.github',
'plugin.install.marketplace',
'plugin.install.local',
'plugin.upgrade',
'plugin.get',
'plugin.list',
'plugin.config.get',
'plugin.config.update',
'plugin.logs',
'plugin.delete',
'provider.list',
'provider.get',
'provider.create',
'provider.update',
'provider.delete',
'provider.scan_models',
'model.llm.list',
'model.llm.get',
'model.llm.create',
'model.llm.update',
'model.llm.delete',
'model.llm.test',
'model.embedding.list',
'model.embedding.get',
'model.embedding.create',
'model.embedding.update',
'model.embedding.delete',
'model.embedding.test',
'model.rerank.list',
'model.rerank.get',
'model.rerank.create',
'model.rerank.update',
'model.rerank.delete',
'model.rerank.test',
'skill.list',
'skill.get',
'skill.create',
'skill.update',
'skill.delete',
'skill.files.list',
'skill.files.read',
'skill.files.write',
'skill.preview',
'skill.install.github',
'skill.install.upload',
'mcp_server.list',
'mcp_server.get',
'mcp_server.create',
'mcp_server.update',
'mcp_server.delete',
'mcp_server.resources',
'mcp_server.resource_templates',
'mcp_server.resource_read',
'mcp_server.logs',
'mcp_server.test',
)
@group.group_class('system', '/api/v1/system')
class SystemRouterGroup(group.RouterGroup):
async def initialize(self) -> None:
@self.route('/context', methods=['GET'], auth_type=group.AuthType.API_KEY)
async def _(request_context: RequestContext) -> str:
return self.success(
data={
'instance_uuid': request_context.instance_uuid,
'workspace_uuid': request_context.workspace_uuid,
'api_key_id': request_context.principal.api_key_uuid,
'permissions': sorted(request_context.workspace.permissions),
}
)
@self.route('/capabilities', methods=['GET'], auth_type=group.AuthType.API_KEY)
async def _() -> str:
return self.success(
data={
'schema_version': 1,
'operations': {operation: {'supported': True} for operation in SYSTEM_CAPABILITY_OPERATIONS},
}
)
@self.route('/info', methods=['GET'], auth_type=group.AuthType.NONE)
async def _() -> str:
# Read wizard_status and wizard_progress from metadata table
@@ -238,7 +340,7 @@ class SystemRouterGroup(group.RouterGroup):
@self.route(
'/tasks',
methods=['GET'],
auth_type=group.AuthType.USER_TOKEN,
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)
async def _(request_context: RequestContext) -> str:
@@ -257,18 +359,23 @@ class SystemRouterGroup(group.RouterGroup):
instance_uuid=request_context.instance_uuid,
workspace_uuid=request_context.workspace_uuid,
placement_generation=request_context.placement_generation,
public=request_context.principal.principal_type == PrincipalType.API_KEY,
)
)
@self.route(
'/tasks/<task_id>',
methods=['GET'],
auth_type=group.AuthType.USER_TOKEN,
auth_type=group.AuthType.USER_TOKEN_OR_API_KEY,
permission=Permission.RESOURCE_VIEW,
)
async def _(task_id: str, request_context: RequestContext) -> str:
try:
task_index = int(task_id)
except (TypeError, ValueError):
return self.http_status(404, 404, 'Task not found')
task = self.ap.task_mgr.get_task_by_id(
int(task_id),
task_index,
instance_uuid=request_context.instance_uuid,
workspace_uuid=request_context.workspace_uuid,
placement_generation=request_context.placement_generation,
@@ -277,6 +384,8 @@ class SystemRouterGroup(group.RouterGroup):
if task is None:
return self.http_status(404, 404, 'Task not found')
if request_context.principal.principal_type == PrincipalType.API_KEY:
return self.success(data=task.to_public_dict())
return self.success(data=task.to_dict())
@self.route(
@@ -1,7 +1,12 @@
from __future__ import annotations
import quart
import argon2
import asyncio
import datetime
import hmac
import time
import typing
import uuid
from urllib.parse import parse_qs, urlsplit
@@ -11,6 +16,33 @@ from ...context import RequestContext
from .....cloud.launch import SpaceLaunchError
from ...service.user import ControlPlaneDirectoryRequiredError, PublicRegistrationClosedError
# Fixed-window admission quota for the unauthenticated reset-password endpoint (#2392).
# The admission check and slot bump share ONE synchronous critical section with no await
# points, so concurrent bursts within a single event loop cannot slip past accounting.
# Every admitted attempt consumes quota (regardless of success), which throttles both the
# legacy 24-bit keyspace exhaustion and brute-force on modern high-entropy keys.
# NOTE: this state is process-local; multi-worker deployments need a shared limiter upstream.
_MAX_RESET_ATTEMPTS_PER_WINDOW = 5
_RESET_WINDOW_SECONDS = 15 * 60
_reset_password_state: dict = {'window_started_at': 0.0, 'attempts': 0}
def _admit_reset_attempt(now: float) -> bool:
"""Atomically reserve one reset-password admission slot.
Must stay await-free: running to completion without suspension makes the
check-and-increment atomic under the single-threaded event loop.
"""
st = _reset_password_state
if now - st['window_started_at'] >= _RESET_WINDOW_SECONDS:
st['window_started_at'] = now
st['attempts'] = 0
if st['attempts'] >= _MAX_RESET_ATTEMPTS_PER_WINDOW:
return False
st['attempts'] += 1
return True
@group.group_class('user', '/api/v1/user')
class UserRouterGroup(group.RouterGroup):
@@ -35,6 +67,22 @@ class UserRouterGroup(group.RouterGroup):
return redirect_uri
def _extract_origin_and_rp_id(self, json_data: dict[str, typing.Any] | None = None) -> tuple[str, str]:
origin = ''
if json_data and isinstance(json_data, dict):
origin = json_data.get('origin', '')
if not origin:
origin = quart.request.headers.get('Origin', '')
if not origin:
origin = quart.request.headers.get('Referer', '')
if not origin:
origin = quart.request.url_root.rstrip('/')
parsed = urlsplit(origin)
rp_id = parsed.hostname or 'localhost'
clean_origin = f'{parsed.scheme}://{parsed.netloc}' if parsed.scheme and parsed.netloc else origin.rstrip('/')
return clean_origin, rp_id
async def initialize(self) -> None:
@self.route('/init', methods=['GET', 'POST'], auth_type=group.AuthType.NONE)
async def _() -> str:
@@ -81,6 +129,12 @@ class UserRouterGroup(group.RouterGroup):
@self.route('/reset-password', methods=['POST'], auth_type=group.AuthType.NONE)
async def _() -> str:
# Admit (or reject) BEFORE touching the body or any service call (#2392):
# rejecting requests never reach the slow path, and quota accounting happens
# synchronously at entry, closing the post-await race of burst requests.
if not _admit_reset_attempt(time.monotonic()):
return self.http_status(429, -1, 'Too many attempts, try again later')
json_data = await quart.request.json
user_email = json_data['user']
@@ -98,7 +152,18 @@ class UserRouterGroup(group.RouterGroup):
if user_obj is None:
return self.http_status(400, -1, 'User not found')
if recovery_key != self.ap.instance_config.data['system']['recovery_key']:
stored_key = self.ap.instance_config.data['system']['recovery_key']
try:
key_matches = (
isinstance(recovery_key, str)
and isinstance(stored_key, str)
and hmac.compare_digest(recovery_key.encode(), stored_key.encode())
)
except UnicodeEncodeError:
# JSON can contain lone surrogates, which are not valid UTF-8.
key_matches = False
if not key_matches:
return self.http_status(403, -1, 'Invalid recovery key')
await self.ap.user_service.reset_password(user_email, new_password)
@@ -186,6 +251,9 @@ class UserRouterGroup(group.RouterGroup):
json_data = await quart.request.json
code = json_data.get('code')
state = json_data.get('state')
redirect_uri = json_data.get('redirect_uri') or (
quart.request.url_root.rstrip('/') + '/auth/space/callback'
)
launch_assertion = json_data.get('launch_assertion')
workspace_uuid = json_data.get('workspace_uuid')
@@ -199,8 +267,11 @@ class UserRouterGroup(group.RouterGroup):
return self.fail(1, 'Missing authorization code')
if not state:
return self.fail(1, 'Missing state parameter')
if not str(code).startswith('v4_'):
return self.fail(1, 'Unsupported Space OAuth code contract')
try:
redirect_uri = self._validate_space_redirect_uri(str(redirect_uri), bind=False)
consumed_state = await self.ap.user_service.consume_space_oauth_state_details(state, 'login')
# Exchange code for tokens
launch_workspace_uuid = consumed_state.launch_workspace_uuid
@@ -218,24 +289,36 @@ class UserRouterGroup(group.RouterGroup):
code,
workspace_uuids,
workspace_created_ats,
redirect_uri=redirect_uri,
)
access_token = token_data.get('access_token')
refresh_token = token_data.get('refresh_token')
expires_in = token_data.get('expires_in', 0)
cloud_workspace_uuid = token_data.get('cloud_workspace_uuid')
if not access_token:
return self.fail(1, 'Failed to get access token from Space')
# Authenticate and create/update local user
cloud_mode = getattr(getattr(self.ap, 'deployment', None), 'mode', 'oss') == 'cloud'
if cloud_mode and launch_workspace_uuid and launch_workspace_uuid != cloud_workspace_uuid:
return self.fail(1, 'Space OAuth Workspace binding mismatch')
target_workspace_uuid = launch_workspace_uuid or cloud_workspace_uuid
if cloud_mode:
if not target_workspace_uuid:
return self.fail(1, 'Space OAuth response is missing the Cloud Workspace binding')
await self.ap.directory_projection_service.reconcile_workspaces((target_workspace_uuid,))
# Authenticate only after the signed, exact Workspace delta has
# established the Account and membership runtime shadow rows.
jwt_token, user_obj = await self.ap.user_service.authenticate_space_user(
access_token, refresh_token, expires_in
)
if launch_workspace_uuid:
if target_workspace_uuid:
try:
access = await self.ap.workspace_collaboration_service.resolve_account_workspace(
user_obj.uuid,
launch_workspace_uuid,
target_workspace_uuid,
)
except Exception:
self.ap.logger.warning('Rejected Space OAuth launch for unauthorized Workspace')
@@ -323,6 +406,8 @@ class UserRouterGroup(group.RouterGroup):
capabilities['password_login_enabled'] = False
capabilities['authenticated_invitation_acceptance_enabled'] = cloud_mode
capabilities['invitation_registration_enabled'] = not cloud_mode
capabilities['passkey_login_enabled'] = True
capabilities['passkey_supported'] = True
return self.success(data={'initialized': True, **capabilities})
@self.route('/set-password', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
@@ -367,12 +452,17 @@ class UserRouterGroup(group.RouterGroup):
json_data = await quart.request.json
code = json_data.get('code')
state = json_data.get('state')
redirect_uri = json_data.get('redirect_uri') or (
quart.request.url_root.rstrip('/') + '/auth/space/callback?mode=bind'
)
if not code:
return self.http_status(400, -1, 'Missing authorization code')
if not state:
return self.http_status(400, -1, 'Missing state parameter')
if not str(code).startswith('v4_'):
return self.http_status(400, -1, 'Unsupported Space OAuth code contract')
try:
user_obj = await self.ap.user_service.consume_space_oauth_state(state, 'bind')
@@ -385,7 +475,10 @@ class UserRouterGroup(group.RouterGroup):
return self.http_status(400, -1, 'Only local accounts can bind to Space')
try:
updated_user = await self.ap.user_service.bind_space_account(user_obj.user, code)
redirect_uri = self._validate_space_redirect_uri(str(redirect_uri), bind=True)
updated_user = await self.ap.user_service.bind_space_account(
user_obj.user, code, redirect_uri=redirect_uri
)
jwt_token = await self.ap.user_service.generate_jwt_token(updated_user)
return self.success(
data={
@@ -405,6 +498,182 @@ class UserRouterGroup(group.RouterGroup):
except Exception:
raise
@self.route('/passkey/register/options', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
async def _(user_email: str) -> str:
"""Generate WebAuthn registration options for current account."""
allow_modify_login_info = self.ap.instance_config.data.get('system', {}).get(
'allow_modify_login_info', True
)
if not allow_modify_login_info:
return self.http_status(403, -1, 'Modifying login info is disabled')
user_obj = await self.ap.user_service.get_user_by_email(user_email)
if user_obj is None:
return self.http_status(404, -1, 'User not found')
json_data = (await quart.request.json) or {}
origin, rp_id = self._extract_origin_and_rp_id(json_data)
try:
options, challenge_token = await self.ap.user_service.generate_passkey_registration_options(
account_uuid=user_obj.uuid,
rp_id=rp_id,
origin=origin,
rp_name='LangBot',
)
return self.success(data={'options': options, 'challenge_token': challenge_token})
except Exception as e:
return self.fail(1, str(e))
@self.route('/passkey/register/verify', methods=['POST'], auth_type=group.AuthType.USER_TOKEN)
async def _(user_email: str) -> str:
"""Verify WebAuthn registration response and save credential."""
allow_modify_login_info = self.ap.instance_config.data.get('system', {}).get(
'allow_modify_login_info', True
)
if not allow_modify_login_info:
return self.http_status(403, -1, 'Modifying login info is disabled')
user_obj = await self.ap.user_service.get_user_by_email(user_email)
if user_obj is None:
return self.http_status(404, -1, 'User not found')
json_data = await quart.request.json
challenge_token = json_data.get('challenge_token')
credential = json_data.get('credential') or json_data.get('response')
name = json_data.get('name')
if not challenge_token or not credential:
return self.fail(1, 'Missing challenge_token or credential')
try:
cred = await self.ap.user_service.verify_and_save_passkey_registration(
challenge_token=challenge_token,
credential_data=credential,
name=name,
)
return self.success(
data={
'uuid': cred.uuid,
'name': cred.name,
'created_at': cred.created_at.isoformat() if cred.created_at else None,
}
)
except Exception as e:
return self.fail(1, str(e))
@self.route('/passkey/auth/options', methods=['POST'], auth_type=group.AuthType.NONE)
async def _() -> str:
"""Generate WebAuthn authentication options for passkey login."""
json_data = (await quart.request.json) or {}
email = json_data.get('email')
origin, rp_id = self._extract_origin_and_rp_id(json_data)
try:
options, challenge_token = await self.ap.user_service.generate_passkey_authentication_options(
rp_id=rp_id,
origin=origin,
email=email,
)
return self.success(data={'options': options, 'challenge_token': challenge_token})
except Exception as e:
return self.fail(1, str(e))
@self.route('/passkey/auth/verify', methods=['POST'], auth_type=group.AuthType.NONE)
async def _() -> str:
"""Verify WebAuthn authentication response and log in."""
json_data = await quart.request.json
challenge_token = json_data.get('challenge_token')
credential = json_data.get('credential') or json_data.get('response')
if not challenge_token or not credential:
return self.fail(1, 'Missing challenge_token or credential')
try:
token, user_obj = await self.ap.user_service.verify_passkey_authentication(
challenge_token=challenge_token,
credential_data=credential,
)
return self.success(
data={
'token': token,
'user': user_obj.user,
}
)
except Exception as e:
return self.fail(1, str(e))
@self.route('/passkeys', methods=['GET'], auth_type=group.AuthType.USER_TOKEN)
async def _(user_email: str) -> str:
"""List registered passkeys for the current user."""
user_obj = await self.ap.user_service.get_user_by_email(user_email)
if user_obj is None:
return self.http_status(404, -1, 'User not found')
passkeys = await self.ap.user_service.get_user_passkeys(user_obj.uuid)
return self.success(
data=[
{
'uuid': pk.uuid,
'name': pk.name,
'aaguid': pk.aaguid,
'transports': pk.transports,
'backed_up': pk.backed_up,
'created_at': pk.created_at.isoformat() if pk.created_at else None,
'last_used_at': pk.last_used_at.isoformat() if pk.last_used_at else None,
}
for pk in passkeys
]
)
@self.route('/passkey/<passkey_uuid>', methods=['PATCH'], auth_type=group.AuthType.USER_TOKEN)
async def _(user_email: str, passkey_uuid: str) -> str:
"""Rename a registered passkey."""
allow_modify_login_info = self.ap.instance_config.data.get('system', {}).get(
'allow_modify_login_info', True
)
if not allow_modify_login_info:
return self.http_status(403, -1, 'Modifying login info is disabled')
user_obj = await self.ap.user_service.get_user_by_email(user_email)
if user_obj is None:
return self.http_status(404, -1, 'User not found')
json_data = await quart.request.json
name = (json_data.get('name') or '').strip()
if not name:
return self.fail(1, 'Passkey name cannot be empty')
updated = await self.ap.user_service.rename_user_passkey(
account_uuid=user_obj.uuid,
passkey_uuid=passkey_uuid,
new_name=name,
)
if not updated:
return self.http_status(404, -1, 'Passkey not found')
return self.success(data={'uuid': updated.uuid, 'name': updated.name})
@self.route('/passkey/<passkey_uuid>', methods=['DELETE'], auth_type=group.AuthType.USER_TOKEN)
async def _(user_email: str, passkey_uuid: str) -> str:
"""Delete/revoke a registered passkey."""
allow_modify_login_info = self.ap.instance_config.data.get('system', {}).get(
'allow_modify_login_info', True
)
if not allow_modify_login_info:
return self.http_status(403, -1, 'Modifying login info is disabled')
user_obj = await self.ap.user_service.get_user_by_email(user_email)
if user_obj is None:
return self.http_status(404, -1, 'User not found')
deleted = await self.ap.user_service.delete_user_passkey(
account_uuid=user_obj.uuid,
passkey_uuid=passkey_uuid,
)
if not deleted:
return self.http_status(404, -1, 'Passkey not found')
return self.success()
async def _handle_space_direct_launch(
self,
launch_assertion: str,
@@ -428,6 +697,10 @@ class UserRouterGroup(group.RouterGroup):
}
)
projection_service = self.ap.directory_projection_service
if projection_service is None:
raise SpaceLaunchError('Cloud directory projection is unavailable')
await projection_service.reconcile_workspaces((launch['workspace_uuid'],))
account = await self.ap.user_service.get_user_by_uuid(launch['account_uuid'])
if account is None:
raise SpaceLaunchError('Launch Account is not projected into Core')
@@ -80,16 +80,40 @@ class MaintenanceService:
DEFAULT_LOG_RETENTION_DAYS,
'storage.cleanup.log_retention_days',
)
media_cfg = self.ap.instance_config.data.get('storage', {}).get('media_cache', {})
media_retention_days = self._positive_int(
media_cfg.get('retention_days'),
30,
'storage.media_cache.retention_days',
)
media_max_size_mb = self._non_negative_int(
media_cfg.get('max_size_mb'),
0,
'storage.media_cache.max_size_mb',
)
media_cache = getattr(getattr(self.ap, 'storage_mgr', None), 'media_cache', None)
is_singleton = await self._is_oss_singleton(context)
media_cleanup = (
await media_cache.cleanup(
media_retention_days,
media_max_size_mb,
)
if media_cache is not None and is_singleton
else {}
)
return {
result = {
'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days),
'log_files': await asyncio.to_thread(
self._cleanup_expired_log_files,
log_retention_days,
)
if await self._is_oss_singleton(context)
if is_singleton
else 0,
}
if media_cache is not None and is_singleton:
result['media_files'] = media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0)
return result
async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]:
require_workspace_uuid(context)
@@ -573,6 +597,17 @@ class MaintenanceService:
count += len(files)
return count
def _non_negative_int(self, value: Any, default: int, name: str) -> int:
try:
parsed = int(value)
except (TypeError, ValueError):
self.ap.logger.warning(f'Invalid {name}: {value!r}, using {default}')
return default
if parsed < 0:
self.ap.logger.warning(f'{name} must be non-negative: {value!r}, using {default}')
return default
return parsed
def _positive_int(self, value: Any, default: int, name: str) -> int:
try:
parsed = int(value)
+16 -9
View File
@@ -446,15 +446,19 @@ class MCPService:
persisted_session = runtime_mcp_session
async def _refresh_and_report() -> None:
needs_start = persisted_session.status == MCPSessionStatus.ERROR or persisted_session.session is None
if needs_start:
await persisted_session.start()
else:
try:
await persisted_session.refresh()
except Exception:
try:
needs_start = (
persisted_session.status == MCPSessionStatus.ERROR or persisted_session.session is None
)
if needs_start:
await persisted_session.start()
ctx.metadata['runtime_info'] = persisted_session.get_runtime_info_dict()
else:
try:
await persisted_session.refresh()
except Exception:
await persisted_session.start()
finally:
ctx.metadata['runtime_info'] = persisted_session.get_runtime_info_dict()
coroutine = _refresh_and_report()
else:
@@ -471,8 +475,11 @@ class MCPService:
async def _run_and_cleanup() -> None:
try:
await test_session.start()
ctx.metadata['runtime_info'] = test_session.get_runtime_info_dict()
finally:
# start() raises for a failed connection. Preserve the
# terminal runtime state so the UI can render actionable
# failure phases such as OAuth-required.
ctx.metadata['runtime_info'] = test_session.get_runtime_info_dict()
try:
await test_session.shutdown()
except Exception as exc:
+112 -23
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import re
import uuid
import datetime
import functools
@@ -29,6 +30,19 @@ _DEFAULT_CLEANUP_BATCHES_PER_TABLE = 4
_HARD_MAX_CLEANUP_BATCHES_PER_TABLE = 100
def _normalize_user_id(value: str | int | None) -> str | None:
"""Convert numeric platform IDs before binding a VARCHAR with asyncpg.
Opaque string IDs (including whitespace and leading zeros) and missing
IDs must remain unchanged. Do not silently stringify unsupported objects.
"""
if value is None or isinstance(value, str):
return value
if isinstance(value, int) and not isinstance(value, bool):
return str(value)
raise TypeError('user_id must be a string, integer, or None')
def _workspace_transaction(method):
"""Run an explicit service entrypoint in one Workspace transaction."""
@@ -281,19 +295,21 @@ class MonitoringService:
for _batch_number in range(max_batches):
async def delete_batch() -> tuple[int, int]:
key_columns = list(model_cls.__table__.primary_key.columns)
select_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(pk_column)
sqlalchemy.select(*key_columns)
.where(model_cls.workspace_uuid == workspace_uuid, ts_column < cutoff)
.limit(batch_size)
)
pk_values = list(select_result.scalars().all())
pk_values = [tuple(row) for row in select_result.all()]
if not pk_values:
return 0, 0
delete_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.delete(model_cls).where(
model_cls.workspace_uuid == workspace_uuid,
pk_column.in_(pk_values),
sqlalchemy.tuple_(*key_columns).in_(pk_values),
ts_column < cutoff,
)
)
return len(pk_values), int(delete_result.rowcount or 0)
@@ -402,6 +418,32 @@ class MonitoringService:
# ========== Recording Methods ==========
def _sanitize_message_content(self, content: str) -> str:
"""Strip raw base64 data to protect database storage from unbounded bloating."""
if not content or len(content) < 10000 or (';base64,' not in content and 'data:image/' not in content):
return content
try:
data = json.loads(content)
def _strip_node(node):
if isinstance(node, list):
return [_strip_node(x) for x in node]
if isinstance(node, dict):
res = dict(node)
if res.get('type') == 'Image' and res.get('base64'):
res['base64'] = None
for k, v in list(res.items()):
if isinstance(v, (list, dict)):
res[k] = _strip_node(v)
return res
return node
return json.dumps(_strip_node(data), ensure_ascii=False)
except Exception:
return re.sub(
r'data:image/[a-zA-Z0-9.+_-]+;base64,[\sA-Za-z0-9+/=]{1000,}', '[base64 image omitted]', content
)
@_workspace_transaction
async def record_message(
self,
@@ -415,7 +457,7 @@ class MonitoringService:
status: str = 'success',
level: str = 'info',
platform: str | None = None,
user_id: str | None = None,
user_id: str | int | None = None,
user_name: str | None = None,
runner_name: str | None = None,
variables: str | None = None,
@@ -424,6 +466,7 @@ class MonitoringService:
"""Record a message"""
workspace_uuid = self._require_write_context(context)
message_id = str(uuid.uuid4())
message_content = self._sanitize_message_content(message_content)
message_data = {
'id': message_id,
'workspace_uuid': workspace_uuid,
@@ -437,7 +480,7 @@ class MonitoringService:
'status': status,
'level': level,
'platform': platform,
'user_id': user_id,
'user_id': _normalize_user_id(user_id),
'user_name': user_name,
'runner_name': runner_name,
'variables': variables,
@@ -610,7 +653,7 @@ class MonitoringService:
pipeline_id: str,
pipeline_name: str,
platform: str | None = None,
user_id: str | None = None,
user_id: str | int | None = None,
user_name: str | None = None,
) -> None:
"""Record a new session"""
@@ -622,17 +665,29 @@ class MonitoringService:
'bot_name': bot_name,
'pipeline_id': pipeline_id,
'pipeline_name': pipeline_name,
'message_count': 0,
'message_count': 1,
'start_time': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
'last_activity': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
'is_active': True,
'platform': platform,
'user_id': user_id,
'user_id': _normalize_user_id(user_id),
'user_name': user_name,
}
model = persistence_monitoring.MonitoringSession
dialect = self.ap.persistence_mgr.get_db_engine().dialect.name
insert = postgresql_dialect.insert if dialect == 'postgresql' else sqlite_dialect.insert
statement = insert(model).values(session_data)
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(persistence_monitoring.MonitoringSession).values(session_data)
statement.on_conflict_do_update(
index_elements=['workspace_uuid', 'bot_id', 'session_id'],
set_={
'message_count': model.message_count + 1,
'last_activity': statement.excluded.last_activity,
'pipeline_id': statement.excluded.pipeline_id,
'pipeline_name': statement.excluded.pipeline_name,
},
)
)
@_workspace_transaction
@@ -642,6 +697,7 @@ class MonitoringService:
session_id: str,
pipeline_id: str | None = None,
pipeline_name: str | None = None,
bot_id: str | None = None,
) -> bool:
"""Update session last activity time and increment message count.
@@ -651,6 +707,9 @@ class MonitoringService:
True if session was found and updated, False if session doesn't exist.
"""
workspace_uuid = self._require_write_context(context)
bot_id = bot_id if bot_id is not None else context.bot_uuid
if not bot_id:
raise ValueError('Session activity requires a bot_id')
update_values = {
'last_activity': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
'message_count': persistence_monitoring.MonitoringSession.message_count + 1,
@@ -667,6 +726,7 @@ class MonitoringService:
.where(
persistence_monitoring.MonitoringSession.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringSession.session_id == session_id,
persistence_monitoring.MonitoringSession.bot_id == bot_id,
)
.values(update_values)
)
@@ -769,13 +829,13 @@ class MonitoringService:
message_conditions.append(persistence_monitoring.MonitoringMessage.timestamp >= start_time)
llm_conditions.append(persistence_monitoring.MonitoringLLMCall.timestamp >= start_time)
embedding_conditions.append(persistence_monitoring.MonitoringEmbeddingCall.timestamp >= start_time)
session_conditions.append(persistence_monitoring.MonitoringSession.start_time >= start_time)
session_conditions.append(persistence_monitoring.MonitoringSession.last_activity >= start_time)
if end_time:
message_conditions.append(persistence_monitoring.MonitoringMessage.timestamp <= end_time)
llm_conditions.append(persistence_monitoring.MonitoringLLMCall.timestamp <= end_time)
embedding_conditions.append(persistence_monitoring.MonitoringEmbeddingCall.timestamp <= end_time)
session_conditions.append(persistence_monitoring.MonitoringSession.start_time <= end_time)
session_conditions.append(persistence_monitoring.MonitoringSession.last_activity <= end_time)
# Total messages
message_query = sqlalchemy.select(sqlalchemy.func.count(persistence_monitoring.MonitoringMessage.id))
@@ -1257,6 +1317,7 @@ class MonitoringService:
pipeline_ids: list[str] | None = None,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
user_query: str | None = None,
is_active: bool | None = None,
limit: int = 100,
offset: int = 0,
@@ -1271,9 +1332,17 @@ class MonitoringService:
if pipeline_ids:
conditions.append(persistence_monitoring.MonitoringSession.pipeline_id.in_(pipeline_ids))
if start_time:
conditions.append(persistence_monitoring.MonitoringSession.start_time >= start_time)
conditions.append(persistence_monitoring.MonitoringSession.last_activity >= start_time)
if end_time:
conditions.append(persistence_monitoring.MonitoringSession.start_time <= end_time)
conditions.append(persistence_monitoring.MonitoringSession.last_activity <= end_time)
if user_query and user_query.strip():
user_pattern = f'%{user_query.strip()}%'
conditions.append(
sqlalchemy.or_(
persistence_monitoring.MonitoringSession.user_id.ilike(user_pattern),
persistence_monitoring.MonitoringSession.user_name.ilike(user_pattern),
)
)
if is_active is not None:
conditions.append(persistence_monitoring.MonitoringSession.is_active == is_active)
@@ -1365,6 +1434,9 @@ class MonitoringService:
self,
context: TenantContext,
session_id: str,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
bot_id: str | None = None,
) -> dict:
"""Get bounded session details with full statistics computed in SQL."""
workspace_uuid = require_workspace_uuid(context)
@@ -1374,8 +1446,13 @@ class MonitoringService:
persistence_monitoring.MonitoringSession.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringSession.session_id == session_id,
)
session_result = await self.ap.persistence_mgr.execute_async(session_query)
session_row = session_result.first()
if bot_id is not None:
session_query = session_query.where(persistence_monitoring.MonitoringSession.bot_id == bot_id)
session_result = await self.ap.persistence_mgr.execute_async(session_query.limit(2))
session_rows = session_result.all()
if len(session_rows) > 1:
return {'session_id': session_id, 'found': False, 'ambiguous': True}
session_row = session_rows[0] if session_rows else None
if not session_row:
return {
@@ -1384,6 +1461,7 @@ class MonitoringService:
}
session = session_row[0] if isinstance(session_row, tuple) else session_row
bot_id = session.bot_id
message_stats_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(
@@ -1411,6 +1489,7 @@ class MonitoringService:
).where(
persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringMessage.session_id == session_id,
persistence_monitoring.MonitoringMessage.bot_id == bot_id,
)
)
message_stats = message_stats_result.one()
@@ -1449,6 +1528,7 @@ class MonitoringService:
).where(
persistence_monitoring.MonitoringLLMCall.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringLLMCall.session_id == session_id,
persistence_monitoring.MonitoringLLMCall.bot_id == bot_id,
)
)
llm_stats = llm_stats_result.one()
@@ -1475,15 +1555,22 @@ class MonitoringService:
).where(
persistence_monitoring.MonitoringToolCall.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringToolCall.session_id == session_id,
persistence_monitoring.MonitoringToolCall.bot_id == bot_id,
)
)
tool_stats = tool_stats_result.one()
tool_conditions = [
persistence_monitoring.MonitoringToolCall.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringToolCall.session_id == session_id,
persistence_monitoring.MonitoringToolCall.bot_id == bot_id,
]
if start_time is not None:
tool_conditions.append(persistence_monitoring.MonitoringToolCall.timestamp >= start_time)
if end_time is not None:
tool_conditions.append(persistence_monitoring.MonitoringToolCall.timestamp <= end_time)
tool_query = (
sqlalchemy.select(persistence_monitoring.MonitoringToolCall)
.where(
persistence_monitoring.MonitoringToolCall.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringToolCall.session_id == session_id,
)
.where(*tool_conditions)
.order_by(persistence_monitoring.MonitoringToolCall.timestamp.asc())
.limit(detail_limit + 1)
)
@@ -1504,6 +1591,7 @@ class MonitoringService:
.where(
persistence_monitoring.MonitoringError.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringError.session_id == session_id,
persistence_monitoring.MonitoringError.bot_id == bot_id,
)
.order_by(persistence_monitoring.MonitoringError.timestamp.desc())
.limit(detail_limit + 1)
@@ -1988,9 +2076,9 @@ class MonitoringService:
if pipeline_ids:
conditions.append(persistence_monitoring.MonitoringSession.pipeline_id.in_(pipeline_ids))
if start_time:
conditions.append(persistence_monitoring.MonitoringSession.start_time >= start_time)
conditions.append(persistence_monitoring.MonitoringSession.last_activity >= start_time)
if end_time:
conditions.append(persistence_monitoring.MonitoringSession.start_time <= end_time)
conditions.append(persistence_monitoring.MonitoringSession.last_activity <= end_time)
query = sqlalchemy.select(persistence_monitoring.MonitoringSession).order_by(
persistence_monitoring.MonitoringSession.last_activity.desc()
@@ -2024,6 +2112,7 @@ class MonitoringService:
# ========== Feedback Methods ==========
@_workspace_transaction
async def record_feedback(
self,
context: ExecutionContext,
@@ -2038,7 +2127,7 @@ class MonitoringService:
session_id: str | None = None,
message_id: str | None = None,
stream_id: str | None = None,
user_id: str | None = None,
user_id: str | int | None = None,
platform: str | None = None,
) -> str | None:
"""Record user feedback (like/dislike) from AI Bot conversation.
@@ -2094,7 +2183,7 @@ class MonitoringService:
'session_id': session_id,
'message_id': message_id,
'stream_id': stream_id,
'user_id': user_id,
'user_id': _normalize_user_id(user_id),
'platform': platform,
}
dialect_name = self.ap.persistence_mgr.get_db_engine().dialect.name
@@ -0,0 +1,83 @@
"""Bounded traffic aggregation, independent of record-list pagination."""
from __future__ import annotations
import datetime
import typing
import sqlalchemy
from ....entity.persistence.monitoring import MonitoringLLMCall, MonitoringMessage
from .tenant import TenantContext, require_workspace_uuid
if typing.TYPE_CHECKING:
from ....core.app import Application
MAX_TRAFFIC_POINTS = 1000
async def get_traffic_series(
ap: Application,
context: TenantContext,
*,
bot_ids: list[str] | None = None,
pipeline_ids: list[str] | None = None,
start_time: datetime.datetime | None = None,
end_time: datetime.datetime | None = None,
) -> dict:
"""Count all matching records in UTC buckets, returning at most 1000 points."""
workspace_uuid = require_workspace_uuid(context)
bucket = 'hour' if start_time and end_time and end_time - start_time <= datetime.timedelta(days=7) else 'day'
step = datetime.timedelta(hours=1) if bucket == 'hour' else datetime.timedelta(days=1)
postgres = ap.persistence_mgr.get_db_engine().dialect.name == 'postgresql'
points: dict[datetime.datetime, dict[str, int]] = {}
truncated = False
for model, field in ((MonitoringMessage, 'messages'), (MonitoringLLMCall, 'llm_calls')):
timestamp = model.timestamp
if postgres:
time_bucket = sqlalchemy.func.date_trunc(bucket, timestamp)
else:
pattern = '%Y-%m-%dT%H:00:00' if bucket == 'hour' else '%Y-%m-%dT00:00:00'
time_bucket = sqlalchemy.func.strftime(pattern, timestamp)
conditions = [model.workspace_uuid == workspace_uuid]
if bot_ids:
conditions.append(model.bot_id.in_(bot_ids))
if pipeline_ids:
conditions.append(model.pipeline_id.in_(pipeline_ids))
if start_time is not None:
conditions.append(timestamp >= start_time)
if end_time is not None:
conditions.append(timestamp <= end_time)
statement = (
sqlalchemy.select(time_bucket.label('bucket'), sqlalchemy.func.count(model.id).label('count'))
.where(*conditions)
.group_by(time_bucket)
.order_by(time_bucket)
.limit(MAX_TRAFFIC_POINTS + 1)
)
result = await ap.persistence_mgr.execute_async(statement)
rows = result.all()
truncated = truncated or len(rows) > MAX_TRAFFIC_POINTS
for timestamp_value, count in rows[:MAX_TRAFFIC_POINTS]:
key = (
datetime.datetime.fromisoformat(timestamp_value)
if isinstance(timestamp_value, str)
else timestamp_value
)
points.setdefault(key, {'messages': 0, 'llm_calls': 0})[field] = int(count)
def floor(value: datetime.datetime) -> datetime.datetime:
return value.replace(minute=0, second=0, microsecond=0, **({'hour': 0} if bucket == 'day' else {}))
first = floor(start_time) if start_time is not None else min(points, default=None)
last = floor(end_time) if end_time is not None else max(points, default=None)
series = []
if first is not None and last is not None:
cursor = first
while cursor <= last and len(series) < MAX_TRAFFIC_POINTS:
series.append(
{'timestamp': cursor.isoformat() + 'Z', **points.get(cursor, {'messages': 0, 'llm_calls': 0})}
)
cursor += step
truncated = truncated or cursor <= last
return {'bucket': bucket, 'points': series, 'truncated': truncated}
+129 -50
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
import uuid
import traceback
@@ -7,8 +8,10 @@ import sqlalchemy
from ....cloud.model_catalog import LANGBOT_MODELS_PROVIDER_REQUESTER
from ....core import app
from ....core.task_boundary import create_detached_task
from ....entity.persistence import model as persistence_model
from ....workspace.errors import WorkspaceNotFoundError
from ....provider.modelmgr.codex_auth import CodexAuth, REQUESTER as CODEX_REQUESTER, validate_config
from .secrets import contains_secret_placeholder, redact_secrets, restore_secret_placeholders
from .tenant import TenantContext, require_workspace_uuid, scope_statement
@@ -20,6 +23,8 @@ class ModelProviderService:
def __init__(self, ap: app.Application) -> None:
self.ap = ap
self.codex_auth = CodexAuth(ap)
self._deletion_tasks: set[asyncio.Task[None]] = set()
def _is_cloud_runtime(self) -> bool:
mode = getattr(self.ap.persistence_mgr, 'mode', None)
@@ -116,14 +121,30 @@ class ModelProviderService:
provider_data = provider_data.copy()
if self._system_requester_is_reserved(provider_data.get('requester')):
raise ValueError('space-chat-completions is reserved for the Cloud-managed LangBot Models provider')
validate_config(provider_data)
provider_data['uuid'] = str(uuid.uuid4())
provider_data['workspace_uuid'] = require_workspace_uuid(context)
provider_data['api_keys'] = self._normalize_api_keys(
restore_secret_placeholders(provider_data.get('api_keys'), sensitive=True)
)
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(persistence_model.ModelProvider).values(**provider_data)
)
if provider_data.get('requester') == CODEX_REQUESTER:
async with self.ap.persistence_mgr.tenant_uow(provider_data['workspace_uuid']):
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(persistence_model.ModelProvider).values(**provider_data)
)
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(persistence_model.CodexCredential).values(
workspace_uuid=provider_data['workspace_uuid'],
provider_uuid=provider_data['uuid'],
payload={},
version=0,
lease_until=0,
)
)
else:
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(persistence_model.ModelProvider).values(**provider_data)
)
# load to runtime
runtime_provider = await self.ap.model_mgr.load_provider(context, provider_data)
@@ -138,6 +159,17 @@ class ModelProviderService:
raise ValueError('space-chat-completions is reserved for the Cloud-managed LangBot Models provider')
provider_data.pop('uuid', None)
provider_data.pop('workspace_uuid', None)
if {'requester', 'base_url', 'api_keys'} & provider_data.keys():
current = await self.get_provider(context, provider_uuid, include_secret=True)
if current is None:
raise WorkspaceNotFoundError('Provider not found')
if CODEX_REQUESTER in (current.get('requester'), provider_data.get('requester')):
if provider_data.get('requester', current.get('requester')) != current.get('requester'):
raise ValueError('Create a separate provider to change the ChatGPT authentication type')
merged = {**current, **provider_data}
validate_config(merged)
provider_data['base_url'] = merged['base_url']
provider_data['api_keys'] = []
if 'api_keys' in provider_data:
submitted_keys = provider_data.get('api_keys')
if contains_secret_placeholder(submitted_keys, sensitive=True):
@@ -163,60 +195,107 @@ class ModelProviderService:
raise WorkspaceNotFoundError('Provider not found')
await self.ap.model_mgr.reload_provider(context, provider_uuid)
async def delete_provider(self, context: TenantContext, provider_uuid: str) -> None:
"""Delete a provider (only if no models reference it)"""
await self._assert_provider_mutable(context, provider_uuid)
async def delete_provider(self, context: TenantContext, provider_uuid: str, cascade: bool = False) -> None:
"""Delete a provider, optionally deleting all its Workspace-scoped models."""
workspace_uuid = require_workspace_uuid(context)
# Check if any models use this provider
llm_result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_model.LLMModel).where(
persistence_model.LLMModel.provider_uuid == provider_uuid
),
persistence_model.LLMModel,
workspace_uuid,
)
persistence = self.ap.persistence_mgr
model_types = (
(persistence_model.LLMModel, 'LLM', 'remove_llm_model'),
(persistence_model.EmbeddingModel, 'Embedding', 'remove_embedding_model'),
(persistence_model.RerankModel, 'Rerank', 'remove_rerank_model'),
)
if llm_result.first() is not None:
raise ValueError('Cannot delete provider: LLM models still reference it')
embedding_result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_model.EmbeddingModel).where(
persistence_model.EmbeddingModel.provider_uuid == provider_uuid
),
persistence_model.EmbeddingModel,
workspace_uuid,
deleted_models: list[tuple[str, list[str]]] = []
async with persistence.tenant_uow(workspace_uuid):
# Check ownership before touching children. Lock the provider on PostgreSQL
# so concurrent model inserts cannot race the reference check/deletion.
provider_result = await persistence.execute_async(
scope_statement(
sqlalchemy.select(persistence_model.ModelProvider.requester)
.where(persistence_model.ModelProvider.uuid == provider_uuid)
.with_for_update(),
persistence_model.ModelProvider,
workspace_uuid,
)
)
)
if embedding_result.first() is not None:
raise ValueError('Cannot delete provider: Embedding models still reference it')
provider = provider_result.first()
if provider is None:
raise WorkspaceNotFoundError('Provider not found')
if self._system_requester_is_reserved(provider.requester):
raise ValueError('LangBot Models is managed by Cloud and cannot be modified')
rerank_result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.select(persistence_model.RerankModel).where(
persistence_model.RerankModel.provider_uuid == provider_uuid
),
persistence_model.RerankModel,
workspace_uuid,
for model_type, label, remover in model_types:
result = await persistence.execute_async(
scope_statement(
sqlalchemy.select(model_type.uuid).where(model_type.provider_uuid == provider_uuid),
model_type,
workspace_uuid,
)
)
model_uuids = list(result.scalars())
if model_uuids and not cascade:
raise ValueError(f'Cannot delete provider: {label} models still reference it')
if model_uuids:
# Model services have no pipeline/KB deletion side effects: they
# delete the scoped row and evict its runtime cache. Defer eviction
# here rather than calling those services before our commit.
await persistence.execute_async(
scope_statement(
sqlalchemy.delete(model_type).where(model_type.provider_uuid == provider_uuid),
model_type,
workspace_uuid,
)
)
deleted_models.append((remover, model_uuids))
# Explicit cleanup also works on legacy SQLite connections without FK
# enforcement; never load or serialize the private credential payload.
await persistence.execute_async(
scope_statement(
sqlalchemy.delete(persistence_model.CodexCredential).where(
persistence_model.CodexCredential.provider_uuid == provider_uuid
),
persistence_model.CodexCredential,
workspace_uuid,
)
)
)
if rerank_result.first() is not None:
raise ValueError('Cannot delete provider: Rerank models still reference it')
result = await self.ap.persistence_mgr.execute_async(
scope_statement(
sqlalchemy.delete(persistence_model.ModelProvider).where(
persistence_model.ModelProvider.uuid == provider_uuid
),
persistence_model.ModelProvider,
workspace_uuid,
result = await persistence.execute_async(
scope_statement(
sqlalchemy.delete(persistence_model.ModelProvider).where(
persistence_model.ModelProvider.uuid == provider_uuid
),
persistence_model.ModelProvider,
workspace_uuid,
)
)
)
if getattr(result, 'rowcount', None) == 0:
raise WorkspaceNotFoundError('Provider not found')
if result.rowcount == 0:
raise WorkspaceNotFoundError('Provider not found')
await self.ap.model_mgr.remove_provider(context, provider_uuid)
async def remove_runtime() -> None:
async with persistence.tenant_scope(workspace_uuid):
for remover, model_uuids in deleted_models:
for model_uuid in model_uuids:
await getattr(self.ap.model_mgr, remover)(context, model_uuid)
# This also closes the requester's HTTP client; models go first.
await self.ap.model_mgr.remove_provider(context, provider_uuid)
if persistence.current_session() is None:
await remove_runtime()
else:
# A nested UoW has not committed yet. Reuse the rollback-cancelled gate
# and detached context boundary instead of evicting uncommitted data.
task = create_detached_task(
remove_runtime(),
after_commit_manager=persistence,
workspace_uuid=workspace_uuid,
)
self._deletion_tasks.add(task)
def completed(task: asyncio.Task[None]) -> None:
self._deletion_tasks.discard(task)
if not task.cancelled() and task.exception() is not None:
self.ap.logger.error('Failed to remove deleted provider runtime', exc_info=task.exception())
task.add_done_callback(completed)
async def get_provider_model_counts(self, context: TenantContext, provider_uuid: str) -> dict:
"""Get count of models using this provider"""
+4 -1
View File
@@ -119,7 +119,7 @@ class SpaceService:
space_config = self._get_space_config()
authorize_url = space_config['oauth_authorize_url']
params = {'redirect_uri': redirect_uri}
params = {'redirect_uri': redirect_uri, 'code_contract': 'redirect-v1'}
if state:
params['state'] = state
return f'{authorize_url}?{urlencode(params)}'
@@ -129,6 +129,8 @@ class SpaceService:
code: str,
workspace_uuids: list[str] | None = None,
workspace_created_ats: dict[str, int] | None = None,
*,
redirect_uri: str = '',
) -> typing.Dict:
"""Exchange OAuth authorization code for tokens"""
from langbot.pkg.utils import constants
@@ -141,6 +143,7 @@ class SpaceService:
f'{space_url}/api/v1/accounts/oauth/token',
json={
'code': code,
'redirect_uri': redirect_uri,
'instance_id': constants.instance_id,
# Sending an explicit empty list tells new Space servers not to
# synthesize a legacy instance-derived Workspace binding.
+336 -2
View File
@@ -4,6 +4,7 @@ import sqlalchemy
import argon2
import jwt
import datetime
import json
import typing
import asyncio
import dataclasses
@@ -12,10 +13,19 @@ import hashlib
import secrets
import time
import uuid
import webauthn
from webauthn.helpers import bytes_to_base64url, base64url_to_bytes
from webauthn.helpers.structs import (
AuthenticatorSelectionCriteria,
PublicKeyCredentialDescriptor,
ResidentKeyRequirement,
UserVerificationRequirement,
)
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from ....entity.persistence import user
from ....entity.persistence import passkey
from ....entity.persistence.workspace import MembershipRole, MembershipStatus, WorkspaceMembership
from ....utils import constants
from ....entity.errors import account as account_errors
@@ -29,6 +39,9 @@ if typing.TYPE_CHECKING:
_SPACE_OAUTH_STATE_MAX_ENTRIES = 4096
_SPACE_OAUTH_STATE_HEAP_COMPACT_FLOOR = 64
_SPACE_OAUTH_STATE_HEAP_MAX_MULTIPLIER = 4
_PASSKEY_CHALLENGE_MAX_ENTRIES = 4096
_PASSKEY_CHALLENGE_HEAP_COMPACT_FLOOR = 64
_PASSKEY_CHALLENGE_HEAP_MAX_MULTIPLIER = 4
class AccountExistsLoginRequiredError(ValueError):
@@ -54,6 +67,17 @@ class SpaceOAuthStateConsumption:
launch_workspace_uuid: str | None = None
@dataclasses.dataclass(frozen=True, slots=True)
class PasskeyChallengeData:
challenge: bytes
purpose: typing.Literal['register', 'auth']
rp_id: str
origin: str
expires_at: float
account_uuid: str | None = None
user_email: str | None = None
class UserService:
ap: Application
_create_user_lock: asyncio.Lock
@@ -65,6 +89,9 @@ class UserService:
self._space_oauth_state_lock = asyncio.Lock()
self._space_oauth_states: dict[str, tuple[str, str | None, float, str | None]] = {}
self._space_oauth_state_expiry_heap: list[tuple[float, str]] = []
self._passkey_challenge_lock = asyncio.Lock()
self._passkey_challenges: dict[str, PasskeyChallengeData] = {}
self._passkey_challenge_expiry_heap: list[tuple[float, str]] = []
@staticmethod
def _space_oauth_state_digest(state: str) -> str:
@@ -774,7 +801,7 @@ class UserService:
f'email:{normalized_email}',
)
async def bind_space_account(self, user_email: str, code: str) -> user.User:
async def bind_space_account(self, user_email: str, code: str, *, redirect_uri: str = '') -> user.User:
"""Bind Space account to existing local account"""
local_account = await self.get_user_by_email(user_email)
if local_account is None:
@@ -794,12 +821,13 @@ class UserService:
code,
[binding.workspace_uuid],
{binding.workspace_uuid: created_ts},
redirect_uri=redirect_uri,
)
else:
# Compatibility for early/bootstrap call sites that have not wired
# WorkspaceService yet; old Space servers still derive the legacy
# Workspace identity from instance_id when the field is omitted.
token_data = await self.ap.space_service.exchange_oauth_code(code)
token_data = await self.ap.space_service.exchange_oauth_code(code, redirect_uri=redirect_uri)
access_token = token_data.get('access_token')
refresh_token = token_data.get('refresh_token')
expires_in = token_data.get('expires_in', 0)
@@ -849,3 +877,309 @@ class UserService:
await self._update_space_provider_for_account(local_account, api_key)
return await self.get_user_by_email(space_email)
def _prune_passkey_challenges(self, now: float) -> None:
while self._passkey_challenge_expiry_heap:
expires_at, token = self._passkey_challenge_expiry_heap[0]
entry = self._passkey_challenges.get(token)
if entry is None or entry.expires_at != expires_at:
heapq.heappop(self._passkey_challenge_expiry_heap)
continue
if expires_at > now:
break
heapq.heappop(self._passkey_challenge_expiry_heap)
self._passkey_challenges.pop(token, None)
max_heap_entries = max(
_PASSKEY_CHALLENGE_HEAP_COMPACT_FLOOR,
len(self._passkey_challenges) * _PASSKEY_CHALLENGE_HEAP_MAX_MULTIPLIER,
)
if len(self._passkey_challenge_expiry_heap) > max_heap_entries:
self._passkey_challenge_expiry_heap[:] = [
(entry.expires_at, token) for token, entry in self._passkey_challenges.items()
]
heapq.heapify(self._passkey_challenge_expiry_heap)
async def issue_passkey_challenge(
self,
purpose: typing.Literal['register', 'auth'],
rp_id: str,
origin: str,
*,
account_uuid: str | None = None,
user_email: str | None = None,
ttl_seconds: int = 300,
) -> tuple[str, bytes]:
now = time.monotonic()
challenge_bytes = secrets.token_bytes(32)
challenge_token = secrets.token_urlsafe(32)
expires_at = now + ttl_seconds
async with self._passkey_challenge_lock:
self._prune_passkey_challenges(now)
while len(self._passkey_challenges) >= _PASSKEY_CHALLENGE_MAX_ENTRIES:
if not self._passkey_challenge_expiry_heap:
break
_, oldest_token = heapq.heappop(self._passkey_challenge_expiry_heap)
self._passkey_challenges.pop(oldest_token, None)
self._passkey_challenges[challenge_token] = PasskeyChallengeData(
challenge=challenge_bytes,
purpose=purpose,
rp_id=rp_id,
origin=origin,
expires_at=expires_at,
account_uuid=account_uuid,
user_email=user_email,
)
heapq.heappush(self._passkey_challenge_expiry_heap, (expires_at, challenge_token))
return challenge_token, challenge_bytes
async def consume_passkey_challenge(
self,
challenge_token: str,
purpose: typing.Literal['register', 'auth'],
) -> PasskeyChallengeData:
now = time.monotonic()
async with self._passkey_challenge_lock:
self._prune_passkey_challenges(now)
data = self._passkey_challenges.pop(challenge_token, None)
if data is None or data.expires_at < now:
raise ValueError('Invalid or expired passkey challenge')
if data.purpose != purpose:
raise ValueError('Passkey challenge purpose mismatch')
return data
async def get_user_passkeys(self, account_uuid: str) -> list[passkey.PasskeyCredential]:
statement = (
sqlalchemy.select(passkey.PasskeyCredential)
.where(passkey.PasskeyCredential.account_uuid == account_uuid)
.order_by(passkey.PasskeyCredential.created_at.desc())
)
async with self._session_factory()() as session:
result = await session.scalars(statement)
return list(result.all())
async def get_passkey_by_credential_id(self, credential_id: str) -> passkey.PasskeyCredential | None:
statement = sqlalchemy.select(passkey.PasskeyCredential).where(
passkey.PasskeyCredential.credential_id == credential_id
)
async with self._session_factory()() as session:
return await session.scalar(statement)
async def get_passkey_by_uuid(self, passkey_uuid: str) -> passkey.PasskeyCredential | None:
statement = sqlalchemy.select(passkey.PasskeyCredential).where(passkey.PasskeyCredential.uuid == passkey_uuid)
async with self._session_factory()() as session:
return await session.scalar(statement)
async def generate_passkey_registration_options(
self,
account_uuid: str,
rp_id: str,
origin: str,
rp_name: str = 'LangBot',
) -> tuple[dict[str, typing.Any], str]:
account = await self.get_user_by_uuid(account_uuid)
if account is None:
raise ValueError('User not found')
self._require_active_account(account)
challenge_token, challenge_bytes = await self.issue_passkey_challenge(
purpose='register',
rp_id=rp_id,
origin=origin,
account_uuid=account_uuid,
user_email=account.user,
)
existing_passkeys = await self.get_user_passkeys(account_uuid)
exclude_credentials = [
PublicKeyCredentialDescriptor(id=base64url_to_bytes(pk.credential_id)) for pk in existing_passkeys
]
options = webauthn.generate_registration_options(
rp_id=rp_id,
rp_name=rp_name,
user_name=account.user,
user_id=account.uuid.encode('utf-8'),
user_display_name=account.user,
challenge=challenge_bytes,
exclude_credentials=exclude_credentials or None,
authenticator_selection=AuthenticatorSelectionCriteria(
resident_key=ResidentKeyRequirement.PREFERRED,
),
)
options_dict = json.loads(webauthn.options_to_json(options))
return options_dict, challenge_token
async def verify_and_save_passkey_registration(
self,
challenge_token: str,
credential_data: dict[str, typing.Any] | str,
name: str | None = None,
) -> passkey.PasskeyCredential:
challenge_data = await self.consume_passkey_challenge(challenge_token, 'register')
if not challenge_data.account_uuid:
raise ValueError('Registration challenge must be bound to an account')
verification = webauthn.verify_registration_response(
credential=credential_data,
expected_challenge=challenge_data.challenge,
expected_rp_id=challenge_data.rp_id,
expected_origin=challenge_data.origin,
require_user_verification=False,
)
cred_id_str = bytes_to_base64url(verification.credential_id)
pub_key_str = bytes_to_base64url(verification.credential_public_key)
transports = None
if isinstance(credential_data, dict):
resp = credential_data.get('response', {})
if isinstance(resp, dict) and 'transports' in resp:
t_list = resp.get('transports')
if isinstance(t_list, list):
transports = ','.join(str(x) for x in t_list)
credential_name = (name or '').strip()
if not credential_name:
credential_name = f'Passkey ({datetime.datetime.now().strftime("%Y-%m-%d %H:%M")})'
record = passkey.PasskeyCredential(
uuid=str(uuid.uuid4()),
account_uuid=challenge_data.account_uuid,
name=credential_name,
credential_id=cred_id_str,
public_key=pub_key_str,
sign_count=verification.sign_count,
aaguid=verification.aaguid,
transports=transports,
backed_up=verification.credential_backed_up,
)
async with self._session_factory()() as session:
async with session.begin():
session.add(record)
await session.flush()
await session.refresh(record)
return record
async def generate_passkey_authentication_options(
self,
rp_id: str,
origin: str,
email: str | None = None,
) -> tuple[dict[str, typing.Any], str]:
challenge_token, challenge_bytes = await self.issue_passkey_challenge(
purpose='auth',
rp_id=rp_id,
origin=origin,
user_email=email,
)
allow_credentials: list[PublicKeyCredentialDescriptor] | None = None
if email:
user_obj = await self.get_user_by_email(email)
if user_obj:
user_passkeys = await self.get_user_passkeys(user_obj.uuid)
if user_passkeys:
allow_credentials = [
PublicKeyCredentialDescriptor(id=base64url_to_bytes(pk.credential_id)) for pk in user_passkeys
]
options = webauthn.generate_authentication_options(
rp_id=rp_id,
challenge=challenge_bytes,
allow_credentials=allow_credentials or None,
user_verification=UserVerificationRequirement.PREFERRED,
)
options_dict = json.loads(webauthn.options_to_json(options))
return options_dict, challenge_token
async def verify_passkey_authentication(
self,
challenge_token: str,
credential_data: dict[str, typing.Any] | str,
) -> tuple[str, user.User]:
challenge_data = await self.consume_passkey_challenge(challenge_token, 'auth')
raw_id = credential_data.get('id') if isinstance(credential_data, dict) else None
if not raw_id:
raise ValueError('Missing credential id')
stored_credential = await self.get_passkey_by_credential_id(raw_id)
if stored_credential is None:
raise ValueError('Passkey credential not recognized')
user_obj = await self.get_user_by_uuid(stored_credential.account_uuid)
if user_obj is None:
raise ValueError('Associated user not found')
self._require_active_account(user_obj)
verification = webauthn.verify_authentication_response(
credential=credential_data,
expected_challenge=challenge_data.challenge,
expected_rp_id=challenge_data.rp_id,
expected_origin=challenge_data.origin,
credential_public_key=base64url_to_bytes(stored_credential.public_key),
credential_current_sign_count=stored_credential.sign_count,
require_user_verification=False,
)
async with self._session_factory()() as session:
async with session.begin():
record = await session.scalar(
sqlalchemy.select(passkey.PasskeyCredential).where(
passkey.PasskeyCredential.id == stored_credential.id
)
)
if record:
record.sign_count = verification.new_sign_count
record.last_used_at = datetime.datetime.now()
record.backed_up = verification.credential_backed_up
token = await self.generate_jwt_token(user_obj)
return token, user_obj
async def rename_user_passkey(
self,
account_uuid: str,
passkey_uuid: str,
new_name: str,
) -> passkey.PasskeyCredential | None:
async with self._session_factory()() as session:
async with session.begin():
record = await session.scalar(
sqlalchemy.select(passkey.PasskeyCredential).where(
passkey.PasskeyCredential.uuid == passkey_uuid,
passkey.PasskeyCredential.account_uuid == account_uuid,
)
)
if record is None:
return None
record.name = new_name
await session.flush()
await session.refresh(record)
return record
async def delete_user_passkey(
self,
account_uuid: str,
passkey_uuid: str,
) -> bool:
async with self._session_factory()() as session:
async with session.begin():
record = await session.scalar(
sqlalchemy.select(passkey.PasskeyCredential).where(
passkey.PasskeyCredential.uuid == passkey_uuid,
passkey.PasskeyCredential.account_uuid == account_uuid,
)
)
if record is None:
return False
await session.delete(record)
return True
+10 -3
View File
@@ -464,7 +464,9 @@ class BoxService:
async def _require_validated_workspace_sandbox(self, execution_context: ExecutionContext) -> None:
if not self._available:
raise BoxError('Box runtime is not available. Install and start Docker to use sandbox features.')
raise BoxError(
'Box runtime is not available. Configure an available Box backend before using Box features.'
)
if self._cloud_managed:
if self._admission is None:
raise BoxAdmissionError('Cloud Box sandbox admission is unavailable')
@@ -579,7 +581,9 @@ class BoxService:
skip_host_mount_validation: bool = False,
) -> dict:
if not self._available:
raise BoxError('Box runtime is not available. Install and start Docker to use sandbox features.')
raise BoxError(
'Box runtime is not available. Configure an available Box backend before using Box features.'
)
execution_context = await self._validated_execution_context(self._query_execution_context(query))
spec_payload = self._managed_policy_payload(execution_context, spec_payload)
await self._require_validated_workspace_sandbox(execution_context)
@@ -2170,5 +2174,8 @@ class BoxService:
if backend_name:
payload['connector_error'] = f'Configured sandbox backend "{backend_name}" is unavailable'
else:
payload['connector_error'] = 'No supported sandbox backend (Docker / nsjail / E2B) is available'
payload['connector_error'] = (
'No supported sandbox backend (Docker / nsjail / E2B) is available. '
'Trusted local development may explicitly select the unsafe host backend.'
)
return payload
+100 -2
View File
@@ -125,10 +125,21 @@ class DirectoryProjectionService:
# The database cursor remains the shared projection high-water mark,
# while this cursor tracks what this process has actually observed.
self._consumer_cursor: int | None = None
self._sync_lock = asyncio.Lock()
async def initialize(self) -> None:
"""Block Cloud startup until one full signed snapshot is committed."""
async with self._sync_lock:
await self._refresh_snapshot()
async def refresh_snapshot(self) -> None:
"""Refresh from one full signed snapshot within the sync single-flight."""
async with self._sync_lock:
await self._refresh_snapshot()
async def _refresh_snapshot(self) -> None:
last_superseded: _DirectorySnapshotSuperseded | None = None
for _attempt in range(5):
snapshot = await self.provider.fetch_snapshot(self.instance_uuid)
@@ -159,9 +170,84 @@ class DirectoryProjectionService:
delay = min(max(delay * 2, self.sync_interval_seconds), self.max_staleness_seconds / 2)
async def sync_once(self) -> None:
async with self._sync_lock:
await self._sync_once()
async def reconcile_workspaces(self, workspace_uuids: Iterable[str]) -> None:
"""Synchronously project an exact Workspace set without moving the event cursor."""
requested = tuple(sorted({str(value).strip() for value in workspace_uuids if str(value).strip()}))
if not requested:
raise DirectoryProjectionUnavailableError('Targeted directory reconciliation requires a Workspace')
if len(requested) > self.event_limit:
raise DirectoryProjectionUnavailableError('Targeted directory reconciliation exceeds the batch limit')
async with self._sync_lock:
delta = await self.provider.fetch_workspaces(self.instance_uuid, requested)
await self._apply_targeted_delta(delta, requested)
async def _apply_targeted_delta(
self,
delta: DirectoryDelta,
requested_workspace_uuids: tuple[str, ...],
) -> None:
if not isinstance(delta, DirectoryDelta):
raise DirectoryProjectionUnavailableError('Directory provider returned an invalid delta')
workspace_count, membership_count = self._validate_batch_capacity(
delta.workspaces,
full_snapshot=False,
)
delta = DirectoryDelta.model_validate(delta.model_dump())
if delta.instance_uuid != self.instance_uuid:
raise DirectoryProjectionUnavailableError('Directory delta targets another LangBot instance')
requested = set(requested_workspace_uuids)
if set(delta.requested_workspace_uuids) != requested:
raise DirectoryProjectionUnavailableError('Directory delta does not match the requested Workspaces')
if {workspace.uuid for workspace in delta.workspaces} != requested:
raise DirectoryProjectionUnavailableError('Directory delta omitted a requested Workspace')
directory_uow = getattr(self.ap.persistence_mgr, 'directory_projection_uow', None)
if not callable(directory_uow):
raise DirectoryProjectionUnavailableError('Directory projection persistence scope is unavailable')
async with directory_uow(self.instance_uuid) as uow:
session = uow.session
state = await session.scalar(
sqlalchemy.select(DirectoryProjectionState)
.where(DirectoryProjectionState.instance_uuid == self.instance_uuid)
.with_for_update()
)
if state is None:
raise DirectoryProjectionUnavailableError('Directory projection is not initialized')
snapshot = DirectorySnapshot(
instance_uuid=self.instance_uuid,
cursor=state.cursor,
generated_at=delta.generated_at,
workspaces=delta.workspaces,
)
accounts_by_uuid = await self._apply_accounts(session, snapshot, preserve_existing=True)
await self._apply_workspaces(session, snapshot, accounts_by_uuid=accounts_by_uuid)
active_workspace_count = await self._enforce_active_workspace_capacity(session)
await session.flush()
await self._update_entitlement_workspace_activity(
snapshot.workspaces,
requested_workspace_uuids=requested,
)
self._publish_runtime_execution_projection(
snapshot.workspaces,
affected_workspace_uuids=requested,
)
self._request_model_catalog_sync()
self._record_batch_cardinality(
active_workspaces=active_workspace_count,
workspaces=workspace_count,
memberships=membership_count,
)
async def _sync_once(self) -> None:
cursor = self._consumer_cursor
if cursor is None:
await self.initialize()
await self._refresh_snapshot()
return
batch = await self.provider.fetch_events(
self.instance_uuid,
@@ -708,7 +794,13 @@ class DirectoryProjectionService:
for row in inbox_rows:
row.applied_at = now
async def _apply_accounts(self, session: Any, snapshot: DirectorySnapshot) -> dict[str, User]:
async def _apply_accounts(
self,
session: Any,
snapshot: DirectorySnapshot,
*,
preserve_existing: bool = False,
) -> dict[str, User]:
selected: dict[str, DirectoryMember] = {}
emails: dict[str, str] = {}
for workspace in snapshot.workspaces:
@@ -773,6 +865,12 @@ class DirectoryProjectionService:
continue
if account.source != AccountSource.CLOUD_PROJECTION.value:
raise DirectoryProjectionUnavailableError('Directory account UUID collides with a local Core account')
if preserve_existing:
# A targeted Workspace fetch has no independently monotonic
# Account revision. It may create a missing runtime shadow, but
# ordered event/snapshot projection remains the only updater of
# existing Account identity and status fields.
continue
if account.projection_revision > snapshot.cursor:
raise DirectoryProjectionUnavailableError('Directory account revision rolled back')
projected_account = self._account_projection(member)
+2 -2
View File
@@ -653,9 +653,9 @@ class Application:
frontend_path = paths.get_frontend_path()
if not os.path.exists(frontend_path):
self.logger.warning('WebUI 文件缺失,请根据文档部署:https://docs.langbot.app/zh')
self.logger.warning('WebUI 文件缺失,请根据文档部署:https://langbot.app/docs/zh')
self.logger.warning(
'WebUI files are missing, please deploy according to the documentation: https://docs.langbot.app/en'
'WebUI files are missing, please deploy according to the documentation: https://langbot.app/docs/en'
)
return
+1
View File
@@ -43,6 +43,7 @@ required_deps = {
'slack_sdk': 'slack_sdk',
'asyncpg': 'asyncpg',
'litellm': 'litellm',
'xxhash': 'xxhash',
}
+20 -1
View File
@@ -1,9 +1,18 @@
from __future__ import annotations
import logging
import secrets
from .. import stage, app
# This stage runs before SetupLoggerStage, so ap.logger is still None here;
# the module logger falls back to the stderr lastResort handler.
_logger = logging.getLogger(__name__)
# 32 symbols without 0/O or 1/I; eight independent draws provide 40 random bits.
_RECOVERY_KEY_ALPHABET = '23456789ABCDEFGHJKLMNPQRSTUVWXYZ'
_RECOVERY_KEY_LENGTH = 8
@stage.stage_class('GenKeysStage')
class GenKeysStage(stage.BootingStage):
@@ -20,5 +29,15 @@ class GenKeysStage(stage.BootingStage):
ap.instance_config.data['system']['recovery_key'] = ''
if not ap.instance_config.data['system']['recovery_key']:
ap.instance_config.data['system']['recovery_key'] = secrets.token_hex(3).upper()
# Keep recovery practical to type. Security also requires the reset
# endpoint's concurrency-safe quota (five admissions per 15 minutes).
ap.instance_config.data['system']['recovery_key'] = ''.join(
secrets.choice(_RECOVERY_KEY_ALPHABET) for _ in range(_RECOVERY_KEY_LENGTH)
)
await ap.instance_config.dump_config()
elif len(ap.instance_config.data['system']['recovery_key']) < _RECOVERY_KEY_LENGTH:
_logger.warning(
'Low-entropy legacy recovery key detected (length < 8); '
'regenerate system.recovery_key in the configuration file '
'with a strong random value (#2392)'
)
+49 -12
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import json
import typing
import datetime
import time
@@ -197,6 +198,41 @@ class TaskWrapper:
},
}
def to_public_dict(self) -> dict:
"""Return the stable task projection exposed to API-key callers."""
if self.task.cancelled():
status = 'cancelled'
error = {'type': 'task_cancelled', 'message': 'Task was cancelled'}
result = None
elif not self.task.done():
status = 'running'
error = None
result = None
else:
exception = self.assume_exception()
if exception is not None:
status = 'failed'
error = {'type': 'task_failed', 'message': 'Task execution failed'}
result = None
else:
status = 'succeeded'
error = None
result = self.assume_result()
try:
json.dumps(result)
except (TypeError, ValueError):
result = None
return {
'id': self.id,
'task_type': self.task_type,
'kind': self.kind,
'status': status,
'error': error,
'result': result,
'created_at': self.created_at,
}
def cancel(self):
self.task.cancel()
@@ -325,19 +361,20 @@ class AsyncTaskManager:
instance_uuid: str | None = None,
workspace_uuid: str | None = None,
placement_generation: int | None = None,
public: bool = False,
) -> dict:
return {
'tasks': [
t.to_dict()
for t in self.tasks
if (type is None or t.task_type == type)
and (kind is None or t.kind == kind)
and (instance_uuid is None or t.instance_uuid == instance_uuid)
and (workspace_uuid is None or t.workspace_uuid == workspace_uuid)
and (placement_generation is None or t.placement_generation == placement_generation)
],
'id_index': TaskWrapper._id_index,
}
tasks = [
t.to_public_dict() if public else t.to_dict()
for t in self.tasks
if (type is None or t.task_type == type)
and (kind is None or t.kind == kind)
and (instance_uuid is None or t.instance_uuid == instance_uuid)
and (workspace_uuid is None or t.workspace_uuid == workspace_uuid)
and (placement_generation is None or t.placement_generation == placement_generation)
]
if public:
return {'tasks': tasks}
return {'tasks': tasks, 'id_index': TaskWrapper._id_index}
def get_stats(self) -> dict:
completed = sum(1 for t in self.tasks if t.task.done())
@@ -33,6 +33,28 @@ class ModelProvider(Base):
)
class CodexCredential(Base):
"""Server-only OAuth state. Never joined into provider/model serialization."""
__tablename__ = 'codex_credentials'
provider_uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
workspace_uuid = sqlalchemy.Column(sqlalchemy.String(36), nullable=False)
payload = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default=dict)
version = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0)
lease_owner = sqlalchemy.Column(sqlalchemy.String(64), nullable=True)
lease_until = sqlalchemy.Column(sqlalchemy.Float, nullable=False, default=0)
__table_args__ = (
sqlalchemy.ForeignKeyConstraint(
['workspace_uuid', 'provider_uuid'],
['model_providers.workspace_uuid', 'model_providers.uuid'],
name='fk_codex_credentials_workspace_provider',
ondelete='CASCADE',
),
sqlalchemy.Index('ix_codex_credentials_workspace', 'workspace_uuid'),
)
class LLMModel(Base):
"""LLM model"""
@@ -111,8 +111,8 @@ class MonitoringSession(Base):
sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'),
primary_key=True,
)
bot_id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, index=True)
session_id = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True)
bot_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, index=True)
bot_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
pipeline_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False, index=True)
pipeline_name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
@@ -0,0 +1,38 @@
from __future__ import annotations
import uuid as uuid_lib
import sqlalchemy
from .base import Base
class PasskeyCredential(Base):
__tablename__ = 'passkey_credentials'
id = sqlalchemy.Column(sqlalchemy.Integer, primary_key=True, autoincrement=True)
uuid = sqlalchemy.Column(
sqlalchemy.String(36),
nullable=False,
default=lambda: str(uuid_lib.uuid4()),
)
account_uuid = sqlalchemy.Column(
sqlalchemy.String(36),
sqlalchemy.ForeignKey('users.uuid', ondelete='CASCADE'),
nullable=False,
)
name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
credential_id = sqlalchemy.Column(sqlalchemy.String(255), nullable=False)
public_key = sqlalchemy.Column(sqlalchemy.Text, nullable=False)
sign_count = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0)
aaguid = sqlalchemy.Column(sqlalchemy.String(64), nullable=True)
transports = sqlalchemy.Column(sqlalchemy.String(255), nullable=True)
backed_up = sqlalchemy.Column(sqlalchemy.Boolean, nullable=False, default=False)
created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now())
last_used_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=True)
__table_args__ = (
sqlalchemy.Index('uq_passkey_credentials_uuid', 'uuid', unique=True),
sqlalchemy.Index('uq_passkey_credentials_cred_id', 'credential_id', unique=True),
sqlalchemy.Index('ix_passkey_credentials_account', 'account_uuid'),
)
@@ -0,0 +1,48 @@
"""Add isolated server-only Codex credentials and tenant RLS.
Revision ID: 0022_codex_credentials
Revises: 0021_merge_reasoning_config
"""
from alembic import op
import sqlalchemy as sa
revision = '0022_codex_credentials'
down_revision = '0021_merge_reasoning_config'
branch_labels = None
depends_on = None
def upgrade() -> None:
conn = op.get_bind()
# Fresh startup creates ORM metadata before running Alembic.
if 'codex_credentials' not in sa.inspect(conn).get_table_names():
op.create_table(
'codex_credentials',
sa.Column('provider_uuid', sa.String(255), primary_key=True),
sa.Column('workspace_uuid', sa.String(36), nullable=False),
sa.Column('payload', sa.JSON(), nullable=False),
sa.Column('version', sa.Integer(), nullable=False),
sa.Column('lease_owner', sa.String(64), nullable=True),
sa.Column('lease_until', sa.Float(), nullable=False),
sa.ForeignKeyConstraint(
['workspace_uuid', 'provider_uuid'],
['model_providers.workspace_uuid', 'model_providers.uuid'],
name='fk_codex_credentials_workspace_provider',
ondelete='CASCADE',
),
)
op.create_index('ix_codex_credentials_workspace', 'codex_credentials', ['workspace_uuid'])
if conn.dialect.name == 'postgresql':
op.execute('ALTER TABLE codex_credentials ENABLE ROW LEVEL SECURITY')
op.execute('ALTER TABLE codex_credentials FORCE ROW LEVEL SECURITY')
op.execute('DROP POLICY IF EXISTS langbot_workspace_isolation ON codex_credentials')
expression = "workspace_uuid::text = NULLIF(current_setting('langbot.workspace_uuid', true), '')"
op.execute(
f'CREATE POLICY langbot_workspace_isolation ON codex_credentials '
f'FOR ALL USING ({expression}) WITH CHECK ({expression})'
)
def downgrade() -> None:
op.drop_table('codex_credentials')
@@ -0,0 +1,104 @@
"""Scope monitoring sessions by bot without changing runtime session IDs.
Revision ID: 0023_bot_scoped_sessions
Revises: 0022_codex_credentials
"""
from alembic import op
import sqlalchemy as sa
from sqlalchemy.dialects import postgresql, sqlite
revision = '0023_bot_scoped_sessions'
down_revision = '0022_codex_credentials'
branch_labels = None
depends_on = None
_TABLE = 'monitoring_sessions'
_KEY = ['workspace_uuid', 'bot_id', 'session_id']
def upgrade() -> None:
conn = op.get_bind()
inspector = sa.inspect(conn)
if _TABLE not in inspector.get_table_names():
return
pk = inspector.get_pk_constraint(_TABLE)
if pk['constrained_columns'] == _KEY:
return
# PostgreSQL alters in place, retaining indexes, grants, policies and RLS.
# SQLite batch reflection retains all existing indexes and foreign keys.
with op.batch_alter_table(_TABLE, naming_convention={'pk': 'pk_%(table_name)s'}) as batch:
batch.drop_constraint(pk['name'] or f'pk_{_TABLE}', type_='primary')
batch.create_primary_key(f'pk_{_TABLE}', _KEY)
metadata = sa.MetaData()
sessions = sa.Table(_TABLE, metadata, autoload_with=conn)
messages = sa.Table('monitoring_messages', metadata, autoload_with=conn)
m = messages.c
collisions = (
sa.select(m.workspace_uuid, m.session_id)
.group_by(m.workspace_uuid, m.session_id)
.having(sa.func.count(sa.distinct(m.bot_id)) > 1)
.subquery()
)
partition = [m.workspace_uuid, m.bot_id, m.session_id]
# Repair only demonstrable collisions. Retention may have removed earlier
# evidence; these summaries describe surviving messages, never invented text.
ranked = (
sa.select(
*[m[name] for name in _KEY],
m.bot_name,
m.pipeline_id,
m.pipeline_name,
m.platform,
m.user_id,
m.user_name,
sa.func.sum(sa.case((sa.or_(m.role == 'user', m.role.is_(None)), 1), else_=0))
.over(partition_by=partition)
.label('message_count'),
sa.func.min(m.timestamp).over(partition_by=partition).label('start_time'),
sa.func.max(m.timestamp).over(partition_by=partition).label('last_activity'),
sa.func.row_number().over(partition_by=partition, order_by=[m.timestamp.desc(), m.id.desc()]).label('rank'),
)
.join(
collisions,
sa.and_(m.workspace_uuid == collisions.c.workspace_uuid, m.session_id == collisions.c.session_id),
)
.subquery()
)
columns = _KEY + [
'bot_name',
'pipeline_id',
'pipeline_name',
'platform',
'user_id',
'user_name',
'message_count',
'start_time',
'last_activity',
'is_active',
]
select = sa.select(*[ranked.c[name] for name in columns[:-1]], sa.literal(True)).where(ranked.c.rank == 1)
insert = postgresql.insert if conn.dialect.name == 'postgresql' else sqlite.insert
statement = insert(sessions).from_select(columns, select)
conn.execute(
statement.on_conflict_do_update(
index_elements=_KEY,
set_={name: statement.excluded[name] for name in columns if name not in _KEY and name != 'is_active'},
)
)
def downgrade() -> None:
conn = op.get_bind()
if _TABLE not in sa.inspect(conn).get_table_names():
return
collisions = conn.execute(
sa.text('SELECT 1 FROM monitoring_sessions GROUP BY workspace_uuid, session_id HAVING COUNT(*) > 1 LIMIT 1')
).first()
if collisions:
raise RuntimeError('Cannot downgrade bot-scoped sessions without losing colliding bot records')
pk = sa.inspect(conn).get_pk_constraint(_TABLE)
with op.batch_alter_table(_TABLE, naming_convention={'pk': 'pk_%(table_name)s'}) as batch:
batch.drop_constraint(pk['name'] or f'pk_{_TABLE}', type_='primary')
batch.create_primary_key(f'pk_{_TABLE}', ['workspace_uuid', 'session_id'])
@@ -0,0 +1,54 @@
"""add passkey credentials table
Revision ID: 0024_passkey_credentials
Revises: 0023_bot_scoped_sessions
Create Date: 2026-09-12
"""
from __future__ import annotations
import sqlalchemy as sa
from alembic import op
revision = '0024_passkey_credentials'
down_revision = '0023_bot_scoped_sessions'
branch_labels = None
depends_on = None
_TABLE_NAME = 'passkey_credentials'
def upgrade() -> None:
conn = op.get_bind()
existing_tables = set(sa.inspect(conn).get_table_names())
if _TABLE_NAME not in existing_tables:
op.create_table(
_TABLE_NAME,
sa.Column('id', sa.Integer(), primary_key=True, autoincrement=True),
sa.Column('uuid', sa.String(36), nullable=False),
sa.Column(
'account_uuid',
sa.String(36),
sa.ForeignKey('users.uuid', ondelete='CASCADE'),
nullable=False,
),
sa.Column('name', sa.String(255), nullable=False),
sa.Column('credential_id', sa.String(255), nullable=False),
sa.Column('public_key', sa.Text(), nullable=False),
sa.Column('sign_count', sa.Integer(), nullable=False, server_default='0'),
sa.Column('aaguid', sa.String(64), nullable=True),
sa.Column('transports', sa.String(255), nullable=True),
sa.Column('backed_up', sa.Boolean(), nullable=False, server_default='0'),
sa.Column('created_at', sa.DateTime(), nullable=False, server_default=sa.func.now()),
sa.Column('last_used_at', sa.DateTime(), nullable=True),
)
op.create_index('uq_passkey_credentials_uuid', _TABLE_NAME, ['uuid'], unique=True)
op.create_index('uq_passkey_credentials_cred_id', _TABLE_NAME, ['credential_id'], unique=True)
op.create_index('ix_passkey_credentials_account', _TABLE_NAME, ['account_uuid'], unique=False)
def downgrade() -> None:
op.drop_index('ix_passkey_credentials_account', table_name=_TABLE_NAME)
op.drop_index('uq_passkey_credentials_cred_id', table_name=_TABLE_NAME)
op.drop_index('uq_passkey_credentials_uuid', table_name=_TABLE_NAME)
op.drop_table(_TABLE_NAME)
@@ -0,0 +1,22 @@
"""Join the published master credentials and 4.11 bot processor branches.
Revision ID: 0026_merge_master_beta
Revises: 0024_passkey_credentials, 0025_bot_plugin_processors
Both parent chains are already published. Keep their identities and schema/data
operations intact; Alembic applies the missing branch before this no-op merge.
"""
revision = '0026_merge_master_beta'
down_revision = ('0024_passkey_credentials', '0025_bot_plugin_processors')
branch_labels = None
depends_on = None
def upgrade() -> None:
pass
def downgrade() -> None:
# Only unmerge the revision graph; neither branch's schema/data is removed.
pass
+2
View File
@@ -64,6 +64,8 @@ _ALEMBIC_TENANT_TABLES = {
'binary_storages',
'mcp_servers',
'model_providers',
'codex_credentials',
'passkey_credentials',
'llm_models',
'embedding_models',
'rerank_models',
+35 -1
View File
@@ -52,6 +52,7 @@ TENANT_TABLE_COLUMNS: dict[str, str] = {
'binary_storages': 'workspace_uuid',
'mcp_servers': 'workspace_uuid',
'model_providers': 'workspace_uuid',
'codex_credentials': 'workspace_uuid',
'llm_models': 'workspace_uuid',
'embedding_models': 'workspace_uuid',
'rerank_models': 'workspace_uuid',
@@ -207,6 +208,8 @@ _SYNC_PROXY_CAPABILITY: contextvars.ContextVar[_ScopedSessionGuardState | None]
_ALLOWED_SCOPED_BUILTIN_FUNCTION_TYPES = {
'coalesce': sqlalchemy.sql.functions.coalesce,
'count': sqlalchemy.sql.functions.count,
'min': sqlalchemy.sql.functions.min,
'max': sqlalchemy.sql.functions.max,
'now': sqlalchemy.sql.functions.now,
'sum': sqlalchemy.sql.functions.sum,
}
@@ -853,7 +856,30 @@ class TenantScopedAsyncSession(sqlalchemy_asyncio.AsyncSession):
self._require_owner_task()
self._enter_internal_access()
try:
await transaction.commit()
# Retain the actual connection before COMMIT: after a failed SQLite
# COMMIT the logical transaction is inactive, but the DBAPI writer
# can still hold PENDING/RESERVED locks. Session.close()/rollback()
# alone can then return that poisoned connection to the pool.
connection = await super().connection()
try:
await transaction.commit()
except BaseException as exc:
cleanup = asyncio.create_task(connection.invalidate())
# Invalidation does not access the task-owned Session. Shield
# physical cleanup, including against repeated cancellation,
# before the owner closes the Session and releases its scope.
while not cleanup.done():
try:
await asyncio.shield(cleanup)
except asyncio.CancelledError:
continue
except BaseException:
break
try:
cleanup.result()
except BaseException as cleanup_error:
exc.add_note(f'Failed to invalidate transaction connection: {cleanup_error!r}')
raise
finally:
self._exit_internal_access()
@@ -1370,6 +1396,7 @@ class TenantUnitOfWork:
state.mark_rollback_only(exc_value)
rollback_only = state.rollback_only
committed = False
transaction_error: BaseException | None = None
try:
if exc_type is None and not rollback_only:
await typing.cast(TenantScopedAsyncSession, session)._commit_owned_transaction(
@@ -1382,6 +1409,9 @@ class TenantUnitOfWork:
_UOW_SESSION_CONTROL_CAPABILITY,
transaction,
)
except BaseException as exc:
transaction_error = exc
raise
finally:
try:
if self._active_transaction is not None and self._context_token is not None:
@@ -1389,6 +1419,10 @@ class TenantUnitOfWork:
await typing.cast(TenantScopedAsyncSession, session)._close_owned_session(
_UOW_SESSION_CONTROL_CAPABILITY
)
except BaseException as cleanup_error:
if transaction_error is None:
raise
transaction_error.add_note(f'Failed to close transaction Session: {cleanup_error!r}')
finally:
if self._database_operation_token is not None:
_DATABASE_OPERATION_TRANSACTION.reset(self._database_operation_token)
@@ -48,7 +48,10 @@ class MonitoringHelper:
# Try to record message
# Use JSON serialization to preserve message chain structure (including image URLs, etc.)
if hasattr(query, 'message_chain') and hasattr(query.message_chain, 'model_dump'):
message_content = json.dumps(query.message_chain.model_dump(), ensure_ascii=False)
chain_dump = query.message_chain.model_dump()
if hasattr(ap, 'storage_mgr') and hasattr(ap.storage_mgr, 'media_cache'):
chain_dump = await ap.storage_mgr.media_cache.externalize_chain_dump(chain_dump)
message_content = json.dumps(chain_dump, ensure_ascii=False)
else:
message_content = str(query)
@@ -79,6 +82,7 @@ class MonitoringHelper:
session_updated = await ap.monitoring_service.update_session_activity(
get_query_execution_context(query),
session_id,
bot_id=bot_id,
pipeline_id=pipeline_id,
pipeline_name=pipeline_name,
)
@@ -167,7 +171,10 @@ class MonitoringHelper:
if hasattr(last_resp, 'get_content_platform_message_chain'):
chain = last_resp.get_content_platform_message_chain()
if hasattr(chain, 'model_dump'):
message_content = json.dumps(chain.model_dump(), ensure_ascii=False)
chain_dump = chain.model_dump()
if hasattr(ap, 'storage_mgr') and hasattr(ap.storage_mgr, 'media_cache'):
chain_dump = await ap.storage_mgr.media_cache.externalize_chain_dump(chain_dump)
message_content = json.dumps(chain_dump, ensure_ascii=False)
else:
message_content = str(chain)
else:
+4 -1
View File
@@ -1144,7 +1144,10 @@ class RuntimeBot:
"""Record a discarded message in the monitoring system."""
try:
if hasattr(message_chain, 'model_dump'):
message_content = json.dumps(message_chain.model_dump(), ensure_ascii=False)
chain_dump = message_chain.model_dump()
if hasattr(self.ap, 'storage_mgr') and hasattr(self.ap.storage_mgr, 'media_cache'):
chain_dump = await self.ap.storage_mgr.media_cache.externalize_chain_dump(chain_dump)
message_content = json.dumps(chain_dump, ensure_ascii=False)
else:
message_content = str(message_chain)
+1 -1
View File
@@ -2675,7 +2675,7 @@ class RuntimeConnectionHandler(handler.Handler):
return None
def require_outbound_installation_context(self) -> InstallationBinding:
binding = self._outbound_installation_context.get()
binding = self._outbound_installation_context.get(None)
if not isinstance(binding, InstallationBinding):
raise ValueError('Host plugin action requires an InstallationBinding scope')
return binding
@@ -0,0 +1,420 @@
"""ChatGPT device auth with server-only credentials and cross-process refresh leases.
Network I/O never holds a DB transaction. A persisted CAS lease serializes refresh
and poll; cancel fences device exchanges but waits for existing-token refreshes.
"""
from __future__ import annotations
import asyncio
import base64
import json
import math
import secrets
import time
from contextlib import asynccontextmanager
from datetime import datetime, timezone
import httpx
import sqlalchemy as sa
from ...entity.persistence.model import CodexCredential, ModelProvider
from ...api.http.context import PrincipalType, RequestContext
from ...api.http.authz import Permission, has_permission
from ...api.http.service.tenant import require_workspace_uuid
from ...workspace.errors import WorkspaceNotFoundError
REQUESTER = 'openai-codex'
BASE_URL = 'https://chatgpt.com/backend-api/codex'
ISSUER = 'https://auth.openai.com'
CLIENT_ID = 'app_EMoamEEZ73f0CkXaXp7hrann'
LOGIN_REQUIRED = 'ChatGPT sign-in required. Open this provider and sign in again.'
LEASE_SECONDS = 90
def validate_config(data: dict) -> None:
if data.get('requester') != REQUESTER:
return
if data.get('base_url') not in (None, '', BASE_URL):
raise ValueError('Codex uses the fixed ChatGPT endpoint; custom base URLs are not supported')
if data.get('api_keys') not in (None, [], ''):
raise ValueError('Codex uses ChatGPT sign-in, not API keys')
data['base_url'] = BASE_URL
data['api_keys'] = []
def _claims(token: str) -> dict:
"""Read routing metadata, NOT trusted LangBot identity, from issuer tokens."""
try:
part = token.split('.')[1]
value = json.loads(base64.urlsafe_b64decode(part + '=' * (-len(part) % 4)))
return value if isinstance(value, dict) else {}
except (ValueError, IndexError, TypeError):
return {}
def _tokens(data: dict, previous: dict | None = None) -> dict:
previous = previous or {}
access = data.get('access_token')
refresh = data.get('refresh_token') or previous.get('refresh_token')
account = None
for token in (access, data.get('id_token')):
namespace = _claims(token or '').get('https://api.openai.com/auth', {})
if isinstance(namespace, dict) and isinstance(namespace.get('chatgpt_account_id'), str):
account = namespace['chatgpt_account_id']
break
account = account or previous.get('account_id')
try:
expires_at = (
time.time() + float(data['expires_in'])
if data.get('expires_in') is not None
else float(_claims(access or '').get('exp', 0))
)
except (TypeError, ValueError):
expires_at = 0
if (
not all(isinstance(v, str) and v for v in (access, refresh, account))
or not math.isfinite(expires_at)
or expires_at <= time.time()
):
raise ValueError('ChatGPT returned an incomplete authorization. Please sign in again.')
return {
'access_token': access,
'refresh_token': refresh,
'account_id': account,
'expires_at': expires_at,
'connection_id': previous.get('connection_id') or secrets.token_urlsafe(24),
}
class CodexAuth:
def __init__(self, ap):
self.ap = ap
def _where(self, workspace: str, provider: str):
return (CodexCredential.workspace_uuid == workspace, CodexCredential.provider_uuid == provider)
async def _execute(self, statement):
# SQLAlchemy/driver/serialization errors may embed the entire secret payload.
try:
return await self.ap.persistence_mgr.execute_async(statement)
except Exception:
raise ValueError('ChatGPT credential storage failed. Please retry.') from None
async def _read(self, workspace: str, provider: str) -> dict | None:
result = await self._execute(sa.select(CodexCredential).where(*self._where(workspace, provider)))
try:
row = result.first()
return dict(row._mapping) if row is not None else None
except Exception:
raise ValueError('ChatGPT credential storage failed. Please retry.') from None
async def _provider(self, context, provider: str, *, user: bool = False) -> str:
workspace = require_workspace_uuid(context)
if user and (
not isinstance(context, RequestContext)
or context.principal.principal_type != PrincipalType.ACCOUNT
or not context.account_uuid
or not has_permission(context, Permission.PROVIDER_SECRET_MANAGE)
):
raise ValueError('ChatGPT authorization requires an authorized workspace user')
result = await self._execute(
sa.select(ModelProvider.requester).where(
ModelProvider.workspace_uuid == workspace, ModelProvider.uuid == provider
)
)
kind = result.scalar()
if kind is None:
raise WorkspaceNotFoundError('Provider not found')
if kind != REQUESTER:
raise ValueError('This provider does not use ChatGPT sign-in')
return workspace
@asynccontextmanager
async def _lease(self, workspace: str, provider: str, *, refresh: bool = False):
owner = ('refresh:' if refresh else 'device:') + secrets.token_urlsafe(32)
deadline = time.monotonic() + 65
while True:
now = time.time()
result = await self._execute(
sa.update(CodexCredential)
.where(
*self._where(workspace, provider),
sa.or_(CodexCredential.lease_owner.is_(None), CodexCredential.lease_until < now),
)
.values(lease_owner=owner, lease_until=now + LEASE_SECONDS)
)
if result.rowcount == 1:
break
if await self._read(workspace, provider) is None:
raise ValueError(LOGIN_REQUIRED)
if time.monotonic() >= deadline:
raise ValueError('ChatGPT authorization is busy. Please retry shortly.')
await asyncio.sleep(0.1)
try:
yield owner
finally:
await self._execute(
sa.update(CodexCredential)
.where(*self._where(workspace, provider), CodexCredential.lease_owner == owner)
.values(lease_owner=None, lease_until=0)
)
async def _save(self, workspace: str, provider: str, owner: str, payload: dict) -> None:
result = await self._execute(
sa.update(CodexCredential)
.where(
*self._where(workspace, provider),
CodexCredential.lease_owner == owner,
CodexCredential.lease_until > time.time(),
)
.values(payload=payload, version=CodexCredential.version + 1)
)
if result.rowcount != 1:
raise ValueError('ChatGPT authorization was cancelled or replaced. Please retry.')
async def _post(self, path: str, *, data=None, json_body=None) -> httpx.Response:
try:
async with httpx.AsyncClient(timeout=20, follow_redirects=False) as client:
return await asyncio.wait_for(
client.post(
ISSUER + path,
data=data,
json=json_body,
headers={'Accept': 'application/json', 'User-Agent': 'LangBot'},
),
25,
)
except (httpx.HTTPError, TimeoutError):
raise ValueError('ChatGPT authorization network error. Please retry.') from None
@staticmethod
def _json(response: httpx.Response) -> dict:
try:
value = response.json()
if not isinstance(value, dict):
raise ValueError
return value
except ValueError:
raise ValueError('ChatGPT returned an invalid authorization response') from None
async def status(self, context, provider: str) -> dict:
workspace = await self._provider(context, provider, user=True)
row = await self._read(workspace, provider)
payload = row['payload'] if row else {}
tokens = payload.get('tokens')
connected = bool(tokens and not payload.get('invalid'))
return {
'status': 'connected' if connected else 'expired' if payload.get('invalid') else 'disconnected',
'connected': connected,
'expires_at': tokens.get('expires_at') if tokens else None,
}
async def start(self, context, provider: str) -> dict:
workspace = await self._provider(context, provider, user=True)
async with self._lease(workspace, provider) as owner:
response = await self._post('/api/accounts/deviceauth/usercode', json_body={'client_id': CLIENT_ID})
if response.status_code != 200:
raise ValueError('Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.')
data = self._json(response)
try:
code = data.get('user_code') or data['usercode']
device = data['device_auth_id']
interval = max(5, min(60, int(data.get('interval') or 5)))
if not isinstance(code, str) or not isinstance(device, str) or not code or not device:
raise ValueError
except (KeyError, ValueError, TypeError):
raise ValueError('ChatGPT returned an invalid device code') from None
now = time.time()
try:
expiry = data.get('expires_at')
if expiry is None:
expiry = now + float(data.get('expires_in', 900))
try:
expires_at = float(expiry)
except ValueError:
parsed = datetime.fromisoformat(expiry.replace('Z', '+00:00'))
if parsed.tzinfo is None:
parsed = parsed.replace(tzinfo=timezone.utc)
expires_at = parsed.timestamp()
if not math.isfinite(expires_at) or expires_at <= now:
raise ValueError
expires_at = min(now + 900, expires_at)
except (ValueError, TypeError):
raise ValueError('ChatGPT returned an invalid device code expiry') from None
pending = {
'authorization_id': secrets.token_urlsafe(32),
'user_code': code,
'device_auth_id': device,
'account_uuid': context.account_uuid,
'interval': interval,
'expires_at': expires_at,
'next_poll_at': now + interval,
}
row = await self._read(workspace, provider)
payload = dict(row['payload'])
payload['pending'] = pending
await self._save(workspace, provider, owner, payload)
return {k: pending[k] for k in ('authorization_id', 'user_code', 'interval', 'expires_at')} | {
'verification_uri': ISSUER + '/codex/device'
}
@staticmethod
def _attempt(payload: dict, context, authorization_id: str) -> dict | None:
pending = payload.get('pending')
if not pending or pending.get('authorization_id') != authorization_id:
return None
if pending.get('account_uuid') != context.account_uuid:
raise WorkspaceNotFoundError('Authorization not found')
return pending
async def poll(self, context, provider: str, authorization_id: str) -> dict:
workspace = await self._provider(context, provider, user=True)
if not isinstance(authorization_id, str) or not authorization_id:
raise ValueError('authorization_id is required')
async with self._lease(workspace, provider) as owner:
row = await self._read(workspace, provider)
payload = dict(row['payload'])
pending = self._attempt(payload, context, authorization_id)
if pending is None:
completed = payload.get('completed', {})
if (
completed.get('authorization_id') == authorization_id
and completed.get('account_uuid') == context.account_uuid
):
return {'status': 'connected'}
return {'status': 'expired'}
now = time.time()
if pending['expires_at'] <= now or pending.get('consumed'):
payload.pop('pending', None)
await self._save(workspace, provider, owner, payload)
return {'status': 'expired'}
if pending['next_poll_at'] > now:
return {'status': 'pending', 'interval': pending['interval']}
pending['next_poll_at'] = now + pending['interval']
await self._save(workspace, provider, owner, payload)
response = await self._post(
'/api/accounts/deviceauth/token',
json_body={'device_auth_id': pending['device_auth_id'], 'user_code': pending['user_code']},
)
if response.status_code in (403, 404, 429):
if response.status_code == 429:
pending['interval'] = min(60, pending['interval'] + 5)
pending['next_poll_at'] = time.time() + pending['interval']
await self._save(workspace, provider, owner, payload)
return {'status': 'pending', 'interval': pending['interval']}
if response.status_code != 200:
payload.pop('pending', None)
await self._save(workspace, provider, owner, payload)
raise ValueError('ChatGPT device authorization failed. Please start again.')
data = self._json(response)
if not data.get('authorization_code') or not data.get('code_verifier'):
payload.pop('pending', None)
await self._save(workspace, provider, owner, payload)
raise ValueError('ChatGPT returned an incomplete device authorization')
# Keep an attempt tombstone so cancel can preempt exchange, but never replay a code.
pending['consumed'] = True
await self._save(workspace, provider, owner, payload)
response = await self._post(
'/oauth/token',
data={
'grant_type': 'authorization_code',
'client_id': CLIENT_ID,
'code': data['authorization_code'],
'code_verifier': data['code_verifier'],
'redirect_uri': ISSUER + '/deviceauth/callback',
},
)
if response.status_code != 200:
raise ValueError('ChatGPT token exchange failed. Please start sign-in again.')
tokens = _tokens(self._json(response))
await self._save(
workspace,
provider,
owner,
{
'tokens': tokens,
'completed': {'authorization_id': authorization_id, 'account_uuid': context.account_uuid},
},
)
return {'status': 'connected'}
async def disconnect(self, context, provider: str) -> None:
workspace = await self._provider(context, provider, user=True)
await self._execute(
sa.update(CodexCredential)
.where(*self._where(workspace, provider))
.values(payload={}, lease_owner=None, lease_until=0, version=CodexCredential.version + 1)
)
async def cancel(self, context, provider: str, authorization_id: str) -> None:
workspace = await self._provider(context, provider, user=True)
deadline = time.monotonic() + 65
while time.monotonic() < deadline:
row = await self._read(workspace, provider)
if row is None:
return
old = row['payload']
if self._attempt(old, context, authorization_id) is None:
return
lease_owner = row['lease_owner']
if lease_owner and lease_owner.startswith('refresh:') and row['lease_until'] > time.time():
# A rotated refresh token must be committed before removing the attempt.
await asyncio.sleep(0.1)
continue
payload = dict(old)
payload.pop('pending', None)
result = await self._execute(
sa.update(CodexCredential)
.where(
*self._where(workspace, provider),
CodexCredential.version == row['version'],
# Lease acquisition does not change version; fence that race too.
CodexCredential.lease_owner == lease_owner,
)
.values(payload=payload, lease_owner=None, lease_until=0, version=CodexCredential.version + 1)
)
if result.rowcount == 1:
return
raise ValueError('Authorization changed concurrently. Please retry cancellation.')
async def access(self, context, provider: str, *, rejected_token: str | None = None) -> dict:
workspace = await self._provider(context, provider)
row = await self._read(workspace, provider)
payload = row['payload'] if row else {}
tokens = payload.get('tokens')
if not tokens or payload.get('invalid'):
raise ValueError(LOGIN_REQUIRED)
if tokens['expires_at'] > time.time() + 120 and tokens['access_token'] != rejected_token:
return tokens
async with self._lease(workspace, provider, refresh=True) as owner:
row = await self._read(workspace, provider)
payload = dict(row['payload'])
tokens = payload.get('tokens')
if not tokens or payload.get('invalid'):
raise ValueError(LOGIN_REQUIRED)
if tokens['expires_at'] > time.time() + 120 and tokens['access_token'] != rejected_token:
return tokens
response = await self._post(
'/oauth/token',
data={'grant_type': 'refresh_token', 'client_id': CLIENT_ID, 'refresh_token': tokens['refresh_token']},
)
error = self._json(response).get('error') if response.status_code in (400, 401, 403) else None
error_code = error.get('code') if isinstance(error, dict) else error
if error_code in (
'invalid_grant',
'refresh_token_reused',
'refresh_token_expired',
'refresh_token_revoked',
):
payload['invalid'] = True
payload.pop('tokens', None)
payload.pop('completed', None)
await self._save(workspace, provider, owner, payload)
raise ValueError(LOGIN_REQUIRED)
if response.status_code != 200:
raise ValueError('ChatGPT token refresh temporarily failed. Please retry.')
refreshed = _tokens(self._json(response), tokens)
payload['tokens'] = refreshed
await self._save(workspace, provider, owner, payload)
return refreshed
@@ -0,0 +1,10 @@
"""Explicitly safe Codex failures; never construct messages from upstream bodies."""
class CodexProviderError(ValueError):
"""A known provider failure safe to expose at the HTTP boundary."""
def __init__(self, message: str, status_code: int = 502, error_code: str = 'codex_upstream_failure'):
super().__init__(message)
self.status_code = status_code
self.error_code = error_code
+31 -5
View File
@@ -18,7 +18,7 @@ from ...discover import engine
from ...entity.errors import provider as provider_errors
from ...entity.persistence import model as persistence_model
from ...workspace.entities import WorkspaceExecutionBinding
from ...workspace.errors import WorkspaceError, WorkspaceInvariantError
from ...workspace.errors import WorkspaceError, WorkspaceInvariantError, WorkspaceNotFoundError
from . import requester, token
@@ -638,10 +638,32 @@ class ModelManager:
) -> requester.RuntimeLLMModel:
execution_context = await self.resolve_execution_context(context)
provider_info = {**model_info.get('provider', {}), 'workspace_uuid': execution_context.workspace_uuid}
runtime_provider = await self._build_provider(
execution_context,
persistence_model.ModelProvider(**provider_info),
)
provider_uuid = model_info.get('provider_uuid') or provider_info.get('uuid')
inline_codex = provider_info.get('requester') == 'openai-codex'
provider_entity = persistence_model.ModelProvider(**provider_info)
if provider_uuid:
if provider_info.get('uuid') and provider_info['uuid'] != provider_uuid:
raise ValueError('Conflicting provider identities')
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(persistence_model.ModelProvider).where(
persistence_model.ModelProvider.workspace_uuid == execution_context.workspace_uuid,
persistence_model.ModelProvider.uuid == provider_uuid,
)
)
saved_provider = result.first()
if saved_provider is None:
if inline_codex or model_info.get('provider_uuid'):
raise WorkspaceNotFoundError('Provider not found')
else:
saved_provider = self._coerce_provider(saved_provider, execution_context)
if saved_provider.requester == 'openai-codex':
# OAuth identity and transport configuration are server-owned.
provider_entity = saved_provider
elif inline_codex:
raise ValueError('This provider does not use ChatGPT sign-in')
elif inline_codex:
raise WorkspaceNotFoundError('Provider not found')
runtime_provider = await self._build_provider(execution_context, provider_entity)
model_entity = persistence_model.LLMModel(
workspace_uuid=execution_context.workspace_uuid,
uuid=model_info.get('uuid', ''),
@@ -723,6 +745,10 @@ class ModelManager:
'requester_name': provider_entity.requester,
}
if provider_entity.requester == 'openai-codex':
config['provider_uuid'] = provider_entity.uuid
config['workspace_uuid'] = context.workspace_uuid
if litellm_provider:
from .requesters import litellmchat
@@ -0,0 +1,423 @@
"""Native ChatGPT Codex Responses/SSE requester (never Chat Completions)."""
from __future__ import annotations
import asyncio
import json
import secrets
import time
from collections import OrderedDict
import httpx
import langbot
import langbot_plugin.api.entities.builtin.provider.message as pm
from .. import requester, reasoning
from ..codex_auth import BASE_URL, CodexAuth, LOGIN_REQUIRED
from ..codex_errors import CodexProviderError
async def sse_events(response):
"""Decode SSE records, including CRLF, comments, and multiline data."""
data = []
size = 0
async for line in response.aiter_lines():
if not line:
if data:
text = '\n'.join(data)
if text == '[DONE]':
return
try:
event = json.loads(text)
if not isinstance(event, dict):
raise ValueError
except ValueError:
raise ValueError('Codex returned an invalid stream event') from None
yield event
data, size = [], 0
elif line.startswith('data:'):
value = line[5:]
if value.startswith(' '):
value = value[1:]
size += len(value)
if size > 4 * 1024 * 1024:
raise ValueError('Codex stream event exceeds the size limit')
data.append(value)
# SSE requires the blank separator; unterminated records cannot prove completion.
def _content(message):
content = message.content
if isinstance(content, str):
return [{'type': 'output_text' if message.role == 'assistant' else 'input_text', 'text': content}]
result = []
for part in content or []:
if part.type == 'text':
result.append(
{'type': 'output_text' if message.role == 'assistant' else 'input_text', 'text': part.text or ''}
)
elif part.type == 'image_url' and part.image_url is not None:
result.append({'type': 'input_image', 'image_url': part.image_url.url})
elif part.type == 'image_base64' and part.image_base64:
value = part.image_base64
result.append(
{
'type': 'input_image',
'image_url': value if value.startswith('data:') else 'data:image/png;base64,' + value,
}
)
else:
raise ValueError('Codex supports text and images only; this message contains unsupported content')
return result
def _tool(item):
try:
return pm.ToolCall(
id=item['call_id'],
type='function',
function=pm.FunctionCall(name=item['name'], arguments=item.get('arguments') or ''),
)
except (KeyError, ValueError, TypeError):
raise ValueError('Codex returned an invalid function call') from None
def _usage(response):
usage = response.get('usage') or {}
return {
'prompt_tokens': usage.get('input_tokens', 0),
'completion_tokens': usage.get('output_tokens', 0),
'total_tokens': usage.get('total_tokens', usage.get('input_tokens', 0) + usage.get('output_tokens', 0)),
'prompt_tokens_details': usage.get('input_tokens_details', {}),
'completion_tokens_details': usage.get('output_tokens_details', {}),
}
class CodexRequester(requester.ProviderAPIRequester):
async def initialize(self):
self.auth = CodexAuth(self.ap)
self.workspace = self.requester_cfg['workspace_uuid']
self.provider = self.requester_cfg['provider_uuid']
# Opaque replay data stays server-side; handles are scoped to the same query,
# model and OAuth connection. No token or encrypted reasoning enters messages.
self._replay = OrderedDict()
async def aclose(self):
self._replay.clear()
def get_reasoning_capabilities(self, model):
return {
'supported': True,
'levels': ['provider_default', 'low', 'medium', 'high', 'xhigh'],
'source': 'provider',
}
@staticmethod
def _headers(tokens, *, stream=False):
return {
'Authorization': 'Bearer ' + tokens['access_token'],
'ChatGPT-Account-ID': tokens['account_id'],
'User-Agent': 'LangBot/' + langbot.__version__,
'originator': 'langbot',
'OpenAI-Beta': 'responses=experimental',
'Accept': 'text/event-stream' if stream else 'application/json',
}
@staticmethod
def _http_error(status):
if status == 401:
# Upstream authentication is not LangBot authentication: HTTP401 would
# make the browser discard its own valid user session.
return CodexProviderError(LOGIN_REQUIRED, 400, 'codex_reauthentication_required')
if status == 429:
return CodexProviderError(
'ChatGPT request was limited (rate limit or usage restriction). Please retry later or check your plan.',
429,
'codex_rate_limited',
)
if status == 403:
return CodexProviderError(
'ChatGPT denied this request. Check subscription and workspace permissions.',
403,
'codex_access_denied',
)
if status == 400:
return CodexProviderError(
'ChatGPT rejected the model or request. Check the selected model and request settings.',
400,
'codex_invalid_request',
)
return CodexProviderError('ChatGPT Codex upstream request failed. Please retry later.')
async def _response_error(self, response):
# Inspect only a bounded 429 error record and an allowlisted machine code.
# Never expose upstream prose, reset metadata, headers or credentials.
if response.status_code == 429:
payload = bytearray()
async for chunk in response.aiter_bytes():
if len(payload) + len(chunk) > 8192:
return self._http_error(429)
payload.extend(chunk)
try:
data = json.loads(payload)
error = data.get('error') if isinstance(data, dict) else None
if isinstance(error, dict) and (
error.get('type') == 'usage_limit_reached' or error.get('code') == 'usage_limit_reached'
):
return CodexProviderError(
'ChatGPT subscription usage limit reached. Please retry later or check your plan.',
429,
'codex_usage_limit_reached',
)
except (ValueError, UnicodeError):
pass
return self._http_error(response.status_code)
def _scope(self, query, model, tokens):
return (
id(query),
getattr(query, 'query_id', None),
model.model_entity.name,
tokens.get('connection_id'),
tokens['account_id'],
)
def _body(self, query, model, messages, funcs, extra_args, tokens):
args = {**(model.model_entity.extra_args or {}), **(extra_args or {})}
# Never permit credentials, transport overrides, store/history or arbitrary
# SDK kwargs to be smuggled through model advanced parameters.
allowed = {'reasoning', 'text', 'parallel_tool_calls', 'tool_choice'}
unknown = set(args) - allowed
if unknown:
raise ValueError('Unsupported Codex advanced parameters: ' + ', '.join(sorted(unknown)))
instructions = []
items = []
scope = self._scope(query, model, tokens)
for message in messages:
if message.role in ('system', 'developer'):
instructions.append('\n'.join(p['text'] for p in _content(message) if 'text' in p))
continue
if message.role == 'tool':
if not message.tool_call_id:
raise ValueError('Codex tool results require a tool_call_id')
output = (
message.content
if isinstance(message.content, str)
else json.dumps([p.model_dump(exclude_none=True) for p in message.content or []])
)
items.append({'type': 'function_call_output', 'call_id': message.tool_call_id, 'output': output or ''})
continue
if message.role not in ('assistant', 'user'):
raise ValueError('Unsupported Codex message role')
handle = (message.provider_specific_fields or {}).get('codex_replay_id')
cached = self._replay.get(handle) if isinstance(handle, str) else None
if query is not None and cached and cached[0] == scope and cached[1] > time.time():
items.extend(cached[2])
continue
content = _content(message)
if content:
items.append({'type': 'message', 'role': message.role, 'content': content})
for call in message.tool_calls or []:
items.append(
{
'type': 'function_call',
'call_id': call.id,
'name': call.function.name,
'arguments': call.function.arguments,
}
)
body = {
**args,
'model': model.model_entity.name,
'instructions': '\n\n'.join(instructions),
'input': items,
'store': False,
'stream': True,
'include': ['reasoning.encrypted_content'],
}
level = reasoning.normalize_reasoning_config(getattr(model.model_entity, 'reasoning_config', None))['level']
if level != 'provider_default':
reasoning.validate_reasoning_capabilities(
{'level': level}, self.get_reasoning_capabilities(model), model.model_entity.name
)
body['reasoning'] = {'effort': level, 'summary': 'auto'}
if funcs:
body['tools'] = [
{
'type': 'function',
'name': f.name,
'description': f.description,
'parameters': f.parameters,
'strict': False,
}
for f in funcs
]
return body
async def _events(self, query, model, messages, funcs, extra_args):
tokens = await self.auth.access(self.workspace, self.provider)
try:
async with asyncio.timeout(300), httpx.AsyncClient(timeout=120, follow_redirects=False) as client:
for attempt in range(2):
body = self._body(query, model, messages, funcs, extra_args, tokens)
async with client.stream(
'POST', BASE_URL + '/responses', json=body, headers=self._headers(tokens, stream=True)
) as response:
if response.status_code == 401 and attempt == 0:
tokens = await self.auth.access(
self.workspace, self.provider, rejected_token=tokens['access_token']
)
continue
if response.status_code != 200:
raise await self._response_error(response)
async for event in sse_events(response):
yield event, tokens
return
except (httpx.HTTPError, TimeoutError):
raise ValueError('ChatGPT Codex network error or timeout. Please retry.') from None
async def _chunks(self, query, model, messages, funcs, extra_args, remove_think, usage_out):
text = ''
seen_calls = set()
output_items = {}
response_id = None
async for event, tokens in self._events(query, model, messages, funcs, extra_args):
kind = event.get('type')
response = event.get('response') or {}
response_id = response.get('id') or response_id
if kind in ('error', 'response.failed', 'response.incomplete'):
raise CodexProviderError('ChatGPT Codex response failed or was incomplete. Please retry.')
if kind == 'response.output_text.delta':
delta = event.get('delta', '')
text += delta
yield pm.MessageChunk(role='assistant', content=delta, resp_message_id=response_id)
elif kind in ('response.reasoning_summary_text.delta', 'response.reasoning_text.delta'):
if not remove_think:
yield pm.MessageChunk(
role='assistant',
content='',
provider_specific_fields={'reasoning_content': event.get('delta', '')},
)
elif kind == 'response.output_item.done':
item = event.get('item') or {}
output_items[event.get('output_index', len(output_items))] = item
if item.get('type') == 'function_call' and item.get('call_id') not in seen_calls:
seen_calls.add(item.get('call_id'))
yield pm.MessageChunk(role='assistant', content='', tool_calls=[_tool(item)])
elif kind in ('response.completed', 'response.done'):
if response.get('status') not in (None, 'completed'):
raise CodexProviderError('ChatGPT Codex response was not completed')
output = response.get('output') or [output_items[k] for k in sorted(output_items)]
for item in output:
if item.get('type') == 'function_call' and item.get('call_id') not in seen_calls:
seen_calls.add(item.get('call_id'))
yield pm.MessageChunk(role='assistant', content='', tool_calls=[_tool(item)])
# Some servers send only the terminal output, without text deltas.
final_text = ''.join(
p.get('text', '')
for item in output
if item.get('type') == 'message'
for p in item.get('content', [])
if p.get('type') == 'output_text'
)
if not text and final_text:
text = final_text
yield pm.MessageChunk(role='assistant', content=text, resp_message_id=response_id)
usage_out.update(_usage(response))
if query is not None:
if query.variables is None:
query.variables = {}
query.variables[requester.STREAM_USAGE_QUERY_VARIABLE] = dict(usage_out)
fields = None
if query is not None and output:
handle = secrets.token_urlsafe(24)
self._replay[handle] = (self._scope(query, model, tokens), time.time() + 3600, output)
while len(self._replay) > 64:
self._replay.popitem(last=False)
fields = {'codex_replay_id': handle}
yield pm.MessageChunk(
role='assistant',
content='',
all_content=text,
is_final=True,
resp_message_id=response_id,
provider_specific_fields=fields,
)
return
raise CodexProviderError('ChatGPT Codex stream ended before completion. Please retry.')
async def invoke_llm_stream(self, query, model, messages, funcs=None, extra_args=None, remove_think=False):
async for chunk in self._chunks(query, model, messages, funcs, extra_args, remove_think, {}):
yield chunk
async def invoke_llm(self, query, model, messages, funcs=None, extra_args=None, remove_think=False):
usage = {}
text = ''
calls = []
fields = {}
response_id = None
async for chunk in self._chunks(query, model, messages, funcs, extra_args, remove_think, usage):
text += chunk.content or ''
calls.extend(chunk.tool_calls or [])
response_id = chunk.resp_message_id or response_id
for key, value in (chunk.provider_specific_fields or {}).items():
fields[key] = fields.get(key, '') + value if key == 'reasoning_content' else value
return pm.Message(
role='assistant',
content=text,
tool_calls=calls or None,
resp_message_id=response_id,
provider_specific_fields=fields or None,
), usage
async def scan_models(self, api_key=None):
tokens = await self.auth.access(self.workspace, self.provider)
try:
async with asyncio.timeout(90), httpx.AsyncClient(timeout=30, follow_redirects=False) as client:
for attempt in range(2):
response = await client.get(
BASE_URL + '/models',
params={'client_version': langbot.__version__},
headers=self._headers(tokens),
)
if response.status_code == 401 and attempt == 0:
tokens = await self.auth.access(
self.workspace, self.provider, rejected_token=tokens['access_token']
)
continue
if response.status_code != 200:
raise await self._response_error(response)
data = response.json()
if not isinstance(data, dict) or not isinstance(data.get('models'), list):
raise ValueError('ChatGPT returned an invalid model catalog')
result = {}
for item in data['models']:
name = item.get('slug') or item.get('id')
if not isinstance(name, str) or not name or item.get('visibility') == 'hide':
continue
modalities = item.get('input_modalities') or ['text']
abilities = ['func_call']
if 'image' in modalities:
abilities.append('vision')
if item.get('supported_reasoning_levels'):
abilities.append('reasoning')
result[name] = {
'id': name,
'name': name,
'type': 'llm',
'abilities': abilities,
'display_name': item.get('display_name'),
'description': item.get('description'),
'context_length': item.get('context_window'),
'input_modalities': modalities,
'output_modalities': ['text'],
'owned_by': 'openai',
}
return {'models': list(result.values()), 'debug': None}
except (httpx.HTTPError, TimeoutError):
raise ValueError('ChatGPT model discovery network error. Please retry.') from None
except (ValueError, TypeError, KeyError, AttributeError) as exc:
# Never echo upstream response bodies (which may contain credentials).
if isinstance(exc, ValueError) and str(exc).startswith(('ChatGPT', 'Codex')):
raise
raise ValueError('ChatGPT returned an invalid model catalog') from None
@@ -0,0 +1,27 @@
apiVersion: v1
kind: LLMAPIRequester
metadata:
name: openai-codex
label:
en_US: OpenAI Codex
zh_Hans: OpenAI Codex
ja_JP: OpenAI Codex
icon: openai.svg
spec:
config:
- name: base_url
label:
en_US: ChatGPT endpoint
zh_Hans: ChatGPT 服务地址
ja_JP: ChatGPT エンドポイント
type: string
required: false
default: https://chatgpt.com/backend-api/codex
alias: "openai codex ChatGPT subscription OAuth 订阅"
support_type:
- llm
provider_category: manufacturer
execution:
python:
path: ./codex.py
attr: CodexRequester
@@ -573,7 +573,7 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
levels = ['provider_default', 'disabled', 'enabled']
elif family == 'doubao':
levels = ['provider_default', 'disabled', 'low', 'medium', 'high']
elif family == 'ollama':
elif family in ('ollama', 'ollama_chat'):
levels = ['provider_default']
levels.append('disabled')
if normalized_name.startswith('gpt-oss') or '/gpt-oss' in normalized_name:
@@ -1379,7 +1379,14 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
extra_args: dict[str, typing.Any] = {},
) -> tuple[list[list[float]], dict]:
"""Invoke embedding and return vectors with usage info."""
model_name = self._build_litellm_model_name(model.model_entity.name)
# litellm's embedding routing has no "ollama_chat" branch (that provider
# exists only for /api/chat completions) — embeddings still go through
# the plain "ollama" provider. Requesters configured for ollama_chat
# (to get native tool-calling on the chat path) must fall back to
# "ollama" here specifically, or embedding calls raise "Unmapped LLM
# provider for this endpoint".
embedding_provider = 'ollama' if self._get_custom_llm_provider() == 'ollama_chat' else None
model_name = self._build_litellm_model_name(model.model_entity.name, embedding_provider)
api_key = model.provider.token_mgr.get_token()
args = {
@@ -1575,6 +1582,12 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
event_hooks=httpclient.httpx_response_limit_hooks(),
) as client:
response = await client.get(models_url, headers=headers)
if response.status_code == 404 and not base_url.rstrip('/').endswith('/v1'):
# Some OpenAI-compatible servers (notably a bare Ollama host,
# e.g. http://host:11434) expose the model list under /v1/models
# rather than /models. Providers whose configured base_url
# already ends in /v1 keep their original (working) URL.
response = await client.get(f'{base_url}/v1/models', headers=headers)
response.raise_for_status()
payload = await httpclient.parse_json_response(response)
@@ -7,7 +7,7 @@ metadata:
zh_Hans: Ollama
icon: ollama.svg
spec:
litellm_provider: ollama
litellm_provider: ollama_chat
config:
- name: base_url
label:
@@ -1,5 +1,8 @@
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64" role="img" aria-label="TokenLab">
<rect width="64" height="64" rx="14" fill="#111827"/>
<path fill="#38bdf8" d="M17 14h30v8H36v28h-8V22H17z"/>
<path fill="#a7f3d0" d="M40 30h8v20H28v-8h12z"/>
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 48 48" role="img" aria-labelledby="tokenlab-symbol-title tokenlab-symbol-desc">
<title id="tokenlab-symbol-title">TokenLab</title>
<desc id="tokenlab-symbol-desc">Specimen Split symbol, positive master for sizes from 32 to 96 pixels.</desc>
<g fill="#151714">
<path d="M5 13h14.25L35 43H5V13Z"/>
<path d="M20.7 5H43v30H35.6L20.7 5Z"/>
</g>
</svg>

Before

Width:  |  Height:  |  Size: 265 B

After

Width:  |  Height:  |  Size: 416 B

+56 -1
View File
@@ -1,5 +1,6 @@
from __future__ import annotations
import dataclasses
import enum
import json
import math
@@ -208,6 +209,13 @@ class MCPSessionStatus(enum.Enum):
ERROR = 'error'
@dataclasses.dataclass(frozen=True)
class MCPOAuthChallenge:
"""Bearer challenge metadata returned by an OAuth-protected MCP server."""
resource_metadata_url: str | None
class _TransportReconnect(Exception):
"""Internal signal: the Box stdio WS transport dropped but the managed
process is still alive. Triggers a lightweight transport reconnect that
@@ -267,6 +275,7 @@ class RuntimeMCPSession:
_ready_event: asyncio.Event
error_message: str | None = None
_public_error_code: str = 'runtime_error'
error_phase: MCPSessionErrorPhase | None = None
@@ -513,6 +522,13 @@ class RuntimeMCPSession:
await self._init_streamable_http_server()
return
except Exception as e:
if self._extract_oauth_challenge(e) is not None:
self.error_phase = MCPSessionErrorPhase.OAUTH_REQUIRED
self.ap.logger.info(
f'MCP server {self.server_name}: remote server requires OAuth authorization; '
'not falling back to SSE'
)
raise
if not self._should_fallback_to_sse(e):
self.ap.logger.info(
f'MCP server {self.server_name}: Streamable HTTP transport failed '
@@ -636,6 +652,7 @@ class RuntimeMCPSession:
except Exception as e:
self.status = MCPSessionStatus.ERROR
self.error_message = str(e)
self._public_error_code = self._classify_public_error(e)
self.ap.logger.error(f'Error in MCP session lifecycle {self.server_name}: {e}\n{traceback.format_exc()}')
# Do NOT set _ready_event here — let _lifecycle_loop_with_retry
# handle retries first. It will set the event when all retries
@@ -759,6 +776,11 @@ class RuntimeMCPSession:
self.retry_count += 1
if self._shutdown_event.is_set():
return # Shutdown requested, don't retry
if self.error_phase == MCPSessionErrorPhase.OAUTH_REQUIRED:
self.retry_count = attempt + 1
self.status = MCPSessionStatus.ERROR
self._ready_event.set()
return
# A lifecycle that reached CONNECTED proved its preceding
# startup attempt was healthy. Give a later runtime failure a
# fresh consecutive-startup retry budget instead of exhausting
@@ -897,6 +919,39 @@ class RuntimeMCPSession:
else:
yield exc
@staticmethod
def _classify_public_error(exc: BaseException) -> str:
"""Expose a safe category without transport URLs, headers, or arguments."""
for leaf in RuntimeMCPSession._iter_exception_leaves(exc):
if isinstance(leaf, httpx.HTTPStatusError):
return f'http_{leaf.response.status_code}'
if isinstance(leaf, (httpx.TimeoutException, TimeoutError)):
return 'connection_timeout'
if isinstance(leaf, httpx.ConnectError):
return 'connection_unreachable'
return 'runtime_error'
@staticmethod
def _extract_oauth_challenge(exc: BaseException) -> MCPOAuthChallenge | None:
"""Extract an OAuth Bearer challenge from a remote MCP connection failure."""
for leaf in RuntimeMCPSession._iter_exception_leaves(exc):
if not isinstance(leaf, httpx.HTTPStatusError) or leaf.response.status_code != 401:
continue
for header in leaf.response.headers.get_list('www-authenticate'):
bearer_match = re.search(r'(?:^|,)\s*Bearer(?:\s|,|$)', header, flags=re.IGNORECASE)
if bearer_match is None:
continue
metadata_match = re.search(
r'(?:^|,)\s*resource_metadata\s*=\s*(?:"([^"]+)"|([^,\s]+))',
header[bearer_match.end() :],
flags=re.IGNORECASE,
)
if metadata_match is None:
continue
resource_metadata_url = metadata_match.group(1) or metadata_match.group(2)
return MCPOAuthChallenge(resource_metadata_url=resource_metadata_url)
return None
@staticmethod
def _should_fallback_to_sse(exc: BaseException) -> bool:
"""Whether a Streamable HTTP failure matches legacy-SSE fallback.
@@ -1439,7 +1494,7 @@ class RuntimeMCPSession:
# environment values. Detailed diagnostics belong in AUDIT_VIEW
# logs; resource-list responses expose only a stable status.
'error_message': 'MCP runtime failed' if self.error_message else None,
'error_code': 'runtime_error' if self.error_message else None,
'error_code': self._public_error_code if self.error_message else None,
'error_phase': self.error_phase.value if self.error_phase else None,
'retry_count': self.retry_count,
'tool_count': len(self.get_tools()),
@@ -52,6 +52,7 @@ class MCPSessionErrorPhase(enum.Enum):
MCP_INIT = 'mcp_init'
RUNTIME = 'runtime'
TOOL_CALL = 'tool_call'
OAUTH_REQUIRED = 'oauth_required'
# Stdio MCP refused because Box is disabled in config or currently
# unavailable. Not transient — retries would be pointless. The frontend
# uses this phase to render a localized actionable message instead of
@@ -223,6 +223,7 @@ class NativeToolLoader(loader.ToolLoader):
self.ap.logger.warning(
'Native sandbox tools (exec/read/write/edit/glob/grep) are NOT available. '
'No sandbox backend (Docker/nsjail/E2B) is ready. '
'Trusted local development may explicitly select box.backend=host. '
'The LLM will not have access to code execution or file operation tools.'
)
@@ -42,7 +42,8 @@ class SkillToolLoader(loader.ToolLoader):
else:
self.ap.logger.info(
'Skill tools (activate/register_skill) are NOT available. '
'No sandbox backend (Docker/nsjail/E2B) is ready.'
'No sandbox backend (Docker/nsjail/E2B) is ready. '
'Trusted local development may explicitly select box.backend=host.'
)
async def _check_sandbox_available(self) -> bool:
+231
View File
@@ -0,0 +1,231 @@
from __future__ import annotations
import asyncio
import base64
import copy
import datetime
import mimetypes
import os
import re
import xxhash
from pathlib import Path
from typing import TYPE_CHECKING, Any
if TYPE_CHECKING:
from ...core import app
from . import mgr as storage_mgr
DEFAULT_RETENTION_DAYS = 30
DEFAULT_MAX_SIZE_MB = 0
MEDIA_DIR = 'media_cache'
SAFE_MEDIA_FILENAME = re.compile(r'^[a-f0-9]{32,64}(\.[a-zA-Z0-9]{1,10})?$')
class MediaCache:
"""Content-addressable storage cache for images and media attachments.
Deduplicates media files using xxHash3-128,
offloads payloads from SQLite to StorageProvider, and implements LRU
and age-based retention cleanup.
"""
def __init__(self, ap: app.Application, storage_mgr: storage_mgr.StorageMgr):
self.ap = ap
self.storage_mgr = storage_mgr
@staticmethod
def hash_bytes(data: bytes) -> str:
"""Compute content-addressable hash for binary data."""
return xxhash.xxh3_128_hexdigest(data)
@staticmethod
def parse_data_url(data_url: str) -> tuple[bytes, str] | None:
"""Parse a data URL or raw base64 string into bytes and mime type."""
if not data_url or not isinstance(data_url, str):
return None
try:
if data_url.startswith('data:'):
split_index = data_url.find(';base64,')
if split_index != -1:
mime_type = data_url[5:split_index]
b64_data = data_url[split_index + 8 :]
return base64.b64decode(b64_data), mime_type
# Try raw base64 if sufficiently long
if len(data_url) > 20 and not data_url.startswith(('http://', 'https://', '/')):
return base64.b64decode(data_url), 'application/octet-stream'
except Exception:
return None
return None
@staticmethod
def guess_extension(mime_type: str | None, default: str = '.jpg') -> str:
"""Guess appropriate file extension from MIME type."""
if not mime_type:
return default
mime_lower = mime_type.lower()
if 'png' in mime_lower:
return '.png'
if 'webp' in mime_lower:
return '.webp'
if 'gif' in mime_lower:
return '.gif'
if 'jpeg' in mime_lower or 'jpg' in mime_lower:
return '.jpg'
ext = mimetypes.guess_extension(mime_type)
if ext == '.jpe':
return '.jpg'
return ext or default
async def save_media(self, data: bytes, mime_type: str | None = None) -> tuple[str, str, int]:
"""Save media bytes into content-addressable storage cache.
Returns:
Tuple of (hash_str, storage_key, byte_size)
"""
hash_str = self.hash_bytes(data)
ext = self.guess_extension(mime_type)
storage_key = f'{MEDIA_DIR}/{hash_str}{ext}'
provider = self.storage_mgr.storage_provider
if not await provider.exists(storage_key):
await provider.save(storage_key, data)
else:
await self.touch(storage_key)
return hash_str, storage_key, len(data)
async def get_media(self, filename_or_key: str) -> tuple[bytes, str] | None:
"""Retrieve media bytes and mime type by key or filename."""
filename = os.path.basename(filename_or_key)
if not SAFE_MEDIA_FILENAME.match(filename):
return None
storage_key = f'{MEDIA_DIR}/{filename}'
provider = self.storage_mgr.storage_provider
if not await provider.exists(storage_key):
return None
data = await self.storage_mgr._load_object_bounded(storage_key)
mime_type = mimetypes.guess_type(filename)[0] or 'application/octet-stream'
await self.touch(storage_key)
return data, mime_type
async def touch(self, storage_key: str) -> None:
"""Update access/modified time of a media file for LRU tracking."""
provider = getattr(self.storage_mgr, 'storage_provider', None)
if provider is not None and provider.__class__.__name__ == 'LocalStorageProvider':
full_path = os.path.join('data', 'storage', storage_key)
if os.path.exists(full_path):
now = datetime.datetime.now().timestamp()
try:
await asyncio.to_thread(os.utime, full_path, (now, now))
except Exception:
# Failures to update mtime are intentionally ignored because LRU touch is opportunistic.
pass
async def cleanup(
self,
retention_days: int = DEFAULT_RETENTION_DAYS,
max_size_mb: int = DEFAULT_MAX_SIZE_MB,
) -> dict[str, int]:
"""Perform age-based and LRU size-based cleanup on media cache.
Args:
retention_days: Retain media accessed within this many days (default 30).
max_size_mb: Maximum total size in MB (0 means unlimited).
Returns:
Dictionary of cleanup metrics.
"""
provider = getattr(self.storage_mgr, 'storage_provider', None)
if provider is None or provider.__class__.__name__ != 'LocalStorageProvider':
return {'expired_deleted': 0, 'size_deleted': 0, 'bytes_freed': 0}
target_dir = Path('data/storage') / MEDIA_DIR
if not target_dir.exists() or not target_dir.is_dir():
return {'expired_deleted': 0, 'size_deleted': 0, 'bytes_freed': 0}
now = datetime.datetime.now().timestamp()
cutoff = (now - retention_days * 86400) if retention_days > 0 else 0
expired_deleted = 0
size_deleted = 0
bytes_freed = 0
remaining: list[tuple[Path, int, float]] = []
for entry in target_dir.iterdir():
if not entry.is_file():
continue
try:
stat = entry.stat()
except OSError:
continue
if cutoff > 0 and stat.st_mtime < cutoff:
try:
entry.unlink(missing_ok=True)
expired_deleted += 1
bytes_freed += stat.st_size
except OSError:
# Best-effort cleanup; file may already be gone or temporarily inaccessible.
pass
else:
remaining.append((entry, stat.st_size, stat.st_mtime))
if max_size_mb > 0:
max_bytes = max_size_mb * 1024 * 1024
total_bytes = sum(item[1] for item in remaining)
if total_bytes > max_bytes:
remaining.sort(key=lambda item: item[2])
for path, size, _ in remaining:
if total_bytes <= max_bytes:
break
try:
path.unlink(missing_ok=True)
size_deleted += 1
bytes_freed += size
total_bytes -= size
except OSError:
# Best-effort cleanup; file may already be gone or temporarily inaccessible.
continue
return {
'expired_deleted': expired_deleted,
'size_deleted': size_deleted,
'bytes_freed': bytes_freed,
}
async def externalize_chain_dump(self, chain_dump: Any) -> Any:
"""Recursively extract raw base64 media into cache and replace with references."""
if isinstance(chain_dump, list):
return [await self.externalize_chain_dump(item) for item in chain_dump]
if isinstance(chain_dump, dict):
node = copy.copy(chain_dump)
node_type = node.get('type')
if node_type == 'Image':
b64 = node.get('base64')
if b64 and isinstance(b64, str):
try:
parsed = self.parse_data_url(b64)
if parsed is not None:
raw_bytes, mime_type = parsed
hash_str, storage_key, size = await self.save_media(raw_bytes, mime_type)
filename = os.path.basename(storage_key)
current_url = node.get('url') or ''
if current_url and not current_url.startswith('data:'):
node['original_url'] = current_url
node['url'] = f'/api/v1/files/media/{filename}'
node['base64'] = None
node['hash'] = hash_str
node['storage_key'] = storage_key
node['size'] = size
node['mime_type'] = mime_type
except Exception as e:
if hasattr(self.ap, 'logger') and self.ap.logger:
self.ap.logger.warning(f'Failed to externalize image to media cache: {e}')
node['base64'] = None
for k, v in list(node.items()):
if isinstance(v, (list, dict)):
node[k] = await self.externalize_chain_dump(v)
return node
return chain_dump
+3
View File
@@ -34,6 +34,9 @@ class StorageMgr:
def __init__(self, ap: app.Application):
self.ap = ap
from . import media
self.media_cache = media.MediaCache(ap, self)
def _object_read_limit(self) -> int:
config = getattr(getattr(self.ap, 'instance_config', None), 'data', {})
+1 -1
View File
@@ -83,7 +83,7 @@ class VersionManager:
try:
if await self.is_new_version_available():
return (
'New version available. Update guide: https://link.langbot.app/en/docs/update',
'New version available. Update guide: https://langbot.app/docs/en/deploy/update',
logging.INFO,
)
except Exception as e:
+25 -26
View File
@@ -101,18 +101,6 @@ class SeekDBVectorDatabase(VectorDatabase):
self._collection_configs: Dict[str, HNSWConfiguration] = {}
self._runtime_cache_limit = runtime_cache_limit(ap)
self._escape_table = str.maketrans(
{
'\x00': '',
'\\': '\\\\',
"'": "''", # Standard SQL escaping (OceanBase NO_BACKSLASH_ESCAPES)
'"': '\\"',
'\n': '\\n',
'\r': '\\r',
'\t': '\\t',
}
)
async def close(self) -> None:
self._collections.clear()
self._collection_configs.clear()
@@ -192,16 +180,22 @@ class SeekDBVectorDatabase(VectorDatabase):
return coll
def _clean_metadata(self, meta: Dict[str, Any]) -> Dict[str, Any]:
"""SeekDB metadata doesn't support \\ and ", insert will error 3104"""
return {
k: v.translate(self._escape_table)
if isinstance(v, str)
else v
if v is None or isinstance(v, (int, float, bool))
else str(v)
for k, v in meta.items()
if v is not None
}
"""Keep supported scalar metadata values without altering strings."""
return {k: v if isinstance(v, (str, int, float, bool)) else str(v) for k, v in meta.items() if v is not None}
@staticmethod
def _relevance_scores_to_distances(results: Dict[str, Any]) -> None:
"""Convert SeekDB hybrid relevance scores to lower-is-better distances."""
distances = results.get('distances')
if not isinstance(distances, list):
return
results['distances'] = [
[1.0 - float(score) if isinstance(score, (int, float)) else score for score in batch]
if isinstance(batch, list)
else batch
for batch in distances
]
async def get_or_create_collection(self, collection: str):
"""Get or create collection (without vector size - will use default)."""
@@ -236,10 +230,10 @@ class SeekDBVectorDatabase(VectorDatabase):
kwargs: Dict[str, Any] = dict(ids=ids, embeddings=embeddings_list, metadatas=cleaned_metadatas)
if documents is not None:
kwargs['documents'] = [doc.translate(self._escape_table) for doc in documents]
await asyncio.to_thread(coll.add, **kwargs)
kwargs['documents'] = documents
await asyncio.to_thread(coll.upsert, **kwargs)
self.ap.logger.info(f"Added {len(ids)} embeddings to SeekDB collection '{collection}'")
self.ap.logger.info(f"Upserted {len(ids)} embeddings into SeekDB collection '{collection}'")
async def search(
self,
@@ -287,7 +281,8 @@ class SeekDBVectorDatabase(VectorDatabase):
# Route by search type.
# pyseekdb's query() always requires embeddings, so full-text and
# hybrid modes use hybrid_search() which supports text-only queries
# and returns the same nested-list format with distances.
# and returns relevance scores in the nested ``distances`` field.
returns_relevance_scores = False
if search_type == SearchType.FULL_TEXT:
if not query_text:
return {'ids': [[]], 'metadatas': [[]], 'distances': [[]]}
@@ -309,6 +304,7 @@ class SeekDBVectorDatabase(VectorDatabase):
n_results=k,
include=['documents', 'metadatas'],
)
returns_relevance_scores = True
elif search_type == SearchType.HYBRID:
if not query_text:
@@ -352,6 +348,7 @@ class SeekDBVectorDatabase(VectorDatabase):
n_results=k,
include=['documents', 'metadatas'],
)
returns_relevance_scores = True
self.ap.logger.info(
f"SeekDB hybrid search in '{collection}' returned {len(results.get('ids', [[]])[0])} results."
)
@@ -363,6 +360,8 @@ class SeekDBVectorDatabase(VectorDatabase):
results = await asyncio.to_thread(coll.query, **query_kwargs)
results = self._json_safe(results)
if returns_relevance_scores:
self._relevance_scores_to_distances(results)
self.ap.logger.info(
f"SeekDB {search_type} search in '{collection}' returned {len(results.get('ids', [[]])[0])} results"
)
@@ -269,7 +269,7 @@ class InvitationDeliveryService:
<table role="presentation" width="100%" cellspacing="0" cellpadding="0" border="0" style="width:100%;max-width:600px;">
<tr>
<td style="padding:0 4px 20px;">
<img src="https://docs.langbot.app/langbot-logo.png" alt="LangBot" width="34" height="34" style="display:inline-block;width:34px;height:34px;border:0;vertical-align:middle;">
<img src="https://langbot.app/docs/langbot-logo.png" alt="LangBot" width="34" height="34" style="display:inline-block;width:34px;height:34px;border:0;vertical-align:middle;">
<span style="display:inline-block;margin-left:10px;vertical-align:middle;font-size:18px;font-weight:700;letter-spacing:-.01em;">LangBot</span>
</td>
</tr>
+10 -1
View File
@@ -227,6 +227,12 @@ storage:
# Bound every object materialized into Core memory. Built-in Local/S3
# providers enforce this while reading (hard cap: 64 MiB).
max_object_read_bytes: 10485760
# Media content cache (images & attachments externalized from monitoring and pipelines)
media_cache:
# Retention period in days for cached media (defaults to 30 days)
retention_days: 30
# Maximum disk storage for cached media in MB (0 means unlimited, defaults to 0)
max_size_mb: 0
cleanup:
# Enable periodic cleanup of local/S3 uploaded files and old log files
enabled: true
@@ -344,7 +350,10 @@ box:
# skill tool, skill add/edit, and stdio-mode MCP servers. Skills can still
# be listed read-only and http/sse MCP servers continue to work.
enabled: true
backend: 'local' # 'local' (Docker/nsjail), 'docker', 'nsjail', or 'e2b'. Can be written via BOX__BACKEND.
# 'host' runs commands directly as the Box Runtime user without sandbox
# isolation. It is never auto-selected and is only for trusted local
# development. Can be written via BOX__BACKEND.
backend: 'local' # 'local' (Docker/nsjail), 'docker', 'nsjail', 'e2b', or explicit unsafe 'host'.
runtime:
# LANGBOT_BOX_CONTROL_TOKEN is optional for OSS external WebSocket
# runtimes. To protect an exposed endpoint, set the same strong secret
+4 -3
View File
@@ -645,9 +645,10 @@
.replace(/\s+/g, " ")
.trim();
if (
prevContent === content ||
prevContent.indexOf(content) >= 0 ||
content.indexOf(prevContent) >= 0
prevContent &&
(prevContent === content ||
prevContent.indexOf(content) >= 0 ||
content.indexOf(prevContent) >= 0)
)
return;
}