feat(cloud): harden multi-tenant runtime resources

This commit is contained in:
Junyan Qin
2026-07-29 11:32:26 +08:00
parent 32abbb636f
commit ae85ac2b16
211 changed files with 14963 additions and 1968 deletions
+15
View File
@@ -27,6 +27,21 @@ class TestStartupFlow:
"""Verify LangBot API is responding."""
assert langbot_process.health_check()
def test_health_check_exposes_bounded_blocking_executor(self, e2e_client):
"""The production startup path installs blocking-work admission."""
response = e2e_client.get('/healthz')
assert response.status_code == 200
executor = response.json()['resources']['blocking_executor']
assert executor['max_workers'] == 8
assert executor['max_pending'] == 128
assert executor['max_inflight_per_scope'] == 4
assert executor['inflight'] >= 0
assert executor['rejected_total'] >= 0
event_loop = response.json()['resources']['event_loop']
assert event_loop['running'] is True
assert event_loop['recent_max_lag_ms'] >= 0
def test_system_info_endpoint(self, e2e_client):
"""Test /api/v1/system/info endpoint."""
response = e2e_client.get('/api/v1/system/info')
+2 -4
View File
@@ -160,7 +160,7 @@ class TestHealthEndpoint:
assert response.status_code == 200
data = await response.get_json()
assert data == {'code': 0, 'msg': 'ok'}
assert data == {'code': 0, 'msg': 'ok', 'resources': {}}
@pytest.mark.asyncio
async def test_healthz_no_auth_required(self, quart_test_client):
@@ -327,9 +327,7 @@ class TestUserInitEndpoint:
)
assert response.status_code == 200
fake_api_app.user_service.reset_password.assert_awaited_once_with(
'member@example.com', 'new-member-password'
)
fake_api_app.user_service.reset_password.assert_awaited_once_with('member@example.com', 'new-member-password')
@pytest.mark.usefixtures('mock_circular_import_chain')
@@ -20,6 +20,7 @@ import asyncio
import contextlib
import datetime
import hashlib
import time
import typing
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -72,6 +73,12 @@ from langbot.pkg.api.http.service.user import UserService
from langbot.pkg.api.mcp.context import get_request_context as get_mcp_request_context
from langbot.pkg.api.mcp.mount import MCPMount
from langbot.pkg.entity.persistence.apikey import ApiKey
from langbot.pkg.entity.persistence import bot as persistence_bot
from langbot.pkg.entity.persistence import mcp as persistence_mcp
from langbot.pkg.entity.persistence import model as persistence_model
from langbot.pkg.entity.persistence import pipeline as persistence_pipeline
from langbot.pkg.entity.persistence import plugin as persistence_plugin
from langbot.pkg.entity.persistence import rag as persistence_rag
from langbot.pkg.entity.persistence.metadata import WorkspaceMetadata
from langbot.pkg.entity.persistence.monitoring import MonitoringFeedback
from langbot.pkg.entity.persistence.workspace import (
@@ -81,6 +88,9 @@ from langbot.pkg.entity.persistence.workspace import (
)
from langbot.pkg.platform.botmgr import PlatformManager
from langbot.pkg.pipeline.pipelinemgr import PipelineManager
from langbot.pkg.plugin.connector import PluginRuntimeConnector
from langbot.pkg.provider.modelmgr import requester as model_requester
from langbot.pkg.provider.modelmgr import token as model_token
from langbot.pkg.provider.modelmgr.modelmgr import ModelManager
from langbot.pkg.provider.tools.loaders.mcp import MCPLoader
from langbot.pkg.rag.knowledge.kbmgr import RAGManager
@@ -99,6 +109,39 @@ class _NoopDirectoryProjectionProvider:
raise AssertionError(f'Unexpected delta fetch for {instance_uuid}: {workspace_uuids!r}')
class _CapacityPluginRuntimeHandler:
"""Minimal shared Runtime control-plane surface for the startup probe."""
def __init__(self) -> None:
self.bindings: dict[str, typing.Any] = {}
self.reconciled: tuple[typing.Any, ...] = ()
def register_installation_binding(
self,
binding,
*,
plugin_author: str,
plugin_name: str,
) -> None:
self.bindings[binding.installation_uuid] = (
binding,
plugin_author,
plugin_name,
)
def unregister_installation_binding(self, binding) -> None:
self.bindings.pop(binding.installation_uuid, None)
async def reconcile_plugin_installations(self, desired_states) -> dict:
self.reconciled = tuple(desired_states)
return {
'applied': [],
'removed': [],
'missing_artifacts': [],
'failed_installations': [],
}
def _get_script_head() -> str:
"""Resolve the current Alembic head revision from the script directory.
@@ -653,6 +696,385 @@ class TestPostgreSQLTenantRuntime:
finally:
await _dispose_manager(manager)
@pytest.mark.asyncio
async def test_populated_cloud_startup_is_linear_and_task_bounded(
self,
postgres_url,
postgres_engine,
clean_tables,
clean_alembic_version,
monkeypatch,
):
"""Run the real Cloud startup query graph against populated RLS tenants.
The default is intentionally small enough for CI. Audit runs can raise
``LANGBOT_PG_CAPACITY_WORKSPACES`` without changing the test contract.
Every tenant owns one representative startup resource of each kind.
"""
workspace_count = int(os.environ.get('LANGBOT_PG_CAPACITY_WORKSPACES', '25'))
if not 1 <= workspace_count <= 2_000:
raise ValueError('LANGBOT_PG_CAPACITY_WORKSPACES must be between 1 and 2000')
max_elapsed_raw = os.environ.get(
'LANGBOT_PG_CAPACITY_MAX_SECONDS',
)
max_elapsed = float(max_elapsed_raw) if max_elapsed_raw is not None else None
instance_uuid = 'cloud-populated-startup-capacity-test'
role_name = f'lb_capacity_{uuid.uuid4().hex[:12]}'
role_password = f'Lb{uuid.uuid4().hex}'
quote = postgres_engine.dialect.identifier_preparer.quote
managers: list[PersistenceManager] = []
role_created = False
statement_counts = {
table_name: 0
for table_name in (
'model_providers',
'llm_models',
'embedding_models',
'rerank_models',
'bots',
'legacy_pipelines',
'knowledge_bases',
'mcp_servers',
'plugin_settings',
)
}
measured_engine = None
model_manager = None
platform_manager = None
mcp_loader = None
_restore_postgres_manager_registry(monkeypatch)
monkeypatch.setattr(constants, 'instance_id', instance_uuid)
release_manager = PersistenceManager(
_application_for_postgres_url(
postgres_url,
'postgres-capacity-release-test',
),
mode=PersistenceMode.RELEASE_MIGRATION,
)
managers.append(release_manager)
def role_url() -> str:
return (
sa.engine.make_url(postgres_url)
.set(username=role_name, password=role_password)
.render_as_string(hide_password=False)
)
def count_resource_statements(
_conn,
_cursor,
statement,
_parameters,
_context,
_executemany,
) -> None:
normalized = ' '.join(str(statement).lower().split())
for table_name in statement_counts:
if f' from {table_name}' in normalized or f' from "{table_name}"' in normalized:
statement_counts[table_name] += 1
try:
await release_manager.initialize()
for index in range(workspace_count):
workspace_uuid = f'ca{index:06x}-0000-4000-8000-{index:012x}'
suffix = f'{index:06d}'
provider_uuid = f'capacity-provider-{suffix}'
async with release_manager.tenant_uow(workspace_uuid) as uow:
uow.session.add(
Workspace(
uuid=workspace_uuid,
instance_uuid=instance_uuid,
name=f'Capacity {suffix}',
slug=f'capacity-{suffix}',
type='team',
status='active',
source='cloud_projection',
projection_revision=1,
)
)
await uow.session.flush()
uow.session.add_all(
[
WorkspaceExecutionState(
workspace_uuid=workspace_uuid,
instance_uuid=instance_uuid,
active_generation=1,
state='active',
write_fenced=False,
source='cloud',
desired_state_revision=1,
),
persistence_model.ModelProvider(
uuid=provider_uuid,
workspace_uuid=workspace_uuid,
name='Capacity Provider',
requester='capacity-probe',
base_url='https://capacity.invalid',
api_keys=[],
),
]
)
await uow.session.flush()
uow.session.add_all(
[
persistence_model.LLMModel(
uuid=f'capacity-llm-{suffix}',
workspace_uuid=workspace_uuid,
name='Capacity LLM',
provider_uuid=provider_uuid,
abilities=[],
extra_args={},
),
persistence_model.EmbeddingModel(
uuid=f'capacity-embedding-{suffix}',
workspace_uuid=workspace_uuid,
name='Capacity Embedding',
provider_uuid=provider_uuid,
extra_args={},
),
persistence_model.RerankModel(
uuid=f'capacity-rerank-{suffix}',
workspace_uuid=workspace_uuid,
name='Capacity Rerank',
provider_uuid=provider_uuid,
extra_args={},
),
persistence_bot.Bot(
uuid=f'capacity-bot-{suffix}',
workspace_uuid=workspace_uuid,
name='Capacity Bot',
description='',
adapter='capacity-probe',
adapter_config={},
enable=False,
pipeline_routing_rules=[],
),
persistence_pipeline.LegacyPipeline(
uuid=f'capacity-pipeline-{suffix}',
workspace_uuid=workspace_uuid,
name='Capacity Pipeline',
description='',
for_version='capacity-probe',
is_default=True,
stages=[],
config={},
extensions_preferences={},
),
persistence_rag.KnowledgeBase(
uuid=f'capacity-kb-{suffix}',
workspace_uuid=workspace_uuid,
name='Capacity Knowledge Base',
description='',
collection_id=f'capacity-collection-{suffix}',
legacy_vector_collection=False,
),
persistence_mcp.MCPServer(
uuid=f'capacity-mcp-{suffix}',
workspace_uuid=workspace_uuid,
name=f'capacity-mcp-{suffix}',
enable=False,
mode='remote',
extra_args={},
),
persistence_plugin.PluginSetting(
workspace_uuid=workspace_uuid,
plugin_author='capacity',
plugin_name=f'plugin-{suffix}',
installation_uuid=str(
uuid.uuid5(
uuid.NAMESPACE_URL,
f'langbot-capacity:{workspace_uuid}',
)
),
artifact_digest=hashlib.sha256(workspace_uuid.encode()).hexdigest(),
runtime_revision=1,
enabled=False,
config={},
install_source='github',
install_info={},
),
]
)
async with postgres_engine.connect() as conn:
await conn.execute(text(f"CREATE ROLE {quote(role_name)} LOGIN PASSWORD '{role_password}'"))
role_created = True
await conn.execute(
text(
f'GRANT CONNECT ON DATABASE '
f'{quote(sa.engine.make_url(postgres_url).database)} '
f'TO {quote(role_name)}'
)
)
await conn.execute(text(f'GRANT USAGE ON SCHEMA public TO {quote(role_name)}'))
await _grant_runtime_role_business_objects(
conn,
role_name,
quote,
)
runtime_application = _application_for_postgres_url(
role_url(),
'postgres-capacity-runtime-test',
)
runtime_application.instance_config.data.update(
{
'plugin': {'enable': True},
'mcp': {'lifecycle_concurrency': 8},
}
)
runtime_application.deployment = SimpleNamespace(
mode='cloud',
multi_workspace_enabled=False,
)
runtime_application.task_mgr = SimpleNamespace(
cancel_by_scope=lambda *_args, **_kwargs: None,
)
cloud_manager = PersistenceManager(
runtime_application,
mode=PersistenceMode.CLOUD_RUNTIME,
)
managers.append(cloud_manager)
await cloud_manager.initialize()
runtime_application.persistence_mgr = cloud_manager
cloud_manager.ap = runtime_application
runtime_application.workspace_service = WorkspaceService(
runtime_application,
policy=CloudWorkspacePolicy(),
instance_uuid=instance_uuid,
)
measured_engine = cloud_manager.get_db_engine().sync_engine
sa.event.listen(
measured_engine,
'before_cursor_execute',
count_resource_statements,
)
wall_started = time.monotonic()
cpu_started = time.process_time()
bindings = await runtime_application.workspace_service.prime_startup_execution_bindings()
assert len(bindings) == workspace_count
model_manager = ModelManager(runtime_application)
async def build_capacity_provider(
context,
provider_entity,
):
return model_requester.RuntimeProvider(
context,
provider_entity,
model_token.TokenManager(
provider_entity.uuid,
provider_entity.api_keys or [],
),
SimpleNamespace(aclose=AsyncMock()),
)
model_manager._build_provider = build_capacity_provider
await model_manager.load_models_from_db()
platform_manager = PlatformManager(runtime_application)
platform_manager.load_bot = AsyncMock()
await platform_manager.load_bots_from_db()
pipeline_manager = PipelineManager(runtime_application)
pipeline_manager.stage_dict = {}
await pipeline_manager.load_pipelines_from_db()
rag_manager = RAGManager(runtime_application)
await rag_manager.load_knowledge_bases_from_db()
mcp_loader = MCPLoader(runtime_application)
mcp_loader.host_mcp_server = AsyncMock()
await mcp_loader.load_mcp_servers_from_db()
dispatch_tasks = tuple(mcp_loader._host_dispatch_tasks)
if dispatch_tasks:
await asyncio.gather(*dispatch_tasks)
await asyncio.sleep(0)
plugin_connector = PluginRuntimeConnector(
runtime_application,
AsyncMock(),
)
plugin_handler = _CapacityPluginRuntimeHandler()
plugin_connector.handler = plugin_handler
contexts = [
ExecutionContext(
instance_uuid=binding.instance_uuid,
workspace_uuid=binding.workspace_uuid,
placement_generation=binding.placement_generation,
)
for binding in bindings
]
await plugin_connector.reconcile_projected_workspaces(contexts)
elapsed = time.monotonic() - wall_started
cpu_seconds = time.process_time() - cpu_started
logging.getLogger('postgres-capacity-runtime-test').info(
'Populated Cloud startup capacity: workspaces=%d elapsed=%.3fs cpu=%.3fs statements=%s',
workspace_count,
elapsed,
cpu_seconds,
statement_counts,
)
assert len(model_manager.provider_dict) == workspace_count
assert len(model_manager.llm_model_dict) == workspace_count
assert len(model_manager.embedding_model_dict) == workspace_count
assert len(model_manager.rerank_model_dict) == workspace_count
assert platform_manager.load_bot.await_count == workspace_count
assert len(pipeline_manager.pipelines) == workspace_count
assert len(rag_manager.knowledge_bases) == workspace_count
assert mcp_loader.host_mcp_server.await_count == workspace_count
assert not mcp_loader._host_dispatch_tasks
assert not mcp_loader._hosted_mcp_tasks
assert len(plugin_handler.reconciled) == workspace_count
assert len(plugin_handler.bindings) == workspace_count
assert all(count == workspace_count for count in statement_counts.values()), statement_counts
if max_elapsed is not None:
assert elapsed <= max_elapsed
finally:
cleanup_errors: list[BaseException] = []
if measured_engine is not None:
sa.event.remove(
measured_engine,
'before_cursor_execute',
count_resource_statements,
)
if mcp_loader is not None:
try:
await mcp_loader.shutdown()
except BaseException as exc:
cleanup_errors.append(exc)
if platform_manager is not None:
try:
await platform_manager.shutdown()
except BaseException as exc:
cleanup_errors.append(exc)
if model_manager is not None:
try:
await model_manager.shutdown()
except BaseException as exc:
cleanup_errors.append(exc)
for manager in reversed(managers):
try:
await _dispose_manager(manager)
except BaseException as exc:
cleanup_errors.append(exc)
if role_created:
try:
async with postgres_engine.connect() as conn:
await conn.execute(text(f'DROP OWNED BY {quote(role_name)}'))
await conn.execute(text(f'DROP ROLE {quote(role_name)}'))
except BaseException as exc:
cleanup_errors.append(exc)
if cleanup_errors:
raise cleanup_errors[0]
@pytest.mark.asyncio
async def test_release_bootstrap_and_runtime_isolation(
self,
@@ -367,6 +367,12 @@ async def test_release_entrypoint_holds_lock_migrates_validates_and_disposes(
with pytest.raises(RuntimeError, match="table 'metadata' grants are incomplete"):
await manager._validate_configured_runtime_postgres_role(require_grants=True)
finally:
manager = getattr(ap, 'persistence_mgr', None)
if isinstance(manager, PersistenceManager):
# The one-shot entrypoint disposes its pool before returning. This
# test deliberately reuses the manager for catalog mutation checks,
# which can open a fresh pool and therefore owns a second shutdown.
await manager.shutdown()
async with postgres_engine.connect() as conn:
await conn.execute(text(f'DROP OWNED BY {quote(runtime_role)}'))
await conn.execute(text(f'DROP ROLE IF EXISTS {quote(runtime_role)}'))
@@ -631,6 +637,11 @@ async def test_runtime_role_catalog_validator_rejects_delegation_and_escape_hatc
async with postgres_engine.connect() as conn:
await conn.execute(text(f'ALTER EXTENSION vector DROP FUNCTION public.{quote(security_definer)}()'))
finally:
manager = getattr(ap, 'persistence_mgr', None)
if isinstance(manager, PersistenceManager):
# The entrypoint disposed the release pool before returning; all
# validator calls above happened afterward and can reopen it.
await manager.shutdown()
async with postgres_engine.connect() as conn:
await conn.execute(text(f'ALTER DATABASE {quote(database_name)} RESET search_path'))
await conn.execute(text(f'ALTER ROLE {quote(runtime_role)} RESET search_path'))
@@ -0,0 +1,39 @@
from __future__ import annotations
import quart
from langbot.pkg.api.http.controller import main as controller_main
from langbot.pkg.utils import bounded_executor
async def test_bounded_json_request_decodes_off_loop_in_workspace_scope(
monkeypatch,
):
app = quart.Quart(__name__)
app.request_class = controller_main.BoundedJSONRequest
observed_scopes: list[str | None] = []
async def fake_to_thread(fn, *args, **kwargs):
observed_scopes.append(bounded_executor.current_blocking_work_scope())
return fn(*args, **kwargs)
monkeypatch.setattr(
controller_main.asyncio,
'to_thread',
fake_to_thread,
)
@app.post('/json')
async def parse_json():
with bounded_executor.blocking_work_scope('workspace-a'):
payload = await quart.request.get_json()
return quart.jsonify(payload)
response = await app.test_client().post(
'/json',
json={'nested': {'value': 1}},
)
assert response.status_code == 200
assert await response.get_json() == {'nested': {'value': 1}}
assert observed_scopes == ['workspace-a']
@@ -9,6 +9,10 @@ import quart
from langbot.pkg.api.http.controller import group
from langbot.pkg.api.http.controller.groups.webhooks import WebhookRouterGroup
from langbot.pkg.utils.bounded_executor import (
BlockingWorkCapacityError,
current_blocking_work_scope,
)
pytestmark = pytest.mark.asyncio
@@ -34,6 +38,16 @@ class _AuthenticatedRouterGroup(group.RouterGroup):
return self.success()
class _BlockingCapacityRouterGroup(group.RouterGroup):
name = 'blocking-capacity-test'
path = '/blocking-capacity-test'
async def initialize(self) -> None:
@self.route('', methods=['GET'], auth_type=group.AuthType.NONE)
async def _():
raise BlockingWorkCapacityError('Workspace blocking executor capacity reached')
class _InvalidAccountRouterGroup(group.RouterGroup):
name = 'invalid-account-test'
path = '/invalid-account-test'
@@ -102,6 +116,20 @@ async def test_public_webhook_error_uses_same_generic_error_contract():
assert 'do-not-return' not in (await response.get_data(as_text=True))
async def test_blocking_work_capacity_maps_to_retryable_http_response():
application = SimpleNamespace(logger=Mock())
quart_app = quart.Quart(__name__)
await _BlockingCapacityRouterGroup(application, quart_app).initialize()
response = await quart_app.test_client().get('/blocking-capacity-test')
assert response.status_code == 429
assert await response.get_json() == {
'code': 'blocking_work_capacity_exceeded',
'msg': 'Workspace blocking executor capacity reached',
}
async def test_public_webhook_carries_scope_without_holding_database_session():
class ScopeOnlyPersistenceManager:
mode = SimpleNamespace(value='cloud_runtime')
@@ -128,6 +156,7 @@ async def test_public_webhook_carries_scope_without_holding_database_session():
async def handle_unified_webhook(self, **_kwargs):
assert persistence_mgr.active_workspace == workspace_uuid
assert persistence_mgr.current_session() is None
assert current_blocking_work_scope() == workspace_uuid
return {'ok': True}
async def get_execution_binding(resolved_workspace_uuid, expected_generation=None):
@@ -156,6 +185,41 @@ async def test_public_webhook_carries_scope_without_holding_database_session():
assert persistence_mgr.active_workspace is None
async def test_public_webhook_blocking_capacity_is_retryable():
workspace_uuid = '00000000-0000-0000-0000-00000000000a'
bot_uuid = '11111111-1111-4111-8111-111111111111'
class Adapter:
async def handle_unified_webhook(self, **_kwargs):
raise BlockingWorkCapacityError(
'Workspace blocking executor capacity reached',
scope=workspace_uuid,
)
runtime_bot = SimpleNamespace(
workspace_uuid=workspace_uuid,
placement_generation=4,
enable=True,
adapter=Adapter(),
)
application = SimpleNamespace(
logger=Mock(),
persistence_mgr=SimpleNamespace(mode=SimpleNamespace(value='oss')),
platform_mgr=SimpleNamespace(resolve_public_bot=AsyncMock(return_value=runtime_bot)),
workspace_service=SimpleNamespace(get_execution_binding=AsyncMock(return_value=None)),
)
quart_app = quart.Quart(__name__)
await WebhookRouterGroup(application, quart_app).initialize()
response = await quart_app.test_client().post(f'/bots/{bot_uuid}')
assert response.status_code == 429
assert await response.get_json() == {
'code': 'blocking_work_capacity_exceeded',
'msg': 'Workspace blocking executor capacity reached',
}
async def test_authentication_failure_does_not_return_internal_exception_text():
logger = Mock()
application = SimpleNamespace(
@@ -29,6 +29,7 @@ from langbot.pkg.api.http.context import (
WorkspaceContext,
)
from langbot.pkg.api.http.service.mcp import MCPService, redact_mcp_secrets, restore_mcp_secret_placeholders
from langbot.pkg.core.taskmgr import TaskCapacityError
from langbot.pkg.entity.persistence.mcp import MCPServer
from langbot.pkg.provider.tools.loaders.mcp_policy import MCPStdioDisabledError
from langbot.pkg.workspace.errors import WorkspaceNotFoundError
@@ -1050,3 +1051,23 @@ class TestMCPServiceTestMCPServer:
# Verify - load_mcp_server called
ap.tool_mgr.mcp_tool_loader.load_mcp_server.assert_called_once()
assert task_id == 456
async def test_rejected_transient_test_session_is_shut_down(self):
ap = SimpleNamespace()
mock_session = MagicMock()
mock_session.shutdown = AsyncMock()
ap.tool_mgr = SimpleNamespace(
mcp_tool_loader=SimpleNamespace(load_mcp_server=AsyncMock(return_value=mock_session))
)
def reject(coroutine, **_kwargs):
coroutine.close()
raise TaskCapacityError('capacity')
ap.task_mgr = SimpleNamespace(create_user_task=Mock(side_effect=reject))
service = _service(ap)
with pytest.raises(TaskCapacityError, match='capacity'):
await service.test_mcp_server(_CONTEXT, '_', {'name': 'New Server'})
mock_session.shutdown.assert_awaited_once_with()
@@ -13,6 +13,8 @@ Source: src/langbot/pkg/api/http/service/space.py
from __future__ import annotations
from collections import OrderedDict
import json
from urllib.parse import parse_qs, urlsplit
import pytest
@@ -28,6 +30,23 @@ from langbot.pkg.entity.persistence.user import User
pytestmark = pytest.mark.asyncio
def _set_response_body(response: MagicMock, body: dict | str) -> None:
"""Configure an aiohttp-like streaming body on an HTTP response mock."""
raw_body = body.encode() if isinstance(body, str) else json.dumps(body).encode()
class Content:
async def iter_chunked(self, _chunk_size: int):
midpoint = max(len(raw_body) // 2, 1)
yield raw_body[:midpoint]
if midpoint < len(raw_body):
yield raw_body[midpoint:]
response.headers = {}
response.content = Content()
response.charset = 'utf-8'
def _create_mock_user(
email: str = 'test@example.com',
account_type: str = 'space',
@@ -292,6 +311,40 @@ class TestSpaceServiceGetCredits:
# Verify - returns cached value without API call
assert result == 100
async def test_cached_credit_lookup_does_not_scan_all_users(self):
ap = SimpleNamespace()
ap.instance_config = SimpleNamespace(data={})
ap.persistence_mgr = SimpleNamespace()
service = SpaceService(ap)
class AtMostOneStepOrderedDict(OrderedDict):
def __iter__(self):
iterator = super().__iter__()
yielded = False
def next_entry():
nonlocal yielded
if yielded:
raise AssertionError('credits cache scanned all users')
yielded = True
return next(iterator)
class AtMostOneStepIterator:
def __iter__(self):
return self
def __next__(self):
return next_entry()
return AtMostOneStepIterator()
now = time.time()
service._credits_cache = AtMostOneStepOrderedDict(
(f'user-{index}@example.com', (index, now)) for index in range(512)
)
assert await service.get_credits('user-511@example.com') == 511
async def test_get_credits_cache_expired_refreshes(self):
"""Refreshes expired cache."""
# Setup
@@ -406,6 +459,7 @@ class TestSpaceServiceRefreshToken:
},
}
)
_set_response_body(mock_response, mock_response.json.return_value)
with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
mock_session_obj = MagicMock()
@@ -441,6 +495,7 @@ class TestSpaceServiceRefreshToken:
}
)
mock_response.text = AsyncMock(return_value='{"code":1,"msg":"Invalid refresh token"}')
_set_response_body(mock_response, mock_response.json.return_value)
with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
mock_session_obj = MagicMock()
@@ -467,6 +522,7 @@ class TestSpaceServiceRefreshToken:
mock_response = MagicMock()
mock_response.status = 500
mock_response.text = AsyncMock(return_value='Internal Server Error')
_set_response_body(mock_response, mock_response.text.return_value)
with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
mock_session_obj = MagicMock()
@@ -506,6 +562,7 @@ class TestSpaceServiceExchangeOAuthCode:
},
}
)
_set_response_body(mock_response, mock_response.json.return_value)
with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
mock_session_obj = MagicMock()
@@ -535,6 +592,7 @@ class TestSpaceServiceExchangeOAuthCode:
mock_response.status = 200
mock_response.json = AsyncMock(return_value={'code': 1, 'msg': 'Invalid code'})
mock_response.text = AsyncMock(return_value='{"code":1,"msg":"Invalid code"}')
_set_response_body(mock_response, mock_response.json.return_value)
with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
mock_session_obj = MagicMock()
@@ -573,6 +631,7 @@ class TestSpaceServiceGetUserInfoRaw:
},
}
)
_set_response_body(mock_response, mock_response.json.return_value)
with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
mock_session_obj = MagicMock()
@@ -603,6 +662,7 @@ class TestSpaceServiceGetUserInfoRaw:
mock_response.status = 200
mock_response.json = AsyncMock(return_value={'code': 1, 'msg': 'Unauthorized'})
mock_response.text = AsyncMock(return_value='{"code":1,"msg":"Unauthorized"}')
_set_response_body(mock_response, mock_response.json.return_value)
with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
mock_session_obj = MagicMock()
@@ -703,6 +763,7 @@ class TestSpaceServiceGetModels:
},
}
)
_set_response_body(mock_response, mock_response.json.return_value)
with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
mock_session_obj = MagicMock()
@@ -732,6 +793,7 @@ class TestSpaceServiceGetModels:
mock_response.status = 200
mock_response.json = AsyncMock(return_value={'code': 1, 'msg': 'Unauthorized'})
mock_response.text = AsyncMock(return_value='{"code":1,"msg":"Unauthorized"}')
_set_response_body(mock_response, mock_response.json.return_value)
with patch('langbot.pkg.api.http.service.space.httpclient.get_session') as mock_session:
mock_session_obj = MagicMock()
@@ -29,11 +29,25 @@ from langbot.pkg.entity.errors.account import (
SpaceAccountBindingRequiredError,
SpaceAccountNotRegisteredError,
)
from langbot.pkg.utils.bounded_executor import BlockingWorkCapacityError
pytestmark = pytest.mark.asyncio
async def test_password_hashing_rejects_concurrent_waiters() -> None:
service = UserService(SimpleNamespace())
await service._password_hash_lock.acquire()
try:
with pytest.raises(
BlockingWorkCapacityError,
match='Password hashing capacity reached',
):
await service._hash_password('secret')
finally:
service._password_hash_lock.release()
class TestSpaceOAuthState:
async def test_login_state_is_opaque_single_use(self):
service = UserService(SimpleNamespace())
@@ -91,6 +105,32 @@ class TestSpaceOAuthState:
assert consumed.account is None
assert consumed.launch_workspace_uuid == 'workspace-a'
async def test_issue_state_does_not_scan_all_live_states(self, monkeypatch):
service = UserService(SimpleNamespace())
for _ in range(512):
await service.issue_space_oauth_state('login')
class NoGlobalIterationDict(dict):
def __iter__(self):
raise AssertionError('OAuth state issuance scanned all live states')
def keys(self):
raise AssertionError('OAuth state issuance scanned all live states')
def items(self):
raise AssertionError('OAuth state issuance scanned all live states')
def values(self):
raise AssertionError('OAuth state issuance scanned all live states')
guarded_states = NoGlobalIterationDict(service._space_oauth_states)
monkeypatch.setattr(service, '_space_oauth_states', guarded_states)
state = await service.issue_space_oauth_state('login')
assert await service.consume_space_oauth_state(state, 'login') is None
assert len(guarded_states) == 512
def _create_mock_user(
email: str = 'test@example.com',
@@ -2,7 +2,7 @@ from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
from unittest.mock import AsyncMock, Mock
import lark_oapi
import pytest
@@ -19,7 +19,9 @@ from langbot.pkg.api.http.controller.groups.platform.adapters import (
_AdapterSessionScope,
_bind_session_scope,
_get_owned_session,
_make_room_for_session,
_pop_owned_session,
_start_adapter_session_task,
)
@@ -94,6 +96,10 @@ async def _create_client(*, role: str = 'developer'):
),
)
class TestTaskManager:
def create_user_task(self, coro, **_kwargs):
return SimpleNamespace(task=asyncio.create_task(coro))
application = SimpleNamespace(
user_service=SimpleNamespace(
get_authenticated_account=AsyncMock(side_effect=get_authenticated_account),
@@ -102,6 +108,7 @@ async def _create_client(*, role: str = 'developer'):
resolve_account_workspace=AsyncMock(side_effect=resolve_account_workspace),
),
platform_mgr=SimpleNamespace(),
task_mgr=TestTaskManager(),
)
router = AdaptersRouterGroup(application, quart_app)
await router.initialize()
@@ -142,6 +149,58 @@ async def test_session_scope_matches_exact_tenant_placement_and_principal():
assert sessions == {}
async def test_session_capacity_evicts_oldest_session_in_same_workspace():
owner_context = _request_context()
sessions: dict[str, dict] = {}
tasks = []
for index in range(10):
task = SimpleNamespace(done=Mock(return_value=False), cancel=Mock())
tasks.append(task)
session = {'created_at': float(index), 'task': task}
_bind_session_scope(session, owner_context)
sessions[f'session-{index}'] = session
_make_room_for_session(sessions, owner_context)
assert 'session-0' not in sessions
assert len(sessions) == 9
tasks[0].cancel.assert_called_once_with()
async def test_adapter_session_task_uses_tenant_task_admission():
blocker = asyncio.Event()
async def credential_exchange():
await blocker.wait()
task_manager = SimpleNamespace(create_user_task=Mock())
def create_user_task(coro, **_kwargs):
return SimpleNamespace(task=asyncio.create_task(coro))
task_manager.create_user_task.side_effect = create_user_task
application = SimpleNamespace(task_mgr=task_manager)
request_context = _request_context()
returned = _start_adapter_session_task(
application,
credential_exchange(),
adapter='lark',
session_id='session-test',
request_context=request_context,
)
assert returned is not None
task_manager.create_user_task.assert_called_once()
kwargs = task_manager.create_user_task.call_args.kwargs
assert kwargs['kind'] == 'platform-adapter-credential-exchange'
assert kwargs['instance_uuid'] == request_context.instance_uuid
assert kwargs['workspace_uuid'] == request_context.workspace_uuid
assert kwargs['placement_generation'] == request_context.placement_generation
blocker.set()
await returned
async def test_lark_session_status_and_delete_hide_cross_scope_sessions(monkeypatch):
registration_blocker = asyncio.Event()
@@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
from contextlib import asynccontextmanager
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
@@ -13,6 +14,11 @@ from langbot.pkg.api.http.context import (
WorkspaceContext,
)
from langbot.pkg.api.http.controller.groups.pipelines.websocket_chat import WebSocketChatRouterGroup
from langbot.pkg.api.http.controller.groups.pipelines.websocket_chat import (
create_scoped_duplex_tasks,
)
from langbot.pkg.api.http.controller.groups.pipelines.websocket_chat import wait_for_duplex_tasks
from langbot.pkg.utils.bounded_executor import current_blocking_work_scope
@pytest.mark.asyncio
@@ -67,3 +73,72 @@ async def test_websocket_pipeline_lookup_opens_workspace_uow_after_auth_scope_cl
assert result is adapter
assert scopes == [workspace_uuid]
@pytest.mark.asyncio
async def test_duplex_websocket_tasks_cancel_blocked_peer_when_one_direction_ends() -> None:
blocked = asyncio.Event()
async def receive_forever() -> None:
blocked.set()
await asyncio.Future()
async def send_finishes() -> None:
await blocked.wait()
receive_task = asyncio.create_task(receive_forever())
send_task = asyncio.create_task(send_finishes())
await asyncio.wait_for(
wait_for_duplex_tasks(receive_task, send_task),
timeout=1,
)
assert receive_task.cancelled()
assert send_task.done()
@pytest.mark.asyncio
async def test_duplex_websocket_tasks_allow_terminal_send_to_drain() -> None:
receive_finished = asyncio.Event()
send_drained = asyncio.Event()
async def receive_finishes() -> None:
receive_finished.set()
async def send_terminal_frame() -> None:
await receive_finished.wait()
await asyncio.sleep(0)
send_drained.set()
receive_task = asyncio.create_task(receive_finishes())
send_task = asyncio.create_task(send_terminal_frame())
await wait_for_duplex_tasks(receive_task, send_task)
assert send_drained.is_set()
assert send_task.done()
assert not send_task.cancelled()
@pytest.mark.asyncio
async def test_duplex_websocket_tasks_share_trusted_workspace_budget() -> None:
observed: list[tuple[str, str | None]] = []
async def observe(direction: str) -> None:
await asyncio.sleep(0)
observed.append((direction, current_blocking_work_scope()))
receive_task, send_task = create_scoped_duplex_tasks(
observe('receive'),
observe('send'),
'workspace-a',
)
await asyncio.gather(receive_task, send_task)
assert sorted(observed) == [
('receive', 'workspace-a'),
('send', 'workspace-a'),
]
assert current_blocking_work_scope() is None
+22 -9
View File
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import datetime as dt
import os
import pathlib
import tempfile
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
@@ -2050,6 +2051,17 @@ class TestAttachmentHelpers:
component = SimpleNamespace(base64=None, url=None, path=None)
assert await BoxService._component_to_bytes(component) is None
@pytest.mark.asyncio
async def test_component_to_bytes_rejects_oversized_base64(self, monkeypatch):
monkeypatch.setattr(BoxService, '_ATTACHMENT_MAX_BYTES', 4)
component = SimpleNamespace(
base64='data:application/octet-stream;base64,' + ('A' * 12),
url=None,
path=None,
)
assert await BoxService._component_to_bytes(component) is None
class TestInboundOutboundRoundTrip:
def _service(self) -> BoxService:
@@ -2206,7 +2218,7 @@ class TestAttachmentHostPath:
# File actually landed on the host workspace.
host_file = os.path.join(ws, 'inbox', str(query.query_uuid), d['name'])
assert os.path.isfile(host_file)
assert open(host_file, 'rb').read() == big
assert pathlib.Path(host_file).read_bytes() == big
@pytest.mark.asyncio
async def test_inbound_host_clears_stale_query_dir(self, tmp_path):
@@ -2218,7 +2230,7 @@ class TestAttachmentHostPath:
# Seed a stale file under the same query_id (simulates webchat id reuse).
stale_dir = os.path.join(ws, 'inbox', 'query-42')
os.makedirs(stale_dir, exist_ok=True)
open(os.path.join(stale_dir, 'image_1.png'), 'wb').write(b'STALE-OLD-IMAGE')
pathlib.Path(stale_dir, 'image_1.png').write_bytes(b'STALE-OLD-IMAGE')
new = b'\x89PNG\r\n\x1a\n NEW'
b64 = 'data:image/png;base64,' + base64.b64encode(new).decode()
@@ -2228,9 +2240,10 @@ class TestAttachmentHostPath:
descriptors = await service.materialize_inbound_attachments(query)
# The new write recreated the dir; the stale file is gone, new bytes present.
host_file = os.path.join(stale_dir, descriptors[0]['name'])
assert open(host_file, 'rb').read() == new
host_bytes = pathlib.Path(host_file).read_bytes()
assert host_bytes == new
# No leftover content from the stale image.
assert b'STALE-OLD-IMAGE' not in open(host_file, 'rb').read()
assert b'STALE-OLD-IMAGE' not in host_bytes
@pytest.mark.asyncio
async def test_inbound_host_replaces_query_symlink_without_touching_other_workspace(self, tmp_path):
@@ -2259,7 +2272,7 @@ class TestAttachmentHostPath:
assert descriptors[0]['path'] == '/workspace/inbox/query-42/input.bin'
assert protected.read_bytes() == b'workspace-b-secret'
assert not os.path.islink(os.path.join(inbox, 'query-42'))
assert open(os.path.join(inbox, 'query-42', 'input.bin'), 'rb').read() == payload
assert pathlib.Path(inbox, 'query-42', 'input.bin').read_bytes() == payload
@pytest.mark.asyncio
async def test_outbound_reads_host_and_clears(self, tmp_path):
@@ -2269,8 +2282,8 @@ class TestAttachmentHostPath:
os.makedirs(outbox, exist_ok=True)
# A large file that would be truncated on the exec/stdout path:
big_png = b'\x89PNG\r\n\x1a\n' + b'y' * (400 * 1024)
open(os.path.join(outbox, 'result.png'), 'wb').write(big_png)
open(os.path.join(outbox, 'notes.txt'), 'wb').write(b'hello')
pathlib.Path(outbox, 'result.png').write_bytes(big_png)
pathlib.Path(outbox, 'notes.txt').write_bytes(b'hello')
service.execute_tool = AsyncMock(side_effect=AssertionError('exec must not be used on host path'))
attachments = await service.collect_outbound_attachments(query)
@@ -2354,7 +2367,7 @@ class TestAttachmentHostPath:
# it and then wipe the dir. Simulate "nothing produced this turn" by
# treating any present file as stale and asserting it is not re-sent
# across a second, genuinely-empty collection.
open(os.path.join(outbox, 'stale.png'), 'wb').write(b'\x89PNG\r\n\x1a\n old')
pathlib.Path(outbox, 'stale.png').write_bytes(b'\x89PNG\r\n\x1a\n old')
service.execute_tool = AsyncMock(side_effect=AssertionError('exec must not be used on host path'))
# First collection drains + clears the dir.
@@ -2376,7 +2389,7 @@ class TestAttachmentHostPath:
for sub in ('inbox', 'outbox'):
d = os.path.join(ws, sub, '0')
os.makedirs(d, exist_ok=True)
open(os.path.join(d, 'leftover.bin'), 'wb').write(b'from a previous process')
pathlib.Path(d, 'leftover.bin').write_bytes(b'from a previous process')
service.execute_tool = AsyncMock(side_effect=AssertionError('exec must not be used for host-owned files'))
await service._purge_attachment_dirs()
@@ -1,5 +1,7 @@
from __future__ import annotations
import asyncio
import pytest
from unittest.mock import AsyncMock
@@ -107,3 +109,53 @@ async def test_resolver_checks_deployment_admission_before_and_after_provider_ca
with pytest.raises(RuntimeError, match='expired during provider call'):
await resolver.resolve('workspace-a', now=150)
assert checks == 2
@pytest.mark.asyncio
async def test_directory_activity_reconciliation_drops_historical_snapshots():
provider = AsyncMock()
provider.get_workspace_entitlement = AsyncMock(return_value=_snapshot())
resolver = EntitlementResolver('instance-a', provider)
await resolver.reconcile_active_workspaces({'workspace-a', 'workspace-b'})
await resolver.resolve('workspace-a', now=150)
await resolver.reconcile_active_workspaces({'workspace-b'})
assert resolver.snapshot_counts() == {
'active_workspaces': 1,
'cached_snapshots': 0,
}
with pytest.raises(EntitlementUnavailableError, match='directory projection'):
await resolver.resolve('workspace-a', now=150)
provider.get_workspace_entitlement.assert_awaited_once()
@pytest.mark.asyncio
async def test_directory_fence_wins_race_with_inflight_entitlement_fetch():
provider_started = asyncio.Event()
release_provider = asyncio.Event()
async def fetch(_workspace_uuid: str) -> EntitlementSnapshot:
provider_started.set()
await release_provider.wait()
return _snapshot()
provider = AsyncMock()
provider.get_workspace_entitlement = AsyncMock(side_effect=fetch)
resolver = EntitlementResolver('instance-a', provider)
await resolver.reconcile_active_workspaces({'workspace-a'})
resolve_task = asyncio.create_task(resolver.resolve('workspace-a', now=150))
await provider_started.wait()
await resolver.update_workspace_activity(
active_workspace_uuids=set(),
inactive_workspace_uuids={'workspace-a'},
)
release_provider.set()
with pytest.raises(EntitlementUnavailableError, match='directory projection'):
await resolve_task
assert resolver.snapshot_counts() == {
'active_workspaces': 0,
'cached_snapshots': 0,
}
@@ -86,6 +86,50 @@ async def test_consumes_valid_workspace_launch_assertion_once():
await service.consume_assertion(token, expected_workspace_uuid=WORKSPACE_UUID)
async def test_replay_cache_does_not_scan_all_live_assertions(monkeypatch):
private_key = Ed25519PrivateKey.generate()
now = int(time.time())
service = _service(private_key, now=now)
for index in range(512):
await service._consume_jti(f'jti-{index}', now + 90)
class NoGlobalIterationDict(dict):
def __iter__(self):
raise AssertionError('replay admission scanned all live assertions')
def keys(self):
raise AssertionError('replay admission scanned all live assertions')
def items(self):
raise AssertionError('replay admission scanned all live assertions')
def values(self):
raise AssertionError('replay admission scanned all live assertions')
guarded_jtis = NoGlobalIterationDict(service._consumed_jtis)
monkeypatch.setattr(service, '_consumed_jtis', guarded_jtis)
await service._consume_jti('jti-new', now + 90)
assert len(guarded_jtis) == 513
async def test_replay_cache_fails_closed_at_capacity(monkeypatch):
from langbot.pkg.cloud import launch
private_key = Ed25519PrivateKey.generate()
now = int(time.time())
service = _service(private_key, now=now)
monkeypatch.setattr(launch, '_CONSUMED_JTI_MAX_ENTRIES', 2)
await service._consume_jti('jti-1', now + 90)
await service._consume_jti('jti-2', now + 90)
with pytest.raises(SpaceLaunchError, match='replay cache capacity'):
await service._consume_jti('jti-3', now + 90)
with pytest.raises(SpaceLaunchError, match='already been consumed'):
await service._consume_jti('jti-1', now + 90)
async def test_rejects_expired_wrong_workspace_and_wrong_instance_assertions():
private_key = Ed25519PrivateKey.generate()
now = int(time.time())
+122
View File
@@ -0,0 +1,122 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from langbot.pkg.core.app import Application
@pytest.mark.asyncio
async def test_shutdown_closes_mcp_session_manager_once() -> None:
app = Application()
stop_session_manager = AsyncMock()
app.platform_mgr = SimpleNamespace(shutdown=AsyncMock())
app.tool_mgr = SimpleNamespace(shutdown=AsyncMock())
app.model_mgr = SimpleNamespace(shutdown=AsyncMock())
app.box_service = SimpleNamespace(shutdown=AsyncMock())
app.plugin_connector = SimpleNamespace(aclose=AsyncMock())
app.telemetry = SimpleNamespace(shutdown=AsyncMock())
app.vector_db_mgr = SimpleNamespace(shutdown=AsyncMock())
app.storage_mgr = SimpleNamespace(shutdown=AsyncMock())
manifest_provider = SimpleNamespace(aclose=AsyncMock())
app.deployment = SimpleNamespace(manifest_provider=manifest_provider)
persistence_engine = SimpleNamespace(dispose=AsyncMock())
app.persistence_mgr = SimpleNamespace(db=SimpleNamespace(engine=persistence_engine))
app.http_ctrl = SimpleNamespace(mcp_mount=SimpleNamespace(stop_session_manager=stop_session_manager))
await app.shutdown()
await app.shutdown()
stop_session_manager.assert_awaited_once()
app.platform_mgr.shutdown.assert_awaited_once()
app.tool_mgr.shutdown.assert_awaited_once()
app.model_mgr.shutdown.assert_awaited_once()
app.box_service.shutdown.assert_awaited_once()
app.plugin_connector.aclose.assert_awaited_once()
app.telemetry.shutdown.assert_awaited_once()
app.vector_db_mgr.shutdown.assert_awaited_once()
app.storage_mgr.shutdown.assert_awaited_once()
manifest_provider.aclose.assert_awaited_once()
persistence_engine.dispose.assert_awaited_once()
@pytest.mark.asyncio
async def test_dispose_tracks_only_one_shutdown_task() -> None:
app = Application()
app.event_loop = asyncio.get_running_loop()
app.dispose()
shutdown_task = app._shutdown_task
app.dispose()
assert shutdown_task is not None
assert app._shutdown_task is shutdown_task
await shutdown_task
app.dispose()
assert app._shutdown_task is shutdown_task
@pytest.mark.asyncio
async def test_runtime_resource_stats_are_aggregate_and_constant_time() -> None:
app = Application()
app.event_loop = asyncio.get_running_loop()
app.blocking_executor = SimpleNamespace(
snapshot=lambda: {
'inflight': 3,
'running': 2,
'pending': 1,
'rejected_total': 4,
}
)
app.task_mgr = SimpleNamespace(get_stats=lambda: {'total': 5, 'completed': 2})
app.query_pool = SimpleNamespace(
queries=[object()],
cached_queries={},
active_query_count_by_workspace={'workspace-a': 1},
)
app.model_mgr = SimpleNamespace(
provider_dict={'provider': object()},
llm_model_dict={},
embedding_model_dict={},
rerank_model_dict={},
)
app.platform_mgr = SimpleNamespace(_bots_by_key={})
app.pipeline_mgr = SimpleNamespace(_pipelines_by_key={})
app.rag_mgr = SimpleNamespace(knowledge_bases={})
app.plugin_connector = SimpleNamespace(_known_desired_states={'installation': object()})
app.tool_mgr = SimpleNamespace(
mcp_tool_loader=SimpleNamespace(
_sessions={},
_hosted_mcp_tasks=[],
_host_dispatch_tasks=set(),
)
)
app.telemetry = SimpleNamespace(send_tasks=[])
stats = app.get_runtime_resource_stats()
assert stats['asyncio_tasks'] >= 1
assert stats['event_loop'] == {
'running': False,
'samples_total': 0,
'last_lag_ms': 0,
'recent_p95_lag_ms': 0,
'recent_max_lag_ms': 0,
'max_lag_ms': 0,
}
assert stats['blocking_executor']['rejected_total'] == 4
assert stats['application_tasks'] == {
'total': 5,
'completed': 2,
}
assert stats['query_pool'] == {
'queued': 1,
'cached': 0,
'active_workspaces': 1,
}
assert stats['models']['providers'] == 1
assert stats['runtimes']['plugin_installations'] == 1
+25 -1
View File
@@ -2,13 +2,37 @@ from __future__ import annotations
import signal
from types import SimpleNamespace
from unittest.mock import Mock
from unittest.mock import AsyncMock, Mock
import pytest
from langbot.pkg.core import boot
@pytest.mark.asyncio
async def test_make_app_shuts_down_partially_built_application(monkeypatch):
app_inst = SimpleNamespace(
event_loop=None,
shutdown=AsyncMock(),
initialize=AsyncMock(),
)
class FailingStage:
async def run(self, ap):
assert ap is app_inst
raise RuntimeError('startup failed')
monkeypatch.setattr(boot.app, 'Application', lambda: app_inst)
monkeypatch.setattr(boot, 'stage_order', ['FailingStage'])
monkeypatch.setitem(boot.stage.preregistered_stages, 'FailingStage', FailingStage)
with pytest.raises(RuntimeError, match='startup failed'):
await boot.make_app(SimpleNamespace())
app_inst.shutdown.assert_awaited_once()
app_inst.initialize.assert_not_awaited()
@pytest.mark.asyncio
async def test_main_signal_handler_handles_sigint_before_app_created(monkeypatch):
captured_handler = {}
+12
View File
@@ -333,11 +333,23 @@ class TestApplyEnvOverridesToConfig:
{
'PLUGIN__WORKER__MAX_MEMORY_MB': '768',
'MCP__STDIO__ENABLED': 'false',
'SYSTEM__BLOCKING_EXECUTOR__MAX_WORKERS': '12',
'SYSTEM__BLOCKING_EXECUTOR__MAX_PENDING': '256',
'SYSTEM__BLOCKING_EXECUTOR__MAX_INFLIGHT_PER_SCOPE': '3',
},
clear=True,
):
result = load_config._apply_env_overrides_to_config(completed)
assert result['system']['blocking_executor'] == {
'max_workers': 12,
'max_pending': 256,
'max_inflight_per_scope': 3,
}
assert isinstance(
result['system']['blocking_executor']['max_workers'],
int,
)
assert result['plugin']['worker']['max_memory_mb'] == 768
assert isinstance(result['plugin']['worker']['max_memory_mb'], int)
assert result['mcp']['stdio']['enabled'] is False
+73
View File
@@ -13,6 +13,7 @@ from __future__ import annotations
import pytest
import asyncio
import contextvars
import inspect
import sys
from unittest.mock import Mock, MagicMock
from contextlib import contextmanager
@@ -265,6 +266,28 @@ class TestTaskWrapper:
wrapper.cancel()
@pytest.mark.asyncio
async def test_workspace_task_sets_blocking_work_scope(self):
"""Detached tasks recover tenant fairness from durable ownership."""
_, TaskWrapper, _ = get_taskmgr_classes()
from langbot.pkg.utils.bounded_executor import (
current_blocking_work_scope,
)
mock_app = create_mock_app()
async def read_scope():
return current_blocking_work_scope()
wrapper = TaskWrapper(
mock_app,
read_scope(),
workspace_uuid='workspace-a',
)
assert await wrapper.task == 'workspace-a'
assert current_blocking_work_scope() is None
@pytest.mark.asyncio
async def test_to_dict_serialization(self):
"""Test TaskWrapper.to_dict serialization."""
@@ -530,6 +553,56 @@ class TestAsyncTaskManager:
wrapper.cancel()
@pytest.mark.asyncio
async def test_create_user_task_enforces_workspace_active_limit_and_closes_rejected_coroutine(self):
"""A noisy Workspace cannot accumulate unbounded background work."""
_, _, AsyncTaskManager = get_taskmgr_classes()
mock_app = create_mock_app()
mock_app.instance_config.data['system']['task_retention'].update(
{
'max_active_user_tasks': 10,
'max_active_user_tasks_per_workspace': 1,
}
)
manager = AsyncTaskManager(mock_app)
async def long_coro():
await asyncio.sleep(10)
first = manager.create_user_task(long_coro(), workspace_uuid='workspace-a')
rejected = long_coro()
with pytest.raises(RuntimeError, match='Workspace has too many active user operations'):
manager.create_user_task(rejected, workspace_uuid='workspace-a')
assert inspect.getcoroutinestate(rejected) == inspect.CORO_CLOSED
other_workspace = manager.create_user_task(long_coro(), workspace_uuid='workspace-b')
first.cancel()
other_workspace.cancel()
@pytest.mark.asyncio
async def test_create_user_task_enforces_instance_active_limit(self):
"""The shared process retains a hard cap even across Workspaces."""
_, _, AsyncTaskManager = get_taskmgr_classes()
mock_app = create_mock_app()
mock_app.instance_config.data['system']['task_retention'].update(
{
'max_active_user_tasks': 1,
'max_active_user_tasks_per_workspace': 10,
}
)
manager = AsyncTaskManager(mock_app)
async def long_coro():
await asyncio.sleep(10)
first = manager.create_user_task(long_coro(), workspace_uuid='workspace-a')
rejected = long_coro()
with pytest.raises(RuntimeError, match='instance has too many active user operations'):
manager.create_user_task(rejected, workspace_uuid='workspace-b')
assert inspect.getcoroutinestate(rejected) == inspect.CORO_CLOSED
first.cancel()
@pytest.mark.asyncio
async def test_get_task_by_id(self):
"""Test get_task_by_id returns correct task."""
@@ -13,11 +13,13 @@ from langbot.pkg.persistence.databases import postgresql
@pytest.mark.asyncio
async def test_postgresql_manager_parses_explicit_url_without_string_reassembly(monkeypatch) -> None:
captured = None
captured_options = None
sentinel_engine = object()
def create_engine(url):
nonlocal captured
def create_engine(url, **options):
nonlocal captured, captured_options
captured = url
captured_options = options
return sentinel_engine
monkeypatch.setattr(postgresql.sqlalchemy_asyncio, 'create_async_engine', create_engine)
@@ -40,6 +42,13 @@ async def test_postgresql_manager_parses_explicit_url_without_string_reassembly(
assert captured.password == 'p@ss'
assert captured.query['ssl'] == 'require'
assert 'sslmode' not in captured.query
assert captured_options == {
'pool_size': 10,
'max_overflow': 10,
'pool_timeout': 30,
'pool_recycle': 1800,
'pool_pre_ping': True,
}
assert manager.engine is sentinel_engine
@@ -47,7 +56,7 @@ async def test_postgresql_manager_parses_explicit_url_without_string_reassembly(
async def test_postgresql_manager_builds_structured_url_with_special_password(monkeypatch) -> None:
captured = None
def create_engine(url):
def create_engine(url, **_options):
nonlocal captured
captured = url
return object()
@@ -76,6 +85,60 @@ async def test_postgresql_manager_builds_structured_url_with_special_password(mo
assert captured.database == 'langbot'
@pytest.mark.asyncio
async def test_postgresql_manager_applies_explicit_bounded_pool_options(monkeypatch) -> None:
captured_options = None
def create_engine(_url, **options):
nonlocal captured_options
captured_options = options
return object()
monkeypatch.setattr(postgresql.sqlalchemy_asyncio, 'create_async_engine', create_engine)
ap = SimpleNamespace(
instance_config=SimpleNamespace(
data={
'database': {
'postgresql': {
'pool_size': 24,
'max_overflow': 0,
'pool_timeout_seconds': 7,
'pool_recycle_seconds': 600,
}
}
}
)
)
await postgresql.PostgreSQLDatabaseManager(ap).initialize()
assert captured_options == {
'pool_size': 24,
'max_overflow': 0,
'pool_timeout': 7,
'pool_recycle': 600,
'pool_pre_ping': True,
}
@pytest.mark.asyncio
@pytest.mark.parametrize(
('name', 'value'),
[
('pool_size', 0),
('pool_size', True),
('max_overflow', -1),
('pool_timeout_seconds', 0),
('pool_recycle_seconds', '1800'),
],
)
async def test_postgresql_manager_rejects_invalid_pool_options(name, value) -> None:
ap = SimpleNamespace(instance_config=SimpleNamespace(data={'database': {'postgresql': {name: value}}}))
with pytest.raises(ValueError, match=rf'database\.postgresql\.{name}'):
await postgresql.PostgreSQLDatabaseManager(ap).initialize()
@pytest.mark.asyncio
async def test_postgresql_manager_rejects_non_postgresql_url_without_echoing_secret() -> None:
ap = SimpleNamespace(
@@ -104,6 +104,7 @@ async def test_release_migration_disposes_operator_engine_on_failure(monkeypatch
manager = SimpleNamespace(
db=SimpleNamespace(engine=engine),
initialize=AsyncMock(side_effect=RuntimeError('migration failed')),
shutdown=AsyncMock(side_effect=engine.dispose),
)
def manager_factory(*args, **kwargs):
@@ -121,6 +122,7 @@ async def test_release_migration_disposes_operator_engine_on_failure(monkeypatch
await release_migration.run_cloud_release_migration(ap, environ=_operator_environ())
assert ap.persistence_mgr is manager
manager.shutdown.assert_awaited_once()
engine.dispose.assert_awaited_once()
@@ -464,3 +464,13 @@ class TestChatHandlerHelper:
handler = chat.ChatMessageHandler(fake_app)
result = handler.cut_str('first line\nsecond line')
assert '...' in result
def test_response_size_limit_uses_instance_config(self, fake_app):
from langbot_plugin.api.entities.builtin.provider.message import Message
fake_app.instance_config.data['system'] = {'response_limits': {'max_generated_chars': 4}}
chat = get_chat_handler()
handler = chat.ChatMessageHandler(fake_app)
with pytest.raises(RuntimeError, match='configured limit'):
handler._check_response_size(Message(role='assistant', content='12345'))
@@ -36,6 +36,7 @@ async def test_controller_drops_stale_query_before_pipeline_lookup(
query_pool, session = _prepare_scheduler(mock_app)
mock_app.workspace_service.get_execution_binding.side_effect = WorkspaceGenerationMismatchError('stale generation')
controller = Controller(mock_app)
initial_slots = controller.semaphore._value
await controller._process_query(sample_query)
@@ -47,6 +48,7 @@ async def test_controller_drops_stale_query_before_pipeline_lookup(
query_pool.remove_query.assert_awaited_once_with(sample_query)
session._semaphore.release.assert_called_once_with()
query_pool.condition.notify_all.assert_called_once_with()
assert controller.semaphore._value == initial_slots
@pytest.mark.asyncio
@@ -0,0 +1,17 @@
from unittest.mock import Mock
from langbot.pkg.pipeline.longtext.strategies.image import Text2ImageStrategy
class _WideFont:
def getlength(self, text: str) -> int:
return len(text) * 100
def test_image_strategy_line_split_always_consumes_input():
strategy = Text2ImageStrategy(Mock())
lines = strategy._split_text_lines('abc', 1, _WideFont())
assert lines == ['a', 'b', 'c']
assert ''.join(lines) == 'abc'
+16 -1
View File
@@ -37,7 +37,11 @@ _saved_modules = {name: sys.modules.get(name) for name in _import_stubs}
for _name, _stub in _import_stubs.items():
sys.modules[_name] = _stub
try:
from langbot.pkg.provider.runners.n8nsvapi import N8nServiceAPIRunner
from langbot.pkg.provider.runners.n8nsvapi import (
N8nAPIError,
N8nServiceAPIRunner,
_MAX_N8N_RESPONSE_CHARS,
)
finally:
for _name, _original in _saved_modules.items():
if _original is None:
@@ -230,6 +234,17 @@ async def test_plain_json_non_dict_response():
assert chunks[0].content == '["a", "b"]'
@pytest.mark.asyncio
async def test_response_size_is_bounded():
runner = make_runner()
with pytest.raises(N8nAPIError, match='exceeds the runtime limit'):
await collect_chunks(
runner,
[b'x' * (_MAX_N8N_RESPONSE_CHARS + 1)],
)
@pytest.mark.asyncio
async def test_invalid_json_returns_raw_text():
"""Non-JSON response returns raw text as-is."""
+92 -1
View File
@@ -3,11 +3,13 @@ PipelineManager unit tests
"""
import pytest
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
from importlib import import_module
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.workspace.errors import WorkspaceGenerationMismatchError
from langbot.pkg.workspace.entities import WorkspaceExecutionBinding
from langbot.pkg.workspace.errors import WorkspaceGenerationMismatchError, WorkspaceInvariantError
def _context(pipeline_uuid: str = 'test-uuid') -> ExecutionContext:
@@ -49,6 +51,95 @@ async def test_pipeline_manager_initialize(mock_app):
assert len(manager.pipelines) == 0
@pytest.mark.asyncio
async def test_cloud_startup_reuses_validated_pipeline_binding(mock_app):
class TenantUow:
async def __aenter__(self):
return self
async def __aexit__(self, *_args):
return False
binding = WorkspaceExecutionBinding(
instance_uuid='test-instance',
workspace_uuid='test-workspace',
placement_generation=1,
write_fenced=False,
state='active',
)
pipeline_entity = Mock(
uuid='test-uuid',
workspace_uuid='test-workspace',
stages=[],
config={},
extensions_preferences={},
)
mock_app.persistence_mgr.mode = SimpleNamespace(value='cloud_runtime')
mock_app.persistence_mgr.tenant_uow = lambda _workspace_uuid: TenantUow()
mock_app.persistence_mgr.execute_async = AsyncMock(return_value=Mock(all=Mock(return_value=[pipeline_entity])))
mock_app.workspace_service.list_active_execution_bindings = AsyncMock(return_value=[binding])
mock_app.workspace_service.get_execution_binding = AsyncMock(
side_effect=AssertionError('startup pipeline loader repeated a validated binding lookup')
)
manager = get_pipelinemgr_module().PipelineManager(mock_app)
manager.stage_dict = {}
await manager.load_pipelines_from_db()
assert len(manager.pipelines) == 1
mock_app.workspace_service.get_execution_binding.assert_not_awaited()
def test_generation_advance_prunes_superseded_workspace_pipelines(mock_app):
class NoGlobalIterationDict(dict):
def __iter__(self):
raise AssertionError('generation advance scanned every pipeline')
def items(self):
raise AssertionError('generation advance scanned every pipeline')
def values(self):
raise AssertionError('generation advance scanned every pipeline')
pipelinemgr = get_pipelinemgr_module()
manager = pipelinemgr.PipelineManager(mock_app)
old_context = _context()
next_context = ExecutionContext(
instance_uuid=old_context.instance_uuid,
workspace_uuid=old_context.workspace_uuid,
placement_generation=2,
pipeline_uuid=old_context.pipeline_uuid,
)
old_pipeline = SimpleNamespace(
execution_context=old_context,
workspace_uuid=old_context.workspace_uuid,
placement_generation=old_context.placement_generation,
)
other_pipelines = [
SimpleNamespace(
execution_context=ExecutionContext(
instance_uuid='test-instance',
workspace_uuid=f'workspace-{index}',
placement_generation=1,
pipeline_uuid=f'pipeline-{index}',
),
workspace_uuid=f'workspace-{index}',
placement_generation=1,
)
for index in range(1_000)
]
manager.pipelines = [old_pipeline, *other_pipelines]
manager._observe_execution_context(old_context)
manager._pipelines_by_key = NoGlobalIterationDict(manager._pipelines_by_key)
manager._observe_execution_context(next_context)
manager._pipelines_by_key = dict(manager._pipelines_by_key)
assert manager.pipelines == other_pipelines
with pytest.raises(WorkspaceInvariantError, match='rolled back'):
manager._observe_execution_context(old_context)
@pytest.mark.asyncio
async def test_load_pipeline(mock_app):
"""Test loading a single pipeline"""
+53
View File
@@ -18,6 +18,7 @@ from langbot.pkg.pipeline.pool import (
ExecutionContextRequiredError,
QueryNotFoundError,
QueryPool,
QueryPoolCapacityError,
get_query_execution_context,
)
@@ -433,3 +434,55 @@ class TestQueryPoolWorkspaceIsolation:
assert pool.get_query_count(workspace_b) == 1
assert pool.get_query_count(next_generation) == 0
assert pool.query_id_counter == 3
async def test_workspace_capacity_discards_oldest_queued_query(self):
pool = QueryPool(max_queries=3, max_queries_per_workspace=2)
first = await add_scoped_mock_query(pool, TEST_CONTEXT)
second = await add_scoped_mock_query(pool, TEST_CONTEXT)
third = await add_scoped_mock_query(pool, TEST_CONTEXT)
assert await pool.get_query(TEST_CONTEXT.workspace_uuid, first.query_uuid) is None
assert await pool.get_query(TEST_CONTEXT.workspace_uuid, second.query_uuid) is second
assert await pool.get_query(TEST_CONTEXT.workspace_uuid, third.query_uuid) is third
assert pool.active_query_count_by_workspace == {TEST_CONTEXT.workspace_uuid: 2}
assert pool.get_dropped_query_count(TEST_CONTEXT) == 1
async def test_capacity_rejects_when_every_query_is_already_running(self):
pool = QueryPool(max_queries=1, max_queries_per_workspace=1)
running = await add_scoped_mock_query(pool, TEST_CONTEXT)
async with pool:
pool.mark_query_running_locked(running)
with pytest.raises(QueryPoolCapacityError):
await add_scoped_mock_query(pool, TEST_CONTEXT)
assert pool.active_query_count_by_workspace == {TEST_CONTEXT.workspace_uuid: 1}
async def test_mark_query_running_keeps_active_indexes_but_removes_queue_entry(self):
pool = QueryPool(max_queries=1, max_queries_per_workspace=1)
running = await add_scoped_mock_query(pool, TEST_CONTEXT)
async with pool:
pool.mark_query_running_locked(running)
assert running not in pool.queries
assert await pool.get_query(TEST_CONTEXT.workspace_uuid, running.query_uuid) is running
assert pool.active_query_count_by_workspace == {TEST_CONTEXT.workspace_uuid: 1}
async def test_historical_workspace_counters_are_bounded(self):
pool = QueryPool(max_queries=2, max_queries_per_workspace=1)
contexts = [
ExecutionContext(
instance_uuid='instance-test',
workspace_uuid=f'workspace-{index}',
placement_generation=1,
)
for index in range(3)
]
for context in contexts:
query = await add_scoped_mock_query(pool, context)
await pool.remove_query(query)
assert len(pool.query_count_by_scope) == 2
assert (contexts[0].instance_uuid, contexts[0].workspace_uuid, 1) not in pool.query_count_by_scope
+24 -4
View File
@@ -152,8 +152,18 @@ class TestFixedWindowAlgo:
# First request creates container
await algo.require_access(sample_query_with_rate_limit, provider_session.LauncherTypes.PERSON, '12345')
# Key format: 'LauncherTypes.PERSON_12345' (enum string representation)
expected_key = 'LauncherTypes.PERSON_12345'
context = sample_query_with_rate_limit._execution_context
expected_key = ':'.join(
(
context.instance_uuid,
context.workspace_uuid,
str(context.placement_generation),
str(sample_query_with_rate_limit.bot_uuid),
str(sample_query_with_rate_limit.pipeline_uuid),
str(provider_session.LauncherTypes.PERSON),
'12345',
)
)
assert expected_key in algo.containers
container = algo.containers[expected_key]
@@ -191,8 +201,18 @@ class TestFixedWindowAlgo:
for i in range(5):
await algo.require_access(sample_query, provider_session.LauncherTypes.PERSON, 'test')
# Key format: 'LauncherTypes.PERSON_test'
expected_key = 'LauncherTypes.PERSON_test'
context = sample_query._execution_context
expected_key = ':'.join(
(
context.instance_uuid,
context.workspace_uuid,
str(context.placement_generation),
str(sample_query.bot_uuid),
str(sample_query.pipeline_uuid),
str(provider_session.LauncherTypes.PERSON),
'test',
)
)
container = algo.containers[expected_key]
assert window_start in container.records
assert container.records[window_start] == 5
@@ -68,6 +68,25 @@ async def test_connection_listener_only_suppresses_exact_duplicates():
]
@pytest.mark.asyncio
async def test_connection_event_cache_is_bounded():
adapter, _ = _make_adapter()
for index in range(150):
await adapter._on_websocket_connection(aiocqhttp.Event({'self_id': index, 'time': index}))
assert len(adapter.on_websocket_connection_event_cache) == 100
def test_group_lookup_caches_are_bounded():
converter = AiocqhttpEventConverter()
converter._group_name_cache = {index: (str(index), 10_000.0) for index in range(5000)}
converter._prune_caches(1.0)
assert len(converter._group_name_cache) == 4096
def test_unregister_listener_removes_registered_wrapper():
adapter, _ = _make_adapter()
@@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
import contextlib
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -8,7 +9,10 @@ import pytest
from langbot.pkg.api.http.authz import WorkspaceRequiredError
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.entity.persistence.bot import Bot
from langbot.pkg.platform.botmgr import PlatformManager, RuntimeBot
from langbot.pkg.workspace.entities import WorkspaceExecutionBinding
from langbot.pkg.workspace.errors import WorkspaceInvariantError
import langbot_plugin.api.entities.builtin.platform.events as platform_events
@@ -83,7 +87,220 @@ async def test_public_route_key_resolves_bound_runtime_and_rejects_non_opaque_in
@pytest.mark.asyncio
async def test_stale_runtime_generation_is_not_returned(manager):
assert await manager.get_bot_by_uuid(_context(WORKSPACE_A, BOT_A, generation=5), BOT_A) is None
with pytest.raises(ValueError, match='stale'):
await manager.get_bot_by_uuid(_context(WORKSPACE_A, BOT_A, generation=5), BOT_A)
@pytest.mark.asyncio
async def test_generation_advance_shuts_down_and_prunes_old_workspace_bots():
class NoGlobalIterationDict(dict):
def __iter__(self):
raise AssertionError('generation advance scanned every bot runtime')
def items(self):
raise AssertionError('generation advance scanned every bot runtime')
def values(self):
raise AssertionError('generation advance scanned every bot runtime')
manager = PlatformManager(SimpleNamespace())
old_bot = SimpleNamespace(
workspace_uuid=WORKSPACE_A,
placement_generation=4,
enable=True,
shutdown=AsyncMock(),
)
other_bot = SimpleNamespace(
workspace_uuid=WORKSPACE_B,
placement_generation=4,
enable=True,
shutdown=AsyncMock(),
)
unrelated_bots = [
SimpleNamespace(
workspace_uuid=f'workspace-{index}',
placement_generation=4,
enable=False,
shutdown=AsyncMock(),
)
for index in range(1_000)
]
manager.bots = [old_bot, other_bot, *unrelated_bots]
old_context = _context(WORKSPACE_A, BOT_A, generation=4)
next_context = _context(WORKSPACE_A, BOT_A, generation=5)
await manager._observe_execution_context(old_context)
manager._bots_by_key = NoGlobalIterationDict(manager._bots_by_key)
await manager._observe_execution_context(next_context)
manager._bots_by_key = dict(manager._bots_by_key)
old_bot.shutdown.assert_awaited_once_with()
assert manager.bots == [other_bot, *unrelated_bots]
with pytest.raises(WorkspaceInvariantError, match='rolled back'):
await manager._observe_execution_context(old_context)
@pytest.mark.asyncio
async def test_concurrent_websocket_proxy_creation_reuses_one_runtime():
created_adapters = []
class WebsocketAdapter:
def __init__(self, *_args, **_kwargs):
created_adapters.append(self)
def register_listener(self, *_args):
pass
application = SimpleNamespace(workspace_service=_WorkspaceService())
manager = PlatformManager(application)
manager.adapter_dict = {'websocket': WebsocketAdapter}
context = ExecutionContext(
instance_uuid='instance',
workspace_uuid=WORKSPACE_A,
placement_generation=4,
)
runtimes = await asyncio.gather(*(manager.get_websocket_proxy_bot(context) for _ in range(20)))
assert len(created_adapters) == 1
assert len({id(runtime) for runtime in runtimes}) == 1
assert manager.websocket_proxy_bots == {WORKSPACE_A: runtimes[0]}
@pytest.mark.asyncio
async def test_websocket_proxy_cache_evicts_oldest_idle_workspace():
created_adapters = []
class WebsocketAdapter:
def __init__(self, *_args, **_kwargs):
self.kill = AsyncMock()
self.inbound_listener_tasks = set()
created_adapters.append(self)
def register_listener(self, *_args):
pass
application = SimpleNamespace(
workspace_service=_WorkspaceService(),
instance_config=SimpleNamespace(
data={
'system': {
'websocket_retention': {'max_workspace_proxies': 1},
}
}
),
)
manager = PlatformManager(application)
manager.adapter_dict = {'websocket': WebsocketAdapter}
await manager.get_websocket_proxy_bot(
ExecutionContext(
instance_uuid='instance',
workspace_uuid=WORKSPACE_A,
placement_generation=4,
)
)
second = await manager.get_websocket_proxy_bot(
ExecutionContext(
instance_uuid='instance',
workspace_uuid=WORKSPACE_B,
placement_generation=4,
)
)
created_adapters[0].kill.assert_awaited_once_with()
assert manager.websocket_proxy_bots == {WORKSPACE_B: second}
assert WORKSPACE_A not in manager._proxy_last_accessed
@pytest.mark.asyncio
async def test_reload_stops_and_drops_existing_platform_runtimes():
old_bot = SimpleNamespace(enable=True, shutdown=AsyncMock())
old_proxy = SimpleNamespace(enable=True, shutdown=AsyncMock())
persistence_mgr = SimpleNamespace(
execute_async=AsyncMock(return_value=SimpleNamespace(all=lambda: [])),
)
application = SimpleNamespace(
logger=SimpleNamespace(info=lambda *_args: None, warning=lambda *_args: None),
persistence_mgr=persistence_mgr,
workspace_service=SimpleNamespace(),
)
manager = PlatformManager(application)
manager.bots = [old_bot]
manager.websocket_proxy_bots = {WORKSPACE_A: old_proxy}
manager._scope_generations = {('instance', WORKSPACE_A): 4}
await manager.load_bots_from_db()
old_bot.shutdown.assert_awaited_once_with()
old_proxy.shutdown.assert_awaited_once_with()
assert manager.bots == []
assert manager.websocket_proxy_bots == {}
assert manager._scope_generations == {}
@pytest.mark.asyncio
async def test_cloud_startup_reuses_validated_platform_binding():
class TenantUow:
async def __aenter__(self):
return self
async def __aexit__(self, *_args):
return False
class ProbeAdapter:
def __init__(self, _config, _logger):
self.listeners = []
def register_listener(self, event_type, listener):
self.listeners.append((event_type, listener))
async def kill(self):
return None
binding = WorkspaceExecutionBinding(
instance_uuid='instance',
workspace_uuid=WORKSPACE_A,
placement_generation=4,
write_fenced=False,
state='active',
)
bot = Bot(
uuid=BOT_A,
workspace_uuid=WORKSPACE_A,
name='Probe',
description='',
adapter='probe',
adapter_config={},
enable=False,
pipeline_routing_rules=[],
)
workspace_service = SimpleNamespace(
list_active_execution_bindings=AsyncMock(return_value=[binding]),
get_execution_binding=AsyncMock(
side_effect=AssertionError('startup platform loader repeated a validated binding lookup')
),
)
application = SimpleNamespace(
logger=SimpleNamespace(
info=lambda *_args, **_kwargs: None,
warning=lambda *_args, **_kwargs: None,
error=lambda *_args, **_kwargs: None,
),
persistence_mgr=SimpleNamespace(
mode=SimpleNamespace(value='cloud_runtime'),
tenant_uow=lambda _workspace_uuid: TenantUow(),
execute_async=AsyncMock(return_value=SimpleNamespace(all=lambda: [bot])),
),
workspace_service=workspace_service,
)
manager = PlatformManager(application)
manager.adapter_dict = {'probe': ProbeAdapter}
await manager.load_bots_from_db()
assert len(manager.bots) == 1
workspace_service.get_execution_binding.assert_not_awaited()
@pytest.mark.asyncio
@@ -5,6 +5,7 @@ from unittest.mock import AsyncMock, MagicMock, patch
import pytest
from langbot.pkg.platform import botmgr as _botmgr # noqa: F401
from langbot.pkg.platform.sources.dingtalk import (
DingTalkAdapter,
_dingtalk_card_markdown,
@@ -17,6 +18,17 @@ from langbot.pkg.platform.sources.dingtalk import (
)
def test_dingtalk_auxiliary_tasks_are_bounded():
adapter = DingTalkAdapter.model_construct()
adapter._background_tasks = {MagicMock(done=MagicMock(return_value=False)) for _ in range(100)}
async def callback():
raise AssertionError('rejected callback must not run')
assert adapter._start_background_task(callback()) is False
assert len(adapter._background_tasks) == 100
def test_dingtalk_select_component_params_expose_options():
params = _dingtalk_form_component_params(
{
@@ -0,0 +1,12 @@
from __future__ import annotations
import pytest
from langbot.pkg.platform.sources import discord
def test_discord_base64_decode_is_bounded(monkeypatch):
monkeypatch.setattr(discord, '_MAX_DISCORD_MEDIA_BYTES', 4)
with pytest.raises(ValueError, match='exceeds'):
discord._decode_discord_base64_limited('A' * 12)
@@ -1,11 +1,13 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
import pytest
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.platform.sources.http_bot import HttpBotAdapter
from langbot.pkg.platform.sources import http_bot as http_bot_module
def _session(key):
@@ -22,6 +24,7 @@ def _adapter(app, execution_context) -> HttpBotAdapter:
outbound_states={},
idempotency_cache={},
sync_waiters={},
inbound_tasks=set(),
)
object.__setattr__(adapter, 'ap', app)
return adapter
@@ -64,3 +67,28 @@ async def test_http_bot_reset_fails_closed_without_trusted_scope():
with pytest.raises(RuntimeError, match='trusted execution scope'):
await adapter._reset_session('person', 'shared-session')
@pytest.mark.asyncio
async def test_http_bot_bounds_inbound_listener_tasks(monkeypatch):
monkeypatch.setattr(http_bot_module, '_INBOUND_TASK_MAX', 1)
adapter = _adapter(SimpleNamespace(), None)
started = asyncio.Event()
release = asyncio.Event()
async def blocking_listener():
started.set()
await release.wait()
first = adapter._start_inbound_task(blocking_listener())
await started.wait()
rejected = adapter._start_inbound_task(blocking_listener())
assert first is not None
assert rejected is None
assert len(adapter.inbound_tasks) == 1
release.set()
await first
await asyncio.sleep(0)
assert adapter.inbound_tasks == set()
@@ -0,0 +1,24 @@
from __future__ import annotations
import json
import zlib
import pytest
from langbot.pkg.platform.sources import kook
def test_kook_gateway_decoder_accepts_raw_and_compressed_json():
payload = {'s': 1, 'd': {'session_id': 'session-a'}}
encoded = json.dumps(payload).encode()
assert kook._decode_gateway_message(encoded) == payload
assert kook._decode_gateway_message(zlib.compress(encoded)) == payload
def test_kook_gateway_decoder_rejects_decompression_bomb(monkeypatch):
monkeypatch.setattr(kook, '_KOOK_MAX_GATEWAY_MESSAGE_BYTES', 1024)
compressed = zlib.compress(b'x' * 1025)
with pytest.raises(ValueError, match='decompressed size limit'):
kook._decode_gateway_message(compressed)
@@ -1,7 +1,13 @@
"""Tests for Lark adapter helper behavior."""
import threading
from unittest.mock import MagicMock
import pytest
from langbot.pkg.platform.sources.lark import (
LarkAdapter,
_decode_lark_base64_limited,
_lark_clean_form_content,
_lark_completed_input_lines,
_lark_current_input_defs,
@@ -11,6 +17,27 @@ from langbot.pkg.platform.sources.lark import (
)
def test_lark_base64_decode_is_bounded(monkeypatch):
import langbot.pkg.platform.sources.lark as lark_module
monkeypatch.setattr(lark_module, '_MAX_LARK_MEDIA_BYTES', 4)
with pytest.raises(ValueError, match='exceeds'):
_decode_lark_base64_limited('A' * 12)
def test_lark_threadsafe_callbacks_are_bounded():
adapter = LarkAdapter.model_construct()
adapter.threadsafe_event_lock = threading.Lock()
adapter.threadsafe_event_futures = {MagicMock(done=MagicMock(return_value=False)) for _ in range(100)}
async def callback():
raise AssertionError('rejected callback must not run')
assert adapter._schedule_threadsafe_event(callback()) is None
assert len(adapter.threadsafe_event_futures) == 100
def test_lark_current_input_defs_only_returns_active_stage():
input_defs = [
{'output_variable_name': 'us_input', 'type': 'paragraph'},
@@ -0,0 +1,30 @@
from __future__ import annotations
import pytest
from unittest.mock import MagicMock
from langbot.pkg.platform import botmgr as _botmgr # noqa: F401
from langbot.pkg.platform.sources import line
def test_line_media_content_accepts_limit_boundary(monkeypatch) -> None:
monkeypatch.setattr(line, 'MAX_LINE_MEDIA_BYTES', 4)
content = b'1234'
assert line._validate_line_media_content(content) is content
def test_line_media_content_rejects_oversized_payload(monkeypatch) -> None:
monkeypatch.setattr(line, 'MAX_LINE_MEDIA_BYTES', 4)
with pytest.raises(ValueError, match='LINE media exceeds'):
line._validate_line_media_content(b'12345')
@pytest.mark.asyncio
async def test_line_kill_closes_api_client() -> None:
api_client = MagicMock()
adapter = line.LINEAdapter.model_construct(api_client=api_client)
assert await adapter.kill() is True
api_client.close.assert_called_once_with()
@@ -0,0 +1,21 @@
from __future__ import annotations
import pytest
from langbot.pkg.platform.sources import matrix
def test_matrix_base64_decode_is_bounded(monkeypatch):
monkeypatch.setattr(matrix, '_MAX_MATRIX_MEDIA_BYTES', 4)
with pytest.raises(ValueError, match='exceeds'):
matrix._decode_matrix_base64_limited('A' * 12)
def test_matrix_local_file_read_is_bounded(tmp_path, monkeypatch):
monkeypatch.setattr(matrix, '_MAX_MATRIX_MEDIA_BYTES', 4)
path = tmp_path / 'large.bin'
path.write_bytes(b'12345')
with pytest.raises(ValueError, match='exceeds'):
matrix._read_matrix_file_limited(str(path))
@@ -0,0 +1,31 @@
from __future__ import annotations
import pytest
from langbot.libs.openclaw_weixin_api.client import (
MAX_CDN_MEDIA_BYTES,
OpenClawWeixinClient,
_decrypt_cdn_payload,
_encrypt_cdn_payload,
)
from langbot.libs.openclaw_weixin_api.types import ApiError
def test_cdn_crypto_helpers_round_trip():
original = b'tenant-media' * 128
aes_key_hex, _encoded_key, encrypted, _raw_md5 = _encrypt_cdn_payload(original)
assert _decrypt_cdn_payload(encrypted, bytes.fromhex(aes_key_hex)) == original
@pytest.mark.asyncio
async def test_upload_media_rejects_oversized_input_before_network_access():
client = OpenClawWeixinClient('https://example.invalid', 'token')
with pytest.raises(ApiError, match='exceeds the size limit'):
await client.upload_media(
b'x' * (MAX_CDN_MEDIA_BYTES + 1),
'recipient',
3,
)
@@ -2,8 +2,10 @@ from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
import langbot_plugin.api.entities.builtin.platform.message as platform_message
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.platform.sources import openclaw_weixin
from langbot.pkg.platform.sources.openclaw_weixin import OpenClawWeixinAdapter
@@ -82,3 +84,12 @@ async def test_persist_config_fails_closed_without_matching_execution_context(ex
app.persistence_mgr.execute_async.assert_not_awaited()
logger.warning.assert_awaited_once()
@pytest.mark.asyncio
async def test_component_base64_decode_is_bounded(monkeypatch):
monkeypatch.setattr(openclaw_weixin, '_MAX_OPENCLAW_COMPONENT_BYTES', 4)
component = platform_message.File(base64='MTIzNDU=')
with pytest.raises(ValueError, match='exceeds'):
await OpenClawWeixinAdapter._get_component_bytes(component)
@@ -10,6 +10,7 @@ import langbot_plugin.api.entities.builtin.platform.message as platform_message
from langbot.libs.qq_official_api.api import (
QQ_SELECT_ACTION_PREFIX,
QQOfficialClient,
build_keyboard_from_select_field,
get_select_field_options,
resolve_select_button_action,
@@ -49,6 +50,28 @@ def test_qq_select_button_resolves_field_and_value():
assert resolve_select_button_action(form_data, f'{QQ_SELECT_ACTION_PREFIX}99') is None
@pytest.mark.asyncio
async def test_qq_seed_rejects_empty_secret_without_spinning():
client = QQOfficialClient('', 'token', 'app-id', AsyncMock())
with pytest.raises(ValueError, match='must not be empty'):
await asyncio.wait_for(client.repeat_seed(''), timeout=0.1)
def test_qq_auxiliary_tasks_are_bounded():
import langbot.pkg.core.app # noqa: F401
from langbot.pkg.platform.sources.qqofficial import QQOfficialAdapter
adapter = QQOfficialAdapter.model_construct()
adapter._background_tasks = {MagicMock(done=MagicMock(return_value=False)) for _ in range(100)}
async def callback():
raise AssertionError('rejected callback must not run')
assert adapter._start_background_task(callback()) is False
assert len(adapter._background_tasks) == 100
def test_qq_select_keyboard_fits_twenty_five_options():
form_data = _select_form_data()
form_data['input_defs'][0]['option_source']['value'] = [f'Option {idx}' for idx in range(25)]
@@ -11,11 +11,21 @@ import langbot_plugin.api.entities.builtin.platform.events as platform_events
import langbot_plugin.api.entities.builtin.platform.message as platform_message
from langbot.pkg.platform.sources.telegram import (
TelegramAdapter,
_decode_telegram_base64_limited,
_telegram_form_action_from_callback,
_telegram_select_field_options,
)
def test_telegram_base64_decode_is_bounded(monkeypatch):
import langbot.pkg.platform.sources.telegram as telegram_module
monkeypatch.setattr(telegram_module, '_MAX_TELEGRAM_MEDIA_BYTES', 4)
with pytest.raises(ValueError, match='exceeds'):
_decode_telegram_base64_limited('A' * 12)
def _select_form_data() -> dict:
return {
'_current_input_field': 'choice',
@@ -88,6 +98,18 @@ def test_telegram_form_callback_cache_preserves_pipeline_uuid():
)
def test_telegram_form_callback_cache_is_bounded():
adapter = TelegramAdapter.model_construct()
adapter._form_action_titles = {}
adapter._cache_form_action_titles(
{f'callback-{index}': str(index) for index in range(5000)},
now=100.0,
)
assert len(adapter._form_action_titles) == adapter._MAX_FORM_ACTION_TITLES
@pytest.mark.asyncio
async def test_telegram_select_field_sends_two_column_inline_keyboard():
bot = MagicMock()
@@ -108,6 +108,71 @@ async def test_pipeline_indexes_and_broadcasts_are_workspace_scoped():
assert manager.get_stats(scope=SCOPE_A)['total_connections'] == 1
@pytest.mark.asyncio
async def test_connection_admission_is_bounded_globally_and_per_workspace():
manager = WebSocketConnectionManager()
await manager.add_connection(
websocket=Mock(),
scope=SCOPE_A,
pipeline_uuid='pipeline-1',
session_type='person',
max_connections=2,
max_connections_per_workspace=1,
)
with pytest.raises(RuntimeError, match='Workspace WebSocket'):
await manager.add_connection(
websocket=Mock(),
scope=SCOPE_A,
pipeline_uuid='pipeline-2',
session_type='person',
max_connections=2,
max_connections_per_workspace=1,
)
await manager.add_connection(
websocket=Mock(),
scope=SCOPE_B,
pipeline_uuid='pipeline-1',
session_type='person',
max_connections=2,
max_connections_per_workspace=1,
)
with pytest.raises(RuntimeError, match='WebSocket connection capacity'):
await manager.add_connection(
websocket=Mock(),
scope=WebSocketScope('instance-a', 'workspace-c', 1),
pipeline_uuid='pipeline-1',
session_type='person',
max_connections=2,
max_connections_per_workspace=1,
)
@pytest.mark.asyncio
async def test_close_scope_closes_and_removes_only_matching_connections():
manager = WebSocketConnectionManager()
websocket_a = Mock(close=AsyncMock())
connection_a = await manager.add_connection(
websocket=websocket_a,
scope=SCOPE_A,
pipeline_uuid='pipeline-1',
session_type='person',
)
connection_b = await manager.add_connection(
websocket=Mock(close=AsyncMock()),
scope=SCOPE_B,
pipeline_uuid='pipeline-1',
session_type='person',
)
await manager.close_scope(SCOPE_A)
websocket_a.close.assert_awaited_once()
assert await manager.get_connection(connection_a.connection_id, scope=SCOPE_A) is None
assert await manager.get_connection(connection_b.connection_id, scope=SCOPE_B) is connection_b
@pytest.mark.asyncio
async def test_embed_event_uses_stable_session_launcher(monkeypatch):
manager = WebSocketConnectionManager()
@@ -0,0 +1,39 @@
from __future__ import annotations
import httpx
import pytest
from langbot.libs.wechatpad_api.api import downloadpai
from langbot.libs.wechatpad_api.util import http_util
class _Response:
headers = {}
def __init__(self, chunks: list[bytes]):
self._chunks = chunks
def iter_content(self, chunk_size=None):
del chunk_size
yield from self._chunks
def test_wechatpad_response_reader_is_bounded(monkeypatch):
monkeypatch.setattr(http_util, '_MAX_WECHATPAD_RESPONSE_BYTES', 4)
with pytest.raises(RuntimeError, match='exceeds the runtime limit'):
http_util._read_requests_response_limited(_Response([b'1234', b'5']))
def test_wechatpad_response_reader_requires_json_object():
with pytest.raises(RuntimeError, match='non-object'):
http_util._read_requests_response_limited(_Response([b'[]']))
@pytest.mark.asyncio
async def test_wechatpad_media_reader_is_bounded(monkeypatch):
monkeypatch.setattr(downloadpai, '_MAX_WECHATPAD_MEDIA_BYTES', 4)
response = httpx.Response(200, content=b'oversized')
with pytest.raises(RuntimeError, match='exceeds'):
await downloadpai._read_media_limited(response)
@@ -0,0 +1,31 @@
from __future__ import annotations
import pytest
from langbot.libs.wecom_api.api import (
_EXTENDED_HTTP_TIMEOUT_SECONDS,
_decode_media_base64_limited,
WecomClient,
)
@pytest.mark.asyncio
async def test_wecom_extended_client_timeout_is_still_bounded() -> None:
client = object.__new__(WecomClient)
client._http_clients = {}
try:
async with client._http_client_context(unbounded_timeout=True) as http_client:
assert http_client.timeout.read == _EXTENDED_HTTP_TIMEOUT_SECONDS
finally:
await client.close()
@pytest.mark.asyncio
async def test_wecom_base64_decode_is_bounded(monkeypatch) -> None:
import langbot.libs.wecom_api.api as wecom_api
monkeypatch.setattr(wecom_api, '_MAX_MEDIA_BYTES', 4)
with pytest.raises(ValueError, match='exceeds'):
await _decode_media_base64_limited('MTIzNDU=')
@@ -1,5 +1,6 @@
import sys
import types
from unittest.mock import Mock
import pytest
@@ -24,6 +25,25 @@ from langbot.libs.wecom_ai_bot_api.api import ( # noqa: E402
from langbot.libs.wecom_ai_bot_api.ws_client import WecomBotWsClient # noqa: E402
def test_ws_callback_tasks_are_bounded():
client = WecomBotWsClient('bot-id', 'secret', object())
client._callback_tasks = {Mock(done=Mock(return_value=False)) for _ in range(100)}
async def callback():
raise AssertionError('rejected callback must not run')
assert client._start_callback_task(callback()) is False
assert len(client._callback_tasks) == 100
def test_webhook_dispatch_tasks_are_bounded():
client = WecomBotClient('', '', '', object(), unified_mode=True)
client._dispatch_tasks = {Mock(done=Mock(return_value=False)) for _ in range(100)}
assert client._start_dispatch_task(Mock()) is False
assert len(client._dispatch_tasks) == 100
def test_extract_template_card_action_supports_nested_button_key():
task_id, event_key, card_type = extract_template_card_action(
{
@@ -287,16 +307,9 @@ async def test_webhook_stream_queues_cumulative_snapshots_for_followups():
assert await client.push_stream_chunk('msg-1', '你好', is_final=False)
assert await client.push_stream_chunk('msg-1', '你好', is_final=True)
chunks = [
await client.stream_sessions.consume(session.stream_id),
await client.stream_sessions.consume(session.stream_id),
await client.stream_sessions.consume(session.stream_id),
]
assert [(chunk.content, chunk.is_final) for chunk in chunks] == [
('', False),
('你好', False),
('你好', True),
]
assert session.queue.qsize() == 1
chunk = await client.stream_sessions.consume(session.stream_id)
assert (chunk.content, chunk.is_final) == ('你好', True)
def test_human_input_payload_keeps_action_select_stage_as_buttons():
@@ -12,6 +12,7 @@ import zipfile
from types import SimpleNamespace
from unittest.mock import MagicMock
import httpx
import pytest
@@ -123,6 +124,17 @@ class TestExtractDepsMetadata:
# Should find requirements.txt in subdirectory
assert task_context.metadata['deps_total'] == 2
def test_archive_preview_rejects_extreme_compression_ratio(self):
from langbot.pkg.plugin.connector import inspect_plugin_archive_metadata
zip_buffer = io.BytesIO()
with zipfile.ZipFile(zip_buffer, 'w', compression=zipfile.ZIP_DEFLATED) as zf:
zf.writestr('manifest.yaml', 'kind: Plugin\nmetadata: {}\n')
zf.writestr('bomb.py', b'A' * (1024 * 1024))
with pytest.raises(ValueError, match='compression-ratio limit'):
inspect_plugin_archive_metadata(zip_buffer.getvalue())
class TestParsePluginId:
"""Tests for _parse_plugin_id static method."""
@@ -141,3 +153,13 @@ class TestParsePluginId:
with pytest.raises(ValueError):
PluginRuntimeConnector._parse_plugin_id('')
@pytest.mark.asyncio
async def test_marketplace_response_reader_is_bounded():
from langbot.pkg.plugin.connector import _read_httpx_response_limited
response = httpx.Response(200, content=b'oversized')
with pytest.raises(ValueError, match='exceeds'):
await _read_httpx_response_limited(response, max_bytes=4)
@@ -136,6 +136,23 @@ async def test_shared_reconnect_replays_two_workspaces_and_removes_missing_proje
assert set(connector._known_desired_states) == {setting_a.installation_uuid}
@pytest.mark.asyncio
async def test_empty_projected_workspaces_do_not_retain_installation_sets():
binding_a = execution_binding('workspace-a')
binding_b = execution_binding('workspace-b')
connector = shared_connector(
[[binding_a, binding_b]],
{'workspace-a': [], 'workspace-b': []},
)
connector.handler = runtime_handler()
await connector._prepare_connected_runtime()
assert connector._workspace_installations == {}
assert connector._known_desired_states == {}
connector.handler.reconcile_plugin_installations.assert_awaited_once_with(())
@pytest.mark.asyncio
async def test_fresh_shared_runtime_cache_replays_persisted_local_package():
package = b'local-lbpkg-bytes'
@@ -257,7 +257,7 @@ class TestSetBinaryStorage:
)
assert response.code != 0
assert '2048 > 1024 bytes' in response.message
assert '1024-byte limit' in response.message
app.persistence_mgr.execute_async.assert_not_awaited()
@pytest.mark.asyncio
@@ -312,21 +312,25 @@ class TestSetBinaryStorage:
)
assert response.code != 0
assert '10485761 > 10485760 bytes' in response.message
assert '10485760' in response.message
app.persistence_mgr.execute_async.assert_not_awaited()
@pytest.mark.asyncio
async def test_negative_limit_disables_size_check(self, app):
"""Negative max_value_bytes allows values larger than the normal default."""
async def test_negative_limit_falls_back_to_bounded_default(self, app, monkeypatch):
"""Negative max_value_bytes cannot disable the process memory boundary."""
import langbot.pkg.plugin.handler as handler_module
runtime_handler = make_handler(app)
app.instance_config.data['plugin']['binary_storage']['max_value_bytes'] = -1
monkeypatch.setattr(handler_module, '_DEFAULT_BINARY_STORAGE_VALUE_BYTES', 1024)
response = await runtime_handler.actions[RuntimeToLangBotAction.SET_BINARY_STORAGE.value](
self.payload(b'x' * 2048)
)
assert response.code == 0
assert app.persistence_mgr.execute_async.await_count == 2
assert response.code != 0
assert '1024-byte limit' in response.message
app.persistence_mgr.execute_async.assert_not_awaited()
@pytest.mark.asyncio
async def test_zero_limit_rejects_non_empty_values(self, app):
@@ -6,6 +6,7 @@ Tests the helper methods that don't require real Dify API calls.
from __future__ import annotations
import pytest
import time
from unittest.mock import AsyncMock, MagicMock
import langbot_plugin.api.entities.builtin.platform.message as platform_message
@@ -18,13 +19,12 @@ class TestDifyWorkflowSubmitClient:
class FakeResponse:
status_code = 503
headers = {}
async def aread(self):
return b''
async def aiter_lines(self):
raise AssertionError('error responses must not enter the SSE loop')
yield
async def aiter_bytes(self, chunk_size=None):
del chunk_size
if False:
yield b''
class FakeStreamContext:
async def __aenter__(self):
@@ -66,6 +66,33 @@ class TestDifyWorkflowSubmitClient:
)
)
@pytest.mark.asyncio
async def test_sse_parser_rejects_an_unbounded_line(self):
from langbot.libs.dify_service_api.v1 import client, errors
class FakeResponse:
async def aiter_bytes(self, chunk_size=None):
del chunk_size
for _ in range(129):
yield b'x' * 8192
with pytest.raises(errors.DifyAPIError, match='SSE event exceeds'):
await anext(client._iter_sse_json(FakeResponse()))
@pytest.mark.asyncio
async def test_upload_rejects_oversized_local_file(self, tmp_path):
from langbot.libs.dify_service_api.v1 import client
file_path = tmp_path / 'large.bin'
file_path.write_bytes(b'x' * (client._MAX_DIFY_UPLOAD_BYTES + 1))
dify_client = client.AsyncDifyServiceClient(
'test-key',
'https://dify.example/v1',
)
with pytest.raises(ValueError, match='exceeds the size limit'):
await dify_client.upload_file(file_path, 'person_user-1')
class TestDifyExtractTextOutput:
"""Tests for _extract_dify_text_output method."""
@@ -320,6 +347,130 @@ class TestDifyHumanInputForms:
assert difysvapi._dify_user_from_query(query_a) == difysvapi._dify_user_from_query(query_c)
difysvapi._PENDING_FORMS.clear()
def test_pending_form_lookup_does_not_scan_unrelated_sessions(self, monkeypatch):
from langbot.pkg.provider.runners import difysvapi
def session_key(index: int):
return (
'instance',
f'workspace-{index}',
1,
'bot',
'pipeline',
'adapter',
'person',
f'user-{index}',
)
difysvapi._PENDING_FORMS.clear()
for index in range(512):
difysvapi._set_pending_form(
session_key(index),
{
'form_token': f'token-{index}',
'workflow_run_id': f'run-{index}',
},
)
class NoGlobalIterationDict(dict):
def __iter__(self):
raise AssertionError('pending form lookup scanned all sessions')
def keys(self):
raise AssertionError('pending form lookup scanned all sessions')
def items(self):
raise AssertionError('pending form lookup scanned all sessions')
def values(self):
raise AssertionError('pending form lookup scanned all sessions')
guarded_forms = NoGlobalIterationDict(difysvapi._PENDING_FORMS)
monkeypatch.setattr(difysvapi, '_PENDING_FORMS', guarded_forms)
assert difysvapi._get_pending_form_by_token(session_key(511), 'token-511')['workflow_run_id'] == 'run-511'
difysvapi._set_pending_form(
session_key(512),
{'form_token': 'token-512', 'workflow_run_id': 'run-512'},
)
assert len(guarded_forms) == 513
def test_pending_form_expiry_heap_ignores_stale_overwrite_and_stays_bounded(self):
from langbot.pkg.provider.runners import difysvapi
session_key = (
'instance',
'workspace',
1,
'bot',
'pipeline',
'adapter',
'person',
'user',
)
difysvapi._PENDING_FORMS.clear()
now = time.time()
difysvapi._set_pending_form(
session_key,
{
'form_token': 'token',
'workflow_run_id': 'stale',
'expiration_time': now + 1,
},
)
for revision in range(500):
difysvapi._set_pending_form(
session_key,
{
'form_token': 'token',
'workflow_run_id': f'current-{revision}',
'expiration_time': now + 3600 + revision,
},
)
difysvapi._prune_pending_forms(now + 2)
assert difysvapi._get_pending_form_by_token(session_key, 'token')['workflow_run_id'] == 'current-499'
assert difysvapi._PENDING_FORM_ACTIVE_COUNT == 1
assert len(difysvapi._PENDING_FORM_EXPIRY_HEAP) <= max(
difysvapi._PENDING_FORM_HEAP_COMPACT_FLOOR,
difysvapi._PENDING_FORM_ACTIVE_COUNT * difysvapi._PENDING_FORM_HEAP_MAX_MULTIPLIER,
)
def test_pending_form_capacity_evicts_earliest_session_without_full_scan(
self,
monkeypatch,
):
from langbot.pkg.provider.runners import difysvapi
def session_key(index: int):
return (
'instance',
f'workspace-{index}',
1,
'bot',
'pipeline',
'adapter',
'person',
f'user-{index}',
)
difysvapi._PENDING_FORMS.clear()
monkeypatch.setattr(difysvapi, '_PENDING_FORM_MAX_SESSIONS', 2)
now = time.time()
for index, expires_in in ((1, 300), (2, 100), (3, 200)):
difysvapi._set_pending_form(
session_key(index),
{
'form_token': f'token-{index}',
'expiration_time': now + expires_in,
},
)
assert session_key(1) in difysvapi._PENDING_FORMS
assert session_key(2) not in difysvapi._PENDING_FORMS
assert session_key(3) in difysvapi._PENDING_FORMS
def test_interactive_form_data_preserves_pipeline_uuid(self):
from langbot.pkg.provider.runners import difysvapi
@@ -0,0 +1,49 @@
from __future__ import annotations
import pytest
from langbot.libs.deerflow_api.client import (
ERROR_BODY_MAX_BYTES,
_read_error_body,
)
from langbot.libs.deerflow_api.errors import DeerFlowAPIError
from langbot.pkg.provider.runners.langflowapi import (
_MAX_LANGFLOW_LINE_CHARS,
_MAX_LANGFLOW_RESPONSE_BYTES,
_iter_limited_lines,
_read_limited_response,
)
class _ChunkedResponse:
def __init__(self, chunks: list[bytes]):
self._chunks = chunks
async def aiter_bytes(self, chunk_size=None):
del chunk_size
for chunk in self._chunks:
yield chunk
@pytest.mark.asyncio
async def test_langflow_rejects_oversized_stream_event():
response = _ChunkedResponse([b'x' * (_MAX_LANGFLOW_LINE_CHARS + 1)])
with pytest.raises(ValueError, match='event exceeds'):
await anext(_iter_limited_lines(response))
@pytest.mark.asyncio
async def test_langflow_rejects_oversized_blocking_response():
response = _ChunkedResponse([b'x' * (_MAX_LANGFLOW_RESPONSE_BYTES + 1)])
with pytest.raises(ValueError, match='response exceeds'):
await _read_limited_response(response)
@pytest.mark.asyncio
async def test_deerflow_rejects_oversized_error_body():
response = _ChunkedResponse([b'x' * (ERROR_BODY_MAX_BYTES + 1)])
with pytest.raises(DeerFlowAPIError, match='response exceeds'):
await _read_error_body(response)
@@ -0,0 +1,62 @@
from __future__ import annotations
import asyncio
import threading
from unittest.mock import AsyncMock
import pytest
from langbot.pkg.provider import runner
from langbot.pkg.provider.runners import (
cozeapi,
dashscopeapi,
tboxapi,
weknoraapi,
)
@pytest.mark.asyncio
async def test_blocking_provider_iterator_runs_outside_event_loop():
release = threading.Event()
def values():
release.wait(timeout=2)
yield 'ready'
task = asyncio.create_task(anext(runner.iterate_sync(values())))
await asyncio.sleep(0)
assert not task.done()
release.set()
assert await asyncio.wait_for(task, timeout=1) == 'ready'
@pytest.mark.asyncio
async def test_sync_provider_iterator_has_event_limit():
with pytest.raises(RuntimeError, match='event limit'):
async for _ in runner.iterate_sync(iter([1, 2]), max_items=1):
pass
@pytest.mark.asyncio
async def test_coze_runner_closes_request_scoped_client():
request_runner = object.__new__(cozeapi.CozeAPIRunner)
request_runner.coze = AsyncMock()
await request_runner.aclose()
request_runner.coze.close.assert_awaited_once()
@pytest.mark.parametrize(
('append', 'exception_type'),
[
(cozeapi._append_bounded, ValueError),
(dashscopeapi._append_bounded, dashscopeapi.DashscopeAPIError),
(tboxapi._append_bounded, tboxapi.TboxAPIError),
(weknoraapi._append_bounded, weknoraapi.errors.WeKnoraAPIError),
],
)
def test_provider_accumulators_reject_oversized_output(append, exception_type):
with pytest.raises(exception_type, match='exceeds the runtime limit'):
append('x' * (1024 * 1024), 'y')
+167 -1
View File
@@ -97,7 +97,11 @@ def _query(variables: dict | None = None, context: ExecutionContext = TEST_EXECU
def _register_session(loader: MCPLoader, session: RuntimeMCPSession) -> None:
loader.sessions[loader._session_key(session.execution_context, session.server_name)] = session
loader._register_session(
session.execution_context,
session.server_name,
session,
)
def _http_status_error(status_code: int) -> httpx.HTTPStatusError:
@@ -615,3 +619,165 @@ async def test_mcp_loader_shutdown_cancels_startup_tasks_and_closes_sessions_con
assert started == {'one', 'two'}
assert loader._hosted_mcp_tasks == []
assert loader.sessions == {}
@pytest.mark.asyncio
async def test_completed_mcp_host_tasks_do_not_accumulate():
loader = MCPLoader(_app())
task = asyncio.create_task(asyncio.sleep(0))
loader.track_hosted_task(task, TEST_EXECUTION_CONTEXT)
await task
await asyncio.sleep(0)
assert loader._hosted_mcp_tasks == []
assert loader._hosted_mcp_tasks_by_scope == {}
assert loader._scope_generations == {}
@pytest.mark.asyncio
async def test_generation_advance_cancels_host_tasks_and_closes_old_sessions():
loader = MCPLoader(_app())
old_session = SimpleNamespace(
server_name='old',
shutdown=AsyncMock(),
)
loader._register_session(
TEST_EXECUTION_CONTEXT,
old_session.server_name,
old_session,
)
async def pending_host():
await asyncio.Event().wait()
hosted_task = asyncio.create_task(pending_host())
loader.track_hosted_task(hosted_task, TEST_EXECUTION_CONTEXT)
await asyncio.sleep(0)
next_context = ExecutionContext(
instance_uuid=TEST_EXECUTION_CONTEXT.instance_uuid,
workspace_uuid=TEST_EXECUTION_CONTEXT.workspace_uuid,
placement_generation=2,
)
loader.ap.workspace_service.get_execution_binding = AsyncMock(
return_value=SimpleNamespace(
instance_uuid=next_context.instance_uuid,
workspace_uuid=next_context.workspace_uuid,
placement_generation=next_context.placement_generation,
)
)
await loader._assert_execution_active(next_context)
assert hosted_task.cancelled()
old_session.shutdown.assert_awaited_once_with()
assert loader.sessions == {}
assert loader._session_keys_by_scope == {}
assert loader._hosted_mcp_tasks_by_scope == {}
assert loader._scope_generations == {}
def test_session_lookup_uses_scope_index_without_global_iteration():
class NoGlobalIterationDict(dict):
def __iter__(self):
raise AssertionError('MCP lookup scanned every tenant session')
def items(self):
raise AssertionError('MCP lookup scanned every tenant session')
def values(self):
raise AssertionError('MCP lookup scanned every tenant session')
loader = MCPLoader(_app())
target_context = None
target_session = None
for index in range(1_000):
context = ExecutionContext(
instance_uuid='instance-a',
workspace_uuid=f'workspace-{index}',
placement_generation=1,
)
session = SimpleNamespace(server_name=f'server-{index}')
loader._register_session(context, session.server_name, session)
if index == 777:
target_context = context
target_session = session
loader._sessions = NoGlobalIterationDict(loader._sessions)
assert loader._sessions_for_context(target_context) == [target_session]
@pytest.mark.asyncio
async def test_mcp_startup_concurrency_is_instance_bounded():
app = _app()
app.instance_config = SimpleNamespace(data={'mcp': {'lifecycle_concurrency': 2}})
loader = MCPLoader(app)
active = 0
maximum_active = 0
release = asyncio.Event()
async def fake_host(_context, _config):
nonlocal active, maximum_active
active += 1
maximum_active = max(maximum_active, active)
if maximum_active == 2:
release.set()
await release.wait()
await asyncio.sleep(0)
active -= 1
loader._host_mcp_server = fake_host
await asyncio.gather(
*(
loader.host_mcp_server(
TEST_EXECUTION_CONTEXT,
{'name': f'server-{index}'},
)
for index in range(20)
)
)
assert loader._lifecycle_concurrency == 2
assert maximum_active == 2
@pytest.mark.asyncio
async def test_mcp_startup_dispatcher_does_not_create_every_server_task_at_once():
app = _app()
app.instance_config = SimpleNamespace(data={'mcp': {'lifecycle_concurrency': 2}})
loader = MCPLoader(app)
started = 0
first_batch_started = asyncio.Event()
release = asyncio.Event()
async def fake_host(_context, _config):
nonlocal started
started += 1
if started == 2:
first_batch_started.set()
await release.wait()
loader.host_mcp_server = fake_host
configs = [(TEST_EXECUTION_CONTEXT, {'name': f'server-{index}'}) for index in range(20)]
dispatch_task = asyncio.create_task(loader._host_server_configs_bounded(configs))
await asyncio.wait_for(first_batch_started.wait(), timeout=1)
assert started == 2
assert len(loader._hosted_mcp_tasks) == 2
release.set()
await dispatch_task
await asyncio.sleep(0)
assert started == 20
assert loader._hosted_mcp_tasks == []
assert loader._hosted_mcp_tasks_by_scope == {}
def test_invalid_mcp_lifecycle_concurrency_uses_safe_default():
app = _app()
app.instance_config = SimpleNamespace(data={'mcp': {'lifecycle_concurrency': True}})
assert MCPLoader(app)._lifecycle_concurrency == 16
+124 -1
View File
@@ -7,6 +7,7 @@ and error handling without calling real LLM APIs.
from __future__ import annotations
import dataclasses
import pytest
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
@@ -18,7 +19,7 @@ from langbot.pkg.entity.errors import provider as provider_errors
from langbot.pkg.provider.modelmgr import token
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.workspace.entities import WorkspaceExecutionBinding
from langbot.pkg.workspace.errors import WorkspaceGenerationMismatchError
from langbot.pkg.workspace.errors import WorkspaceGenerationMismatchError, WorkspaceInvariantError
from tests.unit_tests.provider.conftest import (
TEST_EXECUTION_CONTEXT,
TEST_WORKSPACE_UUID,
@@ -125,6 +126,24 @@ async def test_model_manager_load_models_from_db(fake_requester_registry, fake_p
assert len(model_mgr.rerank_model_dict) == 1
@pytest.mark.asyncio
async def test_empty_cloud_workspace_does_not_retain_generation(
mock_app_for_modelmgr,
):
model_mgr = ModelManager(mock_app_for_modelmgr)
await model_mgr._load_workspace_models(TEST_EXECUTION_CONTEXT)
assert model_mgr.provider_dict == {}
assert model_mgr.llm_model_dict == {}
assert model_mgr.embedding_model_dict == {}
assert model_mgr.rerank_model_dict == {}
assert model_mgr._scope_generations == {}
await model_mgr.resolve_execution_context(TEST_EXECUTION_CONTEXT)
assert model_mgr._scope_generations == {}
@pytest.mark.asyncio
async def test_model_manager_load_provider_unknown_requester(mock_app_for_modelmgr):
"""Test ModelManager raises RequesterNotFoundError for unknown requester."""
@@ -939,6 +958,110 @@ async def test_runtime_cache_rejects_stale_placement_generation(fake_requester_r
await model_mgr.get_model_by_uuid(stale_context, 'any-model')
def test_generation_advance_prunes_superseded_model_runtime_objects():
class NoGlobalIterationDict(dict):
def __iter__(self):
raise AssertionError('generation advance scanned every model runtime')
def items(self):
raise AssertionError('generation advance scanned every model runtime')
def keys(self):
raise AssertionError('generation advance scanned every model runtime')
model_mgr = ModelManager(Mock())
old_context = ExecutionContext(
instance_uuid='instance-a',
workspace_uuid='workspace-a',
placement_generation=1,
)
new_context = dataclasses.replace(old_context, placement_generation=2)
model_mgr._observe_execution_context(old_context)
for cache in (
model_mgr.provider_dict,
model_mgr.llm_model_dict,
model_mgr.embedding_model_dict,
model_mgr.rerank_model_dict,
):
model_mgr._cache_set(
cache,
('instance-a', 'workspace-a', 1, 'resource-a'),
object(),
)
model_mgr._cache_set(
cache,
('instance-a', 'workspace-b', 1, 'resource-b'),
object(),
)
model_mgr.provider_dict = NoGlobalIterationDict(model_mgr.provider_dict)
model_mgr.llm_model_dict = NoGlobalIterationDict(model_mgr.llm_model_dict)
model_mgr.embedding_model_dict = NoGlobalIterationDict(model_mgr.embedding_model_dict)
model_mgr.rerank_model_dict = NoGlobalIterationDict(model_mgr.rerank_model_dict)
model_mgr._observe_execution_context(new_context)
for cache in (
model_mgr.provider_dict,
model_mgr.llm_model_dict,
model_mgr.embedding_model_dict,
model_mgr.rerank_model_dict,
):
assert ('instance-a', 'workspace-a', 1, 'resource-a') not in cache
assert ('instance-a', 'workspace-b', 1, 'resource-b') in cache
with pytest.raises(WorkspaceInvariantError, match='rolled back'):
model_mgr._observe_execution_context(old_context)
@pytest.mark.asyncio
async def test_generation_advance_closes_retired_provider_requester(
fake_requester_registry,
runtime_provider,
):
model_mgr = fake_requester_registry
runtime_provider.requester.aclose = AsyncMock()
await model_mgr.cache_provider(TEST_EXECUTION_CONTEXT, runtime_provider)
next_context = dataclasses.replace(
TEST_EXECUTION_CONTEXT,
placement_generation=2,
)
model_mgr.ap.workspace_service.get_execution_binding = AsyncMock(
return_value=WorkspaceExecutionBinding(
instance_uuid=next_context.instance_uuid,
workspace_uuid=next_context.workspace_uuid,
placement_generation=next_context.placement_generation,
write_fenced=False,
state='active',
)
)
await model_mgr.resolve_execution_context(next_context)
runtime_provider.requester.aclose.assert_awaited_once_with()
assert model_mgr.provider_dict == {}
assert model_mgr._scope_generations == {}
@pytest.mark.asyncio
async def test_model_manager_shutdown_closes_all_requesters_once(
fake_requester_registry,
runtime_provider,
):
model_mgr = fake_requester_registry
runtime_provider.requester.aclose = AsyncMock()
await model_mgr.cache_provider(TEST_EXECUTION_CONTEXT, runtime_provider)
await model_mgr.shutdown()
await model_mgr.shutdown()
runtime_provider.requester.aclose.assert_awaited_once_with()
assert model_mgr.provider_dict == {}
assert model_mgr.llm_model_dict == {}
assert model_mgr.embedding_model_dict == {}
assert model_mgr.rerank_model_dict == {}
def test_provider_not_found_error_str():
"""Test ProviderNotFoundError string representation."""
error = provider_errors.ProviderNotFoundError('test-provider')
@@ -16,6 +16,8 @@ from importlib import import_module
import langbot_plugin.api.entities.builtin.provider.session as provider_session
import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query
import langbot_plugin.api.entities.builtin.provider.message as provider_message
import langbot_plugin.api.entities.builtin.provider.prompt as provider_prompt
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.pipeline.pool import (
@@ -426,3 +428,131 @@ class TestSessionManagerWorkspaceIsolation:
with pytest.raises(ExecutionContextMismatchError):
await manager.get_conversation(query, session, [], 'pipeline-1', 'bot-b')
@pytest.mark.asyncio
async def test_per_workspace_capacity_evicts_oldest_idle_session(self):
manager = self.manager()
manager.ap.instance_config.data['system'] = {
'session_retention': {
'max_entries': 10,
'max_entries_per_workspace': 2,
}
}
queries = []
sessions = []
for index in range(3):
query = scoped_query()
query.launcher_id = f'launcher-{index}'
queries.append(query)
sessions.append(await manager.get_session(query))
assert len(manager.session_list) == 2
assert sessions[0] not in manager.session_list
assert sessions[1:] == manager.session_list
assert await manager.get_session(queries[-1]) is manager.session_list[-1]
@pytest.mark.asyncio
async def test_new_session_does_not_scan_other_workspace_sessions(self):
manager = self.manager()
manager.ap.instance_config.data['system'] = {
'session_retention': {
'max_entries': 600,
'max_entries_per_workspace': 2,
}
}
for index in range(512):
query = scoped_query(workspace_uuid=f'workspace-{index}')
query.launcher_id = f'launcher-{index}'
await manager.get_session(query)
class NoGlobalIterationDict(dict):
def __iter__(self):
raise AssertionError('global session index iteration is forbidden')
def items(self):
raise AssertionError('global session index iteration is forbidden')
def values(self):
raise AssertionError('global session index iteration is forbidden')
manager._session_index = NoGlobalIterationDict(manager._session_index)
query = scoped_query(workspace_uuid='workspace-new')
query.launcher_id = 'launcher-new'
session = await manager.get_session(query)
assert session.workspace_uuid == 'workspace-new'
assert len(manager._session_index) == 513
@pytest.mark.asyncio
async def test_stale_expiry_revision_does_not_evict_recent_session(
self,
monkeypatch,
):
sessionmgr = get_session_module()
manager = self.manager()
manager.ap.instance_config.data['system'] = {
'session_retention': {
'max_entries': 10,
'max_entries_per_workspace': 10,
'idle_ttl_seconds': 1,
}
}
clock = [0.0]
monkeypatch.setattr(sessionmgr.time, 'monotonic', lambda: clock[0])
first_query = scoped_query()
first_query.launcher_id = 'first'
first = await manager.get_session(first_query)
clock[0] = 0.5
assert await manager.get_session(first_query) is first
clock[0] = 1.25
second_query = scoped_query()
second_query.launcher_id = 'second'
await manager.get_session(second_query)
assert first in manager.session_list
clock[0] = 2.0
third_query = scoped_query()
third_query.launcher_id = 'third'
await manager.get_session(third_query)
assert first not in manager.session_list
@pytest.mark.asyncio
async def test_access_revision_heap_stays_bounded(self):
manager = self.manager()
query = scoped_query()
await manager.get_session(query)
for _ in range(1000):
await manager.get_session(query)
assert len(manager._session_expiry_heap) <= 64
def test_trim_conversation_drops_retained_binary_payloads(self):
manager = self.manager()
conversation = provider_session.Conversation(
prompt=provider_prompt.Prompt(name='test', messages=[]),
messages=[
provider_message.Message(
role='user',
content=[
provider_message.ContentElement.from_text('hello'),
provider_message.ContentElement.from_image_base64('x' * 1000000),
provider_message.ContentElement.from_file_base64(
'y' * 1000000,
'large.bin',
),
],
)
],
pipeline_uuid='pipeline-1',
bot_uuid=TEST_BOT_UUID,
)
manager.trim_conversation_messages(conversation, max_rounds=10)
content = conversation.messages[0].content
assert content[1].image_base64 is None
assert content[2].file_base64 is None
@@ -4,6 +4,7 @@ import base64
import contextlib
import os
import tempfile
import threading
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
@@ -459,6 +460,24 @@ async def test_edit_rejects_missing_string():
assert 'not found' in result['error'].lower()
@pytest.mark.asyncio
async def test_edit_rejects_oversized_host_file(monkeypatch):
with tempfile.TemporaryDirectory() as tmpdir:
loader, _ = _make_loader_with_workspace(tmpdir)
with open(os.path.join(tmpdir, 'large.txt'), 'wb') as f:
f.write(b'12345')
monkeypatch.setattr(native_loader, '_MAX_HOST_EDIT_FILE_BYTES', 4)
result = await loader.invoke_tool(
'edit',
{'path': '/workspace/large.txt', 'old_string': '1', 'new_string': 'x'},
_make_query(),
)
assert result['ok'] is False
assert 'edit limit' in result['error']
@pytest.mark.asyncio
async def test_path_escape_blocked():
with tempfile.TemporaryDirectory() as tmpdir:
@@ -678,6 +697,34 @@ async def test_glob_caps_match_count_and_returns_preview():
assert result['truncated_by'] == 'matches'
@pytest.mark.asyncio
async def test_glob_runs_off_event_loop_and_caps_directory_walk(monkeypatch):
monkeypatch.setattr(native_loader, '_FILE_WALK_MAX_ENTRIES', 10)
with tempfile.TemporaryDirectory() as tmpdir:
loader, _ = _make_loader_with_workspace(tmpdir)
event_loop_thread = threading.get_ident()
observed_threads: list[int] = []
original = loader._glob_host_location
def observe(*args, **kwargs):
observed_threads.append(threading.get_ident())
return original(*args, **kwargs)
monkeypatch.setattr(loader, '_glob_host_location', observe)
for index in range(12):
with open(os.path.join(tmpdir, f'file-{index:03d}.txt'), 'w', encoding='utf-8') as f:
f.write(str(index))
result = await loader.invoke_tool('glob', {'path': '/workspace', 'pattern': '*.txt'}, _make_query())
assert result['ok'] is True
assert result['total'] == 10
assert result['truncated'] is True
assert result['truncated_by'] == 'scan'
assert observed_threads and observed_threads[0] != event_loop_thread
@pytest.mark.asyncio
async def test_grep_reports_invalid_regex_and_truncates_long_matching_lines():
with tempfile.TemporaryDirectory() as tmpdir:
@@ -695,3 +742,21 @@ async def test_grep_reports_invalid_regex_and_truncates_long_matching_lines():
assert result['truncated_by'] == 'line'
assert result['matches'][0]['file'] == '/workspace/data.txt'
assert result['matches'][0]['content'].endswith('... [truncated]')
@pytest.mark.asyncio
async def test_grep_interrupts_catastrophic_regex(monkeypatch):
monkeypatch.setattr(native_loader, '_GREP_REGEX_TIMEOUT_SECONDS', 0.001)
with tempfile.TemporaryDirectory() as tmpdir:
loader, _ = _make_loader_with_workspace(tmpdir)
with open(os.path.join(tmpdir, 'data.txt'), 'w', encoding='utf-8') as f:
f.write(('a' * 100_000) + '!')
result = await loader.invoke_tool(
'grep',
{'path': '/workspace', 'pattern': r'(a+)+$'},
_make_query(),
)
assert result == {'ok': False, 'error': 'Regex search timed out'}
+54 -2
View File
@@ -12,6 +12,7 @@ from unittest.mock import AsyncMock, Mock
import pytest
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.core.taskmgr import TaskCapacityError
from langbot.pkg.rag.knowledge.kbmgr import RuntimeKnowledgeBase
from langbot.pkg.storage.mgr import StorageMgr
from langbot.pkg.workspace.errors import WorkspaceNotFoundError
@@ -34,9 +35,9 @@ def _upload_key(logical_key: str, *, context: ExecutionContext = CONTEXT) -> str
)
def _make_zip_bytes(entries: dict[str, bytes]) -> bytes:
def _make_zip_bytes(entries: dict[str, bytes], *, compression: int = zipfile.ZIP_STORED) -> bytes:
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, 'w') as zf:
with zipfile.ZipFile(buffer, 'w', compression=compression) as zf:
for name, content in entries.items():
zf.writestr(name, content)
zf.mkdir('emptydir')
@@ -151,6 +152,24 @@ class TestStoreFile:
kb.ap.storage_mgr.storage_provider.exists.assert_not_awaited()
@pytest.mark.asyncio
async def test_store_file_rolls_back_pending_record_when_task_capacity_is_exhausted(self):
kb = _make_kb()
object_key = _upload_key('queued.pdf')
def reject(coro, **_kwargs):
coro.close()
raise TaskCapacityError('capacity')
kb.ap.task_mgr.create_user_task.side_effect = reject
with pytest.raises(TaskCapacityError, match='capacity'):
await kb.store_file(CONTEXT, object_key)
statements = [str(call.args[0]) for call in kb.ap.persistence_mgr.execute_async.await_args_list]
assert any(statement.startswith('INSERT') for statement in statements)
assert any(statement.startswith('DELETE') for statement in statements)
class TestStoreZipFile:
@pytest.mark.asyncio
@@ -202,6 +221,39 @@ class TestStoreZipFile:
kb.store_file.assert_not_awaited()
kb.ap.storage_mgr.storage_provider.delete.assert_awaited_once_with(_upload_key('archive.zip'))
@pytest.mark.asyncio
async def test_store_zip_file_rejects_too_many_documents_before_extracting(self):
kb = _make_kb()
kb.ap.storage_mgr.storage_provider.load = AsyncMock(
return_value=_make_zip_bytes({f'doc-{index}.txt': b'text' for index in range(9)})
)
kb.store_file = AsyncMock()
with pytest.raises(ValueError, match='too many supported documents'):
await kb._store_zip_file(CONTEXT, _upload_key('archive.zip'))
kb.store_file.assert_not_awaited()
kb.ap.storage_mgr.storage_provider.save.assert_not_awaited()
kb.ap.storage_mgr.storage_provider.delete.assert_awaited_once_with(_upload_key('archive.zip'))
@pytest.mark.asyncio
async def test_store_zip_file_rejects_extreme_compression_ratio_before_extracting(self):
kb = _make_kb()
kb.ap.storage_mgr.storage_provider.load = AsyncMock(
return_value=_make_zip_bytes(
{'bomb.txt': b'A' * (1024 * 1024)},
compression=zipfile.ZIP_DEFLATED,
)
)
kb.store_file = AsyncMock()
with pytest.raises(ValueError, match='compression-ratio limit'):
await kb._store_zip_file(CONTEXT, _upload_key('archive.zip'))
kb.store_file.assert_not_awaited()
kb.ap.storage_mgr.storage_provider.save.assert_not_awaited()
kb.ap.storage_mgr.storage_provider.delete.assert_awaited_once_with(_upload_key('archive.zip'))
class TestStoreFileTask:
@pytest.mark.asyncio
+57 -1
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import dataclasses
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
@@ -10,7 +11,8 @@ import pytest
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.entity.persistence.rag import KnowledgeBase
from langbot.pkg.rag.knowledge.kbmgr import RAGManager, RuntimeKnowledgeBase
from langbot.pkg.workspace.errors import WorkspaceNotFoundError
from langbot.pkg.workspace.entities import WorkspaceExecutionBinding
from langbot.pkg.workspace.errors import WorkspaceInvariantError, WorkspaceNotFoundError
CONTEXT_A = ExecutionContext(
@@ -511,6 +513,37 @@ class TestRAGManagerLoadKnowledgeBasesFromDB:
('workspace-a', 'kb-2'),
}
@pytest.mark.asyncio
async def test_cloud_startup_reuses_validated_binding(self):
class TenantUow:
async def __aenter__(self):
return self
async def __aexit__(self, *_args):
return False
app = _app()
binding = WorkspaceExecutionBinding(
instance_uuid='instance-a',
workspace_uuid='workspace-a',
placement_generation=5,
write_fenced=False,
state='active',
)
app.persistence_mgr.mode = SimpleNamespace(value='cloud_runtime')
app.persistence_mgr.tenant_uow = lambda _workspace_uuid: TenantUow()
app.persistence_mgr.execute_async.return_value = _Result([_entity()])
app.workspace_service.list_active_execution_bindings = AsyncMock(return_value=[binding])
app.workspace_service.get_execution_binding = AsyncMock(
side_effect=AssertionError('startup RAG loader repeated a validated binding lookup')
)
manager = RAGManager(app)
await manager.load_knowledge_bases_from_db()
assert set(manager.knowledge_bases) == {('workspace-a', 'kb-a')}
app.workspace_service.get_execution_binding.assert_not_awaited()
@pytest.mark.asyncio
async def test_handles_load_error_gracefully(self):
app = _app()
@@ -643,6 +676,29 @@ class TestRAGManagerInit:
def test_init_creates_empty_knowledge_bases_dict(self):
assert RAGManager(_app()).knowledge_bases == {}
def test_generation_advance_prunes_superseded_runtime_knowledge_bases(self):
class NoGlobalItemsScan(dict):
def items(self):
raise AssertionError('generation advance scanned every knowledge runtime')
app = _app()
manager = RAGManager(app)
manager._cache_runtime(
RuntimeKnowledgeBase(
app,
_entity(),
CONTEXT_A,
)
)
manager.knowledge_bases = NoGlobalItemsScan(manager.knowledge_bases)
next_context = dataclasses.replace(CONTEXT_A, placement_generation=6)
manager._observe_execution_context(next_context)
assert manager.knowledge_bases == {}
with pytest.raises(WorkspaceInvariantError, match='rolled back'):
manager._observe_execution_context(CONTEXT_A)
class TestRAGManagerGetKnowledgeBase:
@pytest.mark.asyncio
@@ -74,6 +74,19 @@ class TestS3StorageProviderInit:
assert provider.s3_client is None
assert provider.bucket_name is None
@pytest.mark.asyncio
async def test_shutdown_closes_client_once(self):
s3storage = get_s3storage_module()
provider = s3storage.S3StorageProvider(Mock())
client = Mock()
provider.s3_client = client
await provider.shutdown()
await provider.shutdown()
client.close.assert_called_once_with()
assert provider.s3_client is None
class TestS3StorageProviderWithMoto:
"""Tests using moto to mock AWS S3."""
@@ -92,6 +92,15 @@ class TestStorageMgr:
await storage_mgr.initialize()
mock_init.assert_called_once()
@pytest.mark.asyncio
async def test_shutdown_delegates_to_active_provider(self):
storage_mgr = StorageMgr(Mock())
storage_mgr.storage_provider = Mock(shutdown=AsyncMock())
await storage_mgr.shutdown()
storage_mgr.storage_provider.shutdown.assert_awaited_once()
class TestStorageProviderBase:
"""Test StorageProvider base class methods."""
@@ -8,6 +8,7 @@ import pytest
from langbot.pkg.api.http.authz import WorkspaceRequiredError
from langbot.pkg.api.http.context import ExecutionContext
from langbot.pkg.storage.mgr import StorageMgr
from langbot.pkg.utils.bounded_executor import current_blocking_work_scope
WORKSPACE_A = '00000000-0000-0000-0000-00000000000a'
@@ -33,14 +34,17 @@ def _context_for_instance(instance_uuid: str) -> ExecutionContext:
class _Provider:
def __init__(self):
self.values: dict[str, bytes] = {}
self.observed_public_scopes: list[str | None] = []
async def save(self, key: str, value: bytes):
self.values[key] = value
async def load(self, key: str) -> bytes:
self.observed_public_scopes.append(current_blocking_work_scope())
return self.values[key]
async def exists(self, key: str) -> bool:
self.observed_public_scopes.append(current_blocking_work_scope())
return key in self.values
async def size(self, key: str) -> int:
@@ -120,6 +124,10 @@ async def test_public_object_route_derives_trusted_workspace(manager):
value=b'image-a',
)
assert await manager.resolve_public_object(object_key, expected_owner_type='upload') == b'image-a'
assert manager.storage_provider.observed_public_scopes[-2:] == [
WORKSPACE_A,
WORKSPACE_A,
]
assert await manager.resolve_public_object(object_key, expected_owner_type='plugin') is None
guessed_workspace_key = object_key.replace(WORKSPACE_A, WORKSPACE_B)
@@ -29,6 +29,13 @@ def create_mock_app():
mock_app.instance_config.data = {'space': {'url': 'https://space.example.com'}}
mock_app.persistence_mgr = AsyncMock()
mock_app.persistence_mgr.execute_async = AsyncMock()
mock_app.workspace_service.instance_uuid = 'instance-test'
def close_scheduled_coroutine(coro, **kwargs):
coro.close()
return Mock()
mock_app.task_mgr.create_task = Mock(side_effect=close_scheduled_coroutine)
return mock_app
@@ -3,6 +3,7 @@
from __future__ import annotations
import json
from types import SimpleNamespace
import pytest
from unittest.mock import AsyncMock, Mock
@@ -96,6 +97,35 @@ class TestBuildHeartbeatPayload:
payload = await heartbeat.build_heartbeat_payload(ap)
assert payload['features']['pipeline_count'] == -1
@pytest.mark.asyncio
async def test_cloud_counts_loaded_registries_without_tenant_sql(self):
heartbeat = get_heartbeat_module()
ap = make_app()
ap.persistence_mgr.mode = SimpleNamespace(value='cloud_runtime')
ap.persistence_mgr.execute_async = AsyncMock(
side_effect=AssertionError('Cloud heartbeat must not issue per-tenant COUNTs')
)
ap.pipeline_mgr = SimpleNamespace(
_pipelines_by_key={'pipeline-a': object(), 'pipeline-b': object()},
)
ap.tool_mgr = SimpleNamespace(
mcp_tool_loader=SimpleNamespace(
_sessions={'mcp-a': object(), 'mcp-b': object(), 'mcp-c': object()},
),
)
ap.rag_mgr = SimpleNamespace(
knowledge_bases={'kb-a': object()},
)
payload = await heartbeat.build_heartbeat_payload(ap)
features = payload['features']
assert features['pipeline_count'] == 2
assert features['mcp_server_count'] == 3
assert features['knowledge_base_count'] == 1
assert features['bot_count'] == 1
ap.persistence_mgr.execute_async.assert_not_awaited()
@pytest.mark.asyncio
async def test_no_user_content_fields(self):
"""The heartbeat must never carry message content / credentials keys."""
+5 -2
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
from types import SimpleNamespace
import asyncio
import pytest
@@ -20,7 +21,9 @@ async def test_send_tasks_are_scoped_to_manager_instance(monkeypatch):
assert first.send_tasks is not second.send_tasks
await first.start_send_task({'event': 'first'})
await first.send_tasks[0]
task = first.send_tasks[0]
await task
await asyncio.sleep(0)
assert len(first.send_tasks) == 1
assert first.send_tasks == []
assert second.send_tasks == []
@@ -0,0 +1,278 @@
from __future__ import annotations
import asyncio
import threading
import pytest
from langbot.pkg.utils.bounded_executor import (
BlockingWorkCapacityError,
BoundedThreadPoolExecutor,
blocking_work_scope,
configure_bounded_default_executor,
run_blocking_atomic,
run_blocking_cleanup,
)
def test_bounded_executor_rejects_instead_of_queueing_without_limit():
executor = BoundedThreadPoolExecutor(
max_workers=1,
max_pending=1,
max_inflight_per_scope=1,
)
started = threading.Event()
release = threading.Event()
def block() -> str:
started.set()
release.wait(timeout=5)
return 'done'
first = executor.submit(block)
assert started.wait(timeout=1)
second = executor.submit(lambda: 'queued')
with pytest.raises(
BlockingWorkCapacityError,
match='capacity reached',
):
executor.submit(lambda: 'rejected')
assert executor.snapshot() == {
'max_workers': 1,
'max_pending': 1,
'max_inflight_per_scope': 1,
'inflight': 2,
'running': 1,
'pending': 1,
'active_scopes': 0,
'submitted_total': 2,
'completed_total': 0,
'rejected_total': 1,
'global_rejected_total': 1,
'scope_rejected_total': 0,
}
release.set()
assert first.result(timeout=1) == 'done'
assert second.result(timeout=1) == 'queued'
assert executor.snapshot()['inflight'] == 0
executor.shutdown()
def test_workspace_scope_cannot_monopolize_global_workers():
executor = BoundedThreadPoolExecutor(
max_workers=2,
max_pending=2,
max_inflight_per_scope=1,
)
release = threading.Event()
workspace_a_started = threading.Event()
workspace_b_started = threading.Event()
def block(started: threading.Event) -> str:
started.set()
release.wait(timeout=5)
return 'done'
try:
with blocking_work_scope('workspace-a'):
workspace_a = executor.submit(block, workspace_a_started)
assert workspace_a_started.wait(timeout=1)
with pytest.raises(
BlockingWorkCapacityError,
match='Workspace blocking executor capacity reached',
):
executor.submit(lambda: 'rejected')
with blocking_work_scope('workspace-b'):
workspace_b = executor.submit(block, workspace_b_started)
assert workspace_b_started.wait(timeout=1)
snapshot = executor.snapshot()
assert snapshot['inflight'] == 2
assert snapshot['active_scopes'] == 2
assert snapshot['scope_rejected_total'] == 1
assert snapshot['global_rejected_total'] == 0
finally:
release.set()
assert workspace_a.result(timeout=1) == 'done'
assert workspace_b.result(timeout=1) == 'done'
executor.shutdown()
def test_default_executor_bounds_asyncio_to_thread():
loop = asyncio.new_event_loop()
executor = configure_bounded_default_executor(
loop,
max_workers=2,
max_pending=3,
)
try:
assert loop.run_until_complete(asyncio.to_thread(lambda: 'bounded')) == 'bounded'
assert executor.snapshot()['completed_total'] == 1
assert (
configure_bounded_default_executor(
loop,
max_workers=2,
max_pending=3,
)
is executor
)
finally:
executor.shutdown()
loop.close()
def test_workspace_scope_is_enforced_for_asyncio_to_thread():
loop = asyncio.new_event_loop()
executor = configure_bounded_default_executor(
loop,
max_workers=2,
max_pending=2,
max_inflight_per_scope=1,
)
started = threading.Event()
release = threading.Event()
def block() -> str:
started.set()
release.wait(timeout=5)
return 'workspace-a'
async def exercise() -> None:
with blocking_work_scope('workspace-a'):
workspace_a = asyncio.create_task(asyncio.to_thread(block))
try:
while not started.is_set():
await asyncio.sleep(0)
with blocking_work_scope('workspace-a'):
with pytest.raises(
BlockingWorkCapacityError,
match='Workspace blocking executor capacity reached',
):
await asyncio.to_thread(lambda: 'rejected')
with blocking_work_scope('workspace-b'):
assert await asyncio.to_thread(lambda: 'workspace-b') == 'workspace-b'
finally:
release.set()
assert await workspace_a == 'workspace-a'
try:
loop.run_until_complete(exercise())
finally:
executor.shutdown()
loop.close()
def test_blocking_cleanup_waits_for_capacity_instead_of_leaking_work():
loop = asyncio.new_event_loop()
executor = configure_bounded_default_executor(
loop,
max_workers=1,
max_pending=0,
max_inflight_per_scope=1,
)
started = threading.Event()
release = threading.Event()
cleaned = threading.Event()
def block() -> None:
started.set()
release.wait(timeout=5)
async def exercise() -> None:
blocker = asyncio.create_task(asyncio.to_thread(block))
while not started.is_set():
await asyncio.sleep(0)
cleanup = asyncio.create_task(run_blocking_cleanup(cleaned.set))
await asyncio.sleep(0.03)
assert not cleanup.done()
release.set()
await blocker
await cleanup
try:
loop.run_until_complete(exercise())
assert cleaned.is_set()
assert executor.snapshot()['global_rejected_total'] >= 1
finally:
release.set()
executor.shutdown()
loop.close()
def test_blocking_atomic_waits_for_thread_before_propagating_cancellation():
loop = asyncio.new_event_loop()
executor = configure_bounded_default_executor(
loop,
max_workers=1,
max_pending=1,
max_inflight_per_scope=1,
)
started = threading.Event()
release = threading.Event()
completed = threading.Event()
def block() -> None:
started.set()
release.wait(timeout=5)
completed.set()
async def exercise() -> None:
operation = asyncio.create_task(run_blocking_atomic(block))
while not started.is_set():
await asyncio.sleep(0)
operation.cancel()
await asyncio.sleep(0)
assert not operation.done()
release.set()
with pytest.raises(asyncio.CancelledError):
await operation
try:
loop.run_until_complete(exercise())
assert completed.is_set()
finally:
release.set()
executor.shutdown()
loop.close()
@pytest.mark.parametrize(
('max_workers', 'max_pending'),
[
(0, 1),
(65, 1),
(1, -1),
(1, 4097),
(True, 1),
],
)
def test_bounded_executor_rejects_unsafe_limits(
max_workers,
max_pending,
):
with pytest.raises(ValueError):
BoundedThreadPoolExecutor(
max_workers=max_workers,
max_pending=max_pending,
)
@pytest.mark.parametrize(
('max_workers', 'max_inflight_per_scope'),
[(8, 0), (8, 4097), (8, True), (8, 5), (2, 2)],
)
def test_bounded_executor_rejects_unsafe_scope_limits(
max_workers,
max_inflight_per_scope,
):
with pytest.raises(ValueError):
BoundedThreadPoolExecutor(
max_workers=max_workers,
max_inflight_per_scope=max_inflight_per_scope,
)
@@ -0,0 +1,446 @@
from __future__ import annotations
import io
import json
from collections import deque
from pathlib import Path
import pytest
from scripts import cloud_runtime_soak as soak
def _write(path: Path, value: str) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(value, encoding='utf-8')
def _process_stat(pid: int, *, user_ticks: int, system_ticks: int) -> str:
fields = [
'S',
'0',
'0',
'0',
'0',
'0',
'0',
'0',
'0',
'0',
'0',
str(user_ticks),
str(system_ticks),
'0',
'0',
'0',
'0',
'0',
'0',
'0',
]
return f'{pid} (worker with spaces) ' + ' '.join(fields)
def _sample(timestamp: float, **metrics: float) -> soak.MetricSample:
return soak.MetricSample(
monotonic_seconds=timestamp,
wall_time=f'sample-{timestamp}',
metrics=metrics,
)
def _state(
kind: str,
samples: list[soak.MetricSample],
*,
baseline: dict[str, float] | None = None,
latest: dict[str, float] | None = None,
) -> soak.TargetState:
return soak.TargetState(
target=soak.Target(name='target', kind=kind, location='/target'),
samples=deque(samples),
baseline_metrics=baseline or dict(samples[0].metrics),
last_metrics=latest or dict(samples[-1].metrics),
attempted_samples=len(samples),
successful_samples=len(samples),
)
def _thresholds(**overrides) -> soak.Thresholds:
values = {
'max_memory_growth_bytes': 64 * soak.BYTES_PER_MIB,
'max_memory_slope_bytes_per_hour': 32 * soak.BYTES_PER_MIB,
'max_tail_cpu_cores': 0.5,
'max_throttled_period_ratio': 0.25,
'allow_rejections': False,
'max_transient_gauge_growth': 0,
'require_hard_limits': False,
'max_event_loop_lag_ms': 1000,
'max_event_loop_p95_lag_ms': 250,
'require_event_loop_metrics': True,
}
values.update(overrides)
return soak.Thresholds(**values)
@pytest.mark.parametrize(
('value', 'expected'),
[
('1', 1),
('1.5s', 1.5),
('2m', 120),
('3H', 10_800),
('1d', 86_400),
],
)
def test_parse_duration(value: str, expected: float) -> None:
assert soak.parse_duration(value) == expected
@pytest.mark.parametrize('value', ['', '0s', '-1s', 'wat'])
def test_parse_duration_rejects_invalid_values(value: str) -> None:
with pytest.raises(Exception):
soak.parse_duration(value)
def test_build_targets_rejects_secret_bearing_health_url() -> None:
with pytest.raises(ValueError, match='must not contain credentials'):
soak.build_targets(
endpoints=['core=https://user:secret@example.test/healthz'],
cgroups=[],
pids=[],
)
with pytest.raises(ValueError, match='query parameters'):
soak.build_targets(
endpoints=['core=https://example.test/healthz?token=secret'],
cgroups=[],
pids=[],
)
def test_read_cgroup_snapshot_reads_v2_pressure_and_limits(tmp_path: Path) -> None:
_write(tmp_path / 'memory.current', '1048576\n')
_write(tmp_path / 'memory.peak', '2097152\n')
_write(tmp_path / 'memory.swap.current', '4096\n')
_write(tmp_path / 'memory.max', '1073741824\n')
_write(tmp_path / 'memory.swap.max', 'max\n')
_write(tmp_path / 'pids.current', '7\n')
_write(tmp_path / 'pids.max', '128\n')
_write(
tmp_path / 'cpu.stat',
'usage_usec 1234\nnr_periods 20\nnr_throttled 2\nthrottled_usec 99\n',
)
_write(tmp_path / 'memory.events', 'high 1\nmax 2\noom 0\noom_kill 0\n')
_write(tmp_path / 'pids.events', 'max 3\n')
_write(tmp_path / 'cpu.max', '100000 100000\n')
metrics = soak.read_cgroup_snapshot(tmp_path)
assert metrics['memory.current_bytes'] == 1_048_576
assert metrics['memory.max_bytes'] == 1_073_741_824
assert 'memory.swap.max_bytes' not in metrics
assert metrics['cpu.usage_usec'] == 1234
assert metrics['cpu.nr_throttled'] == 2
assert metrics['memory.events.max'] == 2
assert metrics['pids.events.max'] == 3
assert metrics['cpu.quota_usec'] == 100_000
def test_read_process_snapshot_aggregates_descendants(tmp_path: Path) -> None:
for pid, rss_kib, threads, user_ticks, system_ticks in (
(100, 1000, 2, 100, 50),
(200, 500, 1, 20, 10),
):
process_root = tmp_path / str(pid)
_write(
process_root / 'status',
f'Name:\tworker\nVmRSS:\t{rss_kib} kB\nThreads:\t{threads}\n',
)
_write(
process_root / 'stat',
_process_stat(
pid,
user_ticks=user_ticks,
system_ticks=system_ticks,
),
)
(process_root / 'fd').mkdir()
(process_root / 'fd' / '0').touch()
(process_root / 'fd' / '1').touch()
(process_root / 'task' / str(pid)).mkdir(parents=True)
_write(tmp_path / '100' / 'task' / '100' / 'children', '200\n')
_write(tmp_path / '200' / 'task' / '200' / 'children', '\n')
metrics = soak.read_process_snapshot(100, proc_root=tmp_path, clock_ticks=100)
assert metrics == {
'rss_bytes': 1500 * 1024,
'cpu_seconds': 1.8,
'threads': 3,
'open_fds': 4,
'processes': 2,
}
class _FakeResponse:
def __init__(self, payload: dict) -> None:
self.status = 200
self.headers = {'Content-Type': 'application/json'}
self._body = json.dumps(payload).encode()
def __enter__(self):
return self
def __exit__(self, *_args):
return None
def getcode(self) -> int:
return self.status
def read(self, limit: int) -> bytes:
return self._body[:limit]
def test_read_endpoint_snapshot_flattens_resource_metrics() -> None:
def opener(_request, *, timeout: float):
assert timeout == 2
return _FakeResponse(
{
'code': 0,
'resources': {
'blocking_executor': {
'pending': 0,
'global_rejected_total': 2,
}
},
}
)
metrics = soak.read_endpoint_snapshot(
'http://langbot.test/healthz',
timeout_seconds=2,
opener=opener,
)
assert metrics['http.ok'] == 1
assert metrics['body.resources.blocking_executor.pending'] == 0
assert metrics['body.resources.blocking_executor.global_rejected_total'] == 2
def test_read_endpoint_snapshot_fails_closed_on_not_ready() -> None:
def opener(_request, *, timeout: float):
return _FakeResponse({'ready': False})
with pytest.raises(RuntimeError, match='not ready'):
soak.read_endpoint_snapshot(
'http://box.test/readyz',
timeout_seconds=2,
opener=opener,
)
def test_evaluate_gate_accepts_stable_process_tail() -> None:
state = _state(
'process',
[
_sample(0, rss_bytes=100 * soak.BYTES_PER_MIB, cpu_seconds=0),
_sample(1800, rss_bytes=101 * soak.BYTES_PER_MIB, cpu_seconds=10),
_sample(3600, rss_bytes=100 * soak.BYTES_PER_MIB, cpu_seconds=20),
],
)
result = soak.evaluate_gate(
[state],
analysis_start_seconds=0,
thresholds=_thresholds(),
)
assert result.passed
assert result.targets['process:target']['cpu.average_cores'] < 0.01
def test_evaluate_gate_detects_material_memory_leak_and_idle_cpu() -> None:
state = _state(
'process',
[
_sample(0, rss_bytes=100 * soak.BYTES_PER_MIB, cpu_seconds=0),
_sample(1800, rss_bytes=150 * soak.BYTES_PER_MIB, cpu_seconds=1800),
_sample(3600, rss_bytes=200 * soak.BYTES_PER_MIB, cpu_seconds=3600),
],
)
result = soak.evaluate_gate(
[state],
analysis_start_seconds=0,
thresholds=_thresholds(),
)
assert not result.passed
assert any('grew 100.00 MiB' in failure for failure in result.failures)
assert any('tail CPU averaged 1.000 cores' in failure for failure in result.failures)
def test_evaluate_gate_counts_oom_and_throttling_across_workload() -> None:
baseline = {
'memory.current_bytes': 100,
'memory.events.high': 0,
'memory.events.max': 0,
'memory.events.oom': 0,
'memory.events.oom_kill': 0,
'pids.events.max': 0,
'cpu.usage_usec': 0,
'cpu.nr_periods': 0,
'cpu.nr_throttled': 0,
}
latest = {
**baseline,
'memory.events.oom_kill': 1,
'pids.events.max': 2,
'cpu.usage_usec': 1_000_000,
'cpu.nr_periods': 100,
'cpu.nr_throttled': 30,
}
state = _state(
'cgroup',
[
_sample(100, **{**baseline, 'cpu.usage_usec': 500_000}),
_sample(200, **latest),
],
baseline=baseline,
latest=latest,
)
result = soak.evaluate_gate(
[state],
analysis_start_seconds=100,
thresholds=_thresholds(max_tail_cpu_cores=100),
)
assert any('memory.events.oom_kill by 1' in failure for failure in result.failures)
assert any('pids.events.max by 2' in failure for failure in result.failures)
assert any('throttled-period ratio 0.300' in failure for failure in result.failures)
def test_evaluate_gate_can_require_all_hard_cgroup_limits() -> None:
metrics = {
'memory.current_bytes': 100,
'memory.max_bytes': 1000,
'memory.events.high': 0,
'memory.events.max': 0,
'memory.events.oom': 0,
'memory.events.oom_kill': 0,
'pids.current': 1,
'pids.events.max': 0,
'cpu.usage_usec': 0,
'cpu.nr_periods': 0,
'cpu.nr_throttled': 0,
}
state = _state(
'cgroup',
[_sample(0, **metrics), _sample(60, **metrics)],
)
result = soak.evaluate_gate(
[state],
analysis_start_seconds=0,
thresholds=_thresholds(require_hard_limits=True),
)
assert any('missing hard cgroup limits: cpu, pids, swap' in failure for failure in result.failures)
def test_evaluate_gate_detects_executor_rejection_and_stuck_pending() -> None:
prefix = 'body.resources.blocking_executor'
state = _state(
'endpoint',
[
_sample(
0,
**{
f'{prefix}.pending': 1,
f'{prefix}.global_rejected_total': 0,
},
),
_sample(
60,
**{
f'{prefix}.pending': 2,
f'{prefix}.global_rejected_total': 1,
},
),
],
)
result = soak.evaluate_gate(
[state],
analysis_start_seconds=0,
thresholds=_thresholds(),
)
assert any('global_rejected_total by 1' in failure for failure in result.failures)
assert any('pending above zero' in failure for failure in result.failures)
def test_evaluate_gate_detects_event_loop_stall_and_sustained_lag() -> None:
prefix = 'body.resources.event_loop'
samples = [
_sample(
0,
**{
f'{prefix}.running': 1,
f'{prefix}.samples_total': 10,
f'{prefix}.recent_max_lag_ms': 20,
f'{prefix}.recent_p95_lag_ms': 10,
},
),
_sample(
60,
**{
f'{prefix}.running': 1,
f'{prefix}.samples_total': 70,
f'{prefix}.recent_max_lag_ms': 1500,
f'{prefix}.recent_p95_lag_ms': 300,
},
),
]
state = _state('endpoint', samples)
state.observed_max_metrics = {
metric: max(sample.metrics[metric] for sample in samples) for metric in samples[0].metrics
}
result = soak.evaluate_gate(
[state],
analysis_start_seconds=0,
thresholds=_thresholds(),
)
assert any('event-loop lag reached 1500.00 ms' in item for item in result.failures)
assert any('recent p95 reached 300.00 ms' in item for item in result.failures)
def test_evaluate_gate_requires_running_event_loop_monitor() -> None:
state = _state(
'endpoint',
[_sample(0, **{'http.ok': 1}), _sample(60, **{'http.ok': 1})],
)
result = soak.evaluate_gate(
[state],
analysis_start_seconds=0,
thresholds=_thresholds(),
)
assert any('did not expose event-loop health metrics' in item for item in result.failures)
def test_write_json_line_streams_one_record() -> None:
stream = io.StringIO()
soak._write_json_line(stream, {'z': 1, 'a': 2})
assert stream.getvalue() == '{"a":2,"z":1}\n'
def test_main_requires_a_target() -> None:
with pytest.raises(SystemExit) as exc_info:
soak.main(['--duration', '2s', '--sample-interval', '1s', '--startup-grace', '1s'])
assert exc_info.value.code == 2
@@ -0,0 +1,75 @@
from __future__ import annotations
import asyncio
import time
import pytest
from langbot.pkg.utils.event_loop_monitor import EventLoopLagMonitor
@pytest.mark.parametrize(
'kwargs',
[
{'sample_interval_seconds': 0},
{'sample_interval_seconds': float('inf')},
{'recent_sample_count': 1},
{'recent_sample_count': 3601},
],
)
def test_event_loop_monitor_rejects_unbounded_configuration(kwargs) -> None:
with pytest.raises(ValueError):
EventLoopLagMonitor(**kwargs)
def test_event_loop_monitor_snapshot_is_bounded_and_reports_p95() -> None:
monitor = EventLoopLagMonitor(recent_sample_count=4)
for lag_seconds in (0.001, 0.002, 0.003, 0.004, 0.100):
monitor._record_lag_seconds(lag_seconds)
snapshot = monitor.snapshot()
assert snapshot == {
'running': False,
'samples_total': 5,
'last_lag_ms': 100,
'recent_p95_lag_ms': 100,
'recent_max_lag_ms': 100,
'max_lag_ms': 100,
}
assert len(monitor._recent_lag_ms) == 4
async def test_event_loop_monitor_start_and_stop_are_idempotent() -> None:
monitor = EventLoopLagMonitor(
sample_interval_seconds=0.001,
recent_sample_count=4,
)
monitor.start()
task = monitor._task
monitor.start()
assert monitor._task is task
await asyncio.sleep(0.005)
assert monitor.snapshot()['samples_total'] > 0
assert monitor.snapshot()['running'] is True
await monitor.stop()
await monitor.stop()
assert monitor.snapshot()['running'] is False
assert task is not None and task.done()
async def test_event_loop_monitor_observes_real_scheduler_stall() -> None:
monitor = EventLoopLagMonitor(
sample_interval_seconds=0.005,
recent_sample_count=8,
)
monitor.start()
try:
await asyncio.sleep(0.01)
time.sleep(0.05)
await asyncio.sleep(0.01)
assert monitor.snapshot()['recent_max_lag_ms'] >= 35
finally:
await monitor.stop()
+88
View File
@@ -6,8 +6,13 @@ Tests session management, reuse, and cleanup.
from __future__ import annotations
import threading
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import aiohttp
import httpx
from aiohttp import web
from langbot.pkg.utils import httpclient
@@ -88,6 +93,89 @@ class TestCloseAll:
assert len(httpclient._sessions) == 0
class TestReadLimited:
async def test_rejects_oversized_content_length_before_reading(self):
content = SimpleNamespace(iter_chunked=None)
response = SimpleNamespace(headers={'Content-Length': '11'}, content=content)
with pytest.raises(httpclient.RemoteResponseTooLargeError):
await httpclient.read_limited(response, max_bytes=10)
async def test_rejects_chunked_body_that_crosses_limit(self):
class Content:
async def iter_chunked(self, _chunk_size):
yield b'12345'
yield b'678901'
response = SimpleNamespace(headers={}, content=Content())
with pytest.raises(httpclient.RemoteResponseTooLargeError):
await httpclient.read_limited(response, max_bytes=10)
async def test_returns_body_within_limit(self):
class Content:
async def iter_chunked(self, _chunk_size):
yield b'12345'
yield b'67890'
response = SimpleNamespace(headers={}, content=Content())
assert await httpclient.read_limited(response, max_bytes=10) == b'1234567890'
async def test_json_reader_uses_same_limit(self):
class Content:
async def iter_chunked(self, _chunk_size):
yield b'{"ok":true}'
response = SimpleNamespace(
headers={},
content=Content(),
)
assert await httpclient.read_json_limited(response, max_bytes=16) == {'ok': True}
async def test_response_json_parse_runs_off_event_loop(self):
event_loop_thread = threading.get_ident()
response = SimpleNamespace(json=lambda: threading.get_ident())
assert await httpclient.parse_json_response(response) != event_loop_thread
async def test_response_json_parse_supports_async_test_doubles(self):
response = SimpleNamespace(json=AsyncMock(return_value={'ok': True}))
assert await httpclient.parse_json_response(response) == {'ok': True}
async def test_response_text_runs_off_loop_and_caps_diagnostics(self):
event_loop_thread = threading.get_ident()
class Response:
@property
def text(self):
return f'{threading.get_ident()}:abcdef'
value = await httpclient.response_text(Response(), max_chars=4)
assert not value.startswith(str(event_loop_thread))
assert value.endswith('[truncated]')
async def test_httpx_hook_rejects_before_automatic_buffer_grows(self):
class Source(httpx.AsyncByteStream):
async def __aiter__(self):
yield b'123'
yield b'45'
async def aclose(self):
return None
transport = httpx.MockTransport(lambda _request: httpx.Response(200, stream=Source()))
async with httpx.AsyncClient(
transport=transport,
event_hooks=httpclient.httpx_response_limit_hooks(max_bytes=4),
) as client:
with pytest.raises(httpclient.RemoteResponseTooLargeError, match='4-byte'):
await client.get('https://example.invalid')
async def test_close_all_handles_already_closed(self):
"""close_all handles already closed sessions gracefully."""
session = httpclient.get_session()
+17
View File
@@ -10,11 +10,28 @@ import pytest
import base64
from langbot.pkg.utils.image import (
decode_base64_limited,
encode_base64,
get_qq_image_downloadable_url,
extract_b64_and_format,
)
@pytest.mark.asyncio
async def test_base64_media_helpers_round_trip_within_limit():
encoded = await encode_base64(b'1234')
assert await decode_base64_limited(encoded, max_bytes=4) == b'1234'
@pytest.mark.asyncio
async def test_base64_media_decode_rejects_oversized_payload():
encoded = base64.b64encode(b'12345').decode()
with pytest.raises(ValueError, match='exceeds'):
await decode_base64_limited(encoded, max_bytes=4)
class TestGetQQImageDownloadableUrl:
"""Tests for get_qq_image_downloadable_url function."""
+15 -1
View File
@@ -7,7 +7,13 @@ Tests log page management and pointer-based retrieval.
from __future__ import annotations
from langbot.pkg.utils.logcache import LogPage, LogCache, LOG_PAGE_SIZE, MAX_CACHED_PAGES
from langbot.pkg.utils.logcache import (
LogPage,
LogCache,
LOG_PAGE_SIZE,
MAX_CACHED_PAGES,
MAX_LOG_LINE_CHARS,
)
class TestLogPage:
@@ -208,3 +214,11 @@ class TestLogCache:
"""LOG_PAGE_SIZE is defined and reasonable."""
assert LOG_PAGE_SIZE > 0
assert LOG_PAGE_SIZE <= 1000 # Reasonable upper bound
def test_single_log_line_is_bounded(self):
cache = LogCache()
cache.add_log('x' * (MAX_LOG_LINE_CHARS * 2))
assert len(cache.log_pages[0].logs[0]) == MAX_LOG_LINE_CHARS
assert cache.log_pages[0].logs[0].endswith('[log truncated]')
+73
View File
@@ -0,0 +1,73 @@
from __future__ import annotations
import threading
import pytest
from langbot.pkg.utils import safe_regex
from langbot.pkg.utils.bounded_executor import blocking_work_scope, current_blocking_work_scope
@pytest.mark.asyncio
async def test_matches_any_runs_off_event_loop_and_preserves_workspace_scope(monkeypatch):
event_loop_thread = threading.get_ident()
observed: dict[str, object] = {}
original = safe_regex._matches_any_sync
def observe(*args, **kwargs):
observed['thread'] = threading.get_ident()
observed['scope'] = current_blocking_work_scope()
return original(*args, **kwargs)
monkeypatch.setattr(safe_regex, '_matches_any_sync', observe)
with blocking_work_scope('workspace-a'):
assert await safe_regex.matches_any(['^hello'], 'hello world') is True
assert observed['scope'] == 'workspace-a'
assert observed['thread'] != event_loop_thread
@pytest.mark.asyncio
async def test_matches_any_interrupts_catastrophic_backtracking():
with pytest.raises(safe_regex.SafeRegexTimeoutError):
await safe_regex.matches_any(
[r'(a+)+$'],
('a' * 100_000) + '!',
timeout_seconds=0.001,
)
@pytest.mark.asyncio
async def test_matches_any_rejects_pattern_and_input_amplification():
with pytest.raises(safe_regex.SafeRegexLimitError):
await safe_regex.matches_any(
['a'] * (safe_regex.MAX_PATTERN_COUNT + 1),
'a',
)
with pytest.raises(safe_regex.SafeRegexLimitError):
await safe_regex.matches_any(
['a'],
'a' * (safe_regex.MAX_INPUT_CHARS + 1),
)
@pytest.mark.asyncio
async def test_mask_patterns_bounds_replacement_growth_and_masks_matches():
found, masked = await safe_regex.mask_patterns(
[r'secret-\d+'],
'a secret-42 value',
mask='*',
mask_word='[hidden]',
)
assert found is True
assert masked == 'a [hidden] value'
with pytest.raises(safe_regex.SafeRegexLimitError):
await safe_regex.mask_patterns(
['a'],
'a' * safe_regex.MAX_INPUT_CHARS,
mask='0123456789',
mask_word='',
)
+21 -1
View File
@@ -6,7 +6,9 @@ based on configuration, without actually creating real VDB instances.
from __future__ import annotations
from unittest.mock import MagicMock
from unittest.mock import AsyncMock, MagicMock
import pytest
from tests.utils.import_isolation import isolated_sys_modules
@@ -383,3 +385,21 @@ class TestVectorDBManagerProxies:
result = mgr.get_supported_search_types()
assert result == ['vector', 'full_text']
@pytest.mark.asyncio
async def test_shutdown_closes_backend_and_releases_reference(self):
mock_app = MagicMock()
mocks = {'langbot.pkg.core.app': MagicMock()}
with isolated_sys_modules(mocks):
from langbot.pkg.vector.mgr import VectorDBManager
mgr = VectorDBManager(mock_app)
backend = MagicMock()
backend.close = AsyncMock()
mgr.vector_db = backend
await mgr.shutdown()
backend.close.assert_awaited_once_with()
assert mgr.vector_db is None
@@ -31,6 +31,7 @@ def make_backend():
# _ensure_client serializes creation through this lock; set it here since
# __init__ (which normally creates it) is bypassed.
backend._client_lock = asyncio.Lock()
backend._runtime_cache_limit = 1024
return backend
+18 -1
View File
@@ -5,7 +5,12 @@ from __future__ import annotations
from unittest.mock import AsyncMock
import pytest
from langbot.pkg.vector.vdb import SearchType, VectorDatabase
from langbot.pkg.vector.vdb import (
SearchType,
VectorDatabase,
remember_bounded_mapping,
remember_bounded_set,
)
class TestSearchType:
@@ -29,6 +34,18 @@ class TestSearchType:
assert SearchType('hybrid') == SearchType.HYBRID
def test_runtime_cache_helpers_bound_mapping_and_set():
mapping = {}
values = set()
for index in range(100):
remember_bounded_mapping(mapping, str(index), object(), 8)
remember_bounded_set(values, str(index), 8)
assert len(mapping) == 8
assert len(values) == 8
class TestVectorDatabaseAbstractMethods:
"""Tests for VectorDatabase abstract methods."""
@@ -2,6 +2,7 @@ from __future__ import annotations
import uuid
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
import sqlalchemy
@@ -18,6 +19,7 @@ from langbot.pkg.entity.persistence.workspace import (
)
from langbot.pkg.workspace import (
WorkspaceExecutionUnavailableError,
WorkspaceExecutionBinding,
WorkspaceGenerationMismatchError,
WorkspaceInvariantError,
WorkspaceLimitExceededError,
@@ -235,3 +237,28 @@ async def test_cloud_policy_never_creates_or_guesses_a_workspace(workspace_test_
await cloud_service.create_local_workspace(name='Forbidden', slug='forbidden')
assert cloud_service.policy.multi_workspace_enabled is True
async def test_startup_binding_snapshot_avoids_repeated_discovery():
service = WorkspaceService(
SimpleNamespace(persistence_mgr=SimpleNamespace()),
policy=CloudWorkspacePolicy(),
instance_uuid='instance-service-test',
)
binding = WorkspaceExecutionBinding(
instance_uuid='instance-service-test',
workspace_uuid='workspace-a',
placement_generation=1,
write_fenced=False,
state='active',
)
service._discover_active_execution_bindings = AsyncMock(return_value=[binding])
assert await service.prime_startup_execution_bindings() == [binding]
assert await service.list_active_execution_bindings() == [binding]
assert await service.list_active_execution_bindings() == [binding]
service._discover_active_execution_bindings.assert_awaited_once()
service.release_startup_execution_bindings()
assert await service.list_active_execution_bindings() == [binding]
assert service._discover_active_execution_bindings.await_count == 2