xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-08-09 04:40:57 +00:00
Code Issues Packages Projects Releases Wiki Activity

chore(merge): sync master into dev/4.11.x

Browse Source
This commit is contained in:
huanghuoguoguo
2026-07-31 19:29:38 +08:00
parent 9c32ea4063 5ec2371879
commit 55f42a4ebc
502 changed files with 77975 additions and 12729 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/unit_tests/core/test_app_resource_maintenance.py
+113
View File
@@ -0,0 +1,113 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
import pytest
from langbot.pkg.core.app import Application
pytestmark = pytest.mark.asyncio
class _TaskManager:
def __init__(self, stop: asyncio.Event) -> None:
self.stop = stop
self.tasks: list[asyncio.Task] = []
def create_task(self, coro, *, name='', **_kwargs):
task = asyncio.create_task(coro, name=name)
self.tasks.append(task)
return SimpleNamespace(task=task)
async def wait_all(self) -> None:
await self.stop.wait()
for task in self.tasks:
task.cancel()
await asyncio.gather(*self.tasks, return_exceptions=True)
async def _wait_forever() -> None:
await asyncio.Event().wait()
async def test_resource_maintenance_waits_and_shares_workspace_discovery() -> None:
stop = asyncio.Event()
completed = asyncio.Event()
discovery_calls = 0
job_calls: list[str] = []
async def list_bindings():
nonlocal discovery_calls
discovery_calls += 1
return [
SimpleNamespace(
instance_uuid='instance',
workspace_uuid='workspace',
placement_generation=1,
)
]
async def cleanup_monitoring(_context, _retention_days, *, batch_size):
assert batch_size == 10
job_calls.append('monitoring')
return {}
async def cleanup_storage(_context):
job_calls.append('storage')
completed.set()
return {}
application = Application()
application.event_loop = asyncio.get_running_loop()
application.event_loop_monitor = SimpleNamespace(start=lambda: None)
application.task_mgr = _TaskManager(stop)
application.plugin_connector = SimpleNamespace(initialize_plugins=lambda: asyncio.sleep(0))
application.platform_mgr = SimpleNamespace(run=_wait_forever)
application.ctrl = SimpleNamespace(run=_wait_forever)
application.http_ctrl = SimpleNamespace(run=_wait_forever)
application.telemetry = None
application.workspace_collaboration_service = None
application.workspace_service = SimpleNamespace(list_active_execution_bindings=list_bindings)
application.monitoring_service = SimpleNamespace(cleanup_expired_records=cleanup_monitoring)
application.maintenance_service = SimpleNamespace(cleanup_expired_files=cleanup_storage)
application.instance_config = SimpleNamespace(
data={
'monitoring': {
'auto_cleanup': {
'enabled': True,
'retention_days': 30,
'delete_batch_size': 10,
'check_interval_hours': 0.00002,
}
},
'storage': {
'cleanup': {
'enabled': True,
'check_interval_hours': 0.00002,
}
},
}
)
application.logger = SimpleNamespace(
info=lambda *_args, **_kwargs: None,
warning=lambda *_args, **_kwargs: None,
error=lambda *_args, **_kwargs: None,
debug=lambda *_args, **_kwargs: None,
)
async def no_web_info() -> None:
return None
application.print_web_access_info = no_web_info
run_task = asyncio.create_task(application.run())
try:
await asyncio.sleep(0.01)
assert discovery_calls == 0
await asyncio.wait_for(completed.wait(), timeout=1)
assert discovery_calls == 1
assert job_calls == ['monitoring', 'storage']
finally:
stop.set()
await asyncio.wait_for(run_task, timeout=1)
tests/unit_tests/core/test_app_shutdown.py
+142
View File
@@ -0,0 +1,142 @@
from __future__ import annotations
import asyncio
from types import SimpleNamespace
from unittest.mock import AsyncMock
import pytest
from langbot.pkg.core.app import Application
@pytest.mark.asyncio
async def test_shutdown_closes_mcp_session_manager_once() -> None:
app = Application()
stop_session_manager = AsyncMock()
app.platform_mgr = SimpleNamespace(shutdown=AsyncMock())
app.tool_mgr = SimpleNamespace(shutdown=AsyncMock())
app.model_mgr = SimpleNamespace(shutdown=AsyncMock())
app.box_service = SimpleNamespace(shutdown=AsyncMock())
app.plugin_connector = SimpleNamespace(aclose=AsyncMock())
app.telemetry = SimpleNamespace(shutdown=AsyncMock())
app.vector_db_mgr = SimpleNamespace(shutdown=AsyncMock())
app.storage_mgr = SimpleNamespace(shutdown=AsyncMock())
manifest_provider = SimpleNamespace(aclose=AsyncMock())
app.deployment = SimpleNamespace(manifest_provider=manifest_provider)
persistence_engine = SimpleNamespace(dispose=AsyncMock())
app.persistence_mgr = SimpleNamespace(db=SimpleNamespace(engine=persistence_engine))
app.http_ctrl = SimpleNamespace(mcp_mount=SimpleNamespace(stop_session_manager=stop_session_manager))
await app.shutdown()
await app.shutdown()
stop_session_manager.assert_awaited_once()
app.platform_mgr.shutdown.assert_awaited_once()
app.tool_mgr.shutdown.assert_awaited_once()
app.model_mgr.shutdown.assert_awaited_once()
app.box_service.shutdown.assert_awaited_once()
app.plugin_connector.aclose.assert_awaited_once()
app.telemetry.shutdown.assert_awaited_once()
app.vector_db_mgr.shutdown.assert_awaited_once()
app.storage_mgr.shutdown.assert_awaited_once()
manifest_provider.aclose.assert_awaited_once()
persistence_engine.dispose.assert_awaited_once()
@pytest.mark.asyncio
async def test_dispose_tracks_only_one_shutdown_task() -> None:
app = Application()
app.event_loop = asyncio.get_running_loop()
app.dispose()
shutdown_task = app._shutdown_task
app.dispose()
assert shutdown_task is not None
assert app._shutdown_task is shutdown_task
await shutdown_task
app.dispose()
assert app._shutdown_task is shutdown_task
@pytest.mark.asyncio
async def test_runtime_resource_stats_are_aggregate_and_constant_time() -> None:
app = Application()
app.event_loop = asyncio.get_running_loop()
app.blocking_executor = SimpleNamespace(
snapshot=lambda: {
'inflight': 3,
'running': 2,
'pending': 1,
'rejected_total': 4,
}
)
app.task_mgr = SimpleNamespace(get_stats=lambda: {'total': 5, 'completed': 2})
app.query_pool = SimpleNamespace(
queries=[object()],
cached_queries={},
active_query_count_by_workspace={'workspace-a': 1},
)
app.model_mgr = SimpleNamespace(
provider_dict={'provider': object()},
llm_model_dict={},
embedding_model_dict={},
rerank_model_dict={},
)
app.platform_mgr = SimpleNamespace(_bots_by_key={})
app.pipeline_mgr = SimpleNamespace(_pipelines_by_key={})
app.rag_mgr = SimpleNamespace(knowledge_bases={})
app.plugin_connector = SimpleNamespace(_known_desired_states={'installation': object()})
app.persistence_mgr = SimpleNamespace(
get_resource_stats=lambda: {
'configured_capacity': 20,
'checked_out': 3,
}
)
app.directory_projection_service = SimpleNamespace(
resource_snapshot=lambda: {
'active_workspaces': 10,
'max_active_workspaces': 1000,
}
)
app.tool_mgr = SimpleNamespace(
mcp_tool_loader=SimpleNamespace(
_sessions={},
_hosted_mcp_tasks=[],
_host_dispatch_tasks=set(),
)
)
app.telemetry = SimpleNamespace(send_tasks=[])
stats = app.get_runtime_resource_stats()
assert stats['asyncio_tasks'] >= 1
assert stats['event_loop'] == {
'running': False,
'samples_total': 0,
'last_lag_ms': 0,
'recent_p95_lag_ms': 0,
'recent_max_lag_ms': 0,
'max_lag_ms': 0,
}
assert stats['blocking_executor']['rejected_total'] == 4
assert stats['application_tasks'] == {
'total': 5,
'completed': 2,
}
assert stats['database_pool'] == {
'configured_capacity': 20,
'checked_out': 3,
}
assert stats['directory'] == {
'active_workspaces': 10,
'max_active_workspaces': 1000,
}
assert stats['query_pool'] == {
'queued': 1,
'cached': 0,
'active_workspaces': 1,
}
assert stats['models']['providers'] == 1
assert stats['runtimes']['plugin_installations'] == 1
tests/unit_tests/core/test_boot.py
+58 -20
View File
@@ -2,12 +2,37 @@ from __future__ import annotations
import signal
from types import SimpleNamespace
from unittest.mock import AsyncMock, Mock
import pytest
from langbot.pkg.core import boot
@pytest.mark.asyncio
async def test_make_app_shuts_down_partially_built_application(monkeypatch):
app_inst = SimpleNamespace(
event_loop=None,
shutdown=AsyncMock(),
initialize=AsyncMock(),
)
class FailingStage:
async def run(self, ap):
assert ap is app_inst
raise RuntimeError('startup failed')
monkeypatch.setattr(boot.app, 'Application', lambda: app_inst)
monkeypatch.setattr(boot, 'stage_order', ['FailingStage'])
monkeypatch.setitem(boot.stage.preregistered_stages, 'FailingStage', FailingStage)
with pytest.raises(RuntimeError, match='startup failed'):
await boot.make_app(SimpleNamespace())
app_inst.shutdown.assert_awaited_once()
app_inst.initialize.assert_not_awaited()
@pytest.mark.asyncio
async def test_main_signal_handler_handles_sigint_before_app_created(monkeypatch):
captured_handler = {}
@@ -18,47 +43,60 @@ async def test_main_signal_handler_handles_sigint_before_app_created(monkeypatch
async def fake_make_app(loop):
captured_handler[signal.SIGINT](signal.SIGINT, None)
def fake_exit(code):
raise SystemExit(code)
monkeypatch.setattr(signal, 'signal', fake_signal)
monkeypatch.setattr(boot, 'make_app', fake_make_app)
monkeypatch.setattr(boot.os, '_exit', fake_exit)
with pytest.raises(SystemExit) as exc_info:
await boot.main(SimpleNamespace())
assert exc_info.value.code == 0
await boot.main(SimpleNamespace())
@pytest.mark.asyncio
async def test_main_signal_handler_disposes_created_app(monkeypatch):
captured_handler = {}
app_inst = SimpleNamespace(disposed=False)
app_inst = SimpleNamespace(shutdown_called=False)
def fake_signal(sig, handler):
captured_handler[sig] = handler
def dispose():
app_inst.disposed = True
async def shutdown():
app_inst.shutdown_called = True
async def run():
captured_handler[signal.SIGINT](signal.SIGINT, None)
async def fake_make_app(loop):
app_inst.dispose = dispose
app_inst.shutdown = shutdown
app_inst.run = run
return app_inst
def fake_exit(code):
raise SystemExit(code)
monkeypatch.setattr(signal, 'signal', fake_signal)
monkeypatch.setattr(boot, 'make_app', fake_make_app)
monkeypatch.setattr(boot.os, '_exit', fake_exit)
with pytest.raises(SystemExit) as exc_info:
await boot.main(SimpleNamespace())
await boot.main(SimpleNamespace())
assert exc_info.value.code == 0
assert app_inst.disposed is True
assert app_inst.shutdown_called is True
@pytest.mark.asyncio
async def test_main_reports_app_run_failure_and_still_shuts_down(monkeypatch):
app_inst = SimpleNamespace(shutdown_called=False)
async def shutdown():
app_inst.shutdown_called = True
async def run():
raise RuntimeError('run failed')
async def fake_make_app(loop):
app_inst.shutdown = shutdown
app_inst.run = run
return app_inst
print_exc = Mock()
monkeypatch.setattr(signal, 'signal', lambda *_args: None)
monkeypatch.setattr(boot, 'make_app', fake_make_app)
monkeypatch.setattr(boot.traceback, 'print_exc', print_exc)
await boot.main(SimpleNamespace())
print_exc.assert_called_once()
assert app_inst.shutdown_called is True
tests/unit_tests/core/test_load_config.py
+134
View File
@@ -35,6 +35,22 @@ class TestApplyEnvOverridesToConfig:
assert result['system']['name'] == 'custom_name'
def test_override_log_never_prints_secret_value(self, capsys):
"""Environment-backed credentials must not be copied into logs."""
load_config = get_load_config_module()
secret = 'database-password-that-must-not-leak'
cfg = {'database': {'postgresql': {'password': ''}}}
env = {'DATABASE__POSTGRESQL__PASSWORD': secret}
with patch.dict(os.environ, env, clear=True):
result = load_config._apply_env_overrides_to_config(cfg)
captured = capsys.readouterr().out
assert result['database']['postgresql']['password'] == secret
assert 'DATABASE__POSTGRESQL__PASSWORD' in captured
assert secret not in captured
def test_override_int_value(self):
"""Test overriding an int value with proper conversion."""
load_config = get_load_config_module()
@@ -48,6 +64,20 @@ class TestApplyEnvOverridesToConfig:
assert result['concurrency']['pipeline'] == 10
assert isinstance(result['concurrency']['pipeline'], int)
def test_cloud_directory_limit_override_keeps_integer_type_on_upgraded_config(self):
load_config = get_load_config_module()
cfg = load_config._complete_runtime_policy_defaults({})
with patch.dict(
os.environ,
{'CLOUD__DIRECTORY__MAX_ACTIVE_WORKSPACES': '250'},
clear=True,
):
result = load_config._apply_env_overrides_to_config(cfg)
assert result['cloud']['directory']['max_active_workspaces'] == 250
assert isinstance(result['cloud']['directory']['max_active_workspaces'], int)
def test_override_int_value_invalid_conversion(self):
"""Test that invalid int conversion keeps string value."""
load_config = get_load_config_module()
@@ -122,6 +152,19 @@ class TestApplyEnvOverridesToConfig:
assert result['system']['disabled_adapters'] == ['aiocqhttp', 'dingtalk', 'telegram']
def test_override_integer_list_preserves_item_type(self):
"""Comma-separated overrides inherit the existing list item type."""
load_config = get_load_config_module()
cfg = {'vdb': {'pgvector': {'allowed_dimensions': [384, 512]}}}
env = {'VDB__PGVECTOR__ALLOWED_DIMENSIONS': '384,512,768'}
with patch.dict(os.environ, env, clear=True):
result = load_config._apply_env_overrides_to_config(cfg)
assert result['vdb']['pgvector']['allowed_dimensions'] == [384, 512, 768]
assert all(isinstance(item, int) for item in result['vdb']['pgvector']['allowed_dimensions'])
def test_override_list_value_empty_items(self):
"""Test that empty items in comma-separated list are filtered."""
load_config = get_load_config_module()
@@ -196,6 +239,19 @@ class TestApplyEnvOverridesToConfig:
assert result['system']['name'] == 'default'
def test_skip_env_vars_with_empty_path_segments(self, capsys):
"""Platform variables such as __CF_USER_TEXT_ENCODING are not config."""
load_config = get_load_config_module()
cfg = {'system': {'name': 'default'}}
env = {'__CF_USER_TEXT_ENCODING': '0x1F5:0x0:0x64'}
with patch.dict(os.environ, env, clear=True):
result = load_config._apply_env_overrides_to_config(cfg)
assert result == cfg
assert capsys.readouterr().out == ''
def test_nested_config_path(self):
"""Test overriding deeply nested config."""
load_config = get_load_config_module()
@@ -259,6 +315,84 @@ class TestApplyEnvOverridesToConfig:
assert result['system']['enable'] is False
assert result['concurrency']['pipeline'] == 10
def test_plugin_worker_and_stdio_policy_native_env_overrides(self):
load_config = get_load_config_module()
cfg = {
'plugin': {
'worker': {
'max_cpus': 1.0,
'max_memory_mb': 512,
'max_pids': 128,
'max_open_files': 256,
'max_file_size_mb': 512,
'max_concurrent_restarts': 1,
'restart_failure_threshold': 8,
'restart_failure_window_seconds': 30.0,
'restart_circuit_open_seconds': 60.0,
}
},
'mcp': {'stdio': {'enabled': True}},
}
env = {
'PLUGIN__WORKER__MAX_CPUS': '2.5',
'PLUGIN__WORKER__MAX_MEMORY_MB': '1024',
'PLUGIN__WORKER__MAX_PIDS': '64',
'PLUGIN__WORKER__MAX_OPEN_FILES': '128',
'PLUGIN__WORKER__MAX_FILE_SIZE_MB': '256',
'PLUGIN__WORKER__MAX_CONCURRENT_RESTARTS': '2',
'PLUGIN__WORKER__RESTART_FAILURE_THRESHOLD': '12',
'PLUGIN__WORKER__RESTART_FAILURE_WINDOW_SECONDS': '45.5',
'PLUGIN__WORKER__RESTART_CIRCUIT_OPEN_SECONDS': '90.0',
'MCP__STDIO__ENABLED': 'false',
}
with patch.dict(os.environ, env, clear=True):
result = load_config._apply_env_overrides_to_config(cfg)
assert result['plugin']['worker'] == {
'max_cpus': 2.5,
'max_memory_mb': 1024,
'max_pids': 64,
'max_open_files': 128,
'max_file_size_mb': 256,
'max_concurrent_restarts': 2,
'restart_failure_threshold': 12,
'restart_failure_window_seconds': 45.5,
'restart_circuit_open_seconds': 90.0,
}
assert result['mcp']['stdio']['enabled'] is False
def test_runtime_policy_defaults_preserve_env_types_for_upgraded_config(self):
load_config = get_load_config_module()
cfg = {'plugin': {'enable': True}}
completed = load_config._complete_runtime_policy_defaults(cfg)
with patch.dict(
os.environ,
{
'PLUGIN__WORKER__MAX_MEMORY_MB': '768',
'MCP__STDIO__ENABLED': 'false',
'SYSTEM__BLOCKING_EXECUTOR__MAX_WORKERS': '12',
'SYSTEM__BLOCKING_EXECUTOR__MAX_PENDING': '256',
'SYSTEM__BLOCKING_EXECUTOR__MAX_INFLIGHT_PER_SCOPE': '3',
},
clear=True,
):
result = load_config._apply_env_overrides_to_config(completed)
assert result['system']['blocking_executor'] == {
'max_workers': 12,
'max_pending': 256,
'max_inflight_per_scope': 3,
}
assert isinstance(
result['system']['blocking_executor']['max_workers'],
int,
)
assert result['plugin']['worker']['max_memory_mb'] == 768
assert isinstance(result['plugin']['worker']['max_memory_mb'], int)
assert result['mcp']['stdio']['enabled'] is False
def test_webhook_prefix_override(self):
"""Test overriding webhook_prefix via environment variable."""
load_config = get_load_config_module()
tests/unit_tests/core/test_taskmgr.py
+121
View File
@@ -12,6 +12,8 @@ from __future__ import annotations
import pytest
import asyncio
import contextvars
import inspect
import sys
from unittest.mock import Mock, MagicMock
from contextlib import contextmanager
@@ -264,6 +266,28 @@ class TestTaskWrapper:
wrapper.cancel()
@pytest.mark.asyncio
async def test_workspace_task_sets_blocking_work_scope(self):
"""Detached tasks recover tenant fairness from durable ownership."""
_, TaskWrapper, _ = get_taskmgr_classes()
from langbot.pkg.utils.bounded_executor import (
current_blocking_work_scope,
)
mock_app = create_mock_app()
async def read_scope():
return current_blocking_work_scope()
wrapper = TaskWrapper(
mock_app,
read_scope(),
workspace_uuid='workspace-a',
)
assert await wrapper.task == 'workspace-a'
assert current_blocking_work_scope() is None
@pytest.mark.asyncio
async def test_to_dict_serialization(self):
"""Test TaskWrapper.to_dict serialization."""
@@ -360,6 +384,53 @@ class TestAsyncTaskManager:
wrapper.cancel()
@pytest.mark.asyncio
async def test_create_task_does_not_inherit_request_context(self):
"""Long-lived tasks must receive identity through explicit arguments."""
_, _, AsyncTaskManager = get_taskmgr_classes()
mock_app = create_mock_app()
manager = AsyncTaskManager(mock_app)
request_value = contextvars.ContextVar('request_value', default=None)
token = request_value.set('request-scoped-transaction')
observed = []
async def detached_task(captured_workspace: str) -> None:
observed.append((request_value.get(), captured_workspace))
try:
wrapper = manager.create_task(detached_task('workspace-a'))
await wrapper.task
finally:
request_value.reset(token)
assert observed == [(None, 'workspace-a')]
@pytest.mark.asyncio
async def test_create_task_waits_for_registered_transaction_commit(self):
_, _, AsyncTaskManager = get_taskmgr_classes()
mock_app = create_mock_app()
gate = asyncio.get_running_loop().create_future()
class PersistenceManagerStub:
def create_after_commit_gate(self):
return gate
mock_app.persistence_mgr = PersistenceManagerStub()
manager = AsyncTaskManager(mock_app)
observed = []
async def background_work() -> None:
observed.append('started')
wrapper = manager.create_task(background_work())
await asyncio.sleep(0)
assert observed == []
gate.set_result(None)
await wrapper.task
assert observed == ['started']
@pytest.mark.asyncio
async def test_get_stats_counts_correctly(self):
"""Test get_stats returns correct counts."""
@@ -482,6 +553,56 @@ class TestAsyncTaskManager:
wrapper.cancel()
@pytest.mark.asyncio
async def test_create_user_task_enforces_workspace_active_limit_and_closes_rejected_coroutine(self):
"""A noisy Workspace cannot accumulate unbounded background work."""
_, _, AsyncTaskManager = get_taskmgr_classes()
mock_app = create_mock_app()
mock_app.instance_config.data['system']['task_retention'].update(
{
'max_active_user_tasks': 10,
'max_active_user_tasks_per_workspace': 1,
}
)
manager = AsyncTaskManager(mock_app)
async def long_coro():
await asyncio.sleep(10)
first = manager.create_user_task(long_coro(), workspace_uuid='workspace-a')
rejected = long_coro()
with pytest.raises(RuntimeError, match='Workspace has too many active user operations'):
manager.create_user_task(rejected, workspace_uuid='workspace-a')
assert inspect.getcoroutinestate(rejected) == inspect.CORO_CLOSED
other_workspace = manager.create_user_task(long_coro(), workspace_uuid='workspace-b')
first.cancel()
other_workspace.cancel()
@pytest.mark.asyncio
async def test_create_user_task_enforces_instance_active_limit(self):
"""The shared process retains a hard cap even across Workspaces."""
_, _, AsyncTaskManager = get_taskmgr_classes()
mock_app = create_mock_app()
mock_app.instance_config.data['system']['task_retention'].update(
{
'max_active_user_tasks': 1,
'max_active_user_tasks_per_workspace': 10,
}
)
manager = AsyncTaskManager(mock_app)
async def long_coro():
await asyncio.sleep(10)
first = manager.create_user_task(long_coro(), workspace_uuid='workspace-a')
rejected = long_coro()
with pytest.raises(RuntimeError, match='instance has too many active user operations'):
manager.create_user_task(rejected, workspace_uuid='workspace-b')
assert inspect.getcoroutinestate(rejected) == inspect.CORO_CLOSED
first.cancel()
@pytest.mark.asyncio
async def test_get_task_by_id(self):
"""Test get_task_by_id returns correct task."""