mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-08 20:30:59 +00:00
feat(cloud): harden multi-tenant runtime resources
This commit is contained in:
Junyan Qin
@@ -0,0 +1,278 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from langbot.pkg.utils.bounded_executor import (
|
||||
BlockingWorkCapacityError,
|
||||
BoundedThreadPoolExecutor,
|
||||
blocking_work_scope,
|
||||
configure_bounded_default_executor,
|
||||
run_blocking_atomic,
|
||||
run_blocking_cleanup,
|
||||
)
|
||||
|
||||
|
||||
def test_bounded_executor_rejects_instead_of_queueing_without_limit():
|
||||
executor = BoundedThreadPoolExecutor(
|
||||
max_workers=1,
|
||||
max_pending=1,
|
||||
max_inflight_per_scope=1,
|
||||
)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def block() -> str:
|
||||
started.set()
|
||||
release.wait(timeout=5)
|
||||
return 'done'
|
||||
|
||||
first = executor.submit(block)
|
||||
assert started.wait(timeout=1)
|
||||
second = executor.submit(lambda: 'queued')
|
||||
|
||||
with pytest.raises(
|
||||
BlockingWorkCapacityError,
|
||||
match='capacity reached',
|
||||
):
|
||||
executor.submit(lambda: 'rejected')
|
||||
|
||||
assert executor.snapshot() == {
|
||||
'max_workers': 1,
|
||||
'max_pending': 1,
|
||||
'max_inflight_per_scope': 1,
|
||||
'inflight': 2,
|
||||
'running': 1,
|
||||
'pending': 1,
|
||||
'active_scopes': 0,
|
||||
'submitted_total': 2,
|
||||
'completed_total': 0,
|
||||
'rejected_total': 1,
|
||||
'global_rejected_total': 1,
|
||||
'scope_rejected_total': 0,
|
||||
}
|
||||
|
||||
release.set()
|
||||
assert first.result(timeout=1) == 'done'
|
||||
assert second.result(timeout=1) == 'queued'
|
||||
assert executor.snapshot()['inflight'] == 0
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
def test_workspace_scope_cannot_monopolize_global_workers():
|
||||
executor = BoundedThreadPoolExecutor(
|
||||
max_workers=2,
|
||||
max_pending=2,
|
||||
max_inflight_per_scope=1,
|
||||
)
|
||||
release = threading.Event()
|
||||
workspace_a_started = threading.Event()
|
||||
workspace_b_started = threading.Event()
|
||||
|
||||
def block(started: threading.Event) -> str:
|
||||
started.set()
|
||||
release.wait(timeout=5)
|
||||
return 'done'
|
||||
|
||||
try:
|
||||
with blocking_work_scope('workspace-a'):
|
||||
workspace_a = executor.submit(block, workspace_a_started)
|
||||
assert workspace_a_started.wait(timeout=1)
|
||||
with pytest.raises(
|
||||
BlockingWorkCapacityError,
|
||||
match='Workspace blocking executor capacity reached',
|
||||
):
|
||||
executor.submit(lambda: 'rejected')
|
||||
|
||||
with blocking_work_scope('workspace-b'):
|
||||
workspace_b = executor.submit(block, workspace_b_started)
|
||||
assert workspace_b_started.wait(timeout=1)
|
||||
|
||||
snapshot = executor.snapshot()
|
||||
assert snapshot['inflight'] == 2
|
||||
assert snapshot['active_scopes'] == 2
|
||||
assert snapshot['scope_rejected_total'] == 1
|
||||
assert snapshot['global_rejected_total'] == 0
|
||||
finally:
|
||||
release.set()
|
||||
assert workspace_a.result(timeout=1) == 'done'
|
||||
assert workspace_b.result(timeout=1) == 'done'
|
||||
executor.shutdown()
|
||||
|
||||
|
||||
def test_default_executor_bounds_asyncio_to_thread():
|
||||
loop = asyncio.new_event_loop()
|
||||
executor = configure_bounded_default_executor(
|
||||
loop,
|
||||
max_workers=2,
|
||||
max_pending=3,
|
||||
)
|
||||
try:
|
||||
assert loop.run_until_complete(asyncio.to_thread(lambda: 'bounded')) == 'bounded'
|
||||
assert executor.snapshot()['completed_total'] == 1
|
||||
assert (
|
||||
configure_bounded_default_executor(
|
||||
loop,
|
||||
max_workers=2,
|
||||
max_pending=3,
|
||||
)
|
||||
is executor
|
||||
)
|
||||
finally:
|
||||
executor.shutdown()
|
||||
loop.close()
|
||||
|
||||
|
||||
def test_workspace_scope_is_enforced_for_asyncio_to_thread():
|
||||
loop = asyncio.new_event_loop()
|
||||
executor = configure_bounded_default_executor(
|
||||
loop,
|
||||
max_workers=2,
|
||||
max_pending=2,
|
||||
max_inflight_per_scope=1,
|
||||
)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
|
||||
def block() -> str:
|
||||
started.set()
|
||||
release.wait(timeout=5)
|
||||
return 'workspace-a'
|
||||
|
||||
async def exercise() -> None:
|
||||
with blocking_work_scope('workspace-a'):
|
||||
workspace_a = asyncio.create_task(asyncio.to_thread(block))
|
||||
try:
|
||||
while not started.is_set():
|
||||
await asyncio.sleep(0)
|
||||
|
||||
with blocking_work_scope('workspace-a'):
|
||||
with pytest.raises(
|
||||
BlockingWorkCapacityError,
|
||||
match='Workspace blocking executor capacity reached',
|
||||
):
|
||||
await asyncio.to_thread(lambda: 'rejected')
|
||||
|
||||
with blocking_work_scope('workspace-b'):
|
||||
assert await asyncio.to_thread(lambda: 'workspace-b') == 'workspace-b'
|
||||
finally:
|
||||
release.set()
|
||||
assert await workspace_a == 'workspace-a'
|
||||
|
||||
try:
|
||||
loop.run_until_complete(exercise())
|
||||
finally:
|
||||
executor.shutdown()
|
||||
loop.close()
|
||||
|
||||
|
||||
def test_blocking_cleanup_waits_for_capacity_instead_of_leaking_work():
|
||||
loop = asyncio.new_event_loop()
|
||||
executor = configure_bounded_default_executor(
|
||||
loop,
|
||||
max_workers=1,
|
||||
max_pending=0,
|
||||
max_inflight_per_scope=1,
|
||||
)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
cleaned = threading.Event()
|
||||
|
||||
def block() -> None:
|
||||
started.set()
|
||||
release.wait(timeout=5)
|
||||
|
||||
async def exercise() -> None:
|
||||
blocker = asyncio.create_task(asyncio.to_thread(block))
|
||||
while not started.is_set():
|
||||
await asyncio.sleep(0)
|
||||
cleanup = asyncio.create_task(run_blocking_cleanup(cleaned.set))
|
||||
await asyncio.sleep(0.03)
|
||||
assert not cleanup.done()
|
||||
release.set()
|
||||
await blocker
|
||||
await cleanup
|
||||
|
||||
try:
|
||||
loop.run_until_complete(exercise())
|
||||
assert cleaned.is_set()
|
||||
assert executor.snapshot()['global_rejected_total'] >= 1
|
||||
finally:
|
||||
release.set()
|
||||
executor.shutdown()
|
||||
loop.close()
|
||||
|
||||
|
||||
def test_blocking_atomic_waits_for_thread_before_propagating_cancellation():
|
||||
loop = asyncio.new_event_loop()
|
||||
executor = configure_bounded_default_executor(
|
||||
loop,
|
||||
max_workers=1,
|
||||
max_pending=1,
|
||||
max_inflight_per_scope=1,
|
||||
)
|
||||
started = threading.Event()
|
||||
release = threading.Event()
|
||||
completed = threading.Event()
|
||||
|
||||
def block() -> None:
|
||||
started.set()
|
||||
release.wait(timeout=5)
|
||||
completed.set()
|
||||
|
||||
async def exercise() -> None:
|
||||
operation = asyncio.create_task(run_blocking_atomic(block))
|
||||
while not started.is_set():
|
||||
await asyncio.sleep(0)
|
||||
operation.cancel()
|
||||
await asyncio.sleep(0)
|
||||
assert not operation.done()
|
||||
release.set()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await operation
|
||||
|
||||
try:
|
||||
loop.run_until_complete(exercise())
|
||||
assert completed.is_set()
|
||||
finally:
|
||||
release.set()
|
||||
executor.shutdown()
|
||||
loop.close()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('max_workers', 'max_pending'),
|
||||
[
|
||||
(0, 1),
|
||||
(65, 1),
|
||||
(1, -1),
|
||||
(1, 4097),
|
||||
(True, 1),
|
||||
],
|
||||
)
|
||||
def test_bounded_executor_rejects_unsafe_limits(
|
||||
max_workers,
|
||||
max_pending,
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
BoundedThreadPoolExecutor(
|
||||
max_workers=max_workers,
|
||||
max_pending=max_pending,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('max_workers', 'max_inflight_per_scope'),
|
||||
[(8, 0), (8, 4097), (8, True), (8, 5), (2, 2)],
|
||||
)
|
||||
def test_bounded_executor_rejects_unsafe_scope_limits(
|
||||
max_workers,
|
||||
max_inflight_per_scope,
|
||||
):
|
||||
with pytest.raises(ValueError):
|
||||
BoundedThreadPoolExecutor(
|
||||
max_workers=max_workers,
|
||||
max_inflight_per_scope=max_inflight_per_scope,
|
||||
)
|
||||
@@ -0,0 +1,446 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import json
|
||||
from collections import deque
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from scripts import cloud_runtime_soak as soak
|
||||
|
||||
|
||||
def _write(path: Path, value: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(value, encoding='utf-8')
|
||||
|
||||
|
||||
def _process_stat(pid: int, *, user_ticks: int, system_ticks: int) -> str:
|
||||
fields = [
|
||||
'S',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
str(user_ticks),
|
||||
str(system_ticks),
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
'0',
|
||||
]
|
||||
return f'{pid} (worker with spaces) ' + ' '.join(fields)
|
||||
|
||||
|
||||
def _sample(timestamp: float, **metrics: float) -> soak.MetricSample:
|
||||
return soak.MetricSample(
|
||||
monotonic_seconds=timestamp,
|
||||
wall_time=f'sample-{timestamp}',
|
||||
metrics=metrics,
|
||||
)
|
||||
|
||||
|
||||
def _state(
|
||||
kind: str,
|
||||
samples: list[soak.MetricSample],
|
||||
*,
|
||||
baseline: dict[str, float] | None = None,
|
||||
latest: dict[str, float] | None = None,
|
||||
) -> soak.TargetState:
|
||||
return soak.TargetState(
|
||||
target=soak.Target(name='target', kind=kind, location='/target'),
|
||||
samples=deque(samples),
|
||||
baseline_metrics=baseline or dict(samples[0].metrics),
|
||||
last_metrics=latest or dict(samples[-1].metrics),
|
||||
attempted_samples=len(samples),
|
||||
successful_samples=len(samples),
|
||||
)
|
||||
|
||||
|
||||
def _thresholds(**overrides) -> soak.Thresholds:
|
||||
values = {
|
||||
'max_memory_growth_bytes': 64 * soak.BYTES_PER_MIB,
|
||||
'max_memory_slope_bytes_per_hour': 32 * soak.BYTES_PER_MIB,
|
||||
'max_tail_cpu_cores': 0.5,
|
||||
'max_throttled_period_ratio': 0.25,
|
||||
'allow_rejections': False,
|
||||
'max_transient_gauge_growth': 0,
|
||||
'require_hard_limits': False,
|
||||
'max_event_loop_lag_ms': 1000,
|
||||
'max_event_loop_p95_lag_ms': 250,
|
||||
'require_event_loop_metrics': True,
|
||||
}
|
||||
values.update(overrides)
|
||||
return soak.Thresholds(**values)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
('value', 'expected'),
|
||||
[
|
||||
('1', 1),
|
||||
('1.5s', 1.5),
|
||||
('2m', 120),
|
||||
('3H', 10_800),
|
||||
('1d', 86_400),
|
||||
],
|
||||
)
|
||||
def test_parse_duration(value: str, expected: float) -> None:
|
||||
assert soak.parse_duration(value) == expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize('value', ['', '0s', '-1s', 'wat'])
|
||||
def test_parse_duration_rejects_invalid_values(value: str) -> None:
|
||||
with pytest.raises(Exception):
|
||||
soak.parse_duration(value)
|
||||
|
||||
|
||||
def test_build_targets_rejects_secret_bearing_health_url() -> None:
|
||||
with pytest.raises(ValueError, match='must not contain credentials'):
|
||||
soak.build_targets(
|
||||
endpoints=['core=https://user:secret@example.test/healthz'],
|
||||
cgroups=[],
|
||||
pids=[],
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match='query parameters'):
|
||||
soak.build_targets(
|
||||
endpoints=['core=https://example.test/healthz?token=secret'],
|
||||
cgroups=[],
|
||||
pids=[],
|
||||
)
|
||||
|
||||
|
||||
def test_read_cgroup_snapshot_reads_v2_pressure_and_limits(tmp_path: Path) -> None:
|
||||
_write(tmp_path / 'memory.current', '1048576\n')
|
||||
_write(tmp_path / 'memory.peak', '2097152\n')
|
||||
_write(tmp_path / 'memory.swap.current', '4096\n')
|
||||
_write(tmp_path / 'memory.max', '1073741824\n')
|
||||
_write(tmp_path / 'memory.swap.max', 'max\n')
|
||||
_write(tmp_path / 'pids.current', '7\n')
|
||||
_write(tmp_path / 'pids.max', '128\n')
|
||||
_write(
|
||||
tmp_path / 'cpu.stat',
|
||||
'usage_usec 1234\nnr_periods 20\nnr_throttled 2\nthrottled_usec 99\n',
|
||||
)
|
||||
_write(tmp_path / 'memory.events', 'high 1\nmax 2\noom 0\noom_kill 0\n')
|
||||
_write(tmp_path / 'pids.events', 'max 3\n')
|
||||
_write(tmp_path / 'cpu.max', '100000 100000\n')
|
||||
|
||||
metrics = soak.read_cgroup_snapshot(tmp_path)
|
||||
|
||||
assert metrics['memory.current_bytes'] == 1_048_576
|
||||
assert metrics['memory.max_bytes'] == 1_073_741_824
|
||||
assert 'memory.swap.max_bytes' not in metrics
|
||||
assert metrics['cpu.usage_usec'] == 1234
|
||||
assert metrics['cpu.nr_throttled'] == 2
|
||||
assert metrics['memory.events.max'] == 2
|
||||
assert metrics['pids.events.max'] == 3
|
||||
assert metrics['cpu.quota_usec'] == 100_000
|
||||
|
||||
|
||||
def test_read_process_snapshot_aggregates_descendants(tmp_path: Path) -> None:
|
||||
for pid, rss_kib, threads, user_ticks, system_ticks in (
|
||||
(100, 1000, 2, 100, 50),
|
||||
(200, 500, 1, 20, 10),
|
||||
):
|
||||
process_root = tmp_path / str(pid)
|
||||
_write(
|
||||
process_root / 'status',
|
||||
f'Name:\tworker\nVmRSS:\t{rss_kib} kB\nThreads:\t{threads}\n',
|
||||
)
|
||||
_write(
|
||||
process_root / 'stat',
|
||||
_process_stat(
|
||||
pid,
|
||||
user_ticks=user_ticks,
|
||||
system_ticks=system_ticks,
|
||||
),
|
||||
)
|
||||
(process_root / 'fd').mkdir()
|
||||
(process_root / 'fd' / '0').touch()
|
||||
(process_root / 'fd' / '1').touch()
|
||||
(process_root / 'task' / str(pid)).mkdir(parents=True)
|
||||
_write(tmp_path / '100' / 'task' / '100' / 'children', '200\n')
|
||||
_write(tmp_path / '200' / 'task' / '200' / 'children', '\n')
|
||||
|
||||
metrics = soak.read_process_snapshot(100, proc_root=tmp_path, clock_ticks=100)
|
||||
|
||||
assert metrics == {
|
||||
'rss_bytes': 1500 * 1024,
|
||||
'cpu_seconds': 1.8,
|
||||
'threads': 3,
|
||||
'open_fds': 4,
|
||||
'processes': 2,
|
||||
}
|
||||
|
||||
|
||||
class _FakeResponse:
|
||||
def __init__(self, payload: dict) -> None:
|
||||
self.status = 200
|
||||
self.headers = {'Content-Type': 'application/json'}
|
||||
self._body = json.dumps(payload).encode()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *_args):
|
||||
return None
|
||||
|
||||
def getcode(self) -> int:
|
||||
return self.status
|
||||
|
||||
def read(self, limit: int) -> bytes:
|
||||
return self._body[:limit]
|
||||
|
||||
|
||||
def test_read_endpoint_snapshot_flattens_resource_metrics() -> None:
|
||||
def opener(_request, *, timeout: float):
|
||||
assert timeout == 2
|
||||
return _FakeResponse(
|
||||
{
|
||||
'code': 0,
|
||||
'resources': {
|
||||
'blocking_executor': {
|
||||
'pending': 0,
|
||||
'global_rejected_total': 2,
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
metrics = soak.read_endpoint_snapshot(
|
||||
'http://langbot.test/healthz',
|
||||
timeout_seconds=2,
|
||||
opener=opener,
|
||||
)
|
||||
|
||||
assert metrics['http.ok'] == 1
|
||||
assert metrics['body.resources.blocking_executor.pending'] == 0
|
||||
assert metrics['body.resources.blocking_executor.global_rejected_total'] == 2
|
||||
|
||||
|
||||
def test_read_endpoint_snapshot_fails_closed_on_not_ready() -> None:
|
||||
def opener(_request, *, timeout: float):
|
||||
return _FakeResponse({'ready': False})
|
||||
|
||||
with pytest.raises(RuntimeError, match='not ready'):
|
||||
soak.read_endpoint_snapshot(
|
||||
'http://box.test/readyz',
|
||||
timeout_seconds=2,
|
||||
opener=opener,
|
||||
)
|
||||
|
||||
|
||||
def test_evaluate_gate_accepts_stable_process_tail() -> None:
|
||||
state = _state(
|
||||
'process',
|
||||
[
|
||||
_sample(0, rss_bytes=100 * soak.BYTES_PER_MIB, cpu_seconds=0),
|
||||
_sample(1800, rss_bytes=101 * soak.BYTES_PER_MIB, cpu_seconds=10),
|
||||
_sample(3600, rss_bytes=100 * soak.BYTES_PER_MIB, cpu_seconds=20),
|
||||
],
|
||||
)
|
||||
|
||||
result = soak.evaluate_gate(
|
||||
[state],
|
||||
analysis_start_seconds=0,
|
||||
thresholds=_thresholds(),
|
||||
)
|
||||
|
||||
assert result.passed
|
||||
assert result.targets['process:target']['cpu.average_cores'] < 0.01
|
||||
|
||||
|
||||
def test_evaluate_gate_detects_material_memory_leak_and_idle_cpu() -> None:
|
||||
state = _state(
|
||||
'process',
|
||||
[
|
||||
_sample(0, rss_bytes=100 * soak.BYTES_PER_MIB, cpu_seconds=0),
|
||||
_sample(1800, rss_bytes=150 * soak.BYTES_PER_MIB, cpu_seconds=1800),
|
||||
_sample(3600, rss_bytes=200 * soak.BYTES_PER_MIB, cpu_seconds=3600),
|
||||
],
|
||||
)
|
||||
|
||||
result = soak.evaluate_gate(
|
||||
[state],
|
||||
analysis_start_seconds=0,
|
||||
thresholds=_thresholds(),
|
||||
)
|
||||
|
||||
assert not result.passed
|
||||
assert any('grew 100.00 MiB' in failure for failure in result.failures)
|
||||
assert any('tail CPU averaged 1.000 cores' in failure for failure in result.failures)
|
||||
|
||||
|
||||
def test_evaluate_gate_counts_oom_and_throttling_across_workload() -> None:
|
||||
baseline = {
|
||||
'memory.current_bytes': 100,
|
||||
'memory.events.high': 0,
|
||||
'memory.events.max': 0,
|
||||
'memory.events.oom': 0,
|
||||
'memory.events.oom_kill': 0,
|
||||
'pids.events.max': 0,
|
||||
'cpu.usage_usec': 0,
|
||||
'cpu.nr_periods': 0,
|
||||
'cpu.nr_throttled': 0,
|
||||
}
|
||||
latest = {
|
||||
**baseline,
|
||||
'memory.events.oom_kill': 1,
|
||||
'pids.events.max': 2,
|
||||
'cpu.usage_usec': 1_000_000,
|
||||
'cpu.nr_periods': 100,
|
||||
'cpu.nr_throttled': 30,
|
||||
}
|
||||
state = _state(
|
||||
'cgroup',
|
||||
[
|
||||
_sample(100, **{**baseline, 'cpu.usage_usec': 500_000}),
|
||||
_sample(200, **latest),
|
||||
],
|
||||
baseline=baseline,
|
||||
latest=latest,
|
||||
)
|
||||
|
||||
result = soak.evaluate_gate(
|
||||
[state],
|
||||
analysis_start_seconds=100,
|
||||
thresholds=_thresholds(max_tail_cpu_cores=100),
|
||||
)
|
||||
|
||||
assert any('memory.events.oom_kill by 1' in failure for failure in result.failures)
|
||||
assert any('pids.events.max by 2' in failure for failure in result.failures)
|
||||
assert any('throttled-period ratio 0.300' in failure for failure in result.failures)
|
||||
|
||||
|
||||
def test_evaluate_gate_can_require_all_hard_cgroup_limits() -> None:
|
||||
metrics = {
|
||||
'memory.current_bytes': 100,
|
||||
'memory.max_bytes': 1000,
|
||||
'memory.events.high': 0,
|
||||
'memory.events.max': 0,
|
||||
'memory.events.oom': 0,
|
||||
'memory.events.oom_kill': 0,
|
||||
'pids.current': 1,
|
||||
'pids.events.max': 0,
|
||||
'cpu.usage_usec': 0,
|
||||
'cpu.nr_periods': 0,
|
||||
'cpu.nr_throttled': 0,
|
||||
}
|
||||
state = _state(
|
||||
'cgroup',
|
||||
[_sample(0, **metrics), _sample(60, **metrics)],
|
||||
)
|
||||
|
||||
result = soak.evaluate_gate(
|
||||
[state],
|
||||
analysis_start_seconds=0,
|
||||
thresholds=_thresholds(require_hard_limits=True),
|
||||
)
|
||||
|
||||
assert any('missing hard cgroup limits: cpu, pids, swap' in failure for failure in result.failures)
|
||||
|
||||
|
||||
def test_evaluate_gate_detects_executor_rejection_and_stuck_pending() -> None:
|
||||
prefix = 'body.resources.blocking_executor'
|
||||
state = _state(
|
||||
'endpoint',
|
||||
[
|
||||
_sample(
|
||||
0,
|
||||
**{
|
||||
f'{prefix}.pending': 1,
|
||||
f'{prefix}.global_rejected_total': 0,
|
||||
},
|
||||
),
|
||||
_sample(
|
||||
60,
|
||||
**{
|
||||
f'{prefix}.pending': 2,
|
||||
f'{prefix}.global_rejected_total': 1,
|
||||
},
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
result = soak.evaluate_gate(
|
||||
[state],
|
||||
analysis_start_seconds=0,
|
||||
thresholds=_thresholds(),
|
||||
)
|
||||
|
||||
assert any('global_rejected_total by 1' in failure for failure in result.failures)
|
||||
assert any('pending above zero' in failure for failure in result.failures)
|
||||
|
||||
|
||||
def test_evaluate_gate_detects_event_loop_stall_and_sustained_lag() -> None:
|
||||
prefix = 'body.resources.event_loop'
|
||||
samples = [
|
||||
_sample(
|
||||
0,
|
||||
**{
|
||||
f'{prefix}.running': 1,
|
||||
f'{prefix}.samples_total': 10,
|
||||
f'{prefix}.recent_max_lag_ms': 20,
|
||||
f'{prefix}.recent_p95_lag_ms': 10,
|
||||
},
|
||||
),
|
||||
_sample(
|
||||
60,
|
||||
**{
|
||||
f'{prefix}.running': 1,
|
||||
f'{prefix}.samples_total': 70,
|
||||
f'{prefix}.recent_max_lag_ms': 1500,
|
||||
f'{prefix}.recent_p95_lag_ms': 300,
|
||||
},
|
||||
),
|
||||
]
|
||||
state = _state('endpoint', samples)
|
||||
state.observed_max_metrics = {
|
||||
metric: max(sample.metrics[metric] for sample in samples) for metric in samples[0].metrics
|
||||
}
|
||||
|
||||
result = soak.evaluate_gate(
|
||||
[state],
|
||||
analysis_start_seconds=0,
|
||||
thresholds=_thresholds(),
|
||||
)
|
||||
|
||||
assert any('event-loop lag reached 1500.00 ms' in item for item in result.failures)
|
||||
assert any('recent p95 reached 300.00 ms' in item for item in result.failures)
|
||||
|
||||
|
||||
def test_evaluate_gate_requires_running_event_loop_monitor() -> None:
|
||||
state = _state(
|
||||
'endpoint',
|
||||
[_sample(0, **{'http.ok': 1}), _sample(60, **{'http.ok': 1})],
|
||||
)
|
||||
|
||||
result = soak.evaluate_gate(
|
||||
[state],
|
||||
analysis_start_seconds=0,
|
||||
thresholds=_thresholds(),
|
||||
)
|
||||
|
||||
assert any('did not expose event-loop health metrics' in item for item in result.failures)
|
||||
|
||||
|
||||
def test_write_json_line_streams_one_record() -> None:
|
||||
stream = io.StringIO()
|
||||
soak._write_json_line(stream, {'z': 1, 'a': 2})
|
||||
assert stream.getvalue() == '{"a":2,"z":1}\n'
|
||||
|
||||
|
||||
def test_main_requires_a_target() -> None:
|
||||
with pytest.raises(SystemExit) as exc_info:
|
||||
soak.main(['--duration', '2s', '--sample-interval', '1s', '--startup-grace', '1s'])
|
||||
assert exc_info.value.code == 2
|
||||
@@ -0,0 +1,75 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from langbot.pkg.utils.event_loop_monitor import EventLoopLagMonitor
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'kwargs',
|
||||
[
|
||||
{'sample_interval_seconds': 0},
|
||||
{'sample_interval_seconds': float('inf')},
|
||||
{'recent_sample_count': 1},
|
||||
{'recent_sample_count': 3601},
|
||||
],
|
||||
)
|
||||
def test_event_loop_monitor_rejects_unbounded_configuration(kwargs) -> None:
|
||||
with pytest.raises(ValueError):
|
||||
EventLoopLagMonitor(**kwargs)
|
||||
|
||||
|
||||
def test_event_loop_monitor_snapshot_is_bounded_and_reports_p95() -> None:
|
||||
monitor = EventLoopLagMonitor(recent_sample_count=4)
|
||||
for lag_seconds in (0.001, 0.002, 0.003, 0.004, 0.100):
|
||||
monitor._record_lag_seconds(lag_seconds)
|
||||
|
||||
snapshot = monitor.snapshot()
|
||||
|
||||
assert snapshot == {
|
||||
'running': False,
|
||||
'samples_total': 5,
|
||||
'last_lag_ms': 100,
|
||||
'recent_p95_lag_ms': 100,
|
||||
'recent_max_lag_ms': 100,
|
||||
'max_lag_ms': 100,
|
||||
}
|
||||
assert len(monitor._recent_lag_ms) == 4
|
||||
|
||||
|
||||
async def test_event_loop_monitor_start_and_stop_are_idempotent() -> None:
|
||||
monitor = EventLoopLagMonitor(
|
||||
sample_interval_seconds=0.001,
|
||||
recent_sample_count=4,
|
||||
)
|
||||
|
||||
monitor.start()
|
||||
task = monitor._task
|
||||
monitor.start()
|
||||
assert monitor._task is task
|
||||
await asyncio.sleep(0.005)
|
||||
assert monitor.snapshot()['samples_total'] > 0
|
||||
assert monitor.snapshot()['running'] is True
|
||||
|
||||
await monitor.stop()
|
||||
await monitor.stop()
|
||||
assert monitor.snapshot()['running'] is False
|
||||
assert task is not None and task.done()
|
||||
|
||||
|
||||
async def test_event_loop_monitor_observes_real_scheduler_stall() -> None:
|
||||
monitor = EventLoopLagMonitor(
|
||||
sample_interval_seconds=0.005,
|
||||
recent_sample_count=8,
|
||||
)
|
||||
monitor.start()
|
||||
try:
|
||||
await asyncio.sleep(0.01)
|
||||
time.sleep(0.05)
|
||||
await asyncio.sleep(0.01)
|
||||
assert monitor.snapshot()['recent_max_lag_ms'] >= 35
|
||||
finally:
|
||||
await monitor.stop()
|
||||
@@ -6,8 +6,13 @@ Tests session management, reuse, and cleanup.
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
import aiohttp
|
||||
import httpx
|
||||
from aiohttp import web
|
||||
|
||||
from langbot.pkg.utils import httpclient
|
||||
@@ -88,6 +93,89 @@ class TestCloseAll:
|
||||
|
||||
assert len(httpclient._sessions) == 0
|
||||
|
||||
|
||||
class TestReadLimited:
|
||||
async def test_rejects_oversized_content_length_before_reading(self):
|
||||
content = SimpleNamespace(iter_chunked=None)
|
||||
response = SimpleNamespace(headers={'Content-Length': '11'}, content=content)
|
||||
|
||||
with pytest.raises(httpclient.RemoteResponseTooLargeError):
|
||||
await httpclient.read_limited(response, max_bytes=10)
|
||||
|
||||
async def test_rejects_chunked_body_that_crosses_limit(self):
|
||||
class Content:
|
||||
async def iter_chunked(self, _chunk_size):
|
||||
yield b'12345'
|
||||
yield b'678901'
|
||||
|
||||
response = SimpleNamespace(headers={}, content=Content())
|
||||
|
||||
with pytest.raises(httpclient.RemoteResponseTooLargeError):
|
||||
await httpclient.read_limited(response, max_bytes=10)
|
||||
|
||||
async def test_returns_body_within_limit(self):
|
||||
class Content:
|
||||
async def iter_chunked(self, _chunk_size):
|
||||
yield b'12345'
|
||||
yield b'67890'
|
||||
|
||||
response = SimpleNamespace(headers={}, content=Content())
|
||||
|
||||
assert await httpclient.read_limited(response, max_bytes=10) == b'1234567890'
|
||||
|
||||
async def test_json_reader_uses_same_limit(self):
|
||||
class Content:
|
||||
async def iter_chunked(self, _chunk_size):
|
||||
yield b'{"ok":true}'
|
||||
|
||||
response = SimpleNamespace(
|
||||
headers={},
|
||||
content=Content(),
|
||||
)
|
||||
|
||||
assert await httpclient.read_json_limited(response, max_bytes=16) == {'ok': True}
|
||||
|
||||
async def test_response_json_parse_runs_off_event_loop(self):
|
||||
event_loop_thread = threading.get_ident()
|
||||
response = SimpleNamespace(json=lambda: threading.get_ident())
|
||||
|
||||
assert await httpclient.parse_json_response(response) != event_loop_thread
|
||||
|
||||
async def test_response_json_parse_supports_async_test_doubles(self):
|
||||
response = SimpleNamespace(json=AsyncMock(return_value={'ok': True}))
|
||||
|
||||
assert await httpclient.parse_json_response(response) == {'ok': True}
|
||||
|
||||
async def test_response_text_runs_off_loop_and_caps_diagnostics(self):
|
||||
event_loop_thread = threading.get_ident()
|
||||
|
||||
class Response:
|
||||
@property
|
||||
def text(self):
|
||||
return f'{threading.get_ident()}:abcdef'
|
||||
|
||||
value = await httpclient.response_text(Response(), max_chars=4)
|
||||
|
||||
assert not value.startswith(str(event_loop_thread))
|
||||
assert value.endswith('[truncated]')
|
||||
|
||||
async def test_httpx_hook_rejects_before_automatic_buffer_grows(self):
|
||||
class Source(httpx.AsyncByteStream):
|
||||
async def __aiter__(self):
|
||||
yield b'123'
|
||||
yield b'45'
|
||||
|
||||
async def aclose(self):
|
||||
return None
|
||||
|
||||
transport = httpx.MockTransport(lambda _request: httpx.Response(200, stream=Source()))
|
||||
async with httpx.AsyncClient(
|
||||
transport=transport,
|
||||
event_hooks=httpclient.httpx_response_limit_hooks(max_bytes=4),
|
||||
) as client:
|
||||
with pytest.raises(httpclient.RemoteResponseTooLargeError, match='4-byte'):
|
||||
await client.get('https://example.invalid')
|
||||
|
||||
async def test_close_all_handles_already_closed(self):
|
||||
"""close_all handles already closed sessions gracefully."""
|
||||
session = httpclient.get_session()
|
||||
|
||||
@@ -10,11 +10,28 @@ import pytest
|
||||
import base64
|
||||
|
||||
from langbot.pkg.utils.image import (
|
||||
decode_base64_limited,
|
||||
encode_base64,
|
||||
get_qq_image_downloadable_url,
|
||||
extract_b64_and_format,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base64_media_helpers_round_trip_within_limit():
|
||||
encoded = await encode_base64(b'1234')
|
||||
|
||||
assert await decode_base64_limited(encoded, max_bytes=4) == b'1234'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_base64_media_decode_rejects_oversized_payload():
|
||||
encoded = base64.b64encode(b'12345').decode()
|
||||
|
||||
with pytest.raises(ValueError, match='exceeds'):
|
||||
await decode_base64_limited(encoded, max_bytes=4)
|
||||
|
||||
|
||||
class TestGetQQImageDownloadableUrl:
|
||||
"""Tests for get_qq_image_downloadable_url function."""
|
||||
|
||||
|
||||
@@ -7,7 +7,13 @@ Tests log page management and pointer-based retrieval.
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
from langbot.pkg.utils.logcache import LogPage, LogCache, LOG_PAGE_SIZE, MAX_CACHED_PAGES
|
||||
from langbot.pkg.utils.logcache import (
|
||||
LogPage,
|
||||
LogCache,
|
||||
LOG_PAGE_SIZE,
|
||||
MAX_CACHED_PAGES,
|
||||
MAX_LOG_LINE_CHARS,
|
||||
)
|
||||
|
||||
|
||||
class TestLogPage:
|
||||
@@ -208,3 +214,11 @@ class TestLogCache:
|
||||
"""LOG_PAGE_SIZE is defined and reasonable."""
|
||||
assert LOG_PAGE_SIZE > 0
|
||||
assert LOG_PAGE_SIZE <= 1000 # Reasonable upper bound
|
||||
|
||||
def test_single_log_line_is_bounded(self):
|
||||
cache = LogCache()
|
||||
|
||||
cache.add_log('x' * (MAX_LOG_LINE_CHARS * 2))
|
||||
|
||||
assert len(cache.log_pages[0].logs[0]) == MAX_LOG_LINE_CHARS
|
||||
assert cache.log_pages[0].logs[0].endswith('[log truncated]')
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
|
||||
import pytest
|
||||
|
||||
from langbot.pkg.utils import safe_regex
|
||||
from langbot.pkg.utils.bounded_executor import blocking_work_scope, current_blocking_work_scope
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_matches_any_runs_off_event_loop_and_preserves_workspace_scope(monkeypatch):
|
||||
event_loop_thread = threading.get_ident()
|
||||
observed: dict[str, object] = {}
|
||||
original = safe_regex._matches_any_sync
|
||||
|
||||
def observe(*args, **kwargs):
|
||||
observed['thread'] = threading.get_ident()
|
||||
observed['scope'] = current_blocking_work_scope()
|
||||
return original(*args, **kwargs)
|
||||
|
||||
monkeypatch.setattr(safe_regex, '_matches_any_sync', observe)
|
||||
|
||||
with blocking_work_scope('workspace-a'):
|
||||
assert await safe_regex.matches_any(['^hello'], 'hello world') is True
|
||||
|
||||
assert observed['scope'] == 'workspace-a'
|
||||
assert observed['thread'] != event_loop_thread
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_matches_any_interrupts_catastrophic_backtracking():
|
||||
with pytest.raises(safe_regex.SafeRegexTimeoutError):
|
||||
await safe_regex.matches_any(
|
||||
[r'(a+)+$'],
|
||||
('a' * 100_000) + '!',
|
||||
timeout_seconds=0.001,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_matches_any_rejects_pattern_and_input_amplification():
|
||||
with pytest.raises(safe_regex.SafeRegexLimitError):
|
||||
await safe_regex.matches_any(
|
||||
['a'] * (safe_regex.MAX_PATTERN_COUNT + 1),
|
||||
'a',
|
||||
)
|
||||
|
||||
with pytest.raises(safe_regex.SafeRegexLimitError):
|
||||
await safe_regex.matches_any(
|
||||
['a'],
|
||||
'a' * (safe_regex.MAX_INPUT_CHARS + 1),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_mask_patterns_bounds_replacement_growth_and_masks_matches():
|
||||
found, masked = await safe_regex.mask_patterns(
|
||||
[r'secret-\d+'],
|
||||
'a secret-42 value',
|
||||
mask='*',
|
||||
mask_word='[hidden]',
|
||||
)
|
||||
assert found is True
|
||||
assert masked == 'a [hidden] value'
|
||||
|
||||
with pytest.raises(safe_regex.SafeRegexLimitError):
|
||||
await safe_regex.mask_patterns(
|
||||
['a'],
|
||||
'a' * safe_regex.MAX_INPUT_CHARS,
|
||||
mask='0123456789',
|
||||
mask_word='',
|
||||
)
|
||||
Reference in New Issue
Block a user