xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-08-08 20:30:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix(cloud): bound runtime restart storms

Browse Source
This commit is contained in:
Junyan Qin
2026-07-29 12:14:08 +08:00
parent ae85ac2b16
commit aa342d9347
16 changed files with 266 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/cloud_runtime_soak.py
+23 -5
View File
@@ -72,7 +72,13 @@ REJECTION_SUFFIXES = (
'.blocking_executor.global_rejected_total',
'.blocking_executor.scope_rejected_total',
)
PENDING_SUFFIX = '.blocking_executor.pending'
RUNTIME_FAILURE_COUNTER_SUFFIXES = ('.restart_coordinator.circuit_open_total',)
DRAIN_GAUGE_SUFFIXES = (
'.blocking_executor.pending',
'.restart_coordinator.active_launches',
'.restart_coordinator.half_open_probe_inflight',
'.restart_coordinator.open_remaining_seconds',
)
TRANSIENT_GAUGE_SUFFIXES = (
'.resources.telemetry_tasks',
'.resources.query_pool.queued',
@@ -714,10 +720,22 @@ def evaluate_gate(
failures.append(f'{target_id} increased {metric} by {delta:g}')
elif delta is not None and delta < 0:
failures.append(f'{target_id} monotonic counter {metric} reset; the runtime may have restarted')
for metric in sorted(key for key in metric_keys if key.endswith(PENDING_SUFFIX)):
values = [sample.metrics[metric] for sample in tail_samples if metric in sample.metrics]
if values and min(values) > 0:
failures.append(f'{target_id} kept {metric} above zero for the entire tail')
for suffix in RUNTIME_FAILURE_COUNTER_SUFFIXES:
for metric in sorted(key for key in metric_keys if key.endswith(suffix)):
delta = _counter_delta(
state.baseline_metrics or first,
state.last_metrics or last,
metric,
)
if delta is not None and delta > 0:
failures.append(f'{target_id} increased {metric} by {delta:g}')
elif delta is not None and delta < 0:
failures.append(f'{target_id} monotonic counter {metric} reset; the runtime may have restarted')
for suffix in DRAIN_GAUGE_SUFFIXES:
for metric in sorted(key for key in metric_keys if key.endswith(suffix)):
values = [sample.metrics[metric] for sample in tail_samples if metric in sample.metrics]
if values and min(values) > 0:
failures.append(f'{target_id} kept {metric} above zero for the entire tail')
for suffix in TRANSIENT_GAUGE_SUFFIXES:
for metric in sorted(key for key in metric_keys if key.endswith(suffix)):
growth = _counter_delta(first, last, metric)