mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-29 20:05:25 +08:00
ci(regression): extend daily endurance to thirty minutes
This commit is contained in:
@@ -15,9 +15,9 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
endurance:
|
||||
name: Daily endurance (10m)
|
||||
name: Daily endurance (30m)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -77,7 +77,7 @@ jobs:
|
||||
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
|
||||
out.write(f"Endurance: {'PASS' if gate.get('passed') is True else 'FAIL / incomplete'}\n\n")
|
||||
out.write(f"Core: `{os.environ['GITHUB_SHA']}`; plugins: `{os.environ['PLUGIN_REVISION']}`.\n\n")
|
||||
out.write('540s offered load + drain + 60s cooldown. Real Core/Runtime/plugins/Box; mock generation and embedding.\n\n')
|
||||
out.write('1740s offered load + drain + 60s cooldown. Real Core/Runtime/plugins/Box; mock generation and embedding.\n\n')
|
||||
out.write('Observational run, not a required PR check. Budgets are initial QA limits, not product SLOs. Hosted runners are not a controlled performance baseline.\n\n')
|
||||
for phase in suite.get('phases', []):
|
||||
out.write(f"Successful throughput: {phase['achieved_rps']:.3f} RPS.\n\n")
|
||||
@@ -91,7 +91,7 @@ jobs:
|
||||
out.write(f"| {target} | {metric} | {values['growth']} | {values['allowance']} |\n")
|
||||
errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', []))
|
||||
if gate.get('timed_out'):
|
||||
errors.append('Suite exceeded its 900-second execution budget.')
|
||||
errors.append('Suite exceeded its 2100-second execution budget.')
|
||||
if gate.get('infrastructure_error'):
|
||||
errors.append(gate['infrastructure_error'])
|
||||
if not gate or not suite:
|
||||
|
||||
+3
-3
@@ -412,9 +412,9 @@ merge check.
|
||||
|
||||
The workflow reuses the disposable smoke stack, locked SDK, pinned official
|
||||
plugins and lbctl v0.3.0. Generation and embedding are deterministic mocks.
|
||||
It runs the endurance profile below, with a 15-minute execution budget for
|
||||
It runs the endurance profile below, with a 35-minute execution budget for
|
||||
resource preparation, load, drain, cooldown and API cleanup. Service preparation
|
||||
is outside that budget; the whole job has a 25-minute limit. Service processes
|
||||
is outside that budget; the whole job has a 45-minute limit. Service processes
|
||||
and owned Box containers are stopped afterward, including after a failed suite.
|
||||
|
||||
Actions summaries show throughput, per-case P95 and retained resource growth.
|
||||
@@ -433,7 +433,7 @@ uv run --no-sync python tools/regression_ci.py --suite endurance \
|
||||
```
|
||||
|
||||
[`regression_profiles/endurance.json`](regression_profiles/endurance.json) runs
|
||||
nine minutes of mixed chat/Box/RAG traffic at 1 request/second across four stable
|
||||
29 minutes of mixed chat/Box/RAG traffic at 1 request/second across four stable
|
||||
groups, followed by one minute without traffic to check recovery. Configure its
|
||||
CLI context, plugin inputs, model endpoints, Core SQLite path and Core/Runtime/Box
|
||||
PID files as for smoke. Set `plugins: []` to reuse initialized plugins. Configure
|
||||
|
||||
@@ -193,7 +193,7 @@ def run(args):
|
||||
str(root / 'gate/suite'),
|
||||
],
|
||||
root / 'gate',
|
||||
60 if args.suite == 'smoke' else 900,
|
||||
60 if args.suite == 'smoke' else 2100,
|
||||
pids,
|
||||
env,
|
||||
)
|
||||
|
||||
@@ -444,7 +444,7 @@
|
||||
"phases": [
|
||||
{
|
||||
"name": "sustained",
|
||||
"duration_seconds": 540,
|
||||
"duration_seconds": 1740,
|
||||
"rps": 1,
|
||||
"workers": 4,
|
||||
"cooldown_seconds": 60,
|
||||
|
||||
@@ -32,7 +32,7 @@ class DailySummaryTest(unittest.TestCase):
|
||||
for gate, expected in [
|
||||
({'passed': True}, 'Endurance: PASS'),
|
||||
({'passed': False, 'infrastructure_error': 'teardown failed'}, 'teardown failed'),
|
||||
({'passed': False, 'timed_out': True}, '900-second execution budget'),
|
||||
({'passed': False, 'timed_out': True}, '2100-second execution budget'),
|
||||
(None, 'Missing verdict'),
|
||||
]:
|
||||
with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory:
|
||||
|
||||
Reference in New Issue
Block a user