diff --git a/.github/workflows/daily-endurance.yml b/.github/workflows/daily-endurance.yml new file mode 100644 index 000000000..5eed78da3 --- /dev/null +++ b/.github/workflows/daily-endurance.yml @@ -0,0 +1,121 @@ +name: Daily Endurance + +on: + schedule: + # 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late. + - cron: '23 19 * * *' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: daily-endurance-${{ github.ref }} + cancel-in-progress: false + +jobs: + endurance: + name: Daily endurance (10m) + runs-on: ubuntu-latest + timeout-minutes: 25 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Install locked dependencies + run: uv sync --dev --frozen + - name: Read official plugin source pin + id: plugins + run: | + python - <<'PY' + import json, os + from pathlib import Path + revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision'] + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(f'revision={revision}\n') + PY + - uses: actions/checkout@v4 + with: + repository: langbot-app/langbot-plugins + ref: ${{ steps.plugins.outputs.revision }} + path: .endurance-plugins + persist-credentials: false + - name: Prepare CLI and sandbox image + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p .endurance-bin + gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .endurance-bin \ + --pattern lbctl_linux_amd64 --pattern checksums.txt + cd .endurance-bin + sha256sum --check --ignore-missing checksums.txt + chmod +x lbctl_linux_amd64 + docker pull python:3.12-alpine + - name: Run sustained traffic and recovery checks + run: | + uv run --no-sync python tools/regression_ci.py \ + --suite endurance --lbctl .endurance-bin/lbctl_linux_amd64 \ + --plugins .endurance-plugins --output .qa/endurance + - name: Publish observations + if: always() + env: + PLUGIN_REVISION: ${{ steps.plugins.outputs.revision }} + run: | + python - <<'PY_SUMMARY' + import json, os + from pathlib import Path + + root = Path('.qa/endurance/gate') + gate = json.loads((root / 'gate.json').read_text()) if (root / 'gate.json').exists() else {} + suite = json.loads((root / 'suite/summary.json').read_text()) if (root / 'suite/summary.json').exists() else {} + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: + out.write(f"Endurance: {'PASS' if gate.get('passed') is True else 'FAIL / incomplete'}\n\n") + out.write(f"Core: `{os.environ['GITHUB_SHA']}`; plugins: `{os.environ['PLUGIN_REVISION']}`.\n\n") + out.write('540s offered load + drain + 60s cooldown. Real Core/Runtime/plugins/Box; mock generation and embedding.\n\n') + out.write('Observational run, not a required PR check. Budgets are initial QA limits, not product SLOs. Hosted runners are not a controlled performance baseline.\n\n') + for phase in suite.get('phases', []): + out.write(f"Successful throughput: {phase['achieved_rps']:.3f} RPS.\n\n") + out.write(f"Completed runs: {phase.get('verified_runs')} / {phase.get('runner_requests')}; ledger reconciliation: {phase.get('ledger_closeout', {}).get('matched')}.\n\n") + out.write('| Case | Requests | Status | P95 (s) |\n|---|---:|---|---:|\n') + for name, case in phase['cases'].items(): + out.write(f"| {name} | {case.get('requests', '?')} | {case.get('status', {})} | {case.get('p95_seconds')} |\n") + out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n') + for target, metrics in phase.get('resources', {}).get('targets', {}).items(): + for metric, values in metrics.items(): + out.write(f"| {target} | {metric} | {values['growth']} | {values['allowance']} |\n") + errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + if gate.get('timed_out'): + errors.append('Suite exceeded its 900-second execution budget.') + if gate.get('infrastructure_error'): + errors.append(gate['infrastructure_error']) + if not gate or not suite: + errors.append('Missing verdict; inspect preparation and service logs.') + for error in errors: + out.write(f"\n- {error}\n") + PY_SUMMARY + - name: Preserve verdicts and observations + if: always() + uses: actions/upload-artifact@v4 + with: + name: endurance-${{ github.run_id }}-${{ github.run_attempt }} + path: | + .qa/endurance/gate/gate.json + .qa/endurance/gate/resources.jsonl + .qa/endurance/gate/suite.log + .qa/endurance/gate/suite/summary.json + .qa/endurance/gate/suite/workload.json + .qa/endurance/gate/suite/health.json + .qa/endurance/gate/suite/cleanup-checks.json + .qa/endurance/gate/suite/sustained.log + .qa/endurance/gate/suite/sustained/summary.json + .qa/endurance/gate/suite/sustained/requests.jsonl + .qa/endurance/gate/suite/sustained/resources.jsonl + .qa/endurance/*.log + if-no-files-found: warn + retention-days: 30 diff --git a/tools/README.md b/tools/README.md index 1b3c00e82..5addffd54 100644 --- a/tools/README.md +++ b/tools/README.md @@ -404,6 +404,34 @@ budgets; the one-minute smoke does not replace them. ## Sustained regression and baseline comparison +`Daily Endurance` runs independently of PR checks at 19:23 UTC (03:23 +Asia/Shanghai) on the default branch. GitHub may delay scheduled starts. It also +supports **Run workflow** with a selected branch. The schedule becomes active +only after the workflow is merged into the default branch; it is not a required +merge check. + +The workflow reuses the disposable smoke stack, locked SDK, pinned official +plugins and lbctl v0.3.0. Generation and embedding are deterministic mocks. +It runs the endurance profile below, with a 15-minute execution budget for +resource preparation, load, drain, cooldown and API cleanup. Service preparation +is outside that budget; the whole job has a 25-minute limit. Service processes +and owned Box containers are stopped afterward, including after a failed suite. + +Actions summaries show throughput, per-case P95 and retained resource growth. +Verdicts, individual requests, resource samples and service logs are retained +for 30 days, including failures. Initial QA budget violations make this +observational job red without blocking PRs. GitHub-hosted runners vary between +runs: these reports do not establish a controlled baseline or prove a performance +regression. No automatic cross-run performance comparison is enabled. + +To run the same entrypoint on a disposable Linux host with an idle Docker daemon: + +```sh +uv run --no-sync python tools/regression_ci.py --suite endurance \ + --lbctl /path/to/lbctl --plugins /path/to/pinned/langbot-plugins \ + --output .qa/endurance-001 +``` + [`regression_profiles/endurance.json`](regression_profiles/endurance.json) runs nine minutes of mixed chat/Box/RAG traffic at 1 request/second across four stable groups, followed by one minute without traffic to check recovery. Configure its diff --git a/tools/regression_ci.py b/tools/regression_ci.py index 0d57d4585..783adc478 100644 --- a/tools/regression_ci.py +++ b/tools/regression_ci.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Prepare a disposable Linux CI stack from this checkout, then run the 60s gate.""" +"""Prepare a disposable Linux CI stack from this checkout, then run a regression suite.""" import argparse import json @@ -27,7 +27,7 @@ def ready(url, processes, timeout=120): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if any(p.poll() is not None for p in processes): - raise RuntimeError('A smoke dependency exited; inspect service logs') + raise RuntimeError('A regression dependency exited; inspect service logs') try: read_endpoint_snapshot(url, timeout_seconds=1) return @@ -78,7 +78,7 @@ def run(args): host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine' ) config_path.write_text(yaml.safe_dump(config)) - profile_path = ROOT / 'tools/regression_profiles/smoke.json' + profile_path = ROOT / f'tools/regression_profiles/{args.suite}.json' profile = json.loads(profile_path.read_text()) cli_config = private / 'lbctl.yaml' cli_config.write_text( @@ -105,8 +105,9 @@ def run(args): mock_records=str(root / 'mock-requests.jsonl'), ) profile['plugins'] = [] - # Two turns retain the history/isolation oracle while leaving room for setup and cleanup. - profile['smoke']['group_turns'] = 2 + if args.suite == 'smoke': + # Two turns retain the history/isolation oracle within the 60-second budget. + profile['smoke']['group_turns'] = 2 for resource in profile['resources']: if resource['kind'] == 'model' and resource.get('type') == 'embedding': resource['body']['name'] = 'mock-embedding' @@ -161,6 +162,8 @@ def run(args): }, 'pids': pids, } + if args.suite == 'endurance': + profile['observe']['box_data_root'] = str(private / 'box') for url in profile['observe']['health'].values(): ready(url, processes) preparation = root / 'preparation' @@ -177,7 +180,7 @@ def run(args): shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__')) archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source) suite.call(['plugin', 'install', 'local', '--file', archive, '--wait']) - generated = private / 'smoke.json' + generated = private / f'{args.suite}.json' generated.write_text(json.dumps(profile, indent=2)) return supervise( [ @@ -190,7 +193,7 @@ def run(args): str(root / 'gate/suite'), ], root / 'gate', - 60, + 60 if args.suite == 'smoke' else 900, pids, env, ) @@ -215,6 +218,7 @@ def main(): parser.add_argument('--output', type=Path, required=True) parser.add_argument('--lbctl', type=Path, required=True) parser.add_argument('--plugins', type=Path, required=True) + parser.add_argument('--suite', choices=('smoke', 'endurance'), default='smoke') args = parser.parse_args() if args.output.exists(): parser.error('--output must be a new directory') diff --git a/tools/test_regression_ci.py b/tools/test_regression_ci.py new file mode 100644 index 000000000..0e4f65bbe --- /dev/null +++ b/tools/test_regression_ci.py @@ -0,0 +1,65 @@ +"""Exercise the daily workflow's published verdict, including incomplete runs.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import yaml + + +class DailySummaryTest(unittest.TestCase): + def test_summary_reports_observations_without_hiding_outer_failures(self): + workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml' + steps = yaml.safe_load(workflow.read_text())['jobs']['endurance']['steps'] + command = next(step['run'] for step in steps if step.get('name') == 'Publish observations') + suite = { + 'passed': True, + 'errors': [], + 'phases': [ + { + 'achieved_rps': 1.0, + 'verified_runs': 540, + 'runner_requests': 540, + 'ledger_closeout': {'matched': True}, + 'cases': {'chat': {'requests': 270, 'status': {'ok': 270}, 'p95_seconds': 0.5}}, + } + ], + } + for gate, expected in [ + ({'passed': True}, 'Endurance: PASS'), + ({'passed': False, 'infrastructure_error': 'teardown failed'}, 'teardown failed'), + ({'passed': False, 'timed_out': True}, '900-second execution budget'), + (None, 'Missing verdict'), + ]: + with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + if gate is not None: + target = root / '.qa/endurance/gate' + (target / 'suite').mkdir(parents=True) + (target / 'gate.json').write_text(json.dumps(gate)) + (target / 'suite/summary.json').write_text(json.dumps(suite)) + summary = root / 'summary.md' + env = { + **os.environ, + 'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'], + 'GITHUB_STEP_SUMMARY': str(summary), + 'GITHUB_SHA': 'core-revision', + 'PLUGIN_REVISION': 'plugin-revision', + } + subprocess.run(['bash', '-e', '-c', command], cwd=root, env=env, check=True, capture_output=True) + text = summary.read_text() + self.assertIn(expected, text) + if gate and gate.get('passed'): + self.assertIn('Completed runs: 540 / 540', text) + self.assertIn('1.000 RPS', text) + else: + self.assertIn('Endurance: FAIL / incomplete', text) + self.assertNotIn('Endurance: PASS', text) + + +if __name__ == '__main__': + unittest.main()