ci(regression): add independent daily endurance observations

.github/workflows/daily-endurance.yml

@@ -0,0 +1,121 @@
+name: Daily Endurance
+
+on:
+  schedule:
+    # 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late.
+    - cron: '23 19 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: daily-endurance-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  endurance:
+    name: Daily endurance (10m)
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+      - name: Install locked dependencies
+        run: uv sync --dev --frozen
+      - name: Read official plugin source pin
+        id: plugins
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision']
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
+              stream.write(f'revision={revision}\n')
+          PY
+      - uses: actions/checkout@v4
+        with:
+          repository: langbot-app/langbot-plugins
+          ref: ${{ steps.plugins.outputs.revision }}
+          path: .endurance-plugins
+          persist-credentials: false
+      - name: Prepare CLI and sandbox image
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          mkdir -p .endurance-bin
+          gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .endurance-bin \
+            --pattern lbctl_linux_amd64 --pattern checksums.txt
+          cd .endurance-bin
+          sha256sum --check --ignore-missing checksums.txt
+          chmod +x lbctl_linux_amd64
+          docker pull python:3.12-alpine
+      - name: Run sustained traffic and recovery checks
+        run: |
+          uv run --no-sync python tools/regression_ci.py \
+            --suite endurance --lbctl .endurance-bin/lbctl_linux_amd64 \
+            --plugins .endurance-plugins --output .qa/endurance
+      - name: Publish observations
+        if: always()
+        env:
+          PLUGIN_REVISION: ${{ steps.plugins.outputs.revision }}
+        run: |
+          python - <<'PY_SUMMARY'
+          import json, os
+          from pathlib import Path
+
+          root = Path('.qa/endurance/gate')
+          gate = json.loads((root / 'gate.json').read_text()) if (root / 'gate.json').exists() else {}
+          suite = json.loads((root / 'suite/summary.json').read_text()) if (root / 'suite/summary.json').exists() else {}
+          with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
+              out.write(f"Endurance: {'PASS' if gate.get('passed') is True else 'FAIL / incomplete'}\n\n")
+              out.write(f"Core: `{os.environ['GITHUB_SHA']}`; plugins: `{os.environ['PLUGIN_REVISION']}`.\n\n")
+              out.write('540s offered load + drain + 60s cooldown. Real Core/Runtime/plugins/Box; mock generation and embedding.\n\n')
+              out.write('Observational run, not a required PR check. Budgets are initial QA limits, not product SLOs. Hosted runners are not a controlled performance baseline.\n\n')
+              for phase in suite.get('phases', []):
+                  out.write(f"Successful throughput: {phase['achieved_rps']:.3f} RPS.\n\n")
+                  out.write(f"Completed runs: {phase.get('verified_runs')} / {phase.get('runner_requests')}; ledger reconciliation: {phase.get('ledger_closeout', {}).get('matched')}.\n\n")
+                  out.write('| Case | Requests | Status | P95 (s) |\n|---|---:|---|---:|\n')
+                  for name, case in phase['cases'].items():
+                      out.write(f"| {name} | {case.get('requests', '?')} | {case.get('status', {})} | {case.get('p95_seconds')} |\n")
+                  out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n')
+                  for target, metrics in phase.get('resources', {}).get('targets', {}).items():
+                      for metric, values in metrics.items():
+                          out.write(f"| {target} | {metric} | {values['growth']} | {values['allowance']} |\n")
+              errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', []))
+              if gate.get('timed_out'):
+                  errors.append('Suite exceeded its 900-second execution budget.')
+              if gate.get('infrastructure_error'):
+                  errors.append(gate['infrastructure_error'])
+              if not gate or not suite:
+                  errors.append('Missing verdict; inspect preparation and service logs.')
+              for error in errors:
+                  out.write(f"\n- {error}\n")
+          PY_SUMMARY
+      - name: Preserve verdicts and observations
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: endurance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            .qa/endurance/gate/gate.json
+            .qa/endurance/gate/resources.jsonl
+            .qa/endurance/gate/suite.log
+            .qa/endurance/gate/suite/summary.json
+            .qa/endurance/gate/suite/workload.json
+            .qa/endurance/gate/suite/health.json
+            .qa/endurance/gate/suite/cleanup-checks.json
+            .qa/endurance/gate/suite/sustained.log
+            .qa/endurance/gate/suite/sustained/summary.json
+            .qa/endurance/gate/suite/sustained/requests.jsonl
+            .qa/endurance/gate/suite/sustained/resources.jsonl
+            .qa/endurance/*.log
+          if-no-files-found: warn
+          retention-days: 30

tools/README.md

@@ -404,6 +404,34 @@ budgets; the one-minute smoke does not replace them.
 
 ## Sustained regression and baseline comparison
 
+`Daily Endurance` runs independently of PR checks at 19:23 UTC (03:23
+Asia/Shanghai) on the default branch. GitHub may delay scheduled starts. It also
+supports **Run workflow** with a selected branch. The schedule becomes active
+only after the workflow is merged into the default branch; it is not a required
+merge check.
+
+The workflow reuses the disposable smoke stack, locked SDK, pinned official
+plugins and lbctl v0.3.0. Generation and embedding are deterministic mocks.
+It runs the endurance profile below, with a 15-minute execution budget for
+resource preparation, load, drain, cooldown and API cleanup. Service preparation
+is outside that budget; the whole job has a 25-minute limit. Service processes
+and owned Box containers are stopped afterward, including after a failed suite.
+
+Actions summaries show throughput, per-case P95 and retained resource growth.
+Verdicts, individual requests, resource samples and service logs are retained
+for 30 days, including failures. Initial QA budget violations make this
+observational job red without blocking PRs. GitHub-hosted runners vary between
+runs: these reports do not establish a controlled baseline or prove a performance
+regression. No automatic cross-run performance comparison is enabled.
+
+To run the same entrypoint on a disposable Linux host with an idle Docker daemon:
+
+```sh
+uv run --no-sync python tools/regression_ci.py --suite endurance \
+  --lbctl /path/to/lbctl --plugins /path/to/pinned/langbot-plugins \
+  --output .qa/endurance-001
+```
+
 [`regression_profiles/endurance.json`](regression_profiles/endurance.json) runs
 nine minutes of mixed chat/Box/RAG traffic at 1 request/second across four stable
 groups, followed by one minute without traffic to check recovery. Configure its

tools/regression_ci.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-"""Prepare a disposable Linux CI stack from this checkout, then run the 60s gate."""
+"""Prepare a disposable Linux CI stack from this checkout, then run a regression suite."""
 
 import argparse
 import json
@@ -27,7 +27,7 @@ def ready(url, processes, timeout=120):
     deadline = time.monotonic() + timeout
     while time.monotonic() < deadline:
         if any(p.poll() is not None for p in processes):
-            raise RuntimeError('A smoke dependency exited; inspect service logs')
+            raise RuntimeError('A regression dependency exited; inspect service logs')
         try:
             read_endpoint_snapshot(url, timeout_seconds=1)
             return
@@ -78,7 +78,7 @@ def run(args):
             host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine'
         )
         config_path.write_text(yaml.safe_dump(config))
-        profile_path = ROOT / 'tools/regression_profiles/smoke.json'
+        profile_path = ROOT / f'tools/regression_profiles/{args.suite}.json'
         profile = json.loads(profile_path.read_text())
         cli_config = private / 'lbctl.yaml'
         cli_config.write_text(
@@ -105,8 +105,9 @@ def run(args):
             mock_records=str(root / 'mock-requests.jsonl'),
         )
         profile['plugins'] = []
-        # Two turns retain the history/isolation oracle while leaving room for setup and cleanup.
-        profile['smoke']['group_turns'] = 2
+        if args.suite == 'smoke':
+            # Two turns retain the history/isolation oracle within the 60-second budget.
+            profile['smoke']['group_turns'] = 2
         for resource in profile['resources']:
             if resource['kind'] == 'model' and resource.get('type') == 'embedding':
                 resource['body']['name'] = 'mock-embedding'
@@ -161,6 +162,8 @@ def run(args):
             },
             'pids': pids,
         }
+        if args.suite == 'endurance':
+            profile['observe']['box_data_root'] = str(private / 'box')
         for url in profile['observe']['health'].values():
             ready(url, processes)
         preparation = root / 'preparation'
@@ -177,7 +180,7 @@ def run(args):
             shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__'))
             archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source)
             suite.call(['plugin', 'install', 'local', '--file', archive, '--wait'])
-        generated = private / 'smoke.json'
+        generated = private / f'{args.suite}.json'
         generated.write_text(json.dumps(profile, indent=2))
         return supervise(
             [
@@ -190,7 +193,7 @@ def run(args):
                 str(root / 'gate/suite'),
             ],
             root / 'gate',
-            60,
+            60 if args.suite == 'smoke' else 900,
             pids,
             env,
         )
@@ -215,6 +218,7 @@ def main():
     parser.add_argument('--output', type=Path, required=True)
     parser.add_argument('--lbctl', type=Path, required=True)
     parser.add_argument('--plugins', type=Path, required=True)
+    parser.add_argument('--suite', choices=('smoke', 'endurance'), default='smoke')
     args = parser.parse_args()
     if args.output.exists():
         parser.error('--output must be a new directory')

tools/test_regression_ci.py

@@ -0,0 +1,65 @@
+"""Exercise the daily workflow's published verdict, including incomplete runs."""
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import tempfile
+import unittest
+
+import yaml
+
+
+class DailySummaryTest(unittest.TestCase):
+    def test_summary_reports_observations_without_hiding_outer_failures(self):
+        workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml'
+        steps = yaml.safe_load(workflow.read_text())['jobs']['endurance']['steps']
+        command = next(step['run'] for step in steps if step.get('name') == 'Publish observations')
+        suite = {
+            'passed': True,
+            'errors': [],
+            'phases': [
+                {
+                    'achieved_rps': 1.0,
+                    'verified_runs': 540,
+                    'runner_requests': 540,
+                    'ledger_closeout': {'matched': True},
+                    'cases': {'chat': {'requests': 270, 'status': {'ok': 270}, 'p95_seconds': 0.5}},
+                }
+            ],
+        }
+        for gate, expected in [
+            ({'passed': True}, 'Endurance: PASS'),
+            ({'passed': False, 'infrastructure_error': 'teardown failed'}, 'teardown failed'),
+            ({'passed': False, 'timed_out': True}, '900-second execution budget'),
+            (None, 'Missing verdict'),
+        ]:
+            with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                if gate is not None:
+                    target = root / '.qa/endurance/gate'
+                    (target / 'suite').mkdir(parents=True)
+                    (target / 'gate.json').write_text(json.dumps(gate))
+                    (target / 'suite/summary.json').write_text(json.dumps(suite))
+                summary = root / 'summary.md'
+                env = {
+                    **os.environ,
+                    'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'],
+                    'GITHUB_STEP_SUMMARY': str(summary),
+                    'GITHUB_SHA': 'core-revision',
+                    'PLUGIN_REVISION': 'plugin-revision',
+                }
+                subprocess.run(['bash', '-e', '-c', command], cwd=root, env=env, check=True, capture_output=True)
+                text = summary.read_text()
+                self.assertIn(expected, text)
+                if gate and gate.get('passed'):
+                    self.assertIn('Completed runs: 540 / 540', text)
+                    self.assertIn('1.000 RPS', text)
+                else:
+                    self.assertIn('Endurance: FAIL / incomplete', text)
+                    self.assertNotIn('Endurance: PASS', text)
+
+
+if __name__ == '__main__':
+    unittest.main()