ci(regression): add independent daily endurance observations

2026-11-01 05:15:22 +08:00 · 2026-10-27 17:03:11 +08:00
parent 407352cc51
commit a943ba3412
4 changed files with 225 additions and 7 deletions
@@ -0,0 +1,121 @@
+name: Daily Endurance
+
+on:
+  schedule:
+    # 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late.
+    - cron: '23 19 * * *'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: daily-endurance-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  endurance:
+    name: Daily endurance (10m)
+    runs-on: ubuntu-latest
+    timeout-minutes: 25
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+      - name: Install locked dependencies
+        run: uv sync --dev --frozen
+      - name: Read official plugin source pin
+        id: plugins
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision']
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
+              stream.write(f'revision={revision}\n')
+          PY
+      - uses: actions/checkout@v4
+        with:
+          repository: langbot-app/langbot-plugins
+          ref: ${{ steps.plugins.outputs.revision }}
+          path: .endurance-plugins
+          persist-credentials: false
+      - name: Prepare CLI and sandbox image
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          mkdir -p .endurance-bin
+          gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .endurance-bin \
+            --pattern lbctl_linux_amd64 --pattern checksums.txt
+          cd .endurance-bin
+          sha256sum --check --ignore-missing checksums.txt
+          chmod +x lbctl_linux_amd64
+          docker pull python:3.12-alpine
+      - name: Run sustained traffic and recovery checks
+        run: |
+          uv run --no-sync python tools/regression_ci.py \
+            --suite endurance --lbctl .endurance-bin/lbctl_linux_amd64 \
+            --plugins .endurance-plugins --output .qa/endurance
+      - name: Publish observations
+        if: always()
+        env:
+          PLUGIN_REVISION: ${{ steps.plugins.outputs.revision }}
+        run: |
+          python - <<'PY_SUMMARY'
+          import json, os
+          from pathlib import Path
+
+          root = Path('.qa/endurance/gate')
+          gate = json.loads((root / 'gate.json').read_text()) if (root / 'gate.json').exists() else {}
+          suite = json.loads((root / 'suite/summary.json').read_text()) if (root / 'suite/summary.json').exists() else {}
+          with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
+              out.write(f"Endurance: {'PASS' if gate.get('passed') is True else 'FAIL / incomplete'}\n\n")
+              out.write(f"Core: `{os.environ['GITHUB_SHA']}`; plugins: `{os.environ['PLUGIN_REVISION']}`.\n\n")
+              out.write('540s offered load + drain + 60s cooldown. Real Core/Runtime/plugins/Box; mock generation and embedding.\n\n')
+              out.write('Observational run, not a required PR check. Budgets are initial QA limits, not product SLOs. Hosted runners are not a controlled performance baseline.\n\n')
+              for phase in suite.get('phases', []):
+                  out.write(f"Successful throughput: {phase['achieved_rps']:.3f} RPS.\n\n")
+                  out.write(f"Completed runs: {phase.get('verified_runs')} / {phase.get('runner_requests')}; ledger reconciliation: {phase.get('ledger_closeout', {}).get('matched')}.\n\n")
+                  out.write('| Case | Requests | Status | P95 (s) |\n|---|---:|---|---:|\n')
+                  for name, case in phase['cases'].items():
+                      out.write(f"| {name} | {case.get('requests', '?')} | {case.get('status', {})} | {case.get('p95_seconds')} |\n")
+                  out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n')
+                  for target, metrics in phase.get('resources', {}).get('targets', {}).items():
+                      for metric, values in metrics.items():
+                          out.write(f"| {target} | {metric} | {values['growth']} | {values['allowance']} |\n")
+              errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', []))
+              if gate.get('timed_out'):
+                  errors.append('Suite exceeded its 900-second execution budget.')
+              if gate.get('infrastructure_error'):
+                  errors.append(gate['infrastructure_error'])
+              if not gate or not suite:
+                  errors.append('Missing verdict; inspect preparation and service logs.')
+              for error in errors:
+                  out.write(f"\n- {error}\n")
+          PY_SUMMARY
+      - name: Preserve verdicts and observations
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: endurance-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            .qa/endurance/gate/gate.json
+            .qa/endurance/gate/resources.jsonl
+            .qa/endurance/gate/suite.log
+            .qa/endurance/gate/suite/summary.json
+            .qa/endurance/gate/suite/workload.json
+            .qa/endurance/gate/suite/health.json
+            .qa/endurance/gate/suite/cleanup-checks.json
+            .qa/endurance/gate/suite/sustained.log
+            .qa/endurance/gate/suite/sustained/summary.json
+            .qa/endurance/gate/suite/sustained/requests.jsonl
+            .qa/endurance/gate/suite/sustained/resources.jsonl
+            .qa/endurance/*.log
+          if-no-files-found: warn
+          retention-days: 30