mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-11-01 13:25:18 +08:00
ci(regression): add independent daily endurance observations
This commit is contained in:
huanghuoguoguo
@@ -0,0 +1,121 @@
|
||||
name: Daily Endurance
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late.
|
||||
- cron: '23 19 * * *'
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: daily-endurance-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
endurance:
|
||||
name: Daily endurance (10m)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 25
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- uses: astral-sh/setup-uv@v4
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Install locked dependencies
|
||||
run: uv sync --dev --frozen
|
||||
- name: Read official plugin source pin
|
||||
id: plugins
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision']
|
||||
with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
|
||||
stream.write(f'revision={revision}\n')
|
||||
PY
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
repository: langbot-app/langbot-plugins
|
||||
ref: ${{ steps.plugins.outputs.revision }}
|
||||
path: .endurance-plugins
|
||||
persist-credentials: false
|
||||
- name: Prepare CLI and sandbox image
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
mkdir -p .endurance-bin
|
||||
gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .endurance-bin \
|
||||
--pattern lbctl_linux_amd64 --pattern checksums.txt
|
||||
cd .endurance-bin
|
||||
sha256sum --check --ignore-missing checksums.txt
|
||||
chmod +x lbctl_linux_amd64
|
||||
docker pull python:3.12-alpine
|
||||
- name: Run sustained traffic and recovery checks
|
||||
run: |
|
||||
uv run --no-sync python tools/regression_ci.py \
|
||||
--suite endurance --lbctl .endurance-bin/lbctl_linux_amd64 \
|
||||
--plugins .endurance-plugins --output .qa/endurance
|
||||
- name: Publish observations
|
||||
if: always()
|
||||
env:
|
||||
PLUGIN_REVISION: ${{ steps.plugins.outputs.revision }}
|
||||
run: |
|
||||
python - <<'PY_SUMMARY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path('.qa/endurance/gate')
|
||||
gate = json.loads((root / 'gate.json').read_text()) if (root / 'gate.json').exists() else {}
|
||||
suite = json.loads((root / 'suite/summary.json').read_text()) if (root / 'suite/summary.json').exists() else {}
|
||||
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
|
||||
out.write(f"Endurance: {'PASS' if gate.get('passed') is True else 'FAIL / incomplete'}\n\n")
|
||||
out.write(f"Core: `{os.environ['GITHUB_SHA']}`; plugins: `{os.environ['PLUGIN_REVISION']}`.\n\n")
|
||||
out.write('540s offered load + drain + 60s cooldown. Real Core/Runtime/plugins/Box; mock generation and embedding.\n\n')
|
||||
out.write('Observational run, not a required PR check. Budgets are initial QA limits, not product SLOs. Hosted runners are not a controlled performance baseline.\n\n')
|
||||
for phase in suite.get('phases', []):
|
||||
out.write(f"Successful throughput: {phase['achieved_rps']:.3f} RPS.\n\n")
|
||||
out.write(f"Completed runs: {phase.get('verified_runs')} / {phase.get('runner_requests')}; ledger reconciliation: {phase.get('ledger_closeout', {}).get('matched')}.\n\n")
|
||||
out.write('| Case | Requests | Status | P95 (s) |\n|---|---:|---|---:|\n')
|
||||
for name, case in phase['cases'].items():
|
||||
out.write(f"| {name} | {case.get('requests', '?')} | {case.get('status', {})} | {case.get('p95_seconds')} |\n")
|
||||
out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n')
|
||||
for target, metrics in phase.get('resources', {}).get('targets', {}).items():
|
||||
for metric, values in metrics.items():
|
||||
out.write(f"| {target} | {metric} | {values['growth']} | {values['allowance']} |\n")
|
||||
errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', []))
|
||||
if gate.get('timed_out'):
|
||||
errors.append('Suite exceeded its 900-second execution budget.')
|
||||
if gate.get('infrastructure_error'):
|
||||
errors.append(gate['infrastructure_error'])
|
||||
if not gate or not suite:
|
||||
errors.append('Missing verdict; inspect preparation and service logs.')
|
||||
for error in errors:
|
||||
out.write(f"\n- {error}\n")
|
||||
PY_SUMMARY
|
||||
- name: Preserve verdicts and observations
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: endurance-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
.qa/endurance/gate/gate.json
|
||||
.qa/endurance/gate/resources.jsonl
|
||||
.qa/endurance/gate/suite.log
|
||||
.qa/endurance/gate/suite/summary.json
|
||||
.qa/endurance/gate/suite/workload.json
|
||||
.qa/endurance/gate/suite/health.json
|
||||
.qa/endurance/gate/suite/cleanup-checks.json
|
||||
.qa/endurance/gate/suite/sustained.log
|
||||
.qa/endurance/gate/suite/sustained/summary.json
|
||||
.qa/endurance/gate/suite/sustained/requests.jsonl
|
||||
.qa/endurance/gate/suite/sustained/resources.jsonl
|
||||
.qa/endurance/*.log
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
Reference in New Issue
Block a user