xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-11-01 21:35:25 +08:00
Code Issues Packages Projects Releases Wiki Activity

ci(regression): consume published plugins and shared daily scenarios

Browse Source
This commit is contained in:
huanghuoguoguo
2026-10-28 12:26:30 +08:00
parent 10e4755f4f
commit 833c1feece
6 changed files with 268 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/workflows/core-smoke.yml
+14 -18
View File
@@ -46,22 +46,6 @@ jobs:
enable-cache: true
- name: Install locked dependencies
run: uv sync --dev --frozen
- name: Read official plugin source pin
id: plugins
run: |
python - <<'PY'
import json, os
from pathlib import Path
revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision']
with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
stream.write(f'revision={revision}\n')
PY
- uses: actions/checkout@v4
with:
repository: langbot-app/langbot-plugins
ref: ${{ steps.plugins.outputs.revision }}
path: .smoke-plugins
persist-credentials: false
- name: Prepare CLI and sandbox image
env:
GH_TOKEN: ${{ github.token }}
@@ -76,21 +60,28 @@ jobs:
- name: Prepare real services and run the 60-second smoke
run: |
uv run --no-sync python tools/regression_ci.py \
--lbctl .smoke-bin/lbctl_linux_amd64 --plugins .smoke-plugins --output .qa/core-smoke
--lbctl .smoke-bin/lbctl_linux_amd64 --output .qa/core-smoke
- name: Publish gate result
if: always()
run: |
python - <<'PY'
import json, os
from pathlib import Path
path = Path('.qa/core-smoke/gate/gate.json')
root = Path('.qa/core-smoke')
path = root / 'gate/gate.json'
manifest = root / 'plugins/manifest.json'
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
if manifest.exists():
for plugin in json.loads(manifest.read_text())['plugins']:
out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
if path.exists():
data = json.loads(path.read_text())
out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; "
f"{data.get('elapsed_seconds', 0):.2f}/60 seconds; {data.get('requests', 0)} requests.\n")
if data.get('infrastructure_error'):
out.write('Preparation or teardown failed; inspect service logs.\n')
for error in data.get('coverage_errors', []):
out.write(f"Coverage evidence error: {error}\n")
out.write('Service/dependency preparation is outside the 60-second execution budget.\n')
else:
out.write('Core smoke did not produce a verdict; inspect preparation failure.\n')
@@ -108,6 +99,11 @@ jobs:
.qa/core-smoke/gate/suite/summary.json
.qa/core-smoke/gate/suite/health.json
.qa/core-smoke/gate/suite/cleanup-checks.json
.qa/core-smoke/gate/suite/coverage.json
.qa/core-smoke/gate/suite/coverage.md
.qa/core-smoke/gate/suite/cycles.json
.qa/core-smoke/gate/suite/execution.json
.qa/core-smoke/plugins/manifest.json
.qa/core-smoke/*.log
if-no-files-found: warn
retention-days: 7
.github/workflows/daily-endurance.yml
+54 -52
View File
@@ -15,7 +15,7 @@ concurrency:
jobs:
endurance:
name: Daily endurance (30m)
name: Daily regression (30m)
runs-on: ubuntu-latest
timeout-minutes: 45
steps:
@@ -30,22 +30,6 @@ jobs:
enable-cache: true
- name: Install locked dependencies
run: uv sync --dev --frozen
- name: Read official plugin source pin
id: plugins
run: |
python - <<'PY'
import json, os
from pathlib import Path
revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision']
with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
stream.write(f'revision={revision}\n')
PY
- uses: actions/checkout@v4
with:
repository: langbot-app/langbot-plugins
ref: ${{ steps.plugins.outputs.revision }}
path: .endurance-plugins
persist-credentials: false
- name: Prepare CLI and sandbox image
env:
GH_TOKEN: ${{ github.token }}
@@ -57,41 +41,57 @@ jobs:
sha256sum --check --ignore-missing checksums.txt
chmod +x lbctl_linux_amd64
docker pull python:3.12-alpine
- name: Run sustained traffic and recovery checks
- name: Run repeated scenarios and recovery checks
run: |
uv run --no-sync python tools/regression_ci.py \
--suite endurance --lbctl .endurance-bin/lbctl_linux_amd64 \
--plugins .endurance-plugins --output .qa/endurance
--suite daily --lbctl .endurance-bin/lbctl_linux_amd64 --output .qa/daily
- name: Publish observations
if: always()
env:
PLUGIN_REVISION: ${{ steps.plugins.outputs.revision }}
run: |
python - <<'PY_SUMMARY'
import json, os
from pathlib import Path
root = Path('.qa/endurance/gate')
gate = json.loads((root / 'gate.json').read_text()) if (root / 'gate.json').exists() else {}
suite = json.loads((root / 'suite/summary.json').read_text()) if (root / 'suite/summary.json').exists() else {}
root = Path('.qa/daily')
def read(path):
source = root / path
return json.loads(source.read_text()) if source.exists() else {}
gate = read('gate/gate.json')
suite = read('gate/suite/summary.json')
manifest = read('plugins/manifest.json')
coverage = read('gate/suite/coverage.json')
passed = gate.get('passed') is True and suite.get('passed') is True
outer_failed = any(gate.get(key) for key in ('infrastructure_error', 'timed_out', 'resource_errors', 'coverage_errors'))
outcome = 'PASS' if passed else 'INCOMPLETE' if suite.get('outcome') == 'INCOMPLETE' and not outer_failed else 'FAIL / incomplete'
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
out.write(f"Endurance: {'PASS' if gate.get('passed') is True else 'FAIL / incomplete'}\n\n")
out.write(f"Core: `{os.environ['GITHUB_SHA']}`; plugins: `{os.environ['PLUGIN_REVISION']}`.\n\n")
out.write('1740s offered load + drain + 60s cooldown. Real Core/Runtime/plugins/Box; mock generation and embedding.\n\n')
out.write('Observational run, not a required PR check. Budgets are initial QA limits, not product SLOs. Hosted runners are not a controlled performance baseline.\n\n')
for phase in suite.get('phases', []):
out.write(f"Successful throughput: {phase['achieved_rps']:.3f} RPS.\n\n")
out.write(f"Completed runs: {phase.get('verified_runs')} / {phase.get('runner_requests')}; ledger reconciliation: {phase.get('ledger_closeout', {}).get('matched')}.\n\n")
out.write('| Case | Requests | Status | P95 (s) |\n|---|---:|---|---:|\n')
for name, case in phase['cases'].items():
out.write(f"| {name} | {case.get('requests', '?')} | {case.get('status', {})} | {case.get('p95_seconds')} |\n")
out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n")
out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n')
out.write('Real Core/Runtime/published plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n')
for plugin in manifest.get('plugins', []):
out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n")
out.write(f"Final ledger reconciliation: {suite.get('ledger_closeout', {}).get('matched', 'not reported')}.\n\n")
data = suite.get('test_data', {})
out.write(f"Observed conversations: {data.get('observed_conversations', 'not reported')}; AgentRuns: {data.get('observed_agent_runs', 'not reported')}; group sessions: {data.get('group_sessions', 'not reported')}; owned API resources: {data.get('owned_api_resources', 'not reported')}.\n\n")
out.write('One Workspace; each pipeline reuses one Box per CI run. This does not test cross-group Box file isolation or tenant isolation.\n\n')
observations = suite.get('observations', {})
if 'successful_requests_per_second' in observations:
out.write(f"Successful throughput: {observations['successful_requests_per_second']:.3f} RPS.\n\n")
if observations.get('step_p95_seconds'):
out.write('| Case | Full-step P95 (s) |\n|---|---:|\n')
for name, p95 in observations['step_p95_seconds'].items():
out.write(f"| {name} | {p95} |\n")
for kind, values in coverage.get('kinds', {}).items():
out.write(f"\n{kind} behavior evidence: {values['decided']} / {values['total']}; statuses: {values['status']}.\n")
if suite.get('resources', {}).get('targets'):
out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n')
for target, metrics in phase.get('resources', {}).get('targets', {}).items():
for target, metrics in suite['resources']['targets'].items():
for metric, values in metrics.items():
out.write(f"| {target} | {metric} | {values['growth']} | {values['allowance']} |\n")
errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', []))
out.write(f"| {target} | {metric} | {values.get('growth', 'unavailable')} | {values.get('allowance', 'unavailable')} |\n")
errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', []))
if gate.get('timed_out'):
errors.append('Suite exceeded its 2100-second execution budget.')
errors.append('Suite exceeded its 1860-second outer process guard.')
if gate.get('infrastructure_error'):
errors.append(gate['infrastructure_error'])
if not gate or not suite:
@@ -103,19 +103,21 @@ jobs:
if: always()
uses: actions/upload-artifact@v4
with:
name: endurance-${{ github.run_id }}-${{ github.run_attempt }}
name: daily-${{ github.run_id }}-${{ github.run_attempt }}
path: |
.qa/endurance/gate/gate.json
.qa/endurance/gate/resources.jsonl
.qa/endurance/gate/suite.log
.qa/endurance/gate/suite/summary.json
.qa/endurance/gate/suite/workload.json
.qa/endurance/gate/suite/health.json
.qa/endurance/gate/suite/cleanup-checks.json
.qa/endurance/gate/suite/sustained.log
.qa/endurance/gate/suite/sustained/summary.json
.qa/endurance/gate/suite/sustained/requests.jsonl
.qa/endurance/gate/suite/sustained/resources.jsonl
.qa/endurance/*.log
.qa/daily/gate/gate.json
.qa/daily/gate/resources.jsonl
.qa/daily/gate/suite.log
.qa/daily/gate/suite/summary.json
.qa/daily/gate/suite/cases.json
.qa/daily/gate/suite/health.json
.qa/daily/gate/suite/cleanup-checks.json
.qa/daily/gate/suite/resources.jsonl
.qa/daily/gate/suite/cycles.json
.qa/daily/gate/suite/execution.json
.qa/daily/gate/suite/coverage.json
.qa/daily/gate/suite/coverage.md
.qa/daily/plugins/manifest.json
.qa/daily/*.log
if-no-files-found: warn
retention-days: 30