mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-30 20:35:22 +08:00
Merge pull request #2784 from langbot-app/ci/daily-endurance
ci(regression): exercise monorepo plugins in daily functional cycles
This commit is contained in:
huanghuoguoguo
@@ -67,14 +67,21 @@ jobs:
|
||||
python - <<'PY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
path = Path('.qa/core-smoke/gate/gate.json')
|
||||
root = Path('.qa/core-smoke')
|
||||
path = root / 'gate/gate.json'
|
||||
manifest = root / 'plugins/manifest.json'
|
||||
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
|
||||
if manifest.exists():
|
||||
for plugin in json.loads(manifest.read_text())['plugins']:
|
||||
out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
|
||||
if path.exists():
|
||||
data = json.loads(path.read_text())
|
||||
out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; "
|
||||
f"{data.get('elapsed_seconds', 0):.2f}/60 seconds; {data.get('requests', 0)} requests.\n")
|
||||
if data.get('infrastructure_error'):
|
||||
out.write('Preparation or teardown failed; inspect service logs.\n')
|
||||
for error in data.get('coverage_errors', []):
|
||||
out.write(f"Coverage evidence error: {error}\n")
|
||||
out.write('Service/dependency preparation is outside the 60-second execution budget.\n')
|
||||
else:
|
||||
out.write('Core smoke did not produce a verdict; inspect preparation failure.\n')
|
||||
@@ -92,6 +99,12 @@ jobs:
|
||||
.qa/core-smoke/gate/suite/summary.json
|
||||
.qa/core-smoke/gate/suite/health.json
|
||||
.qa/core-smoke/gate/suite/cleanup-checks.json
|
||||
.qa/core-smoke/gate/suite/coverage.json
|
||||
.qa/core-smoke/gate/suite/coverage.md
|
||||
.qa/core-smoke/gate/suite/cycles.json
|
||||
.qa/core-smoke/gate/suite/execution.json
|
||||
.qa/core-smoke/plugins/manifest.json
|
||||
.qa/core-smoke/plugins/*-build.log
|
||||
.qa/core-smoke/*.log
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
|
||||
@@ -0,0 +1,160 @@
|
||||
name: Daily Endurance
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late.
|
||||
- cron: '23 19 * * *'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
scenario_groups:
|
||||
description: 'Optional local groups, space separated; empty runs only base cases'
|
||||
type: string
|
||||
default: 'faults resources lifecycle plugins'
|
||||
extra_plugins:
|
||||
description: 'Additional source plugin paths; clear with the plugins group to run only base plugins'
|
||||
type: string
|
||||
default: 'misc/EssentialCommands misc/FAQManager'
|
||||
restart_services:
|
||||
description: 'Check owned Core and Runtime restarts once before resource sampling'
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: daily-endurance-${{ github.ref }}
|
||||
cancel-in-progress: false
|
||||
|
||||
jobs:
|
||||
endurance:
|
||||
name: Daily regression (30m)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- uses: astral-sh/setup-uv@v4
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Install locked dependencies
|
||||
run: uv sync --dev --frozen
|
||||
- name: Prepare CLI and sandbox image
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
mkdir -p .endurance-bin
|
||||
gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .endurance-bin \
|
||||
--pattern lbctl_linux_amd64 --pattern checksums.txt
|
||||
cd .endurance-bin
|
||||
sha256sum --check --ignore-missing checksums.txt
|
||||
chmod +x lbctl_linux_amd64
|
||||
docker pull python:3.12-alpine
|
||||
- name: Run repeated scenarios and recovery checks
|
||||
env:
|
||||
SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle plugins' || inputs.scenario_groups }}
|
||||
EXTRA_PLUGINS: ${{ github.event_name == 'schedule' && 'misc/EssentialCommands misc/FAQManager' || inputs.extra_plugins }}
|
||||
RESTART_SERVICES: ${{ github.event_name == 'schedule' || inputs.restart_services }}
|
||||
run: |
|
||||
read -r -a groups <<< "$SCENARIO_GROUPS"
|
||||
read -r -a extras <<< "$EXTRA_PLUGINS"
|
||||
args=()
|
||||
if [[ "$RESTART_SERVICES" == true ]]; then args+=(--restart-services); fi
|
||||
uv run --no-sync python tools/regression_ci.py \
|
||||
--suite daily --scenario-groups "${groups[@]}" "${args[@]}" \
|
||||
--lbctl .endurance-bin/lbctl_linux_amd64 --plugins plugins --extra-plugins "${extras[@]}" --output .qa/daily
|
||||
- name: Publish observations
|
||||
if: always()
|
||||
run: |
|
||||
python - <<'PY_SUMMARY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path('.qa/daily')
|
||||
def read(path):
|
||||
source = root / path
|
||||
return json.loads(source.read_text()) if source.exists() else {}
|
||||
|
||||
gate = read('gate/gate.json')
|
||||
suite = read('gate/suite/summary.json')
|
||||
manifest = read('plugins/manifest.json')
|
||||
coverage = read('gate/suite/coverage.json')
|
||||
passed = gate.get('passed') is True and suite.get('passed') is True
|
||||
outer_failed = any(gate.get(key) for key in ('infrastructure_error', 'timed_out', 'resource_errors', 'coverage_errors'))
|
||||
outcome = 'PASS' if passed else 'INCOMPLETE' if suite.get('outcome') == 'INCOMPLETE' and not outer_failed else 'FAIL / incomplete'
|
||||
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
|
||||
out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n")
|
||||
out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n')
|
||||
out.write('Real Core/Runtime/source plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n')
|
||||
out.write(f"Enabled local groups: {gate.get('scenario_groups', 'not reported')}.\n\n")
|
||||
restart = gate.get('restart_checks')
|
||||
if restart:
|
||||
out.write(f"Owned-service restart checks: {restart.get('outcome', 'not reported')}; {restart.get('elapsed_seconds', 0):.2f}s, deducted from the shared budget. Resource baselines begin after restarts.\n\n")
|
||||
for error in restart.get('errors', []):
|
||||
out.write(f"Restart check error: {error}\n")
|
||||
out.write(f"Plugin selection: `{manifest.get('selection', 'not reported')}`.\n\n")
|
||||
for plugin in manifest.get('plugins', []):
|
||||
out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
|
||||
out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n")
|
||||
out.write(f"Final ledger reconciliation: {suite.get('ledger_closeout', {}).get('matched', 'not reported')}.\n\n")
|
||||
data = suite.get('test_data', {})
|
||||
out.write(f"Observed conversations: {data.get('observed_conversations', 'not reported')}; AgentRuns: {data.get('observed_agent_runs', 'not reported')}; group sessions: {data.get('group_sessions', 'not reported')}; owned API resources: {data.get('owned_api_resources', 'not reported')}.\n\n")
|
||||
out.write('One Workspace; each pipeline reuses one Box per CI run. This does not test cross-group Box file isolation or tenant isolation.\n\n')
|
||||
observations = suite.get('observations', {})
|
||||
if 'successful_requests_per_second' in observations:
|
||||
out.write(f"Successful throughput: {observations['successful_requests_per_second']:.3f} RPS.\n\n")
|
||||
if observations.get('step_p95_seconds'):
|
||||
out.write('| Case | Full-step P95 (s) |\n|---|---:|\n')
|
||||
for name, p95 in observations['step_p95_seconds'].items():
|
||||
out.write(f"| {name} | {p95} |\n")
|
||||
for kind, values in coverage.get('kinds', {}).items():
|
||||
out.write(f"\n{kind} behavior evidence: {values['decided']} / {values['total']}; statuses: {values['status']}.\n")
|
||||
if suite.get('resources', {}).get('targets'):
|
||||
out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n')
|
||||
for target, metrics in suite['resources']['targets'].items():
|
||||
for metric, values in metrics.items():
|
||||
out.write(f"| {target} | {metric} | {values.get('growth', 'unavailable')} | {values.get('allowance', 'unavailable')} |\n")
|
||||
errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', []))
|
||||
if gate.get('timed_out'):
|
||||
errors.append('Suite exceeded its 1860-second outer process guard.')
|
||||
if gate.get('infrastructure_error'):
|
||||
errors.append(gate['infrastructure_error'])
|
||||
if not gate or not suite:
|
||||
errors.append('Missing verdict; inspect preparation and service logs.')
|
||||
for error in errors:
|
||||
out.write(f"\n- {error}\n")
|
||||
PY_SUMMARY
|
||||
- name: Preserve verdicts and observations
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: daily-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
.qa/daily/gate/gate.json
|
||||
.qa/daily/gate/resources.jsonl
|
||||
.qa/daily/gate/suite.log
|
||||
.qa/daily/gate/suite/summary.json
|
||||
.qa/daily/gate/suite/cases.json
|
||||
.qa/daily/gate/suite/health.json
|
||||
.qa/daily/gate/suite/cleanup-checks.json
|
||||
.qa/daily/gate/suite/resources.jsonl
|
||||
.qa/daily/gate/suite/cycles.json
|
||||
.qa/daily/gate/suite/execution.json
|
||||
.qa/daily/gate/suite/coverage.json
|
||||
.qa/daily/gate/suite/coverage.md
|
||||
.qa/daily/plugins/manifest.json
|
||||
.qa/daily/plugins/*-build.log
|
||||
.qa/daily/restart-checks/profile.json
|
||||
.qa/daily/restart-checks/cases.json
|
||||
.qa/daily/restart-checks/summary.json
|
||||
.qa/daily/restart-checks/coverage.json
|
||||
.qa/daily/restart-checks/coverage.md
|
||||
.qa/daily/restart-checks/cleanup-checks.json
|
||||
.qa/daily/restart-checks/owned.json
|
||||
.qa/daily/*.log
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
Reference in New Issue
Block a user