From 833c1feece43f12fd47e911d8673d7bd4fc68ae9 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Wed, 28 Oct 2026 12:26:30 +0800 Subject: [PATCH] ci(regression): consume published plugins and shared daily scenarios --- .github/workflows/core-smoke.yml | 32 +++--- .github/workflows/daily-endurance.yml | 106 +++++++++--------- tools/README.md | 69 ++++++++---- tools/regression_ci.py | 49 ++++++-- tools/regression_profiles/endurance.json | 2 +- tools/test_regression_ci.py | 135 +++++++++++++++++++---- 6 files changed, 268 insertions(+), 125 deletions(-) diff --git a/.github/workflows/core-smoke.yml b/.github/workflows/core-smoke.yml index 722fb3ae7..cafbba15e 100644 --- a/.github/workflows/core-smoke.yml +++ b/.github/workflows/core-smoke.yml @@ -46,22 +46,6 @@ jobs: enable-cache: true - name: Install locked dependencies run: uv sync --dev --frozen - - name: Read official plugin source pin - id: plugins - run: | - python - <<'PY' - import json, os - from pathlib import Path - revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision'] - with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: - stream.write(f'revision={revision}\n') - PY - - uses: actions/checkout@v4 - with: - repository: langbot-app/langbot-plugins - ref: ${{ steps.plugins.outputs.revision }} - path: .smoke-plugins - persist-credentials: false - name: Prepare CLI and sandbox image env: GH_TOKEN: ${{ github.token }} @@ -76,21 +60,28 @@ jobs: - name: Prepare real services and run the 60-second smoke run: | uv run --no-sync python tools/regression_ci.py \ - --lbctl .smoke-bin/lbctl_linux_amd64 --plugins .smoke-plugins --output .qa/core-smoke + --lbctl .smoke-bin/lbctl_linux_amd64 --output .qa/core-smoke - name: Publish gate result if: always() run: | python - <<'PY' import json, os from pathlib import Path - path = Path('.qa/core-smoke/gate/gate.json') + root = Path('.qa/core-smoke') + path = root / 'gate/gate.json' + manifest = root / 'plugins/manifest.json' with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: + if manifest.exists(): + for plugin in json.loads(manifest.read_text())['plugins']: + out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") if path.exists(): data = json.loads(path.read_text()) out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; " f"{data.get('elapsed_seconds', 0):.2f}/60 seconds; {data.get('requests', 0)} requests.\n") if data.get('infrastructure_error'): out.write('Preparation or teardown failed; inspect service logs.\n') + for error in data.get('coverage_errors', []): + out.write(f"Coverage evidence error: {error}\n") out.write('Service/dependency preparation is outside the 60-second execution budget.\n') else: out.write('Core smoke did not produce a verdict; inspect preparation failure.\n') @@ -108,6 +99,11 @@ jobs: .qa/core-smoke/gate/suite/summary.json .qa/core-smoke/gate/suite/health.json .qa/core-smoke/gate/suite/cleanup-checks.json + .qa/core-smoke/gate/suite/coverage.json + .qa/core-smoke/gate/suite/coverage.md + .qa/core-smoke/gate/suite/cycles.json + .qa/core-smoke/gate/suite/execution.json + .qa/core-smoke/plugins/manifest.json .qa/core-smoke/*.log if-no-files-found: warn retention-days: 7 diff --git a/.github/workflows/daily-endurance.yml b/.github/workflows/daily-endurance.yml index 20c888927..ada43e53a 100644 --- a/.github/workflows/daily-endurance.yml +++ b/.github/workflows/daily-endurance.yml @@ -15,7 +15,7 @@ concurrency: jobs: endurance: - name: Daily endurance (30m) + name: Daily regression (30m) runs-on: ubuntu-latest timeout-minutes: 45 steps: @@ -30,22 +30,6 @@ jobs: enable-cache: true - name: Install locked dependencies run: uv sync --dev --frozen - - name: Read official plugin source pin - id: plugins - run: | - python - <<'PY' - import json, os - from pathlib import Path - revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision'] - with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: - stream.write(f'revision={revision}\n') - PY - - uses: actions/checkout@v4 - with: - repository: langbot-app/langbot-plugins - ref: ${{ steps.plugins.outputs.revision }} - path: .endurance-plugins - persist-credentials: false - name: Prepare CLI and sandbox image env: GH_TOKEN: ${{ github.token }} @@ -57,41 +41,57 @@ jobs: sha256sum --check --ignore-missing checksums.txt chmod +x lbctl_linux_amd64 docker pull python:3.12-alpine - - name: Run sustained traffic and recovery checks + - name: Run repeated scenarios and recovery checks run: | uv run --no-sync python tools/regression_ci.py \ - --suite endurance --lbctl .endurance-bin/lbctl_linux_amd64 \ - --plugins .endurance-plugins --output .qa/endurance + --suite daily --lbctl .endurance-bin/lbctl_linux_amd64 --output .qa/daily - name: Publish observations if: always() - env: - PLUGIN_REVISION: ${{ steps.plugins.outputs.revision }} run: | python - <<'PY_SUMMARY' import json, os from pathlib import Path - root = Path('.qa/endurance/gate') - gate = json.loads((root / 'gate.json').read_text()) if (root / 'gate.json').exists() else {} - suite = json.loads((root / 'suite/summary.json').read_text()) if (root / 'suite/summary.json').exists() else {} + root = Path('.qa/daily') + def read(path): + source = root / path + return json.loads(source.read_text()) if source.exists() else {} + + gate = read('gate/gate.json') + suite = read('gate/suite/summary.json') + manifest = read('plugins/manifest.json') + coverage = read('gate/suite/coverage.json') + passed = gate.get('passed') is True and suite.get('passed') is True + outer_failed = any(gate.get(key) for key in ('infrastructure_error', 'timed_out', 'resource_errors', 'coverage_errors')) + outcome = 'PASS' if passed else 'INCOMPLETE' if suite.get('outcome') == 'INCOMPLETE' and not outer_failed else 'FAIL / incomplete' with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: - out.write(f"Endurance: {'PASS' if gate.get('passed') is True else 'FAIL / incomplete'}\n\n") - out.write(f"Core: `{os.environ['GITHUB_SHA']}`; plugins: `{os.environ['PLUGIN_REVISION']}`.\n\n") - out.write('1740s offered load + drain + 60s cooldown. Real Core/Runtime/plugins/Box; mock generation and embedding.\n\n') - out.write('Observational run, not a required PR check. Budgets are initial QA limits, not product SLOs. Hosted runners are not a controlled performance baseline.\n\n') - for phase in suite.get('phases', []): - out.write(f"Successful throughput: {phase['achieved_rps']:.3f} RPS.\n\n") - out.write(f"Completed runs: {phase.get('verified_runs')} / {phase.get('runner_requests')}; ledger reconciliation: {phase.get('ledger_closeout', {}).get('matched')}.\n\n") - out.write('| Case | Requests | Status | P95 (s) |\n|---|---:|---|---:|\n') - for name, case in phase['cases'].items(): - out.write(f"| {name} | {case.get('requests', '?')} | {case.get('status', {})} | {case.get('p95_seconds')} |\n") + out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n") + out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n') + out.write('Real Core/Runtime/published plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n') + for plugin in manifest.get('plugins', []): + out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") + out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n") + out.write(f"Final ledger reconciliation: {suite.get('ledger_closeout', {}).get('matched', 'not reported')}.\n\n") + data = suite.get('test_data', {}) + out.write(f"Observed conversations: {data.get('observed_conversations', 'not reported')}; AgentRuns: {data.get('observed_agent_runs', 'not reported')}; group sessions: {data.get('group_sessions', 'not reported')}; owned API resources: {data.get('owned_api_resources', 'not reported')}.\n\n") + out.write('One Workspace; each pipeline reuses one Box per CI run. This does not test cross-group Box file isolation or tenant isolation.\n\n') + observations = suite.get('observations', {}) + if 'successful_requests_per_second' in observations: + out.write(f"Successful throughput: {observations['successful_requests_per_second']:.3f} RPS.\n\n") + if observations.get('step_p95_seconds'): + out.write('| Case | Full-step P95 (s) |\n|---|---:|\n') + for name, p95 in observations['step_p95_seconds'].items(): + out.write(f"| {name} | {p95} |\n") + for kind, values in coverage.get('kinds', {}).items(): + out.write(f"\n{kind} behavior evidence: {values['decided']} / {values['total']}; statuses: {values['status']}.\n") + if suite.get('resources', {}).get('targets'): out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n') - for target, metrics in phase.get('resources', {}).get('targets', {}).items(): + for target, metrics in suite['resources']['targets'].items(): for metric, values in metrics.items(): - out.write(f"| {target} | {metric} | {values['growth']} | {values['allowance']} |\n") - errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + out.write(f"| {target} | {metric} | {values.get('growth', 'unavailable')} | {values.get('allowance', 'unavailable')} |\n") + errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', [])) if gate.get('timed_out'): - errors.append('Suite exceeded its 2100-second execution budget.') + errors.append('Suite exceeded its 1860-second outer process guard.') if gate.get('infrastructure_error'): errors.append(gate['infrastructure_error']) if not gate or not suite: @@ -103,19 +103,21 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: endurance-${{ github.run_id }}-${{ github.run_attempt }} + name: daily-${{ github.run_id }}-${{ github.run_attempt }} path: | - .qa/endurance/gate/gate.json - .qa/endurance/gate/resources.jsonl - .qa/endurance/gate/suite.log - .qa/endurance/gate/suite/summary.json - .qa/endurance/gate/suite/workload.json - .qa/endurance/gate/suite/health.json - .qa/endurance/gate/suite/cleanup-checks.json - .qa/endurance/gate/suite/sustained.log - .qa/endurance/gate/suite/sustained/summary.json - .qa/endurance/gate/suite/sustained/requests.jsonl - .qa/endurance/gate/suite/sustained/resources.jsonl - .qa/endurance/*.log + .qa/daily/gate/gate.json + .qa/daily/gate/resources.jsonl + .qa/daily/gate/suite.log + .qa/daily/gate/suite/summary.json + .qa/daily/gate/suite/cases.json + .qa/daily/gate/suite/health.json + .qa/daily/gate/suite/cleanup-checks.json + .qa/daily/gate/suite/resources.jsonl + .qa/daily/gate/suite/cycles.json + .qa/daily/gate/suite/execution.json + .qa/daily/gate/suite/coverage.json + .qa/daily/gate/suite/coverage.md + .qa/daily/plugins/manifest.json + .qa/daily/*.log if-no-files-found: warn retention-days: 30 diff --git a/tools/README.md b/tools/README.md index 236ede5dc..efff0880e 100644 --- a/tools/README.md +++ b/tools/README.md @@ -367,9 +367,13 @@ phases. The dedicated Docker daemon requirement above still applies. `Core Smoke` runs regression-tool tests and a real Core smoke in parallel on GitHub-hosted Linux runners. The smoke starts Core from the checked-out candidate, -uses the SDK from `uv.lock`, the existing official-plugin source pin, `lbctl` +uses the SDK from `uv.lock`, the latest published official plugins, `lbctl` v0.3.0, a Docker Box and mock generation/embedding. It needs no model credentials or access to a developer's running environment. +Plugin versions are resolved once per run, installed from the downloaded release +archives and recorded with hashes and installation readback in `plugins/manifest.json`. +An unavailable or incompatible published package fails preparation; see +[published plugin preparation and replay](published-plugins.md). Dependency installation, service startup, sandbox image download and plugin installation are preparation. After readiness, the gate allows **60 seconds total** @@ -396,13 +400,12 @@ inspect its retained ownership journal before cleanup. CI tears down its own service processes and owned sandboxes afterward. The workflow always creates its two checks on PRs and cancels superseded runs. -After the workflow is merged, configure `Regression tools` and `Core smoke (60s)` -as required status checks in the repository ruleset. A green workflow alone does -not enforce merge protection. Longer load and soak tests use the existing Suite -and Job entry points with separately calibrated latency, throughput and resource -budgets; the one-minute smoke does not replace them. +The existing required check names remain `Regression tools` and `Core smoke (60s)`. +`Daily Endurance` is independent and is not required for merging. Longer load and +soak tests use the existing Suite and Job entry points with separately calibrated +latency, throughput and resource budgets; the one-minute smoke does not replace them. -## Sustained regression and baseline comparison +## Daily regression `Daily Endurance` runs independently of PR checks at 19:23 UTC (03:23 Asia/Shanghai) on the default branch. GitHub may delay scheduled starts. It also @@ -410,30 +413,50 @@ supports **Run workflow** with a selected branch. The schedule becomes active only after the workflow is merged into the default branch; it is not a required merge check. -The workflow reuses the disposable smoke stack, locked SDK, pinned official -plugins and lbctl v0.3.0. Generation and embedding are deterministic mocks. -It runs the endurance profile below, with a 35-minute execution budget for -resource preparation, load, drain, cooldown and API cleanup. Service preparation -is outside that budget; the whole job has a 45-minute limit. Service processes -and owned Box containers are stopped afterward, including after a failed suite. +The workflow reuses the disposable smoke stack and latest published plugins. +`--suite daily` consumes the shared 46-case profile, requires its first pass, then +repeats ordered cases within one **1800-second** window. The clock starts before +creating scenario resources and includes a 120-second reserve for drain/API +cleanup and a 60-second cooldown. Service and plugin preparation precede this +clock. A separate 1860-second process guard allows startup and final reporting; +it does not grant extra scenario time. The whole job retains its 45-minute limit. -Actions summaries show throughput, per-case P95 and retained resource growth. -Verdicts, individual requests, resource samples and service logs are retained -for 30 days, including failures. Initial QA budget violations make this -observational job red without blocking PRs. GitHub-hosted runners vary between -runs: these reports do not establish a controlled baseline or prove a performance -regression. No automatic cross-run performance comparison is enabled. +The daily profile uses the existing endurance resource allowances with a +180-second warmup before baseline sampling. Each pipeline that enables Box uses +one stable reuse key unique to this CI run, so conversations created by repeated +cycles reuse that pipeline's sandbox. This measures repeated tool use; it does not test +cross-group Box file isolation. The profile uses one Workspace and does not +establish isolation between tenants. Conversation history and AgentRuns still +accumulate and are reported. Resource excesses remain failures. + +Throughput and full-step P95 are observations. Business assertions, unknown +results, health failures, incomplete evidence and resource limits determine the +verdict. Summaries include first-pass completion, cycles, plugin installation +readback, final ledger reconciliation and coverage. Manifests, case results, +coverage, cycles, execution stages, resource samples and service logs are retained +for 30 days, including failures. Hosted runners do not provide a controlled +performance baseline. No automatic comparison between runs is enabled. To run the same entrypoint on a disposable Linux host with an idle Docker daemon: ```sh -uv run --no-sync python tools/regression_ci.py --suite endurance \ - --lbctl /path/to/lbctl --plugins /path/to/pinned/langbot-plugins \ - --output .qa/endurance-001 +uv run --no-sync python tools/regression_ci.py --suite daily \ + --lbctl /path/to/lbctl --output .qa/daily-001 ``` +`--suite smoke` retains the 60-second gate. `--suite scenarios` runs the shared +expanded profile once under a 300-second guard. `--suite endurance` retains the +separate ten-minute load profile under a 900-second guard; its CI entry point also +reports throughput/P95 without applying their thresholds. All four modes use +published plugins by default. Explicit `--plugins` remains available for source +development; neither workflow selects it. The suite attempts owned API cleanup +even after failures. The CI owner then stops its service processes and removes +its Box containers; a process guard timeout can interrupt API cleanup. + +## Sustained regression and baseline comparison + [`regression_profiles/endurance.json`](regression_profiles/endurance.json) runs -29 minutes of mixed chat/Box/RAG traffic at 1 request/second across four stable +nine minutes of mixed chat/Box/RAG traffic at 1 request/second across four stable groups, followed by one minute without traffic to check recovery. Configure its CLI context, plugin inputs, model endpoints, Core SQLite path and Core/Runtime/Box PID files as for smoke. Set `plugins: []` to reuse initialized plugins. Configure diff --git a/tools/regression_ci.py b/tools/regression_ci.py index a50706480..26661f067 100644 --- a/tools/regression_ci.py +++ b/tools/regression_ci.py @@ -15,15 +15,45 @@ import time import yaml from regression_gate import supervise +from regression_coverage import write_report from regression_plugins import install_published_plugins, prepare_published_plugins from regression_suite import Suite from regression_traffic import owned_box_containers, read_endpoint_snapshot ROOT = Path(__file__).resolve().parents[1] +SUITE_BUDGETS = {'smoke': 60, 'scenarios': 300, 'daily': 1860, 'endurance': 900} sys.path.insert(0, str(ROOT)) from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402 +def suite_profile(name, run_id): + profile_path = ROOT / 'tools/regression_profiles' / ('endurance.json' if name == 'endurance' else 'smoke.json') + if name in ('scenarios', 'daily'): + from regression_scenarios import expanded_profile + + profile = expanded_profile() + else: + profile = json.loads(profile_path.read_text()) + if name == 'smoke': + profile['smoke']['group_turns'] = 2 + if name == 'daily': + limits = json.loads((profile_path.parent / 'endurance.json').read_text())['resource_limits'] + limits['warmup_seconds'] = 180 + profile.update( + execution={'total_seconds': 1800, 'drain_seconds': 120, 'cooldown_seconds': 60}, + resource_limits=limits, + ) + for resource in profile['resources']: + if resource['kind'] != 'pipeline': + continue + for config in resource['body']['config']['ai']['runner_config'].values(): + if config.get('box-enabled'): + # This workload measures repeated tool use; each pipeline owns one sandbox per CI run. + config['box-session-id-template'] = f'qa-{run_id}-{resource["id"]}' + profile.update(cleanup='always', performance_mode='observe') + return profile, profile_path + + def ready(url, processes, timeout=120): deadline = time.monotonic() + timeout while time.monotonic() < deadline: @@ -82,8 +112,7 @@ def run(args): host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine' ) config_path.write_text(yaml.safe_dump(config)) - profile_path = ROOT / f'tools/regression_profiles/{args.suite}.json' - profile = json.loads(profile_path.read_text()) + profile, profile_path = suite_profile(args.suite, secrets.token_hex(8)) cli_config = private / 'lbctl.yaml' cli_config.write_text( yaml.safe_dump( @@ -109,9 +138,6 @@ def run(args): mock_records=str(root / 'mock-requests.jsonl'), ) profile['plugins'] = [] - if args.suite == 'smoke': - # Two turns retain the history/isolation oracle within the 60-second budget. - profile['smoke']['group_turns'] = 2 for resource in profile['resources']: if resource['kind'] == 'model' and resource.get('type') == 'embedding': resource['body']['name'] = 'mock-embedding' @@ -166,7 +192,7 @@ def run(args): }, 'pids': pids, } - if args.suite == 'endurance': + if args.suite in ('daily', 'endurance'): profile['observe']['box_data_root'] = str(private / 'box') for url in profile['observe']['health'].values(): ready(url, processes) @@ -189,7 +215,7 @@ def run(args): suite.call(['plugin', 'install', 'local', '--file', archive, '--wait']) generated = private / f'{args.suite}.json' generated.write_text(json.dumps(profile, indent=2)) - return supervise( + result = supervise( [ sys.executable, str(ROOT / 'tools/regression_suite.py'), @@ -200,10 +226,15 @@ def run(args): str(root / 'gate/suite'), ], root / 'gate', - 60 if args.suite == 'smoke' else 2100, + SUITE_BUDGETS[args.suite], pids, env, ) + coverage = write_report([root / 'gate/suite'], root / 'gate/suite', http_logs=[root / 'core.log']) + result['coverage_errors'] = coverage['evidence_errors'] + result['passed'] = result['passed'] and not result['coverage_errors'] + (root / 'gate/gate.json').write_text(json.dumps(result, indent=2) + '\n') + return result finally: for process in reversed(processes): if process.poll() is None: @@ -227,7 +258,7 @@ def main(): plugins = parser.add_mutually_exclusive_group() plugins.add_argument('--plugins', type=Path, help='Explicit developer override: package a plugin source checkout') plugins.add_argument('--plugin-manifest', type=Path, help='Replay published versions and hashes from a prior run') - parser.add_argument('--suite', choices=('smoke', 'endurance'), default='smoke') + parser.add_argument('--suite', choices=tuple(SUITE_BUDGETS), default='smoke') args = parser.parse_args() if args.output.exists(): parser.error('--output must be a new directory') diff --git a/tools/regression_profiles/endurance.json b/tools/regression_profiles/endurance.json index 6ba9ce173..5b87ddea7 100644 --- a/tools/regression_profiles/endurance.json +++ b/tools/regression_profiles/endurance.json @@ -444,7 +444,7 @@ "phases": [ { "name": "sustained", - "duration_seconds": 1740, + "duration_seconds": 540, "rps": 1, "workers": 4, "cooldown_seconds": 60, diff --git a/tools/test_regression_ci.py b/tools/test_regression_ci.py index cfbbaface..5cb929cb9 100644 --- a/tools/test_regression_ci.py +++ b/tools/test_regression_ci.py @@ -10,6 +10,59 @@ import unittest import yaml +from regression_ci import SUITE_BUDGETS, suite_profile +from regression_suite import validate + + +class CIProfileTest(unittest.TestCase): + def test_suite_selection_uses_shared_cases_and_one_daily_budget(self): + for name in ('smoke', 'scenarios', 'daily', 'endurance'): + with self.subTest(suite=name): + profile, path = suite_profile(name, 'test-run') + profile['observe'] = { + 'health': {'core': 'http://localhost/healthz'}, + 'pids': {'core': 1, 'runtime': 2, 'box': 3}, + 'box_data_root': '/tmp/test-box', + } + validate(profile) + self.assertTrue(path.exists()) + self.assertEqual(profile['cleanup'], 'always') + self.assertEqual(profile['performance_mode'], 'observe') + if name == 'smoke': + self.assertEqual(profile['smoke']['group_turns'], 2) + self.assertEqual(SUITE_BUDGETS[name], 60) + elif name in ('scenarios', 'daily'): + self.assertEqual(len(profile['cases']), 46) + self.assertEqual(profile['coverage_suite'], 'scenarios') + self.assertEqual(profile['smoke']['group_turns'], 3) + else: + self.assertEqual(profile['phases'][0]['duration_seconds'], 540) + self.assertEqual('execution' in profile, name == 'daily') + daily, _ = suite_profile('daily', 'test-run') + self.assertEqual(daily['execution'], {'total_seconds': 1800, 'drain_seconds': 120, 'cooldown_seconds': 60}) + endurance, _ = suite_profile('endurance', 'test-run') + self.assertEqual(daily['resource_limits']['warmup_seconds'], 180) + self.assertEqual(daily['resource_limits']['targets'], endurance['resource_limits']['targets']) + self.assertEqual(daily['resource_limits']['containers'], endurance['resource_limits']['containers']) + + def test_daily_reuses_boxes_per_pipeline_and_separates_runs(self): + def keys(name, run_id): + profile, _ = suite_profile(name, run_id) + return { + resource['id']: runner['box-session-id-template'] + for resource in profile['resources'] + if resource['kind'] == 'pipeline' + for runner in resource['body']['config']['ai']['runner_config'].values() + if runner.get('box-enabled') + } + + first, second = keys('daily', 'run-a'), keys('daily', 'run-b') + self.assertTrue(first) + self.assertEqual(len(set(first.values())), len(first)) + self.assertFalse(set(first.values()) & set(second.values())) + self.assertEqual(first, keys('daily', 'run-a')) + self.assertTrue(all(value == '{launcher_type}_{launcher_id}' for value in keys('scenarios', 'run-a').values())) + class DailySummaryTest(unittest.TestCase): def test_summary_reports_observations_without_hiding_outer_failures(self): @@ -18,47 +71,85 @@ class DailySummaryTest(unittest.TestCase): command = next(step['run'] for step in steps if step.get('name') == 'Publish observations') suite = { 'passed': True, + 'outcome': 'PASS', 'errors': [], - 'phases': [ - { - 'achieved_rps': 1.0, - 'verified_runs': 540, - 'runner_requests': 540, - 'ledger_closeout': {'matched': True}, - 'cases': {'chat': {'requests': 270, 'status': {'ok': 270}, 'p95_seconds': 0.5}}, - } - ], + 'first_pass_complete': True, + 'cycles': [{'complete': True}], + 'requests': 86, + 'ledger_closeout': {'matched': True}, + 'observations': {'successful_requests_per_second': 1.0, 'step_p95_seconds': {'chat': 0.5}}, } - for gate, expected in [ - ({'passed': True}, 'Endurance: PASS'), - ({'passed': False, 'infrastructure_error': 'teardown failed'}, 'teardown failed'), - ({'passed': False, 'timed_out': True}, '2100-second execution budget'), - (None, 'Missing verdict'), + for gate, actual_suite, expected in [ + ({'passed': True}, suite, 'Daily regression: PASS'), + ({'passed': False, 'infrastructure_error': 'teardown failed'}, suite, 'teardown failed'), + ({'passed': False, 'infrastructure_error': 'published plugin install failed'}, None, 'not confirmed'), + ({'passed': False, 'timed_out': True}, suite, '1860-second outer process guard'), + ({'passed': False, 'coverage_errors': ['missing access records']}, suite, 'missing access records'), + ({'passed': False}, {**suite, 'passed': False, 'outcome': 'INCOMPLETE'}, 'Daily regression: INCOMPLETE'), + (None, None, 'Missing verdict'), ]: with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory: root = Path(directory) + target = root / '.qa/daily' + (target / 'plugins').mkdir(parents=True) + (target / 'plugins/manifest.json').write_text( + json.dumps({'plugins': [{'id': 'langbot-team/LocalAgent', 'version': '1.2.3'}]}) + ) if gate is not None: - target = root / '.qa/endurance/gate' - (target / 'suite').mkdir(parents=True) - (target / 'gate.json').write_text(json.dumps(gate)) - (target / 'suite/summary.json').write_text(json.dumps(suite)) + (target / 'gate/suite').mkdir(parents=True) + (target / 'gate/gate.json').write_text(json.dumps(gate)) + if actual_suite is not None: + (target / 'gate/suite/summary.json').write_text(json.dumps(actual_suite)) summary = root / 'summary.md' env = { **os.environ, 'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'], 'GITHUB_STEP_SUMMARY': str(summary), 'GITHUB_SHA': 'core-revision', - 'PLUGIN_REVISION': 'plugin-revision', } subprocess.run(['bash', '-e', '-c', command], cwd=root, env=env, check=True, capture_output=True) text = summary.read_text() self.assertIn(expected, text) + self.assertIn('langbot-team/LocalAgent@1.2.3', text) + self.assertIn('not confirmed', text) if gate and gate.get('passed'): - self.assertIn('Completed runs: 540 / 540', text) + self.assertIn('Final ledger reconciliation: True', text) self.assertIn('1.000 RPS', text) else: - self.assertIn('Endurance: FAIL / incomplete', text) - self.assertNotIn('Endurance: PASS', text) + self.assertNotIn('Daily regression: PASS', text) + + def test_core_summary_records_preparation_failure_and_selected_plugins(self): + workflow = Path(__file__).resolve().parents[1] / '.github/workflows/core-smoke.yml' + steps = yaml.safe_load(workflow.read_text())['jobs']['smoke']['steps'] + command = next(step['run'] for step in steps if step.get('name') == 'Publish gate result') + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / '.qa/core-smoke' + (target / 'plugins').mkdir(parents=True) + (target / 'gate').mkdir() + (target / 'plugins/manifest.json').write_text( + json.dumps({'plugins': [{'id': 'langbot-team/LocalAgent', 'version': '1.2.3'}]}) + ) + (target / 'gate/gate.json').write_text( + json.dumps({'passed': False, 'infrastructure_error': 'incompatible'}) + ) + summary = root / 'summary.md' + subprocess.run( + ['bash', '-e', '-c', command], + cwd=root, + check=True, + capture_output=True, + env={ + **os.environ, + 'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'], + 'GITHUB_STEP_SUMMARY': str(summary), + }, + ) + text = summary.read_text() + self.assertIn('Core smoke: FAIL', text) + self.assertIn('Preparation or teardown failed', text) + self.assertIn('langbot-team/LocalAgent@1.2.3', text) + self.assertIn('not confirmed', text) if __name__ == '__main__':