From f684d7f3f105c35c9d97e5a95522ca88ecf288b1 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Wed, 28 Oct 2026 19:59:24 +0800 Subject: [PATCH] ci(regression): run daily functional cycles with selectable scenarios --- .github/workflows/core-smoke.yml | 14 +- .github/workflows/daily-endurance.yml | 152 +++++++++++++++++++++ tools/README.md | 99 +++++++++++++- tools/regression_ci.py | 153 ++++++++++++++++++++-- tools/test_regression_ci.py | 181 ++++++++++++++++++++++++++ 5 files changed, 582 insertions(+), 17 deletions(-) create mode 100644 .github/workflows/daily-endurance.yml create mode 100644 tools/test_regression_ci.py diff --git a/.github/workflows/core-smoke.yml b/.github/workflows/core-smoke.yml index 76a66256d..a3169692b 100644 --- a/.github/workflows/core-smoke.yml +++ b/.github/workflows/core-smoke.yml @@ -67,14 +67,21 @@ jobs: python - <<'PY' import json, os from pathlib import Path - path = Path('.qa/core-smoke/gate/gate.json') + root = Path('.qa/core-smoke') + path = root / 'gate/gate.json' + manifest = root / 'plugins/manifest.json' with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: + if manifest.exists(): + for plugin in json.loads(manifest.read_text())['plugins']: + out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") if path.exists(): data = json.loads(path.read_text()) out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; " f"{data.get('elapsed_seconds', 0):.2f}/60 seconds; {data.get('requests', 0)} requests.\n") if data.get('infrastructure_error'): out.write('Preparation or teardown failed; inspect service logs.\n') + for error in data.get('coverage_errors', []): + out.write(f"Coverage evidence error: {error}\n") out.write('Service/dependency preparation is outside the 60-second execution budget.\n') else: out.write('Core smoke did not produce a verdict; inspect preparation failure.\n') @@ -92,6 +99,11 @@ jobs: .qa/core-smoke/gate/suite/summary.json .qa/core-smoke/gate/suite/health.json .qa/core-smoke/gate/suite/cleanup-checks.json + .qa/core-smoke/gate/suite/coverage.json + .qa/core-smoke/gate/suite/coverage.md + .qa/core-smoke/gate/suite/cycles.json + .qa/core-smoke/gate/suite/execution.json + .qa/core-smoke/plugins/manifest.json .qa/core-smoke/*.log if-no-files-found: warn retention-days: 7 diff --git a/.github/workflows/daily-endurance.yml b/.github/workflows/daily-endurance.yml new file mode 100644 index 000000000..d6dc3eb41 --- /dev/null +++ b/.github/workflows/daily-endurance.yml @@ -0,0 +1,152 @@ +name: Daily Endurance + +on: + schedule: + # 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late. + - cron: '23 19 * * *' + workflow_dispatch: + inputs: + scenario_groups: + description: 'Optional local groups, space separated; empty runs only base cases' + type: string + default: 'faults resources lifecycle' + restart_services: + description: 'Check owned Core and Runtime restarts once before resource sampling' + type: boolean + default: true + +permissions: + contents: read + +concurrency: + group: daily-endurance-${{ github.ref }} + cancel-in-progress: false + +jobs: + endurance: + name: Daily regression (30m) + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Install locked dependencies + run: uv sync --dev --frozen + - name: Prepare CLI and sandbox image + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p .endurance-bin + gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .endurance-bin \ + --pattern lbctl_linux_amd64 --pattern checksums.txt + cd .endurance-bin + sha256sum --check --ignore-missing checksums.txt + chmod +x lbctl_linux_amd64 + docker pull python:3.12-alpine + - name: Run repeated scenarios and recovery checks + env: + SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle' || inputs.scenario_groups }} + RESTART_SERVICES: ${{ github.event_name == 'schedule' || inputs.restart_services }} + run: | + read -r -a groups <<< "$SCENARIO_GROUPS" + args=() + if [[ "$RESTART_SERVICES" == true ]]; then args+=(--restart-services); fi + uv run --no-sync python tools/regression_ci.py \ + --suite daily --scenario-groups "${groups[@]}" "${args[@]}" \ + --lbctl .endurance-bin/lbctl_linux_amd64 --output .qa/daily + - name: Publish observations + if: always() + run: | + python - <<'PY_SUMMARY' + import json, os + from pathlib import Path + + root = Path('.qa/daily') + def read(path): + source = root / path + return json.loads(source.read_text()) if source.exists() else {} + + gate = read('gate/gate.json') + suite = read('gate/suite/summary.json') + manifest = read('plugins/manifest.json') + coverage = read('gate/suite/coverage.json') + passed = gate.get('passed') is True and suite.get('passed') is True + outer_failed = any(gate.get(key) for key in ('infrastructure_error', 'timed_out', 'resource_errors', 'coverage_errors')) + outcome = 'PASS' if passed else 'INCOMPLETE' if suite.get('outcome') == 'INCOMPLETE' and not outer_failed else 'FAIL / incomplete' + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: + out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n") + out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n') + out.write('Real Core/Runtime/published plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n') + out.write(f"Enabled local groups: {gate.get('scenario_groups', 'not reported')}.\n\n") + restart = gate.get('restart_checks') + if restart: + out.write(f"Owned-service restart checks: {restart.get('outcome', 'not reported')}; {restart.get('elapsed_seconds', 0):.2f}s, deducted from the shared budget. Resource baselines begin after restarts.\n\n") + for error in restart.get('errors', []): + out.write(f"Restart check error: {error}\n") + for plugin in manifest.get('plugins', []): + out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") + out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n") + out.write(f"Final ledger reconciliation: {suite.get('ledger_closeout', {}).get('matched', 'not reported')}.\n\n") + data = suite.get('test_data', {}) + out.write(f"Observed conversations: {data.get('observed_conversations', 'not reported')}; AgentRuns: {data.get('observed_agent_runs', 'not reported')}; group sessions: {data.get('group_sessions', 'not reported')}; owned API resources: {data.get('owned_api_resources', 'not reported')}.\n\n") + out.write('One Workspace; each pipeline reuses one Box per CI run. This does not test cross-group Box file isolation or tenant isolation.\n\n') + observations = suite.get('observations', {}) + if 'successful_requests_per_second' in observations: + out.write(f"Successful throughput: {observations['successful_requests_per_second']:.3f} RPS.\n\n") + if observations.get('step_p95_seconds'): + out.write('| Case | Full-step P95 (s) |\n|---|---:|\n') + for name, p95 in observations['step_p95_seconds'].items(): + out.write(f"| {name} | {p95} |\n") + for kind, values in coverage.get('kinds', {}).items(): + out.write(f"\n{kind} behavior evidence: {values['decided']} / {values['total']}; statuses: {values['status']}.\n") + if suite.get('resources', {}).get('targets'): + out.write('\n| Resource | Metric | Retained growth | Allowance |\n|---|---|---:|---:|\n') + for target, metrics in suite['resources']['targets'].items(): + for metric, values in metrics.items(): + out.write(f"| {target} | {metric} | {values.get('growth', 'unavailable')} | {values.get('allowance', 'unavailable')} |\n") + errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', [])) + if gate.get('timed_out'): + errors.append('Suite exceeded its 1860-second outer process guard.') + if gate.get('infrastructure_error'): + errors.append(gate['infrastructure_error']) + if not gate or not suite: + errors.append('Missing verdict; inspect preparation and service logs.') + for error in errors: + out.write(f"\n- {error}\n") + PY_SUMMARY + - name: Preserve verdicts and observations + if: always() + uses: actions/upload-artifact@v4 + with: + name: daily-${{ github.run_id }}-${{ github.run_attempt }} + path: | + .qa/daily/gate/gate.json + .qa/daily/gate/resources.jsonl + .qa/daily/gate/suite.log + .qa/daily/gate/suite/summary.json + .qa/daily/gate/suite/cases.json + .qa/daily/gate/suite/health.json + .qa/daily/gate/suite/cleanup-checks.json + .qa/daily/gate/suite/resources.jsonl + .qa/daily/gate/suite/cycles.json + .qa/daily/gate/suite/execution.json + .qa/daily/gate/suite/coverage.json + .qa/daily/gate/suite/coverage.md + .qa/daily/plugins/manifest.json + .qa/daily/restart-checks/profile.json + .qa/daily/restart-checks/cases.json + .qa/daily/restart-checks/summary.json + .qa/daily/restart-checks/coverage.json + .qa/daily/restart-checks/coverage.md + .qa/daily/restart-checks/cleanup-checks.json + .qa/daily/restart-checks/owned.json + .qa/daily/*.log + if-no-files-found: warn + retention-days: 30 diff --git a/tools/README.md b/tools/README.md index 1ec7156ad..193643ef1 100644 --- a/tools/README.md +++ b/tools/README.md @@ -367,9 +367,13 @@ phases. The dedicated Docker daemon requirement above still applies. `Core Smoke` runs regression-tool tests and a real Core smoke in parallel on GitHub-hosted Linux runners. The smoke starts Core from the checked-out candidate, -uses the SDK from `uv.lock`, the existing official-plugin source pin, `lbctl` +uses the SDK from `uv.lock`, the latest published official plugins, `lbctl` v0.3.0, a Docker Box and mock generation/embedding. It needs no model credentials or access to a developer's running environment. +Plugin versions are resolved once per run, installed from the downloaded release +archives and recorded with hashes and installation readback in `plugins/manifest.json`. +An unavailable or incompatible published package fails preparation; see +[published plugin preparation and replay](published-plugins.md). Dependency installation, service startup, sandbox image download and plugin installation are preparation. After readiness, the gate allows **60 seconds total** @@ -396,11 +400,94 @@ inspect its retained ownership journal before cleanup. CI tears down its own service processes and owned sandboxes afterward. The workflow always creates its two checks on PRs and cancels superseded runs. -After the workflow is merged, configure `Regression tools` and `Core smoke (60s)` -as required status checks in the repository ruleset. A green workflow alone does -not enforce merge protection. Longer load and soak tests use the existing Suite -and Job entry points with separately calibrated latency, throughput and resource -budgets; the one-minute smoke does not replace them. +The existing required check names remain `Regression tools` and `Core smoke (60s)`. +`Daily Endurance` is independent and is not required for merging. Longer load and +soak tests use the existing Suite and Job entry points with separately calibrated +latency, throughput and resource budgets; the one-minute smoke does not replace them. + +## Daily regression + +`Daily Endurance` runs independently of PR checks at 19:23 UTC (03:23 +Asia/Shanghai) on the default branch. GitHub may delay scheduled starts. It also +supports **Run workflow** with a selected branch. The schedule becomes active +only after the workflow is merged into the default branch; it is not a required +merge check. + +The workflow reuses the disposable smoke stack and latest published plugins. +`--suite daily` consumes 46 base cases plus the selected local groups, requires its first pass, then +repeats ordered cases within one **1800-second** window. The clock starts before +creating scenario resources and includes a 120-second reserve for drain/API +cleanup and a 60-second cooldown. Service and plugin preparation precede this +clock. A separate 1860-second process guard allows startup and final reporting; +it does not grant extra scenario time. The whole job retains its 45-minute limit. + +The scheduled job enables all local groups and the optional one-time restart +checks. Manual dispatch exposes both switches. `--scenario-groups` accepts a +space-separated selection; supplying the flag with no values runs only base +cases. `scenarios` and `daily` enable all three groups when the flag is omitted. + +| Group | Additional behavior | +|---|---| +| `faults` | Cancel/timeout cleanup, model retry side effects, partial stream failure, exhausted fallback, Box path rejection | +| `resources` | API-key revocation, direct Agent execution, document deletion with a surviving retrieval control | +| `lifecycle` | Signed HTTP callback 5xx retry identity and subsequent delivery | + +For example, select resource and callback cases with +`--suite scenarios --scenario-groups resources lifecycle`. Add +`--restart-services` to check Core history after restart and a Runtime restart +during an active request. These checks run only against services owned by this +disposable runner, once before the fixed-PID resource sampler starts. Their +elapsed time is deducted from both scenario and outer process budgets; the +resource baseline never joins measurements from old and new PIDs. Failed restart +checks retain their own evidence and fail the run before repeated traffic starts. + +Credential revocation and restart diagnostics use an ephemeral user created +through the normal initialization/login APIs in the runner's new private database. +Generic Suite callers must provide `LANGBOT_QA_USER_TOKEN` for credential tests; +missing prerequisites are skipped and produce an incomplete result. Disabled +groups remain unexecuted in coverage, and a saved profile cannot execute a case +whose group is disabled. Expected failed/cancelled/timeout Runs are recorded by +exact ID and status; extra, missing or nonterminal Runs still fail reconciliation. + +Fault cases reuse one Box per case and suite, with unique files per invocation. +This bounds normal sandbox growth while preserving cancellation side-effect +checks. The callback receiver uses the standard library and loopback; Core and +the runner must share a network namespace. Document deletion and callback retry +identity have separate narrow contracts: they do not establish document update +semantics or exactly-once effects at an arbitrary callback receiver. Plugin +upgrade rollback and cases needing additional environments remain explicit gaps. + +The daily profile uses the existing endurance resource allowances with a +180-second warmup before baseline sampling. Each pipeline that enables Box uses +one stable reuse key unique to this CI run, so conversations created by repeated +cycles reuse that pipeline's sandbox. This measures repeated tool use; it does not test +cross-group Box file isolation. The profile uses one Workspace and does not +establish isolation between tenants. Conversation history and AgentRuns still +accumulate and are reported. Resource excesses remain failures. + +Throughput and full-step P95 are observations. Business assertions, unknown +results, health failures, incomplete evidence and resource limits determine the +verdict. Summaries include first-pass completion, cycles, plugin installation +readback, final ledger reconciliation and coverage. Manifests, case results, +coverage, cycles, execution stages, resource samples and service logs are retained +for 30 days, including failures. Hosted runners do not provide a controlled +performance baseline. No automatic comparison between runs is enabled. + +To run the same entrypoint on a disposable Linux host with an idle Docker daemon: + +```sh +uv run --no-sync python tools/regression_ci.py --suite daily --restart-services \ + --lbctl /path/to/lbctl --output .qa/daily-001 +``` + +`--suite smoke` retains the 60-second gate. `--suite scenarios` runs the shared +expanded profile once under a 300-second guard. `--suite endurance` retains the +separate ten-minute load profile under a 900-second guard; its CI entry point also +reports throughput/P95 without applying their thresholds. All four modes use +published plugins by default. Explicit `--plugins` remains available for source +development; neither workflow selects it. The suite attempts owned API cleanup +even after failures. The CI owner then stops its service processes and removes +its Box containers; a process guard timeout can interrupt API cleanup. ## Sustained regression and baseline comparison diff --git a/tools/regression_ci.py b/tools/regression_ci.py index 36c8fa396..facb0517b 100644 --- a/tools/regression_ci.py +++ b/tools/regression_ci.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Prepare a disposable Linux CI stack from this checkout, then run the 60s gate.""" +"""Prepare a disposable Linux CI stack from this checkout, then run a regression suite.""" import argparse import json @@ -15,20 +15,51 @@ import time import yaml from regression_gate import supervise +from regression_coverage import write_report from regression_plugins import install_published_plugins, prepare_published_plugins +from regression_local_cases import GROUPS, api, require from regression_suite import Suite from regression_traffic import owned_box_containers, read_endpoint_snapshot ROOT = Path(__file__).resolve().parents[1] +SUITE_BUDGETS = {'smoke': 60, 'scenarios': 300, 'daily': 1860, 'endurance': 900} sys.path.insert(0, str(ROOT)) from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402 +def suite_profile(name, run_id, groups=()): + profile_path = ROOT / 'tools/regression_profiles' / ('endurance.json' if name == 'endurance' else 'smoke.json') + if name in ('scenarios', 'daily'): + from regression_scenarios import expanded_profile + + profile = expanded_profile(groups) + else: + profile = json.loads(profile_path.read_text()) + if name == 'smoke': + profile['smoke']['group_turns'] = 2 + if name == 'daily': + limits = json.loads((profile_path.parent / 'endurance.json').read_text())['resource_limits'] + limits['warmup_seconds'] = 180 + profile.update( + execution={'total_seconds': 1800, 'drain_seconds': 120, 'cooldown_seconds': 60}, + resource_limits=limits, + ) + for resource in profile['resources']: + if resource['kind'] != 'pipeline': + continue + for config in resource['body']['config']['ai']['runner_config'].values(): + if config.get('box-enabled'): + # This workload measures repeated tool use; each pipeline owns one sandbox per CI run. + config['box-session-id-template'] = f'qa-{run_id}-{resource["id"]}' + profile.update(cleanup='always', performance_mode='observe') + return profile, profile_path + + def ready(url, processes, timeout=120): deadline = time.monotonic() + timeout while time.monotonic() < deadline: if any(p.poll() is not None for p in processes): - raise RuntimeError('A smoke dependency exited; inspect service logs') + raise RuntimeError('A regression dependency exited; inspect service logs') try: read_endpoint_snapshot(url, timeout_seconds=1) return @@ -47,6 +78,7 @@ def run(args): private = root / '.private' private.mkdir(mode=0o700) processes, logs = [], [] + owned_services = {} key, runtime_token, box_token = (secrets.token_urlsafe(32) for _ in range(3)) env = dict( os.environ, @@ -60,14 +92,60 @@ def run(args): ) def start(name, command): - log = (root / f'{name}.log').open('w') + log = (root / f'{name}.log').open('a') logs.append(log) process = subprocess.Popen( command, cwd=private, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True ) processes.append(process) + owned_services[name] = (process, command) return process.pid + def restart_owned(name, timeout=60): + deadline = time.monotonic() + timeout + + def remaining(): + value = deadline - time.monotonic() + if value <= 0: + raise TimeoutError(f'{name} restart exceeded its deadline') + return value + + require(name in {'core', 'runtime'}, "Only this runner's Core and Runtime can be restarted") + process, command = owned_services[name] + old_pid = process.pid + require(process.poll() is None, f'{name} exited before the planned restart') + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=min(5, remaining())) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=remaining()) + processes.remove(process) + pids[name] = start(name, command) + if name == 'runtime': + ready('http://127.0.0.1:15340/healthz', processes, timeout=remaining()) + ready(profile['observe']['health']['core'], processes, timeout=remaining()) + while True: + previous_timeout = suite.timeout + try: + suite.timeout = lambda maximum: min(maximum, remaining()) + listed = suite.call(['plugin', 'list']) + except RuntimeError: + remaining() + time.sleep(min(0.2, remaining())) + continue + finally: + suite.timeout = previous_timeout + initialized = { + f'{item["author"]}/{item["name"]}' + for item in listed['plugins'] + if item.get('enabled') and item.get('status') == 'initialized' + } + if set(profile['smoke']['required_plugins']) <= initialized: + break + time.sleep(min(0.2, remaining())) + return {'service': name, 'old_pid': old_pid, 'new_pid': pids[name], 'old_exit_code': process.returncode} + try: published = None if args.plugins is None: @@ -82,8 +160,10 @@ def run(args): host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine' ) config_path.write_text(yaml.safe_dump(config)) - profile_path = ROOT / 'tools/regression_profiles/smoke.json' - profile = json.loads(profile_path.read_text()) + groups = args.scenario_groups + if groups is None: + groups = list(GROUPS) if args.suite in ('scenarios', 'daily') else [] + profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups) cli_config = private / 'lbctl.yaml' cli_config.write_text( yaml.safe_dump( @@ -109,8 +189,6 @@ def run(args): mock_records=str(root / 'mock-requests.jsonl'), ) profile['plugins'] = [] - # Two turns retain the history/isolation oracle while leaving room for setup and cleanup. - profile['smoke']['group_turns'] = 2 for resource in profile['resources']: if resource['kind'] == 'model' and resource.get('type') == 'embedding': resource['body']['name'] = 'mock-embedding' @@ -165,16 +243,30 @@ def run(args): }, 'pids': pids, } + if args.suite in ('daily', 'endurance'): + profile['observe']['box_data_root'] = str(private / 'box') for url in profile['observe']['health'].values(): ready(url, processes) preparation = root / 'preparation' preparation.mkdir() suite = Suite(profile, profile_path, preparation, env) identity = suite.call(['whoami']) + suite.variables['workspace_uuid'] = identity['workspace_uuid'] # Bind the CLI to the newly bootstrapped Workspace before installing plugins. cli_data = yaml.safe_load(cli_config.read_text()) cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid'] cli_config.write_text(yaml.safe_dump(cli_data)) + if 'resources' in groups or args.restart_services: + # Account creation is restricted to this runner's newly created private database. + credentials = {'user': 'regression@langbot.test', 'password': secrets.token_urlsafe(32)} + suite.secrets.append(credentials['password']) + status, response = api(suite, 'POST', '/api/v1/user/init', credentials, headers={}) + require(status == 200 and response.get('code') == 0, 'Disposable test account initialization failed') + status, response = api(suite, 'POST', '/api/v1/user/auth', credentials, headers={}) + require(status == 200 and response.get('code') == 0, 'Disposable test account authentication failed') + env['LANGBOT_QA_USER_TOKEN'] = response['data']['token'] + suite.env['LANGBOT_QA_USER_TOKEN'] = env['LANGBOT_QA_USER_TOKEN'] + suite.secrets.append(env['LANGBOT_QA_USER_TOKEN']) if published is not None: install_published_plugins(suite.call, root / 'plugins', published) else: @@ -184,9 +276,26 @@ def run(args): shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__')) archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source) suite.call(['plugin', 'install', 'local', '--file', archive, '--wait']) - generated = private / 'smoke.json' + restart_result = None + restart_elapsed = 0 + if args.restart_services: + from regression_lifecycle_cases import run_restart_checks + + started = time.monotonic() + restart_result = run_restart_checks(profile, profile_path, root / 'restart-checks', env, restart_owned) + restart_elapsed = time.monotonic() - started + if not restart_result['passed']: + raise RuntimeError('Owned-service restart checks failed; inspect restart-checks/summary.json') + if 'execution' in profile: + profile['execution']['total_seconds'] -= restart_elapsed + require( + profile['execution']['total_seconds'] + > profile['execution']['drain_seconds'] + profile['execution']['cooldown_seconds'], + 'Restart checks consumed the scenario execution budget', + ) + generated = private / f'{args.suite}.json' generated.write_text(json.dumps(profile, indent=2)) - return supervise( + result = supervise( [ sys.executable, str(ROOT / 'tools/regression_suite.py'), @@ -197,10 +306,21 @@ def run(args): str(root / 'gate/suite'), ], root / 'gate', - 60, + SUITE_BUDGETS[args.suite] - restart_elapsed, pids, env, ) + evidence = [root / 'gate/suite'] + if restart_result is not None: + evidence.append(root / 'restart-checks') + result['restart_checks'] = restart_result + result['restart_elapsed_seconds'] = restart_elapsed + coverage = write_report(evidence, root / 'gate/suite', http_logs=[root / 'core.log']) + result['coverage_errors'] = coverage['evidence_errors'] + result['scenario_groups'] = groups + result['passed'] = result['passed'] and not result['coverage_errors'] + (root / 'gate/gate.json').write_text(json.dumps(result, indent=2) + '\n') + return result finally: for process in reversed(processes): if process.poll() is None: @@ -224,7 +344,17 @@ def main(): plugins = parser.add_mutually_exclusive_group() plugins.add_argument('--plugins', type=Path, help='Explicit developer override: package a plugin source checkout') plugins.add_argument('--plugin-manifest', type=Path, help='Replay published versions and hashes from a prior run') + parser.add_argument('--suite', choices=tuple(SUITE_BUDGETS), default='smoke') + parser.add_argument( + '--scenario-groups', + nargs='*', + choices=tuple(GROUPS), + help='Local groups for scenarios/daily (default: all); an empty list selects only the base cases', + ) + parser.add_argument('--restart-services', action='store_true', help='Check owned Core/Runtime restarts once') args = parser.parse_args() + if args.suite not in ('scenarios', 'daily') and (args.scenario_groups is not None or args.restart_services): + parser.error('Optional scenarios and restarts require --suite scenarios or --suite daily') if args.output.exists(): parser.error('--output must be a new directory') try: @@ -233,6 +363,9 @@ def main(): verdict = args.output / 'gate' / 'gate.json' result = json.loads(verdict.read_text()) if verdict.exists() else {} result.update(passed=False, infrastructure_error=f'{type(exc).__name__}: {exc}') + restart_summary = args.output / 'restart-checks/summary.json' + if restart_summary.exists(): + result['restart_checks'] = json.loads(restart_summary.read_text()) verdict.parent.mkdir(parents=True, exist_ok=True) verdict.write_text(json.dumps(result, indent=2) + '\n') raise diff --git a/tools/test_regression_ci.py b/tools/test_regression_ci.py new file mode 100644 index 000000000..354835fcb --- /dev/null +++ b/tools/test_regression_ci.py @@ -0,0 +1,181 @@ +"""Exercise the daily workflow's published verdict, including incomplete runs.""" + +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +import yaml + +from regression_ci import SUITE_BUDGETS, suite_profile +from regression_suite import validate + + +class CIProfileTest(unittest.TestCase): + def test_local_groups_are_explicit_and_smoke_rejects_them(self): + profile, _ = suite_profile('daily', 'test-run', ['resources']) + self.assertEqual(profile['scenario_groups'], ['resources']) + self.assertIn('auth_revocation', {case['name'] for case in profile['cases']}) + with tempfile.TemporaryDirectory() as directory: + process = subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name('regression_ci.py')), + '--suite', + 'smoke', + '--scenario-groups', + 'resources', + '--lbctl', + 'unused', + '--output', + str(Path(directory) / 'not-created'), + ], + text=True, + capture_output=True, + ) + self.assertEqual(process.returncode, 2) + self.assertIn('require --suite scenarios or --suite daily', process.stderr) + self.assertFalse((Path(directory) / 'not-created').exists()) + + def test_suite_selection_uses_shared_cases_and_one_daily_budget(self): + for name in ('smoke', 'scenarios', 'daily', 'endurance'): + with self.subTest(suite=name): + profile, path = suite_profile(name, 'test-run') + profile['observe'] = { + 'health': {'core': 'http://localhost/healthz'}, + 'pids': {'core': 1, 'runtime': 2, 'box': 3}, + 'box_data_root': '/tmp/test-box', + } + validate(profile) + self.assertTrue(path.exists()) + self.assertEqual(profile['cleanup'], 'always') + self.assertEqual(profile['performance_mode'], 'observe') + if name == 'smoke': + self.assertEqual(profile['smoke']['group_turns'], 2) + self.assertEqual(SUITE_BUDGETS[name], 60) + elif name in ('scenarios', 'daily'): + self.assertEqual(len(profile['cases']), 46) + self.assertEqual(profile['coverage_suite'], 'scenarios') + self.assertEqual(profile['smoke']['group_turns'], 3) + else: + self.assertEqual(profile['phases'][0]['duration_seconds'], 540) + self.assertEqual('execution' in profile, name == 'daily') + daily, _ = suite_profile('daily', 'test-run') + self.assertEqual(daily['execution'], {'total_seconds': 1800, 'drain_seconds': 120, 'cooldown_seconds': 60}) + endurance, _ = suite_profile('endurance', 'test-run') + self.assertEqual(daily['resource_limits']['warmup_seconds'], 180) + self.assertEqual(daily['resource_limits']['targets'], endurance['resource_limits']['targets']) + self.assertEqual(daily['resource_limits']['containers'], endurance['resource_limits']['containers']) + + def test_daily_reuses_boxes_per_pipeline_and_separates_runs(self): + def keys(name, run_id): + profile, _ = suite_profile(name, run_id) + return { + resource['id']: runner['box-session-id-template'] + for resource in profile['resources'] + if resource['kind'] == 'pipeline' + for runner in resource['body']['config']['ai']['runner_config'].values() + if runner.get('box-enabled') + } + + first, second = keys('daily', 'run-a'), keys('daily', 'run-b') + self.assertTrue(first) + self.assertEqual(len(set(first.values())), len(first)) + self.assertFalse(set(first.values()) & set(second.values())) + self.assertEqual(first, keys('daily', 'run-a')) + self.assertTrue(all(value == '{launcher_type}_{launcher_id}' for value in keys('scenarios', 'run-a').values())) + + +class DailySummaryTest(unittest.TestCase): + def test_summary_reports_observations_without_hiding_outer_failures(self): + workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml' + steps = yaml.safe_load(workflow.read_text())['jobs']['endurance']['steps'] + command = next(step['run'] for step in steps if step.get('name') == 'Publish observations') + suite = { + 'passed': True, + 'outcome': 'PASS', + 'errors': [], + 'first_pass_complete': True, + 'cycles': [{'complete': True}], + 'requests': 86, + 'ledger_closeout': {'matched': True}, + 'observations': {'successful_requests_per_second': 1.0, 'step_p95_seconds': {'chat': 0.5}}, + } + for gate, actual_suite, expected in [ + ({'passed': True}, suite, 'Daily regression: PASS'), + ({'passed': False, 'infrastructure_error': 'teardown failed'}, suite, 'teardown failed'), + ({'passed': False, 'infrastructure_error': 'published plugin install failed'}, None, 'not confirmed'), + ({'passed': False, 'timed_out': True}, suite, '1860-second outer process guard'), + ({'passed': False, 'coverage_errors': ['missing access records']}, suite, 'missing access records'), + ({'passed': False}, {**suite, 'passed': False, 'outcome': 'INCOMPLETE'}, 'Daily regression: INCOMPLETE'), + (None, None, 'Missing verdict'), + ]: + with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / '.qa/daily' + (target / 'plugins').mkdir(parents=True) + (target / 'plugins/manifest.json').write_text( + json.dumps({'plugins': [{'id': 'langbot-team/LocalAgent', 'version': '1.2.3'}]}) + ) + if gate is not None: + (target / 'gate/suite').mkdir(parents=True) + (target / 'gate/gate.json').write_text(json.dumps(gate)) + if actual_suite is not None: + (target / 'gate/suite/summary.json').write_text(json.dumps(actual_suite)) + summary = root / 'summary.md' + env = { + **os.environ, + 'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'], + 'GITHUB_STEP_SUMMARY': str(summary), + 'GITHUB_SHA': 'core-revision', + } + subprocess.run(['bash', '-e', '-c', command], cwd=root, env=env, check=True, capture_output=True) + text = summary.read_text() + self.assertIn(expected, text) + self.assertIn('langbot-team/LocalAgent@1.2.3', text) + self.assertIn('not confirmed', text) + if gate and gate.get('passed'): + self.assertIn('Final ledger reconciliation: True', text) + self.assertIn('1.000 RPS', text) + else: + self.assertNotIn('Daily regression: PASS', text) + + def test_core_summary_records_preparation_failure_and_selected_plugins(self): + workflow = Path(__file__).resolve().parents[1] / '.github/workflows/core-smoke.yml' + steps = yaml.safe_load(workflow.read_text())['jobs']['smoke']['steps'] + command = next(step['run'] for step in steps if step.get('name') == 'Publish gate result') + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / '.qa/core-smoke' + (target / 'plugins').mkdir(parents=True) + (target / 'gate').mkdir() + (target / 'plugins/manifest.json').write_text( + json.dumps({'plugins': [{'id': 'langbot-team/LocalAgent', 'version': '1.2.3'}]}) + ) + (target / 'gate/gate.json').write_text( + json.dumps({'passed': False, 'infrastructure_error': 'incompatible'}) + ) + summary = root / 'summary.md' + subprocess.run( + ['bash', '-e', '-c', command], + cwd=root, + check=True, + capture_output=True, + env={ + **os.environ, + 'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'], + 'GITHUB_STEP_SUMMARY': str(summary), + }, + ) + text = summary.read_text() + self.assertIn('Core smoke: FAIL', text) + self.assertIn('Preparation or teardown failed', text) + self.assertIn('langbot-team/LocalAgent@1.2.3', text) + self.assertIn('not confirmed', text) + + +if __name__ == '__main__': + unittest.main()