diff --git a/.github/workflows/daily-endurance.yml b/.github/workflows/daily-endurance.yml index ada43e53a..d6dc3eb41 100644 --- a/.github/workflows/daily-endurance.yml +++ b/.github/workflows/daily-endurance.yml @@ -5,6 +5,15 @@ on: # 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late. - cron: '23 19 * * *' workflow_dispatch: + inputs: + scenario_groups: + description: 'Optional local groups, space separated; empty runs only base cases' + type: string + default: 'faults resources lifecycle' + restart_services: + description: 'Check owned Core and Runtime restarts once before resource sampling' + type: boolean + default: true permissions: contents: read @@ -42,9 +51,16 @@ jobs: chmod +x lbctl_linux_amd64 docker pull python:3.12-alpine - name: Run repeated scenarios and recovery checks + env: + SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle' || inputs.scenario_groups }} + RESTART_SERVICES: ${{ github.event_name == 'schedule' || inputs.restart_services }} run: | + read -r -a groups <<< "$SCENARIO_GROUPS" + args=() + if [[ "$RESTART_SERVICES" == true ]]; then args+=(--restart-services); fi uv run --no-sync python tools/regression_ci.py \ - --suite daily --lbctl .endurance-bin/lbctl_linux_amd64 --output .qa/daily + --suite daily --scenario-groups "${groups[@]}" "${args[@]}" \ + --lbctl .endurance-bin/lbctl_linux_amd64 --output .qa/daily - name: Publish observations if: always() run: | @@ -68,6 +84,12 @@ jobs: out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n") out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n') out.write('Real Core/Runtime/published plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n') + out.write(f"Enabled local groups: {gate.get('scenario_groups', 'not reported')}.\n\n") + restart = gate.get('restart_checks') + if restart: + out.write(f"Owned-service restart checks: {restart.get('outcome', 'not reported')}; {restart.get('elapsed_seconds', 0):.2f}s, deducted from the shared budget. Resource baselines begin after restarts.\n\n") + for error in restart.get('errors', []): + out.write(f"Restart check error: {error}\n") for plugin in manifest.get('plugins', []): out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n") @@ -118,6 +140,13 @@ jobs: .qa/daily/gate/suite/coverage.json .qa/daily/gate/suite/coverage.md .qa/daily/plugins/manifest.json + .qa/daily/restart-checks/profile.json + .qa/daily/restart-checks/cases.json + .qa/daily/restart-checks/summary.json + .qa/daily/restart-checks/coverage.json + .qa/daily/restart-checks/coverage.md + .qa/daily/restart-checks/cleanup-checks.json + .qa/daily/restart-checks/owned.json .qa/daily/*.log if-no-files-found: warn retention-days: 30 diff --git a/tools/README.md b/tools/README.md index efff0880e..193643ef1 100644 --- a/tools/README.md +++ b/tools/README.md @@ -414,13 +414,49 @@ only after the workflow is merged into the default branch; it is not a required merge check. The workflow reuses the disposable smoke stack and latest published plugins. -`--suite daily` consumes the shared 46-case profile, requires its first pass, then +`--suite daily` consumes 46 base cases plus the selected local groups, requires its first pass, then repeats ordered cases within one **1800-second** window. The clock starts before creating scenario resources and includes a 120-second reserve for drain/API cleanup and a 60-second cooldown. Service and plugin preparation precede this clock. A separate 1860-second process guard allows startup and final reporting; it does not grant extra scenario time. The whole job retains its 45-minute limit. +The scheduled job enables all local groups and the optional one-time restart +checks. Manual dispatch exposes both switches. `--scenario-groups` accepts a +space-separated selection; supplying the flag with no values runs only base +cases. `scenarios` and `daily` enable all three groups when the flag is omitted. + +| Group | Additional behavior | +|---|---| +| `faults` | Cancel/timeout cleanup, model retry side effects, partial stream failure, exhausted fallback, Box path rejection | +| `resources` | API-key revocation, direct Agent execution, document deletion with a surviving retrieval control | +| `lifecycle` | Signed HTTP callback 5xx retry identity and subsequent delivery | + +For example, select resource and callback cases with +`--suite scenarios --scenario-groups resources lifecycle`. Add +`--restart-services` to check Core history after restart and a Runtime restart +during an active request. These checks run only against services owned by this +disposable runner, once before the fixed-PID resource sampler starts. Their +elapsed time is deducted from both scenario and outer process budgets; the +resource baseline never joins measurements from old and new PIDs. Failed restart +checks retain their own evidence and fail the run before repeated traffic starts. + +Credential revocation and restart diagnostics use an ephemeral user created +through the normal initialization/login APIs in the runner's new private database. +Generic Suite callers must provide `LANGBOT_QA_USER_TOKEN` for credential tests; +missing prerequisites are skipped and produce an incomplete result. Disabled +groups remain unexecuted in coverage, and a saved profile cannot execute a case +whose group is disabled. Expected failed/cancelled/timeout Runs are recorded by +exact ID and status; extra, missing or nonterminal Runs still fail reconciliation. + +Fault cases reuse one Box per case and suite, with unique files per invocation. +This bounds normal sandbox growth while preserving cancellation side-effect +checks. The callback receiver uses the standard library and loopback; Core and +the runner must share a network namespace. Document deletion and callback retry +identity have separate narrow contracts: they do not establish document update +semantics or exactly-once effects at an arbitrary callback receiver. Plugin +upgrade rollback and cases needing additional environments remain explicit gaps. + The daily profile uses the existing endurance resource allowances with a 180-second warmup before baseline sampling. Each pipeline that enables Box uses one stable reuse key unique to this CI run, so conversations created by repeated @@ -440,7 +476,7 @@ performance baseline. No automatic comparison between runs is enabled. To run the same entrypoint on a disposable Linux host with an idle Docker daemon: ```sh -uv run --no-sync python tools/regression_ci.py --suite daily \ +uv run --no-sync python tools/regression_ci.py --suite daily --restart-services \ --lbctl /path/to/lbctl --output .qa/daily-001 ``` diff --git a/tools/regression_ci.py b/tools/regression_ci.py index 26661f067..facb0517b 100644 --- a/tools/regression_ci.py +++ b/tools/regression_ci.py @@ -17,6 +17,7 @@ import yaml from regression_gate import supervise from regression_coverage import write_report from regression_plugins import install_published_plugins, prepare_published_plugins +from regression_local_cases import GROUPS, api, require from regression_suite import Suite from regression_traffic import owned_box_containers, read_endpoint_snapshot @@ -26,12 +27,12 @@ sys.path.insert(0, str(ROOT)) from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402 -def suite_profile(name, run_id): +def suite_profile(name, run_id, groups=()): profile_path = ROOT / 'tools/regression_profiles' / ('endurance.json' if name == 'endurance' else 'smoke.json') if name in ('scenarios', 'daily'): from regression_scenarios import expanded_profile - profile = expanded_profile() + profile = expanded_profile(groups) else: profile = json.loads(profile_path.read_text()) if name == 'smoke': @@ -77,6 +78,7 @@ def run(args): private = root / '.private' private.mkdir(mode=0o700) processes, logs = [], [] + owned_services = {} key, runtime_token, box_token = (secrets.token_urlsafe(32) for _ in range(3)) env = dict( os.environ, @@ -90,14 +92,60 @@ def run(args): ) def start(name, command): - log = (root / f'{name}.log').open('w') + log = (root / f'{name}.log').open('a') logs.append(log) process = subprocess.Popen( command, cwd=private, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True ) processes.append(process) + owned_services[name] = (process, command) return process.pid + def restart_owned(name, timeout=60): + deadline = time.monotonic() + timeout + + def remaining(): + value = deadline - time.monotonic() + if value <= 0: + raise TimeoutError(f'{name} restart exceeded its deadline') + return value + + require(name in {'core', 'runtime'}, "Only this runner's Core and Runtime can be restarted") + process, command = owned_services[name] + old_pid = process.pid + require(process.poll() is None, f'{name} exited before the planned restart') + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=min(5, remaining())) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait(timeout=remaining()) + processes.remove(process) + pids[name] = start(name, command) + if name == 'runtime': + ready('http://127.0.0.1:15340/healthz', processes, timeout=remaining()) + ready(profile['observe']['health']['core'], processes, timeout=remaining()) + while True: + previous_timeout = suite.timeout + try: + suite.timeout = lambda maximum: min(maximum, remaining()) + listed = suite.call(['plugin', 'list']) + except RuntimeError: + remaining() + time.sleep(min(0.2, remaining())) + continue + finally: + suite.timeout = previous_timeout + initialized = { + f'{item["author"]}/{item["name"]}' + for item in listed['plugins'] + if item.get('enabled') and item.get('status') == 'initialized' + } + if set(profile['smoke']['required_plugins']) <= initialized: + break + time.sleep(min(0.2, remaining())) + return {'service': name, 'old_pid': old_pid, 'new_pid': pids[name], 'old_exit_code': process.returncode} + try: published = None if args.plugins is None: @@ -112,7 +160,10 @@ def run(args): host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine' ) config_path.write_text(yaml.safe_dump(config)) - profile, profile_path = suite_profile(args.suite, secrets.token_hex(8)) + groups = args.scenario_groups + if groups is None: + groups = list(GROUPS) if args.suite in ('scenarios', 'daily') else [] + profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups) cli_config = private / 'lbctl.yaml' cli_config.write_text( yaml.safe_dump( @@ -200,10 +251,22 @@ def run(args): preparation.mkdir() suite = Suite(profile, profile_path, preparation, env) identity = suite.call(['whoami']) + suite.variables['workspace_uuid'] = identity['workspace_uuid'] # Bind the CLI to the newly bootstrapped Workspace before installing plugins. cli_data = yaml.safe_load(cli_config.read_text()) cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid'] cli_config.write_text(yaml.safe_dump(cli_data)) + if 'resources' in groups or args.restart_services: + # Account creation is restricted to this runner's newly created private database. + credentials = {'user': 'regression@langbot.test', 'password': secrets.token_urlsafe(32)} + suite.secrets.append(credentials['password']) + status, response = api(suite, 'POST', '/api/v1/user/init', credentials, headers={}) + require(status == 200 and response.get('code') == 0, 'Disposable test account initialization failed') + status, response = api(suite, 'POST', '/api/v1/user/auth', credentials, headers={}) + require(status == 200 and response.get('code') == 0, 'Disposable test account authentication failed') + env['LANGBOT_QA_USER_TOKEN'] = response['data']['token'] + suite.env['LANGBOT_QA_USER_TOKEN'] = env['LANGBOT_QA_USER_TOKEN'] + suite.secrets.append(env['LANGBOT_QA_USER_TOKEN']) if published is not None: install_published_plugins(suite.call, root / 'plugins', published) else: @@ -213,6 +276,23 @@ def run(args): shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__')) archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source) suite.call(['plugin', 'install', 'local', '--file', archive, '--wait']) + restart_result = None + restart_elapsed = 0 + if args.restart_services: + from regression_lifecycle_cases import run_restart_checks + + started = time.monotonic() + restart_result = run_restart_checks(profile, profile_path, root / 'restart-checks', env, restart_owned) + restart_elapsed = time.monotonic() - started + if not restart_result['passed']: + raise RuntimeError('Owned-service restart checks failed; inspect restart-checks/summary.json') + if 'execution' in profile: + profile['execution']['total_seconds'] -= restart_elapsed + require( + profile['execution']['total_seconds'] + > profile['execution']['drain_seconds'] + profile['execution']['cooldown_seconds'], + 'Restart checks consumed the scenario execution budget', + ) generated = private / f'{args.suite}.json' generated.write_text(json.dumps(profile, indent=2)) result = supervise( @@ -226,12 +306,18 @@ def run(args): str(root / 'gate/suite'), ], root / 'gate', - SUITE_BUDGETS[args.suite], + SUITE_BUDGETS[args.suite] - restart_elapsed, pids, env, ) - coverage = write_report([root / 'gate/suite'], root / 'gate/suite', http_logs=[root / 'core.log']) + evidence = [root / 'gate/suite'] + if restart_result is not None: + evidence.append(root / 'restart-checks') + result['restart_checks'] = restart_result + result['restart_elapsed_seconds'] = restart_elapsed + coverage = write_report(evidence, root / 'gate/suite', http_logs=[root / 'core.log']) result['coverage_errors'] = coverage['evidence_errors'] + result['scenario_groups'] = groups result['passed'] = result['passed'] and not result['coverage_errors'] (root / 'gate/gate.json').write_text(json.dumps(result, indent=2) + '\n') return result @@ -259,7 +345,16 @@ def main(): plugins.add_argument('--plugins', type=Path, help='Explicit developer override: package a plugin source checkout') plugins.add_argument('--plugin-manifest', type=Path, help='Replay published versions and hashes from a prior run') parser.add_argument('--suite', choices=tuple(SUITE_BUDGETS), default='smoke') + parser.add_argument( + '--scenario-groups', + nargs='*', + choices=tuple(GROUPS), + help='Local groups for scenarios/daily (default: all); an empty list selects only the base cases', + ) + parser.add_argument('--restart-services', action='store_true', help='Check owned Core/Runtime restarts once') args = parser.parse_args() + if args.suite not in ('scenarios', 'daily') and (args.scenario_groups is not None or args.restart_services): + parser.error('Optional scenarios and restarts require --suite scenarios or --suite daily') if args.output.exists(): parser.error('--output must be a new directory') try: @@ -268,6 +363,9 @@ def main(): verdict = args.output / 'gate' / 'gate.json' result = json.loads(verdict.read_text()) if verdict.exists() else {} result.update(passed=False, infrastructure_error=f'{type(exc).__name__}: {exc}') + restart_summary = args.output / 'restart-checks/summary.json' + if restart_summary.exists(): + result['restart_checks'] = json.loads(restart_summary.read_text()) verdict.parent.mkdir(parents=True, exist_ok=True) verdict.write_text(json.dumps(result, indent=2) + '\n') raise diff --git a/tools/test_regression_ci.py b/tools/test_regression_ci.py index 5cb929cb9..354835fcb 100644 --- a/tools/test_regression_ci.py +++ b/tools/test_regression_ci.py @@ -15,6 +15,31 @@ from regression_suite import validate class CIProfileTest(unittest.TestCase): + def test_local_groups_are_explicit_and_smoke_rejects_them(self): + profile, _ = suite_profile('daily', 'test-run', ['resources']) + self.assertEqual(profile['scenario_groups'], ['resources']) + self.assertIn('auth_revocation', {case['name'] for case in profile['cases']}) + with tempfile.TemporaryDirectory() as directory: + process = subprocess.run( + [ + sys.executable, + str(Path(__file__).with_name('regression_ci.py')), + '--suite', + 'smoke', + '--scenario-groups', + 'resources', + '--lbctl', + 'unused', + '--output', + str(Path(directory) / 'not-created'), + ], + text=True, + capture_output=True, + ) + self.assertEqual(process.returncode, 2) + self.assertIn('require --suite scenarios or --suite daily', process.stderr) + self.assertFalse((Path(directory) / 'not-created').exists()) + def test_suite_selection_uses_shared_cases_and_one_daily_budget(self): for name in ('smoke', 'scenarios', 'daily', 'endurance'): with self.subTest(suite=name):