ci(regression): expose local scenario groups and owned restart checks

.github/workflows/daily-endurance.yml

@@ -5,6 +5,15 @@ on:
     # 03:23 Asia/Shanghai; GitHub schedules use UTC and may start late.
     - cron: '23 19 * * *'
   workflow_dispatch:
+    inputs:
+      scenario_groups:
+        description: 'Optional local groups, space separated; empty runs only base cases'
+        type: string
+        default: 'faults resources lifecycle'
+      restart_services:
+        description: 'Check owned Core and Runtime restarts once before resource sampling'
+        type: boolean
+        default: true
 
 permissions:
   contents: read
@@ -42,9 +51,16 @@ jobs:
           chmod +x lbctl_linux_amd64
           docker pull python:3.12-alpine
       - name: Run repeated scenarios and recovery checks
+        env:
+          SCENARIO_GROUPS: ${{ github.event_name == 'schedule' && 'faults resources lifecycle' || inputs.scenario_groups }}
+          RESTART_SERVICES: ${{ github.event_name == 'schedule' || inputs.restart_services }}
         run: |
+          read -r -a groups <<< "$SCENARIO_GROUPS"
+          args=()
+          if [[ "$RESTART_SERVICES" == true ]]; then args+=(--restart-services); fi
           uv run --no-sync python tools/regression_ci.py \
-            --suite daily --lbctl .endurance-bin/lbctl_linux_amd64 --output .qa/daily
+            --suite daily --scenario-groups "${groups[@]}" "${args[@]}" \
+            --lbctl .endurance-bin/lbctl_linux_amd64 --output .qa/daily
       - name: Publish observations
         if: always()
         run: |
@@ -68,6 +84,12 @@ jobs:
               out.write(f"Daily regression: {outcome}\n\nCore: `{os.environ['GITHUB_SHA']}`.\n\n")
               out.write('One 1800-second scenario/resource budget, including 120 seconds reserved for drain/cleanup and 60 seconds for cooldown. The outer process guard is 1860 seconds; deployment and plugin installation precede the scenario clock.\n\n')
               out.write('Real Core/Runtime/published plugins/Box; mock generation and embedding. Throughput/P95 are observations; business, health and resource failures still fail this non-required job.\n\n')
+              out.write(f"Enabled local groups: {gate.get('scenario_groups', 'not reported')}.\n\n")
+              restart = gate.get('restart_checks')
+              if restart:
+                  out.write(f"Owned-service restart checks: {restart.get('outcome', 'not reported')}; {restart.get('elapsed_seconds', 0):.2f}s, deducted from the shared budget. Resource baselines begin after restarts.\n\n")
+                  for error in restart.get('errors', []):
+                      out.write(f"Restart check error: {error}\n")
               for plugin in manifest.get('plugins', []):
                   out.write(f"Published plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
               out.write(f"Required first pass complete: {suite.get('first_pass_complete', 'not reported')}; cycles: {len(suite.get('cycles', []))}; requests: {suite.get('requests', 'not reported')}.\n\n")
@@ -118,6 +140,13 @@ jobs:
             .qa/daily/gate/suite/coverage.json
             .qa/daily/gate/suite/coverage.md
             .qa/daily/plugins/manifest.json
+            .qa/daily/restart-checks/profile.json
+            .qa/daily/restart-checks/cases.json
+            .qa/daily/restart-checks/summary.json
+            .qa/daily/restart-checks/coverage.json
+            .qa/daily/restart-checks/coverage.md
+            .qa/daily/restart-checks/cleanup-checks.json
+            .qa/daily/restart-checks/owned.json
             .qa/daily/*.log
           if-no-files-found: warn
           retention-days: 30

tools/README.md

@@ -414,13 +414,49 @@ only after the workflow is merged into the default branch; it is not a required
 merge check.
 
 The workflow reuses the disposable smoke stack and latest published plugins.
-`--suite daily` consumes the shared 46-case profile, requires its first pass, then
+`--suite daily` consumes 46 base cases plus the selected local groups, requires its first pass, then
 repeats ordered cases within one **1800-second** window. The clock starts before
 creating scenario resources and includes a 120-second reserve for drain/API
 cleanup and a 60-second cooldown. Service and plugin preparation precede this
 clock. A separate 1860-second process guard allows startup and final reporting;
 it does not grant extra scenario time. The whole job retains its 45-minute limit.
 
+The scheduled job enables all local groups and the optional one-time restart
+checks. Manual dispatch exposes both switches. `--scenario-groups` accepts a
+space-separated selection; supplying the flag with no values runs only base
+cases. `scenarios` and `daily` enable all three groups when the flag is omitted.
+
+| Group | Additional behavior |
+|---|---|
+| `faults` | Cancel/timeout cleanup, model retry side effects, partial stream failure, exhausted fallback, Box path rejection |
+| `resources` | API-key revocation, direct Agent execution, document deletion with a surviving retrieval control |
+| `lifecycle` | Signed HTTP callback 5xx retry identity and subsequent delivery |
+
+For example, select resource and callback cases with
+`--suite scenarios --scenario-groups resources lifecycle`. Add
+`--restart-services` to check Core history after restart and a Runtime restart
+during an active request. These checks run only against services owned by this
+disposable runner, once before the fixed-PID resource sampler starts. Their
+elapsed time is deducted from both scenario and outer process budgets; the
+resource baseline never joins measurements from old and new PIDs. Failed restart
+checks retain their own evidence and fail the run before repeated traffic starts.
+
+Credential revocation and restart diagnostics use an ephemeral user created
+through the normal initialization/login APIs in the runner's new private database.
+Generic Suite callers must provide `LANGBOT_QA_USER_TOKEN` for credential tests;
+missing prerequisites are skipped and produce an incomplete result. Disabled
+groups remain unexecuted in coverage, and a saved profile cannot execute a case
+whose group is disabled. Expected failed/cancelled/timeout Runs are recorded by
+exact ID and status; extra, missing or nonterminal Runs still fail reconciliation.
+
+Fault cases reuse one Box per case and suite, with unique files per invocation.
+This bounds normal sandbox growth while preserving cancellation side-effect
+checks. The callback receiver uses the standard library and loopback; Core and
+the runner must share a network namespace. Document deletion and callback retry
+identity have separate narrow contracts: they do not establish document update
+semantics or exactly-once effects at an arbitrary callback receiver. Plugin
+upgrade rollback and cases needing additional environments remain explicit gaps.
+
 The daily profile uses the existing endurance resource allowances with a
 180-second warmup before baseline sampling. Each pipeline that enables Box uses
 one stable reuse key unique to this CI run, so conversations created by repeated
@@ -440,7 +476,7 @@ performance baseline. No automatic comparison between runs is enabled.
 To run the same entrypoint on a disposable Linux host with an idle Docker daemon:
 
 ```sh
-uv run --no-sync python tools/regression_ci.py --suite daily \
+uv run --no-sync python tools/regression_ci.py --suite daily --restart-services \
   --lbctl /path/to/lbctl --output .qa/daily-001
 ```
 

tools/regression_ci.py

@@ -17,6 +17,7 @@ import yaml
 from regression_gate import supervise
 from regression_coverage import write_report
 from regression_plugins import install_published_plugins, prepare_published_plugins
+from regression_local_cases import GROUPS, api, require
 from regression_suite import Suite
 from regression_traffic import owned_box_containers, read_endpoint_snapshot
 
@@ -26,12 +27,12 @@ sys.path.insert(0, str(ROOT))
 from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories  # noqa: E402
 
 
-def suite_profile(name, run_id):
+def suite_profile(name, run_id, groups=()):
     profile_path = ROOT / 'tools/regression_profiles' / ('endurance.json' if name == 'endurance' else 'smoke.json')
     if name in ('scenarios', 'daily'):
         from regression_scenarios import expanded_profile
 
-        profile = expanded_profile()
+        profile = expanded_profile(groups)
     else:
         profile = json.loads(profile_path.read_text())
     if name == 'smoke':
@@ -77,6 +78,7 @@ def run(args):
     private = root / '.private'
     private.mkdir(mode=0o700)
     processes, logs = [], []
+    owned_services = {}
     key, runtime_token, box_token = (secrets.token_urlsafe(32) for _ in range(3))
     env = dict(
         os.environ,
@@ -90,14 +92,60 @@ def run(args):
     )
 
     def start(name, command):
-        log = (root / f'{name}.log').open('w')
+        log = (root / f'{name}.log').open('a')
         logs.append(log)
         process = subprocess.Popen(
             command, cwd=private, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True
         )
         processes.append(process)
+        owned_services[name] = (process, command)
         return process.pid
 
+    def restart_owned(name, timeout=60):
+        deadline = time.monotonic() + timeout
+
+        def remaining():
+            value = deadline - time.monotonic()
+            if value <= 0:
+                raise TimeoutError(f'{name} restart exceeded its deadline')
+            return value
+
+        require(name in {'core', 'runtime'}, "Only this runner's Core and Runtime can be restarted")
+        process, command = owned_services[name]
+        old_pid = process.pid
+        require(process.poll() is None, f'{name} exited before the planned restart')
+        os.killpg(process.pid, signal.SIGTERM)
+        try:
+            process.wait(timeout=min(5, remaining()))
+        except subprocess.TimeoutExpired:
+            os.killpg(process.pid, signal.SIGKILL)
+            process.wait(timeout=remaining())
+        processes.remove(process)
+        pids[name] = start(name, command)
+        if name == 'runtime':
+            ready('http://127.0.0.1:15340/healthz', processes, timeout=remaining())
+        ready(profile['observe']['health']['core'], processes, timeout=remaining())
+        while True:
+            previous_timeout = suite.timeout
+            try:
+                suite.timeout = lambda maximum: min(maximum, remaining())
+                listed = suite.call(['plugin', 'list'])
+            except RuntimeError:
+                remaining()
+                time.sleep(min(0.2, remaining()))
+                continue
+            finally:
+                suite.timeout = previous_timeout
+            initialized = {
+                f'{item["author"]}/{item["name"]}'
+                for item in listed['plugins']
+                if item.get('enabled') and item.get('status') == 'initialized'
+            }
+            if set(profile['smoke']['required_plugins']) <= initialized:
+                break
+            time.sleep(min(0.2, remaining()))
+        return {'service': name, 'old_pid': old_pid, 'new_pid': pids[name], 'old_exit_code': process.returncode}
+
     try:
         published = None
         if args.plugins is None:
@@ -112,7 +160,10 @@ def run(args):
             host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine'
         )
         config_path.write_text(yaml.safe_dump(config))
-        profile, profile_path = suite_profile(args.suite, secrets.token_hex(8))
+        groups = args.scenario_groups
+        if groups is None:
+            groups = list(GROUPS) if args.suite in ('scenarios', 'daily') else []
+        profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups)
         cli_config = private / 'lbctl.yaml'
         cli_config.write_text(
             yaml.safe_dump(
@@ -200,10 +251,22 @@ def run(args):
         preparation.mkdir()
         suite = Suite(profile, profile_path, preparation, env)
         identity = suite.call(['whoami'])
+        suite.variables['workspace_uuid'] = identity['workspace_uuid']
         # Bind the CLI to the newly bootstrapped Workspace before installing plugins.
         cli_data = yaml.safe_load(cli_config.read_text())
         cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid']
         cli_config.write_text(yaml.safe_dump(cli_data))
+        if 'resources' in groups or args.restart_services:
+            # Account creation is restricted to this runner's newly created private database.
+            credentials = {'user': '[email protected]', 'password': secrets.token_urlsafe(32)}
+            suite.secrets.append(credentials['password'])
+            status, response = api(suite, 'POST', '/api/v1/user/init', credentials, headers={})
+            require(status == 200 and response.get('code') == 0, 'Disposable test account initialization failed')
+            status, response = api(suite, 'POST', '/api/v1/user/auth', credentials, headers={})
+            require(status == 200 and response.get('code') == 0, 'Disposable test account authentication failed')
+            env['LANGBOT_QA_USER_TOKEN'] = response['data']['token']
+            suite.env['LANGBOT_QA_USER_TOKEN'] = env['LANGBOT_QA_USER_TOKEN']
+            suite.secrets.append(env['LANGBOT_QA_USER_TOKEN'])
         if published is not None:
             install_published_plugins(suite.call, root / 'plugins', published)
         else:
@@ -213,6 +276,23 @@ def run(args):
                 shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__'))
                 archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source)
                 suite.call(['plugin', 'install', 'local', '--file', archive, '--wait'])
+        restart_result = None
+        restart_elapsed = 0
+        if args.restart_services:
+            from regression_lifecycle_cases import run_restart_checks
+
+            started = time.monotonic()
+            restart_result = run_restart_checks(profile, profile_path, root / 'restart-checks', env, restart_owned)
+            restart_elapsed = time.monotonic() - started
+            if not restart_result['passed']:
+                raise RuntimeError('Owned-service restart checks failed; inspect restart-checks/summary.json')
+            if 'execution' in profile:
+                profile['execution']['total_seconds'] -= restart_elapsed
+                require(
+                    profile['execution']['total_seconds']
+                    > profile['execution']['drain_seconds'] + profile['execution']['cooldown_seconds'],
+                    'Restart checks consumed the scenario execution budget',
+                )
         generated = private / f'{args.suite}.json'
         generated.write_text(json.dumps(profile, indent=2))
         result = supervise(
@@ -226,12 +306,18 @@ def run(args):
                 str(root / 'gate/suite'),
             ],
             root / 'gate',
-            SUITE_BUDGETS[args.suite],
+            SUITE_BUDGETS[args.suite] - restart_elapsed,
             pids,
             env,
         )
-        coverage = write_report([root / 'gate/suite'], root / 'gate/suite', http_logs=[root / 'core.log'])
+        evidence = [root / 'gate/suite']
+        if restart_result is not None:
+            evidence.append(root / 'restart-checks')
+            result['restart_checks'] = restart_result
+            result['restart_elapsed_seconds'] = restart_elapsed
+        coverage = write_report(evidence, root / 'gate/suite', http_logs=[root / 'core.log'])
         result['coverage_errors'] = coverage['evidence_errors']
+        result['scenario_groups'] = groups
         result['passed'] = result['passed'] and not result['coverage_errors']
         (root / 'gate/gate.json').write_text(json.dumps(result, indent=2) + '\n')
         return result
@@ -259,7 +345,16 @@ def main():
     plugins.add_argument('--plugins', type=Path, help='Explicit developer override: package a plugin source checkout')
     plugins.add_argument('--plugin-manifest', type=Path, help='Replay published versions and hashes from a prior run')
     parser.add_argument('--suite', choices=tuple(SUITE_BUDGETS), default='smoke')
+    parser.add_argument(
+        '--scenario-groups',
+        nargs='*',
+        choices=tuple(GROUPS),
+        help='Local groups for scenarios/daily (default: all); an empty list selects only the base cases',
+    )
+    parser.add_argument('--restart-services', action='store_true', help='Check owned Core/Runtime restarts once')
     args = parser.parse_args()
+    if args.suite not in ('scenarios', 'daily') and (args.scenario_groups is not None or args.restart_services):
+        parser.error('Optional scenarios and restarts require --suite scenarios or --suite daily')
     if args.output.exists():
         parser.error('--output must be a new directory')
     try:
@@ -268,6 +363,9 @@ def main():
         verdict = args.output / 'gate' / 'gate.json'
         result = json.loads(verdict.read_text()) if verdict.exists() else {}
         result.update(passed=False, infrastructure_error=f'{type(exc).__name__}: {exc}')
+        restart_summary = args.output / 'restart-checks/summary.json'
+        if restart_summary.exists():
+            result['restart_checks'] = json.loads(restart_summary.read_text())
         verdict.parent.mkdir(parents=True, exist_ok=True)
         verdict.write_text(json.dumps(result, indent=2) + '\n')
         raise

tools/test_regression_ci.py

@@ -15,6 +15,31 @@ from regression_suite import validate
 
 
 class CIProfileTest(unittest.TestCase):
+    def test_local_groups_are_explicit_and_smoke_rejects_them(self):
+        profile, _ = suite_profile('daily', 'test-run', ['resources'])
+        self.assertEqual(profile['scenario_groups'], ['resources'])
+        self.assertIn('auth_revocation', {case['name'] for case in profile['cases']})
+        with tempfile.TemporaryDirectory() as directory:
+            process = subprocess.run(
+                [
+                    sys.executable,
+                    str(Path(__file__).with_name('regression_ci.py')),
+                    '--suite',
+                    'smoke',
+                    '--scenario-groups',
+                    'resources',
+                    '--lbctl',
+                    'unused',
+                    '--output',
+                    str(Path(directory) / 'not-created'),
+                ],
+                text=True,
+                capture_output=True,
+            )
+            self.assertEqual(process.returncode, 2)
+            self.assertIn('require --suite scenarios or --suite daily', process.stderr)
+            self.assertFalse((Path(directory) / 'not-created').exists())
+
     def test_suite_selection_uses_shared_cases_and_one_daily_budget(self):
         for name in ('smoke', 'scenarios', 'daily', 'endurance'):
             with self.subTest(suite=name):