ci(daily): run API contracts alongside endurance

.github/workflows/daily-endurance.yml

@@ -158,3 +158,94 @@ jobs:
             .qa/daily/*.log
           if-no-files-found: warn
           retention-days: 30
+
+  api-contracts:
+    name: Daily API contracts
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+      - name: Install locked dependencies
+        run: uv sync --dev --frozen
+      - name: Prepare CLI and sandbox image
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          mkdir -p .api-bin
+          gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .api-bin \
+            --pattern lbctl_linux_amd64 --pattern checksums.txt
+          cd .api-bin
+          sha256sum --check --ignore-missing checksums.txt
+          chmod +x lbctl_linux_amd64
+          docker pull python:3.12-alpine
+      - name: Run API contracts
+        run: |
+          uv run --no-sync python tools/regression_ci.py \
+            --suite api --api-seed "$GITHUB_RUN_ID" \
+            --lbctl .api-bin/lbctl_linux_amd64 --plugins plugins --output .qa/api
+      - name: Publish API result
+        if: always()
+        run: |
+          python - <<'PY_SUMMARY'
+          import json, os
+          from pathlib import Path
+
+          root = Path('.qa/api')
+          def read(path):
+              source = root / path
+              return json.loads(source.read_text()) if source.exists() else {}
+
+          gate = read('gate/gate.json')
+          suite = read('gate/suite/summary.json')
+          coverage = read('gate/suite/coverage.json')
+          manifest = read('plugins/manifest.json')
+          passed = gate.get('passed') is True and suite.get('passed') is True
+          with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
+              out.write(f"Daily API contracts: {'PASS' if passed else 'FAIL / incomplete'}\n\n")
+              out.write(f"Core: `{os.environ['GITHUB_SHA']}`; shuffle seed: `{gate.get('api_seed', os.environ['GITHUB_RUN_ID'])}`.\n\n")
+              out.write('Fixed order followed by a seeded chain shuffle, with sequential operations inside each chain. The execution guard is 600 seconds; dependency and service preparation is separate. This job is not a required PR check.\n\n')
+              out.write(f"Cases: {suite.get('requests', 'not reported')}; statuses: {suite.get('status', 'not reported')}; suite execution: {suite.get('elapsed_seconds', 'not reported')}s.\n\n")
+              out.write(f"API behavior contracts: {coverage.get('areas', {}).get('api', 'not reported')}.\n\n")
+              out.write(f"Observed HTTP route templates (including preparation): {len(coverage['http_observations']['routes']) if 'http_observations' in coverage else 'not reported'}. A route hit does not establish all behavior behind it.\n\n")
+              for plugin in manifest.get('plugins', []):
+                  out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
+              errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', []))
+              if gate.get('timed_out'):
+                  errors.append('API suite exceeded its 600-second outer process guard.')
+              if gate.get('infrastructure_error'):
+                  errors.append(gate['infrastructure_error'])
+              if not gate or not suite:
+                  errors.append('Missing verdict; inspect preparation and service logs.')
+              for error in errors:
+                  out.write(f"- {error}\n")
+          PY_SUMMARY
+      - name: Preserve API verdicts and evidence
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: daily-api-${{ github.run_id }}-${{ github.run_attempt }}
+          path: |
+            .qa/api/gate/gate.json
+            .qa/api/gate/resources.jsonl
+            .qa/api/gate/suite.log
+            .qa/api/gate/suite/summary.json
+            .qa/api/gate/suite/profile.json
+            .qa/api/gate/suite/cases.json
+            .qa/api/gate/suite/health.json
+            .qa/api/gate/suite/cleanup-checks.json
+            .qa/api/gate/suite/resource-case-*.json
+            .qa/api/gate/suite/coverage.json
+            .qa/api/gate/suite/coverage.md
+            .qa/api/plugins/manifest.json
+            .qa/api/plugins/*-build.log
+            .qa/api/*.log
+          if-no-files-found: warn
+          retention-days: 30

tools/README.md

@@ -486,7 +486,7 @@ uv run --no-sync python tools/regression_ci.py --suite daily --restart-services
 `--suite smoke` retains the 60-second gate. `--suite scenarios` runs the shared
 expanded profile once under a 300-second guard. `--suite endurance` retains the
 separate ten-minute load profile under a 900-second guard; its CI entry point also
-reports throughput/P95 without applying their thresholds. All four modes use
+reports throughput/P95 without applying their thresholds. All suites use
 published plugins when `--plugins` is omitted. Both workflows explicitly select
 `--plugins plugins` so source changes are tested in the same PR. Daily additionally
 selects `misc/EssentialCommands misc/FAQManager`; manual dispatch can change the
@@ -498,9 +498,17 @@ bind their own extensions to avoid changing unrelated prompts/tools. The suite a
 even after failures. The CI owner then stops its service processes and removes
 its Box containers; a process guard timeout can interrupt API cleanup.
 
-## Local API contract job
+## Daily API contract job
 
-Run on the same disposable Linux/Docker host used by `regression_ci.py`:
+`daily-endurance.yml` runs API contracts alongside the 30-minute endurance job
+on separate runners, at the existing daily schedule and on manual dispatch.
+The API job uses the GitHub run ID as its shuffle seed, so each daily run explores
+a different order and a rerun keeps the same seed. It has a 15-minute job timeout,
+including preparation, and is not a required PR check. Its summary and separate
+`daily-api-<run_id>-<attempt>` artifact retain failures, order, coverage and cleanup
+evidence for 30 days.
+
+To replay locally on a disposable Linux/Docker host, pass the recorded seed:
 
 ```sh
 uv run --no-sync python tools/regression_ci.py --suite api --api-seed 20261028 \
@@ -527,9 +535,8 @@ Inspect `gate/gate.json` for the verdict, seed and exact case order;
 `gate/suite/cases.json` for HTTP steps, assertions and cleanup results; and
 `gate/suite/coverage.json` for observed route templates and behavior contracts.
 An HTTP route hit includes rejected requests and is not proof of all behavior
-behind that route. The job intentionally fails on invalid MCP modes accepted
-and persisted by Core; fixing that product validation should make the existing
-negative case pass without changing its expected result.
+behind that route. Invalid MCP modes must return HTTP 400 and leave the original
+configuration unchanged. Any failed contract or cleanup makes the job fail.
 
 ## Sustained regression and baseline comparison
 

tools/test_regression_ci.py

@@ -143,6 +143,53 @@ class DailySummaryTest(unittest.TestCase):
                 else:
                     self.assertNotIn('Daily regression: PASS', text)
 
+    def test_api_summary_keeps_preparation_and_outer_failures_visible(self):
+        workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml'
+        steps = yaml.safe_load(workflow.read_text())['jobs']['api-contracts']['steps']
+        command = next(step['run'] for step in steps if step.get('name') == 'Publish API result')
+        suite = {'passed': True, 'requests': 14, 'status': {'passed': 14}, 'elapsed_seconds': 7.3}
+        for gate, actual_suite, expected in [
+            ({'passed': True, 'api_seed': 123}, suite, 'Daily API contracts: PASS'),
+            (
+                {'passed': False, 'infrastructure_error': 'plugin installation failed'},
+                None,
+                'plugin installation failed',
+            ),
+            ({'passed': False, 'timed_out': True}, suite, '600-second outer process guard'),
+            ({'passed': False, 'resource_errors': ['sampling failed']}, suite, 'sampling failed'),
+            ({'passed': False, 'coverage_errors': ['missing evidence']}, suite, 'missing evidence'),
+            ({'passed': False}, {'passed': False, 'errors': ['MCP mode persisted']}, 'MCP mode persisted'),
+            (None, None, 'Missing verdict'),
+        ]:
+            with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory:
+                root = Path(directory)
+                target = root / '.qa/api/gate/suite'
+                target.mkdir(parents=True)
+                if gate is not None:
+                    (target.parent / 'gate.json').write_text(json.dumps(gate))
+                if actual_suite is not None:
+                    (target / 'summary.json').write_text(json.dumps(actual_suite))
+                summary = root / 'summary.md'
+                subprocess.run(
+                    ['bash', '-e', '-c', command],
+                    cwd=root,
+                    check=True,
+                    capture_output=True,
+                    env={
+                        **os.environ,
+                        'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'],
+                        'GITHUB_STEP_SUMMARY': str(summary),
+                        'GITHUB_SHA': 'tested-core',
+                        'GITHUB_RUN_ID': '456',
+                    },
+                )
+                text = summary.read_text()
+                self.assertIn(expected, text)
+                self.assertIn('not reported', text)
+                self.assertIn('shuffle seed: `123`' if gate and gate.get('passed') else 'shuffle seed: `456`', text)
+                if not gate or not gate.get('passed'):
+                    self.assertNotIn('Daily API contracts: PASS', text)
+
     def test_core_summary_records_preparation_failure_and_selected_plugins(self):
         workflow = Path(__file__).resolve().parents[1] / '.github/workflows/core-smoke.yml'
         steps = yaml.safe_load(workflow.read_text())['jobs']['smoke']['steps']