diff --git a/.github/workflows/daily-endurance.yml b/.github/workflows/daily-endurance.yml index f11f86482..bef7c0674 100644 --- a/.github/workflows/daily-endurance.yml +++ b/.github/workflows/daily-endurance.yml @@ -158,3 +158,94 @@ jobs: .qa/daily/*.log if-no-files-found: warn retention-days: 30 + + api-contracts: + name: Daily API contracts + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Install locked dependencies + run: uv sync --dev --frozen + - name: Prepare CLI and sandbox image + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p .api-bin + gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .api-bin \ + --pattern lbctl_linux_amd64 --pattern checksums.txt + cd .api-bin + sha256sum --check --ignore-missing checksums.txt + chmod +x lbctl_linux_amd64 + docker pull python:3.12-alpine + - name: Run API contracts + run: | + uv run --no-sync python tools/regression_ci.py \ + --suite api --api-seed "$GITHUB_RUN_ID" \ + --lbctl .api-bin/lbctl_linux_amd64 --plugins plugins --output .qa/api + - name: Publish API result + if: always() + run: | + python - <<'PY_SUMMARY' + import json, os + from pathlib import Path + + root = Path('.qa/api') + def read(path): + source = root / path + return json.loads(source.read_text()) if source.exists() else {} + + gate = read('gate/gate.json') + suite = read('gate/suite/summary.json') + coverage = read('gate/suite/coverage.json') + manifest = read('plugins/manifest.json') + passed = gate.get('passed') is True and suite.get('passed') is True + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: + out.write(f"Daily API contracts: {'PASS' if passed else 'FAIL / incomplete'}\n\n") + out.write(f"Core: `{os.environ['GITHUB_SHA']}`; shuffle seed: `{gate.get('api_seed', os.environ['GITHUB_RUN_ID'])}`.\n\n") + out.write('Fixed order followed by a seeded chain shuffle, with sequential operations inside each chain. The execution guard is 600 seconds; dependency and service preparation is separate. This job is not a required PR check.\n\n') + out.write(f"Cases: {suite.get('requests', 'not reported')}; statuses: {suite.get('status', 'not reported')}; suite execution: {suite.get('elapsed_seconds', 'not reported')}s.\n\n") + out.write(f"API behavior contracts: {coverage.get('areas', {}).get('api', 'not reported')}.\n\n") + out.write(f"Observed HTTP route templates (including preparation): {len(coverage['http_observations']['routes']) if 'http_observations' in coverage else 'not reported'}. A route hit does not establish all behavior behind it.\n\n") + for plugin in manifest.get('plugins', []): + out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") + errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', [])) + if gate.get('timed_out'): + errors.append('API suite exceeded its 600-second outer process guard.') + if gate.get('infrastructure_error'): + errors.append(gate['infrastructure_error']) + if not gate or not suite: + errors.append('Missing verdict; inspect preparation and service logs.') + for error in errors: + out.write(f"- {error}\n") + PY_SUMMARY + - name: Preserve API verdicts and evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: daily-api-${{ github.run_id }}-${{ github.run_attempt }} + path: | + .qa/api/gate/gate.json + .qa/api/gate/resources.jsonl + .qa/api/gate/suite.log + .qa/api/gate/suite/summary.json + .qa/api/gate/suite/profile.json + .qa/api/gate/suite/cases.json + .qa/api/gate/suite/health.json + .qa/api/gate/suite/cleanup-checks.json + .qa/api/gate/suite/resource-case-*.json + .qa/api/gate/suite/coverage.json + .qa/api/gate/suite/coverage.md + .qa/api/plugins/manifest.json + .qa/api/plugins/*-build.log + .qa/api/*.log + if-no-files-found: warn + retention-days: 30 diff --git a/tools/README.md b/tools/README.md index b5037aa1e..0c3f8d5cb 100644 --- a/tools/README.md +++ b/tools/README.md @@ -486,7 +486,7 @@ uv run --no-sync python tools/regression_ci.py --suite daily --restart-services `--suite smoke` retains the 60-second gate. `--suite scenarios` runs the shared expanded profile once under a 300-second guard. `--suite endurance` retains the separate ten-minute load profile under a 900-second guard; its CI entry point also -reports throughput/P95 without applying their thresholds. All four modes use +reports throughput/P95 without applying their thresholds. All suites use published plugins when `--plugins` is omitted. Both workflows explicitly select `--plugins plugins` so source changes are tested in the same PR. Daily additionally selects `misc/EssentialCommands misc/FAQManager`; manual dispatch can change the @@ -498,9 +498,17 @@ bind their own extensions to avoid changing unrelated prompts/tools. The suite a even after failures. The CI owner then stops its service processes and removes its Box containers; a process guard timeout can interrupt API cleanup. -## Local API contract job +## Daily API contract job -Run on the same disposable Linux/Docker host used by `regression_ci.py`: +`daily-endurance.yml` runs API contracts alongside the 30-minute endurance job +on separate runners, at the existing daily schedule and on manual dispatch. +The API job uses the GitHub run ID as its shuffle seed, so each daily run explores +a different order and a rerun keeps the same seed. It has a 15-minute job timeout, +including preparation, and is not a required PR check. Its summary and separate +`daily-api--` artifact retain failures, order, coverage and cleanup +evidence for 30 days. + +To replay locally on a disposable Linux/Docker host, pass the recorded seed: ```sh uv run --no-sync python tools/regression_ci.py --suite api --api-seed 20261028 \ @@ -527,9 +535,8 @@ Inspect `gate/gate.json` for the verdict, seed and exact case order; `gate/suite/cases.json` for HTTP steps, assertions and cleanup results; and `gate/suite/coverage.json` for observed route templates and behavior contracts. An HTTP route hit includes rejected requests and is not proof of all behavior -behind that route. The job intentionally fails on invalid MCP modes accepted -and persisted by Core; fixing that product validation should make the existing -negative case pass without changing its expected result. +behind that route. Invalid MCP modes must return HTTP 400 and leave the original +configuration unchanged. Any failed contract or cleanup makes the job fail. ## Sustained regression and baseline comparison diff --git a/tools/test_regression_ci.py b/tools/test_regression_ci.py index 354835fcb..83b3ee06c 100644 --- a/tools/test_regression_ci.py +++ b/tools/test_regression_ci.py @@ -143,6 +143,53 @@ class DailySummaryTest(unittest.TestCase): else: self.assertNotIn('Daily regression: PASS', text) + def test_api_summary_keeps_preparation_and_outer_failures_visible(self): + workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml' + steps = yaml.safe_load(workflow.read_text())['jobs']['api-contracts']['steps'] + command = next(step['run'] for step in steps if step.get('name') == 'Publish API result') + suite = {'passed': True, 'requests': 14, 'status': {'passed': 14}, 'elapsed_seconds': 7.3} + for gate, actual_suite, expected in [ + ({'passed': True, 'api_seed': 123}, suite, 'Daily API contracts: PASS'), + ( + {'passed': False, 'infrastructure_error': 'plugin installation failed'}, + None, + 'plugin installation failed', + ), + ({'passed': False, 'timed_out': True}, suite, '600-second outer process guard'), + ({'passed': False, 'resource_errors': ['sampling failed']}, suite, 'sampling failed'), + ({'passed': False, 'coverage_errors': ['missing evidence']}, suite, 'missing evidence'), + ({'passed': False}, {'passed': False, 'errors': ['MCP mode persisted']}, 'MCP mode persisted'), + (None, None, 'Missing verdict'), + ]: + with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / '.qa/api/gate/suite' + target.mkdir(parents=True) + if gate is not None: + (target.parent / 'gate.json').write_text(json.dumps(gate)) + if actual_suite is not None: + (target / 'summary.json').write_text(json.dumps(actual_suite)) + summary = root / 'summary.md' + subprocess.run( + ['bash', '-e', '-c', command], + cwd=root, + check=True, + capture_output=True, + env={ + **os.environ, + 'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'], + 'GITHUB_STEP_SUMMARY': str(summary), + 'GITHUB_SHA': 'tested-core', + 'GITHUB_RUN_ID': '456', + }, + ) + text = summary.read_text() + self.assertIn(expected, text) + self.assertIn('not reported', text) + self.assertIn('shuffle seed: `123`' if gate and gate.get('passed') else 'shuffle seed: `456`', text) + if not gate or not gate.get('passed'): + self.assertNotIn('Daily API contracts: PASS', text) + def test_core_summary_records_preparation_failure_and_selected_plugins(self): workflow = Path(__file__).resolve().parents[1] / '.github/workflows/core-smoke.yml' steps = yaml.safe_load(workflow.read_text())['jobs']['smoke']['steps']