diff --git a/.github/workflows/core-smoke.yml b/.github/workflows/core-smoke.yml new file mode 100644 index 000000000..b9938391b --- /dev/null +++ b/.github/workflows/core-smoke.yml @@ -0,0 +1,111 @@ +name: Core Smoke + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [master, develop] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: core-smoke-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + tools: + name: Regression tools + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install PyYAML==6.0.3 + - name: Check regression tools + run: timeout 60s python -m unittest discover -s tools -p 'test_*.py' -v + + smoke: + name: Core smoke (60s) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Install locked dependencies + run: uv sync --dev --frozen + - name: Read official plugin source pin + id: plugins + run: | + python - <<'PY' + import json, os + from pathlib import Path + revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision'] + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(f'revision={revision}\n') + PY + - uses: actions/checkout@v4 + with: + repository: langbot-app/langbot-plugins + ref: ${{ steps.plugins.outputs.revision }} + path: .smoke-plugins + persist-credentials: false + - name: Prepare CLI and sandbox image + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p .smoke-bin + gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .smoke-bin \ + --pattern lbctl_linux_amd64 --pattern checksums.txt + cd .smoke-bin + sha256sum --check --ignore-missing checksums.txt + chmod +x lbctl_linux_amd64 + docker pull python:3.12-alpine + - name: Prepare real services and run the 60-second smoke + run: | + uv run --no-sync python tools/regression_ci.py \ + --lbctl .smoke-bin/lbctl_linux_amd64 --plugins .smoke-plugins --output .qa/core-smoke + - name: Publish gate result + if: always() + run: | + python - <<'PY' + import json, os + from pathlib import Path + path = Path('.qa/core-smoke/gate/gate.json') + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: + if path.exists(): + data = json.loads(path.read_text()) + out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; " + f"{data['elapsed_seconds']:.2f}/60 seconds; {data['requests']} requests.\n") + out.write('Service/dependency preparation is outside the 60-second execution budget.\n') + else: + out.write('Core smoke did not produce a verdict; inspect preparation failure.\n') + PY + - name: Preserve verdicts and resource samples + if: always() + uses: actions/upload-artifact@v4 + with: + name: core-smoke-${{ github.run_id }} + path: | + .qa/core-smoke/gate/gate.json + .qa/core-smoke/gate/resources.jsonl + .qa/core-smoke/gate/suite.log + .qa/core-smoke/gate/suite/cases.json + .qa/core-smoke/gate/suite/summary.json + .qa/core-smoke/gate/suite/health.json + .qa/core-smoke/gate/suite/cleanup-checks.json + .qa/core-smoke/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f2baae7c7..831f16843 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,6 +9,10 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review] +concurrency: + group: lint-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: ruff: name: Ruff Lint & Format @@ -25,14 +29,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 - - name: Install dependencies - run: uv sync --dev - - name: Run ruff check - run: uv run ruff check src/langbot/ tests/ --output-format=concise + run: uvx ruff@0.14.14 check src/langbot/ tests/ tools/ --output-format=concise - name: Run ruff format - run: uv run ruff format src --check + run: uvx ruff@0.14.14 format src tools --check frontend: name: Frontend Lint diff --git a/tools/README.md b/tools/README.md index ae3a188df..fb86e551f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -362,3 +362,42 @@ cases can specify `exit_code` and `stderr` explicitly. To use a fresh deployment, select this suite in `job.json`; the Job supplies its SQLite/recording paths and uses mock embedding. Smoke skips load warmup and timed phases. The dedicated Docker daemon requirement above still applies. + +## PR smoke gate + +`Core Smoke` runs regression-tool tests and a real Core smoke in parallel on +GitHub-hosted Linux runners. The smoke starts Core from the checked-out candidate, +uses the SDK from `uv.lock`, the existing official-plugin source pin, `lbctl` +v0.3.0, a Docker Box and mock generation/embedding. It needs no model credentials +or access to a developer's running environment. + +Dependency installation, service startup, sandbox image download and plugin +installation are preparation. After readiness, the gate allows **60 seconds total** +for API resource creation, 28 cases / 37 requests (two turns per group), verdict +checks and API cleanup. A missing verdict, failed check or timeout fails the gate. +The ordinary expanded profile keeps three turns / 43 requests. The entire cold +workflow takes longer than a minute; it has a separate 15-minute timeout. + +On an existing prepared stack, use: + +```sh +uv run --no-sync python tools/regression_gate.py \ + --profile /path/to/smoke.json --output .qa/gate-001 +``` + +`gate.json` records the budget, measured duration, verdict and process resource +peaks. `resources.jsonl` samples configured process trees every second (RSS, +cumulative CPU time, threads and file descriptors); suite artifacts contain +per-case timings and business evidence. These are diagnostics, not calibrated +performance-regression thresholds. Docker sandbox resource sampling remains in +the longer load runner. A timeout kills the local Suite/CLI process group without +retrying requests; a reused remote service can still have an in-flight run, so +inspect its retained ownership journal before cleanup. CI tears down its own +service processes and owned sandboxes afterward. + +The workflow always creates its two checks on PRs and cancels superseded runs. +After the workflow is merged, configure `Regression tools` and `Core smoke (60s)` +as required status checks in the repository ruleset. A green workflow alone does +not enforce merge protection. Longer load and soak tests use the existing Suite +and Job entry points with separately calibrated latency, throughput and resource +budgets; the one-minute smoke does not replace them. diff --git a/tools/regression_ci.py b/tools/regression_ci.py new file mode 100644 index 000000000..c61481d7c --- /dev/null +++ b/tools/regression_ci.py @@ -0,0 +1,225 @@ +#!/usr/bin/env python3 +"""Prepare a disposable Linux CI stack from this checkout, then run the 60s gate.""" + +import argparse +import json +import os +from pathlib import Path +import secrets +import shutil +import signal +import subprocess +import sys +import time + +import yaml + +from regression_gate import supervise +from regression_suite import Suite +from regression_traffic import owned_box_containers, read_endpoint_snapshot + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402 + + +def ready(url, processes, timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if any(p.poll() is not None for p in processes): + raise RuntimeError('A smoke dependency exited; inspect service logs') + try: + read_endpoint_snapshot(url, timeout_seconds=1) + return + except (RuntimeError, OSError): + time.sleep(0.2) + raise RuntimeError(f'Service did not become ready: {url}') + + +def run(args): + if subprocess.check_output( + ['docker', 'ps', '-a', '--filter', 'label=langbot.box=true', '--format', '{{.ID}}'], text=True, timeout=10 + ).strip(): + raise RuntimeError('CI needs a dedicated Docker daemon with no existing Box sandboxes') + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=False, mode=0o700) + private = root / '.private' + private.mkdir(mode=0o700) + processes, logs = [], [] + key, runtime_token, box_token = (secrets.token_urlsafe(32) for _ in range(3)) + env = dict( + os.environ, + LANGBOT_QA_API_KEY=key, + LANGBOT_DATA_ROOT=str(private / 'data'), + PYTHONPATH=str(ROOT / 'src'), + LITELLM_LOCAL_MODEL_COST_MAP='True', + PYTHONUNBUFFERED='1', + LANGBOT_PLUGIN_RUNTIME_CONTROL_TOKEN=runtime_token, + LANGBOT_BOX_CONTROL_TOKEN=box_token, + ) + + def start(name, command): + log = (root / f'{name}.log').open('w') + logs.append(log) + process = subprocess.Popen( + command, cwd=private, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True + ) + processes.append(process) + return process.pid + + try: + config_path = create_minimal_config(private, port=15301) + create_test_directories(private) + config = yaml.safe_load(config_path.read_text()) + config['api'].update(global_api_key=key, host='127.0.0.1') + config['plugin'].update(enable=True, runtime_ws_url='ws://127.0.0.1:15340/control/ws') + config['box'].update(enabled=True, backend='local', runtime={'endpoint': 'ws://127.0.0.1:15310/control/ws'}) + config['box']['local'].update( + host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine' + ) + config_path.write_text(yaml.safe_dump(config)) + profile_path = ROOT / 'tools/regression_profiles/smoke.json' + profile = json.loads(profile_path.read_text()) + cli_config = private / 'lbctl.yaml' + cli_config.write_text( + yaml.safe_dump( + { + 'version': 1, + 'current_context': 'smoke', + 'contexts': { + 'smoke': { + 'endpoint': 'http://127.0.0.1:15301', + 'credential': {'type': 'env', 'name': 'LANGBOT_QA_API_KEY'}, + } + }, + } + ) + ) + profile['cli'] = {'executable': str(args.lbctl.resolve()), 'config': str(cli_config)} + profile['vars'].update( + mock_base_url='http://127.0.0.1:18080/v1', + embedding_base_url='http://127.0.0.1:18080/v1', + core_base_url='http://127.0.0.1:15301', + core_health='http://127.0.0.1:15301/healthz', + core_db=str(private / 'data/langbot.db'), + mock_records=str(root / 'mock-requests.jsonl'), + ) + profile['plugins'] = [] + # Two turns retain the history/isolation oracle while leaving room for setup and cleanup. + profile['smoke']['group_turns'] = 2 + for resource in profile['resources']: + if resource['kind'] == 'model' and resource.get('type') == 'embedding': + resource['body']['name'] = 'mock-embedding' + for document in resource.get('documents', []): + document['file'] = str((profile_path.parent / document['file']).resolve()) + pids = {} + pids['mock'] = start( + 'mock', + [ + sys.executable, + str(ROOT / 'tools/mock_llm.py'), + '--port', + '18080', + '--record', + str(root / 'mock-requests.jsonl'), + ], + ) + pids['runtime'] = start( + 'runtime', + [ + sys.executable, + '-m', + 'langbot_plugin.cli.__init__', + 'rt', + '--ws-control-port', + '15340', + '--ws-debug-port', + '15341', + ], + ) + pids['box'] = start( + 'box', + [ + sys.executable, + '-m', + 'langbot_plugin.cli.__init__', + 'box', + '--host', + '127.0.0.1', + '--ws-control-port', + '15310', + ], + ) + pids['core'] = start( + 'core', [sys.executable, str(ROOT / 'main.py'), '--standalone-runtime', '--standalone-box'] + ) + profile['observe'] = { + 'health': { + 'core': 'http://127.0.0.1:15301/healthz', + 'box': 'http://127.0.0.1:15310/readyz', + 'mock': 'http://127.0.0.1:18080/healthz', + }, + 'pids': pids, + } + for url in profile['observe']['health'].values(): + ready(url, processes) + preparation = root / 'preparation' + preparation.mkdir() + suite = Suite(profile, profile_path, preparation, env) + identity = suite.call(['whoami']) + # Bind the CLI to the newly bootstrapped Workspace before installing plugins. + cli_data = yaml.safe_load(cli_config.read_text()) + cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid'] + cli_config.write_text(yaml.safe_dump(cli_data)) + for directory in ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers'): + source = args.plugins / directory + package_source = private / source.name + shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__')) + archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source) + suite.call(['plugin', 'install', 'local', '--file', archive, '--wait']) + generated = private / 'smoke.json' + generated.write_text(json.dumps(profile, indent=2)) + return supervise( + [ + sys.executable, + str(ROOT / 'tools/regression_suite.py'), + 'run', + '--profile', + str(generated), + '--output', + str(root / 'gate/suite'), + ], + root / 'gate', + 60, + pids, + env, + ) + finally: + for process in reversed(processes): + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + for log in logs: + log.close() + # This is a disposable CI daemon; still delete only this run's mounted sandboxes. + for container in owned_box_containers(private / 'box'): + subprocess.run(['docker', 'rm', '-f', container], check=True, capture_output=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--lbctl', type=Path, required=True) + parser.add_argument('--plugins', type=Path, required=True) + args = parser.parse_args() + result = run(args) + print(json.dumps(result, indent=2)) + raise SystemExit(0 if result['passed'] else 1) + + +if __name__ == '__main__': + main() diff --git a/tools/regression_gate.py b/tools/regression_gate.py new file mode 100644 index 000000000..20e2b069a --- /dev/null +++ b/tools/regression_gate.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +"""Bound a prepared-environment smoke run and record process resource samples.""" + +import argparse +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +from regression_suite import validate +from regression_traffic import read_process_snapshot + + +def supervise(command, output, budget, pids, env=None): + output.mkdir(parents=True, exist_ok=False, mode=0o700) + started = time.monotonic() + samples, errors = [], [] + timed_out = False + with (output / 'suite.log').open('w') as log: + process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True) + try: + while process.poll() is None: + remaining = budget - (time.monotonic() - started) + if remaining <= 0: + timed_out = True + os.killpg(process.pid, signal.SIGKILL) + process.wait() + break + sample = {'elapsed_seconds': time.monotonic() - started, 'processes': {}} + for name, pid in pids.items(): + try: + sample['processes'][name] = read_process_snapshot(pid) + except (RuntimeError, OSError) as exc: + errors.append(f'{name}: {exc}') + samples.append(sample) + remaining = budget - (time.monotonic() - started) + try: + process.wait(timeout=max(0, min(1, remaining))) + except subprocess.TimeoutExpired: + pass + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + elapsed = time.monotonic() - started + (output / 'resources.jsonl').write_text(''.join(json.dumps(sample) + '\n' for sample in samples)) + summary_path = output / 'suite' / 'summary.json' + summary = json.loads(summary_path.read_text()) if summary_path.exists() else {} + result = { + 'passed': process.returncode == 0 + and summary.get('passed') is True + and elapsed <= budget + and not timed_out + and not errors, + 'budget_seconds': budget, + 'elapsed_seconds': elapsed, + 'timed_out': timed_out, + 'exit_code': process.returncode, + 'resource_errors': errors, + 'requests': summary.get('requests'), + 'status': summary.get('status'), + 'stages': summary.get('stages'), + 'resource_peaks': { + name: { + metric: max((s['processes'].get(name, {}).get(metric, 0) for s in samples), default=0) + for metric in ('rss_bytes', 'threads', 'open_fds') + } + for name in pids + }, + } + (output / 'gate.json').write_text(json.dumps(result, indent=2) + '\n') + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--profile', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + profile = json.loads(args.profile.read_text()) + validate(profile) + if 'smoke' not in profile: + parser.error('a smoke profile is required') + result = supervise( + [ + sys.executable, + str(Path(__file__).with_name('regression_suite.py')), + 'run', + '--profile', + str(args.profile.resolve()), + '--output', + str(args.output.resolve() / 'suite'), + ], + args.output, + 60, + profile.get('observe', {}).get('pids', {}), + ) + print(json.dumps(result, indent=2)) + raise SystemExit(0 if result['passed'] else 1) + + +if __name__ == '__main__': + main() diff --git a/tools/test_regression_gate.py b/tools/test_regression_gate.py new file mode 100644 index 000000000..95aa14d58 --- /dev/null +++ b/tools/test_regression_gate.py @@ -0,0 +1,40 @@ +"""A gate must fail closed on timeout, missing verdict, and inner failure.""" + +import os +from pathlib import Path +import sys +import tempfile +import unittest + +from regression_gate import supervise + + +class GateTest(unittest.TestCase): + def test_success_records_elapsed_time_and_resources(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'gate' + code = ( + 'from pathlib import Path; import json; ' + f'p=Path({str(output / "suite")!r}); p.mkdir(); ' + '(p/"summary.json").write_text(json.dumps({"passed":True,"requests":1}))' + ) + result = supervise([sys.executable, '-c', code], output, 5, {'test': os.getpid()}) + self.assertTrue(result['passed']) + self.assertLess(result['elapsed_seconds'], 5) + self.assertGreater(result['resource_peaks']['test']['rss_bytes'], 0) + + def test_timeout_is_failed_without_retry(self): + with tempfile.TemporaryDirectory() as directory: + result = supervise([sys.executable, '-c', 'import time; time.sleep(30)'], Path(directory) / 'gate', 0.1, {}) + self.assertFalse(result['passed']) + self.assertTrue(result['timed_out']) + self.assertLess(result['elapsed_seconds'], 2) + + def test_zero_exit_without_business_verdict_is_failed(self): + with tempfile.TemporaryDirectory() as directory: + result = supervise([sys.executable, '-c', 'pass'], Path(directory) / 'gate', 5, {}) + self.assertFalse(result['passed']) + + +if __name__ == '__main__': + unittest.main()