ci: add a timed real core smoke and lightweight tool checks

.github/workflows/core-smoke.yml

@@ -0,0 +1,111 @@
+name: Core Smoke
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened, ready_for_review]
+  push:
+    branches: [master, develop]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+concurrency:
+  group: core-smoke-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  tools:
+    name: Regression tools
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - run: python -m pip install PyYAML==6.0.3
+      - name: Check regression tools
+        run: timeout 60s python -m unittest discover -s tools -p 'test_*.py' -v
+
+  smoke:
+    name: Core smoke (60s)
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+      - uses: astral-sh/setup-uv@v4
+        with:
+          enable-cache: true
+      - name: Install locked dependencies
+        run: uv sync --dev --frozen
+      - name: Read official plugin source pin
+        id: plugins
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision']
+          with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
+              stream.write(f'revision={revision}\n')
+          PY
+      - uses: actions/checkout@v4
+        with:
+          repository: langbot-app/langbot-plugins
+          ref: ${{ steps.plugins.outputs.revision }}
+          path: .smoke-plugins
+          persist-credentials: false
+      - name: Prepare CLI and sandbox image
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          mkdir -p .smoke-bin
+          gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .smoke-bin \
+            --pattern lbctl_linux_amd64 --pattern checksums.txt
+          cd .smoke-bin
+          sha256sum --check --ignore-missing checksums.txt
+          chmod +x lbctl_linux_amd64
+          docker pull python:3.12-alpine
+      - name: Prepare real services and run the 60-second smoke
+        run: |
+          uv run --no-sync python tools/regression_ci.py \
+            --lbctl .smoke-bin/lbctl_linux_amd64 --plugins .smoke-plugins --output .qa/core-smoke
+      - name: Publish gate result
+        if: always()
+        run: |
+          python - <<'PY'
+          import json, os
+          from pathlib import Path
+          path = Path('.qa/core-smoke/gate/gate.json')
+          with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
+              if path.exists():
+                  data = json.loads(path.read_text())
+                  out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; "
+                            f"{data['elapsed_seconds']:.2f}/60 seconds; {data['requests']} requests.\n")
+                  out.write('Service/dependency preparation is outside the 60-second execution budget.\n')
+              else:
+                  out.write('Core smoke did not produce a verdict; inspect preparation failure.\n')
+          PY
+      - name: Preserve verdicts and resource samples
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: core-smoke-${{ github.run_id }}
+          path: |
+            .qa/core-smoke/gate/gate.json
+            .qa/core-smoke/gate/resources.jsonl
+            .qa/core-smoke/gate/suite.log
+            .qa/core-smoke/gate/suite/cases.json
+            .qa/core-smoke/gate/suite/summary.json
+            .qa/core-smoke/gate/suite/health.json
+            .qa/core-smoke/gate/suite/cleanup-checks.json
+            .qa/core-smoke/*.log
+          if-no-files-found: warn
+          retention-days: 7

.github/workflows/lint.yml

@@ -9,6 +9,10 @@ on:
   pull_request:
     types: [opened, synchronize, reopened, ready_for_review]
 
+concurrency:
+  group: lint-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
 jobs:
   ruff:
     name: Ruff Lint & Format
@@ -25,14 +29,11 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v4
 
-      - name: Install dependencies
-        run: uv sync --dev
-
       - name: Run ruff check
-        run: uv run ruff check src/langbot/ tests/ --output-format=concise
+        run: uvx ruff@0.14.14 check src/langbot/ tests/ tools/ --output-format=concise
 
       - name: Run ruff format
-        run: uv run ruff format src --check
+        run: uvx ruff@0.14.14 format src tools --check
 
   frontend:
     name: Frontend Lint

tools/README.md

@@ -362,3 +362,42 @@ cases can specify `exit_code` and `stderr` explicitly.
 To use a fresh deployment, select this suite in `job.json`; the Job supplies its
 SQLite/recording paths and uses mock embedding. Smoke skips load warmup and timed
 phases. The dedicated Docker daemon requirement above still applies.
+
+## PR smoke gate
+
+`Core Smoke` runs regression-tool tests and a real Core smoke in parallel on
+GitHub-hosted Linux runners. The smoke starts Core from the checked-out candidate,
+uses the SDK from `uv.lock`, the existing official-plugin source pin, `lbctl`
+v0.3.0, a Docker Box and mock generation/embedding. It needs no model credentials
+or access to a developer's running environment.
+
+Dependency installation, service startup, sandbox image download and plugin
+installation are preparation. After readiness, the gate allows **60 seconds total**
+for API resource creation, 28 cases / 37 requests (two turns per group), verdict
+checks and API cleanup. A missing verdict, failed check or timeout fails the gate.
+The ordinary expanded profile keeps three turns / 43 requests. The entire cold
+workflow takes longer than a minute; it has a separate 15-minute timeout.
+
+On an existing prepared stack, use:
+
+```sh
+uv run --no-sync python tools/regression_gate.py \
+  --profile /path/to/smoke.json --output .qa/gate-001
+```
+
+`gate.json` records the budget, measured duration, verdict and process resource
+peaks. `resources.jsonl` samples configured process trees every second (RSS,
+cumulative CPU time, threads and file descriptors); suite artifacts contain
+per-case timings and business evidence. These are diagnostics, not calibrated
+performance-regression thresholds. Docker sandbox resource sampling remains in
+the longer load runner. A timeout kills the local Suite/CLI process group without
+retrying requests; a reused remote service can still have an in-flight run, so
+inspect its retained ownership journal before cleanup. CI tears down its own
+service processes and owned sandboxes afterward.
+
+The workflow always creates its two checks on PRs and cancels superseded runs.
+After the workflow is merged, configure `Regression tools` and `Core smoke (60s)`
+as required status checks in the repository ruleset. A green workflow alone does
+not enforce merge protection. Longer load and soak tests use the existing Suite
+and Job entry points with separately calibrated latency, throughput and resource
+budgets; the one-minute smoke does not replace them.

tools/regression_ci.py

@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Prepare a disposable Linux CI stack from this checkout, then run the 60s gate."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import secrets
+import shutil
+import signal
+import subprocess
+import sys
+import time
+
+import yaml
+
+from regression_gate import supervise
+from regression_suite import Suite
+from regression_traffic import owned_box_containers, read_endpoint_snapshot
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories  # noqa: E402
+
+
+def ready(url, processes, timeout=120):
+    deadline = time.monotonic() + timeout
+    while time.monotonic() < deadline:
+        if any(p.poll() is not None for p in processes):
+            raise RuntimeError('A smoke dependency exited; inspect service logs')
+        try:
+            read_endpoint_snapshot(url, timeout_seconds=1)
+            return
+        except (RuntimeError, OSError):
+            time.sleep(0.2)
+    raise RuntimeError(f'Service did not become ready: {url}')
+
+
+def run(args):
+    if subprocess.check_output(
+        ['docker', 'ps', '-a', '--filter', 'label=langbot.box=true', '--format', '{{.ID}}'], text=True, timeout=10
+    ).strip():
+        raise RuntimeError('CI needs a dedicated Docker daemon with no existing Box sandboxes')
+    root = args.output.resolve()
+    root.mkdir(parents=True, exist_ok=False, mode=0o700)
+    private = root / '.private'
+    private.mkdir(mode=0o700)
+    processes, logs = [], []
+    key, runtime_token, box_token = (secrets.token_urlsafe(32) for _ in range(3))
+    env = dict(
+        os.environ,
+        LANGBOT_QA_API_KEY=key,
+        LANGBOT_DATA_ROOT=str(private / 'data'),
+        PYTHONPATH=str(ROOT / 'src'),
+        LITELLM_LOCAL_MODEL_COST_MAP='True',
+        PYTHONUNBUFFERED='1',
+        LANGBOT_PLUGIN_RUNTIME_CONTROL_TOKEN=runtime_token,
+        LANGBOT_BOX_CONTROL_TOKEN=box_token,
+    )
+
+    def start(name, command):
+        log = (root / f'{name}.log').open('w')
+        logs.append(log)
+        process = subprocess.Popen(
+            command, cwd=private, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True
+        )
+        processes.append(process)
+        return process.pid
+
+    try:
+        config_path = create_minimal_config(private, port=15301)
+        create_test_directories(private)
+        config = yaml.safe_load(config_path.read_text())
+        config['api'].update(global_api_key=key, host='127.0.0.1')
+        config['plugin'].update(enable=True, runtime_ws_url='ws://127.0.0.1:15340/control/ws')
+        config['box'].update(enabled=True, backend='local', runtime={'endpoint': 'ws://127.0.0.1:15310/control/ws'})
+        config['box']['local'].update(
+            host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine'
+        )
+        config_path.write_text(yaml.safe_dump(config))
+        profile_path = ROOT / 'tools/regression_profiles/smoke.json'
+        profile = json.loads(profile_path.read_text())
+        cli_config = private / 'lbctl.yaml'
+        cli_config.write_text(
+            yaml.safe_dump(
+                {
+                    'version': 1,
+                    'current_context': 'smoke',
+                    'contexts': {
+                        'smoke': {
+                            'endpoint': 'http://127.0.0.1:15301',
+                            'credential': {'type': 'env', 'name': 'LANGBOT_QA_API_KEY'},
+                        }
+                    },
+                }
+            )
+        )
+        profile['cli'] = {'executable': str(args.lbctl.resolve()), 'config': str(cli_config)}
+        profile['vars'].update(
+            mock_base_url='http://127.0.0.1:18080/v1',
+            embedding_base_url='http://127.0.0.1:18080/v1',
+            core_base_url='http://127.0.0.1:15301',
+            core_health='http://127.0.0.1:15301/healthz',
+            core_db=str(private / 'data/langbot.db'),
+            mock_records=str(root / 'mock-requests.jsonl'),
+        )
+        profile['plugins'] = []
+        # Two turns retain the history/isolation oracle while leaving room for setup and cleanup.
+        profile['smoke']['group_turns'] = 2
+        for resource in profile['resources']:
+            if resource['kind'] == 'model' and resource.get('type') == 'embedding':
+                resource['body']['name'] = 'mock-embedding'
+            for document in resource.get('documents', []):
+                document['file'] = str((profile_path.parent / document['file']).resolve())
+        pids = {}
+        pids['mock'] = start(
+            'mock',
+            [
+                sys.executable,
+                str(ROOT / 'tools/mock_llm.py'),
+                '--port',
+                '18080',
+                '--record',
+                str(root / 'mock-requests.jsonl'),
+            ],
+        )
+        pids['runtime'] = start(
+            'runtime',
+            [
+                sys.executable,
+                '-m',
+                'langbot_plugin.cli.__init__',
+                'rt',
+                '--ws-control-port',
+                '15340',
+                '--ws-debug-port',
+                '15341',
+            ],
+        )
+        pids['box'] = start(
+            'box',
+            [
+                sys.executable,
+                '-m',
+                'langbot_plugin.cli.__init__',
+                'box',
+                '--host',
+                '127.0.0.1',
+                '--ws-control-port',
+                '15310',
+            ],
+        )
+        pids['core'] = start(
+            'core', [sys.executable, str(ROOT / 'main.py'), '--standalone-runtime', '--standalone-box']
+        )
+        profile['observe'] = {
+            'health': {
+                'core': 'http://127.0.0.1:15301/healthz',
+                'box': 'http://127.0.0.1:15310/readyz',
+                'mock': 'http://127.0.0.1:18080/healthz',
+            },
+            'pids': pids,
+        }
+        for url in profile['observe']['health'].values():
+            ready(url, processes)
+        preparation = root / 'preparation'
+        preparation.mkdir()
+        suite = Suite(profile, profile_path, preparation, env)
+        identity = suite.call(['whoami'])
+        # Bind the CLI to the newly bootstrapped Workspace before installing plugins.
+        cli_data = yaml.safe_load(cli_config.read_text())
+        cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid']
+        cli_config.write_text(yaml.safe_dump(cli_data))
+        for directory in ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers'):
+            source = args.plugins / directory
+            package_source = private / source.name
+            shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__'))
+            archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source)
+            suite.call(['plugin', 'install', 'local', '--file', archive, '--wait'])
+        generated = private / 'smoke.json'
+        generated.write_text(json.dumps(profile, indent=2))
+        return supervise(
+            [
+                sys.executable,
+                str(ROOT / 'tools/regression_suite.py'),
+                'run',
+                '--profile',
+                str(generated),
+                '--output',
+                str(root / 'gate/suite'),
+            ],
+            root / 'gate',
+            60,
+            pids,
+            env,
+        )
+    finally:
+        for process in reversed(processes):
+            if process.poll() is None:
+                os.killpg(process.pid, signal.SIGTERM)
+                try:
+                    process.wait(timeout=5)
+                except subprocess.TimeoutExpired:
+                    os.killpg(process.pid, signal.SIGKILL)
+                    process.wait()
+        for log in logs:
+            log.close()
+        # This is a disposable CI daemon; still delete only this run's mounted sandboxes.
+        for container in owned_box_containers(private / 'box'):
+            subprocess.run(['docker', 'rm', '-f', container], check=True, capture_output=True)
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--output', type=Path, required=True)
+    parser.add_argument('--lbctl', type=Path, required=True)
+    parser.add_argument('--plugins', type=Path, required=True)
+    args = parser.parse_args()
+    result = run(args)
+    print(json.dumps(result, indent=2))
+    raise SystemExit(0 if result['passed'] else 1)
+
+
+if __name__ == '__main__':
+    main()

tools/regression_gate.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+"""Bound a prepared-environment smoke run and record process resource samples."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import time
+
+from regression_suite import validate
+from regression_traffic import read_process_snapshot
+
+
+def supervise(command, output, budget, pids, env=None):
+    output.mkdir(parents=True, exist_ok=False, mode=0o700)
+    started = time.monotonic()
+    samples, errors = [], []
+    timed_out = False
+    with (output / 'suite.log').open('w') as log:
+        process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True)
+        try:
+            while process.poll() is None:
+                remaining = budget - (time.monotonic() - started)
+                if remaining <= 0:
+                    timed_out = True
+                    os.killpg(process.pid, signal.SIGKILL)
+                    process.wait()
+                    break
+                sample = {'elapsed_seconds': time.monotonic() - started, 'processes': {}}
+                for name, pid in pids.items():
+                    try:
+                        sample['processes'][name] = read_process_snapshot(pid)
+                    except (RuntimeError, OSError) as exc:
+                        errors.append(f'{name}: {exc}')
+                samples.append(sample)
+                remaining = budget - (time.monotonic() - started)
+                try:
+                    process.wait(timeout=max(0, min(1, remaining)))
+                except subprocess.TimeoutExpired:
+                    pass
+        finally:
+            if process.poll() is None:
+                os.killpg(process.pid, signal.SIGKILL)
+                process.wait()
+    elapsed = time.monotonic() - started
+    (output / 'resources.jsonl').write_text(''.join(json.dumps(sample) + '\n' for sample in samples))
+    summary_path = output / 'suite' / 'summary.json'
+    summary = json.loads(summary_path.read_text()) if summary_path.exists() else {}
+    result = {
+        'passed': process.returncode == 0
+        and summary.get('passed') is True
+        and elapsed <= budget
+        and not timed_out
+        and not errors,
+        'budget_seconds': budget,
+        'elapsed_seconds': elapsed,
+        'timed_out': timed_out,
+        'exit_code': process.returncode,
+        'resource_errors': errors,
+        'requests': summary.get('requests'),
+        'status': summary.get('status'),
+        'stages': summary.get('stages'),
+        'resource_peaks': {
+            name: {
+                metric: max((s['processes'].get(name, {}).get(metric, 0) for s in samples), default=0)
+                for metric in ('rss_bytes', 'threads', 'open_fds')
+            }
+            for name in pids
+        },
+    }
+    (output / 'gate.json').write_text(json.dumps(result, indent=2) + '\n')
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('--profile', type=Path, required=True)
+    parser.add_argument('--output', type=Path, required=True)
+    args = parser.parse_args()
+    profile = json.loads(args.profile.read_text())
+    validate(profile)
+    if 'smoke' not in profile:
+        parser.error('a smoke profile is required')
+    result = supervise(
+        [
+            sys.executable,
+            str(Path(__file__).with_name('regression_suite.py')),
+            'run',
+            '--profile',
+            str(args.profile.resolve()),
+            '--output',
+            str(args.output.resolve() / 'suite'),
+        ],
+        args.output,
+        60,
+        profile.get('observe', {}).get('pids', {}),
+    )
+    print(json.dumps(result, indent=2))
+    raise SystemExit(0 if result['passed'] else 1)
+
+
+if __name__ == '__main__':
+    main()

tools/test_regression_gate.py

@@ -0,0 +1,40 @@
+"""A gate must fail closed on timeout, missing verdict, and inner failure."""
+
+import os
+from pathlib import Path
+import sys
+import tempfile
+import unittest
+
+from regression_gate import supervise
+
+
+class GateTest(unittest.TestCase):
+    def test_success_records_elapsed_time_and_resources(self):
+        with tempfile.TemporaryDirectory() as directory:
+            output = Path(directory) / 'gate'
+            code = (
+                'from pathlib import Path; import json; '
+                f'p=Path({str(output / "suite")!r}); p.mkdir(); '
+                '(p/"summary.json").write_text(json.dumps({"passed":True,"requests":1}))'
+            )
+            result = supervise([sys.executable, '-c', code], output, 5, {'test': os.getpid()})
+            self.assertTrue(result['passed'])
+            self.assertLess(result['elapsed_seconds'], 5)
+            self.assertGreater(result['resource_peaks']['test']['rss_bytes'], 0)
+
+    def test_timeout_is_failed_without_retry(self):
+        with tempfile.TemporaryDirectory() as directory:
+            result = supervise([sys.executable, '-c', 'import time; time.sleep(30)'], Path(directory) / 'gate', 0.1, {})
+            self.assertFalse(result['passed'])
+            self.assertTrue(result['timed_out'])
+            self.assertLess(result['elapsed_seconds'], 2)
+
+    def test_zero_exit_without_business_verdict_is_failed(self):
+        with tempfile.TemporaryDirectory() as directory:
+            result = supervise([sys.executable, '-c', 'pass'], Path(directory) / 'gate', 5, {})
+            self.assertFalse(result['passed'])
+
+
+if __name__ == '__main__':
+    unittest.main()