From 5ec41f15a16ecdf4354e54456243235b4183c27c Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <1051233107@qq.com> Date: Fri, 23 Oct 2026 16:24:26 +0800 Subject: [PATCH] test(ci): add a 60-second core smoke gate and regression jobs (#2711) * test(tools): add deterministic lbctl regression job * test(tools): simulate persistent multi-group conversations * test(tools): add finite core smoke with terminal run and file oracles * test(tools): expand smoke coverage across execution stages * ci: add a timed real core smoke and lightweight tool checks * fix(ci): keep infrastructure failures visible in smoke verdicts --- .github/workflows/core-smoke.yml | 113 ++ .github/workflows/lint.yml | 11 +- tools/README.md | 403 ++++++ tools/fixtures/smoke-isolation.txt | 2 + tools/mock_llm.py | 401 ++++++ tools/mock_llm_scenarios.json | 169 +++ tools/regression_ci.py | 235 ++++ tools/regression_gate.py | 110 ++ tools/regression_job.py | 480 ++++++++ tools/regression_profiles/groups.json | 464 +++++++ tools/regression_profiles/job.json | 15 + tools/regression_profiles/normal.json | 111 ++ tools/regression_profiles/smoke.json | 1644 +++++++++++++++++++++++++ tools/regression_smoke.py | 328 +++++ tools/regression_suite.py | 717 +++++++++++ tools/regression_traffic.py | 449 +++++++ tools/test_mock_llm.py | 209 ++++ tools/test_regression_gate.py | 66 + tools/test_regression_groups.py | 240 ++++ tools/test_regression_job.py | 195 +++ tools/test_regression_smoke.py | 205 +++ tools/test_regression_suite.py | 309 +++++ 22 files changed, 6871 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/core-smoke.yml create mode 100644 tools/README.md create mode 100644 tools/fixtures/smoke-isolation.txt create mode 100644 tools/mock_llm.py create mode 100644 tools/mock_llm_scenarios.json create mode 100644 tools/regression_ci.py create mode 100644 tools/regression_gate.py create mode 100644 tools/regression_job.py create mode 100644 tools/regression_profiles/groups.json create mode 100644 tools/regression_profiles/job.json create mode 100644 tools/regression_profiles/normal.json create mode 100644 tools/regression_profiles/smoke.json create mode 100644 tools/regression_smoke.py create mode 100644 tools/regression_suite.py create mode 100644 tools/regression_traffic.py create mode 100644 tools/test_mock_llm.py create mode 100644 tools/test_regression_gate.py create mode 100644 tools/test_regression_groups.py create mode 100644 tools/test_regression_job.py create mode 100644 tools/test_regression_smoke.py create mode 100644 tools/test_regression_suite.py diff --git a/.github/workflows/core-smoke.yml b/.github/workflows/core-smoke.yml new file mode 100644 index 000000000..722fb3ae7 --- /dev/null +++ b/.github/workflows/core-smoke.yml @@ -0,0 +1,113 @@ +name: Core Smoke + +on: + pull_request: + types: [opened, synchronize, reopened, ready_for_review] + push: + branches: [master, develop] + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: core-smoke-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + tools: + name: Regression tools + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - run: python -m pip install PyYAML==6.0.3 + - name: Check regression tools + run: timeout 60s python -m unittest discover -s tools -p 'test_*.py' -v + + smoke: + name: Core smoke (60s) + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Install locked dependencies + run: uv sync --dev --frozen + - name: Read official plugin source pin + id: plugins + run: | + python - <<'PY' + import json, os + from pathlib import Path + revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision'] + with open(os.environ['GITHUB_OUTPUT'], 'a') as stream: + stream.write(f'revision={revision}\n') + PY + - uses: actions/checkout@v4 + with: + repository: langbot-app/langbot-plugins + ref: ${{ steps.plugins.outputs.revision }} + path: .smoke-plugins + persist-credentials: false + - name: Prepare CLI and sandbox image + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p .smoke-bin + gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .smoke-bin \ + --pattern lbctl_linux_amd64 --pattern checksums.txt + cd .smoke-bin + sha256sum --check --ignore-missing checksums.txt + chmod +x lbctl_linux_amd64 + docker pull python:3.12-alpine + - name: Prepare real services and run the 60-second smoke + run: | + uv run --no-sync python tools/regression_ci.py \ + --lbctl .smoke-bin/lbctl_linux_amd64 --plugins .smoke-plugins --output .qa/core-smoke + - name: Publish gate result + if: always() + run: | + python - <<'PY' + import json, os + from pathlib import Path + path = Path('.qa/core-smoke/gate/gate.json') + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: + if path.exists(): + data = json.loads(path.read_text()) + out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; " + f"{data.get('elapsed_seconds', 0):.2f}/60 seconds; {data.get('requests', 0)} requests.\n") + if data.get('infrastructure_error'): + out.write('Preparation or teardown failed; inspect service logs.\n') + out.write('Service/dependency preparation is outside the 60-second execution budget.\n') + else: + out.write('Core smoke did not produce a verdict; inspect preparation failure.\n') + PY + - name: Preserve verdicts and resource samples + if: always() + uses: actions/upload-artifact@v4 + with: + name: core-smoke-${{ github.run_id }} + path: | + .qa/core-smoke/gate/gate.json + .qa/core-smoke/gate/resources.jsonl + .qa/core-smoke/gate/suite.log + .qa/core-smoke/gate/suite/cases.json + .qa/core-smoke/gate/suite/summary.json + .qa/core-smoke/gate/suite/health.json + .qa/core-smoke/gate/suite/cleanup-checks.json + .qa/core-smoke/*.log + if-no-files-found: warn + retention-days: 7 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index f2baae7c7..831f16843 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -9,6 +9,10 @@ on: pull_request: types: [opened, synchronize, reopened, ready_for_review] +concurrency: + group: lint-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + jobs: ruff: name: Ruff Lint & Format @@ -25,14 +29,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v4 - - name: Install dependencies - run: uv sync --dev - - name: Run ruff check - run: uv run ruff check src/langbot/ tests/ --output-format=concise + run: uvx ruff@0.14.14 check src/langbot/ tests/ tools/ --output-format=concise - name: Run ruff format - run: uv run ruff format src --check + run: uvx ruff@0.14.14 format src tools --check frontend: name: Frontend Lint diff --git a/tools/README.md b/tools/README.md new file mode 100644 index 000000000..fb86e551f --- /dev/null +++ b/tools/README.md @@ -0,0 +1,403 @@ +# LangBot regression tools + +Run the deterministic OpenAI Chat Completions endpoint with Python 3.11+, without +API credentials or third-party dependencies: + +```sh +python3 tools/mock_llm.py --port 18080 --record /tmp/langbot-mock-requests.jsonl +``` + +In LangBot, create an `openai-chat-completions` Provider whose base URL is +`http://127.0.0.1:18080/v1` and whose API key is any test-only value. Use a model +from `GET /v1/models`. The loopback address works when Core and this tool run on +the same host; containers must use an address reachable from their own network. + +| Model | Behavior | +| --- | --- | +| `mock-chat` | Returns `LANGBOT_CHAT_OK`, including SSE streaming | +| `mock-rag` | Requires the golden document sentinel in the model input, then returns `LANGBOT_RAG_OK` | +| `mock-embedding` | Returns deterministic normalized token-hash vectors from `/v1/embeddings` | +| `mock-box` | Calls `exec` twice to write/read `/workspace/mock-llm.txt`, then returns `LANGBOT_BOX_OK` | +| `mock-401`, `mock-429`, `mock-503` | Returns the selected HTTP failure | +| `mock-slow` | Delays the response by two seconds | +| `mock-slow-stream` | Delays each SSE chunk by 0.2 seconds | + +`mock-rag` requires `aurora-parser-rag-9137` in the submitted messages. The +[golden HTML fixture](../skills/skills/langbot-testing/fixtures/rag/parser-golden.html) +contains this marker; retrieve it into the model context without including it +in the user's question. + +For `mock-box`, enable Box in the Runner and expose the native `exec` tool. Each +tool result must contain `LANGBOT_BOX_SENTINEL` before the next step is returned. +Tool-call IDs and fragmented streaming arguments use the normal OpenAI protocol. +Scenario progress comes from each request's current conversation turn, so +concurrent sessions and retries do not consume a shared script counter. +Tagged group traffic additionally requires the previous user turn and assistant +acknowledgement in the model input, and rejects another group's history. Final +responses carry a per-turn acknowledgement checked by the traffic driver. + +Add fixed response/tool sequences with `--scenarios path/to/scenarios.json`, using +`mock_llm_scenarios.json` as the example. `expect_tool_output` checks a substring +in the previous tool results; `expect_prompt` checks the submitted messages. +The server returns tool calls; it never executes +their commands. Request recordings contain prompts, tool results, responses, +status, and disconnects, but no HTTP headers. Use test data only. + +Run the focused HTTP checks on the Linux test host: + +```sh +python3 -m unittest discover -s tools -p test_mock_llm.py -v +``` + +These checks validate the mock server. A LangBot regression must additionally +verify real tool outputs and retrieved document IDs/content. Token usage is +synthetic, and this endpoint does not evaluate answer quality or embedding models. + +`mock-embedding` accepts string/batch inputs, float/base64 encoding, and optional +dimensions (default 384). It tests the embedding protocol and RAG plumbing; +token hashes do not establish semantic retrieval quality. + +## Normal traffic and resource collection + +`regression_traffic.py` executes a JSON list of normal `lbctl` cases at a selected +rate. Each case has a name, CLI arguments, and an expected response substring: + +```json +[{"name":"chat","args":["pipeline","run","PIPELINE_UUID","--message","Hello"],"expect":"LANGBOT_CHAT_OK"}] +``` + +Prepare a Workspace-bound CLI context and its API key environment variable, then +run on the Linux test host: + +```sh +python3 tools/regression_traffic.py \ + --lbctl /path/to/lbctl --config /path/to/lbctl.yaml \ + --cases /path/to/cases.json --output /path/to/new-results-directory \ + --duration 600 --rps 2 --workers 4 --cooldown 30 \ + --pid core=CORE_PID --pid mock=MOCK_PID \ + --health core=http://127.0.0.1:5300/healthz \ + --health mock=http://127.0.0.1:18080/healthz +``` + +Add `--container CONTAINER_ID` for Docker CPU/memory/PID statistics. The executing +account must be able to inspect the selected process trees and Docker daemon. +Resource collection reuses `scripts/cloud_runtime_soak.py`; it does not require +cgroup v2. PID resource totals include descendants, so select disjoint trees. + +The result directory contains `requests.jsonl`, `resources.jsonl`, `progress.json`, +and `summary.json`. The requested rate is a target: four workers bound concurrency, +and slow requests lower the achieved rate. Requests with unknown outcomes are +recorded without retry. A command fails its business check if it returns an error +or omits the expected response. Health/resource collection failures also make the +run fail. API key values are redacted from recorded CLI output. + +Latency percentiles include the entire CLI invocation and use successful +requests only. Achieved request rate is `requests / load_elapsed_seconds`. +`health_samples` counts sampling rounds; `health_failures` counts rounds with +any failed health probe or process/container resource read. + +## Configured regression suites + +[`regression_suite.py`](regression_suite.py) turns a JSON profile into a repeatable +resource preparation, traffic, observation, verdict, and cleanup sequence. Start +with [`regression_profiles/groups.json`](regression_profiles/groups.json) for +continuous group conversations. The earlier +[`normal.json`](regression_profiles/normal.json) profile retains independent CLI +requests; parallel CLI workers alone do not simulate persistent groups. + +Use a dedicated QA Workspace. First provision the instance with `lbctl install` +or your source deployment, enable Docker Box, start the mock and embedding +backends, and prepare a Workspace-bound `lbctl` config. Set the profile's CLI +config path, backend URLs, and three plugin archive paths. The sample embedding +endpoint must serve `all-MiniLM-L6-v2`; the mock serves generation only. Relative +paths resolve against the profile directory, so adjust them if you move it. + +Run on the Linux test host: + +```sh +python3 tools/regression_suite.py validate --profile tools/regression_profiles/groups.json +python3 tools/regression_suite.py run \ + --profile tools/regression_profiles/groups.json --output .qa/regression/run-001 +``` + +`validate` checks profile syntax and references without contacting the instance. +The output directory for `run` must be new. CLI options `--duration`, `--rps`, and +`--workers` override those values in every phase. No additional Python dependency +is needed. + +The profile controls: + +- `plugins`: local archives or GitHub/marketplace request files, installed with + `lbctl plugin install ... --wait`. +- `resources`: Providers, models, knowledge bases, document ingestion, Pipelines, + and Bots, created in dependency order. `${resource_id}` refers to a UUID + returned by an earlier creation; `${workspace_uuid}` is the current Workspace. + Pipeline config patches merge with actual server defaults. Independent CLI + cases can use existing resources supplied through `vars`; group mode creates + its own signed HTTP Bots. +- `cases`: CLI argument arrays or `http_group` settings and expected business text. Integer `weight` + values set the mix; `2/1/1/1` means `40/20/20/20`. An optional `body` object + becomes the case's request file. All workers share the weighted sequence. +- `phases`: each phase has a name, duration, target RPS, worker limit, and optional + cooldown. Multiple phases provide a step load. In-flight requests drain at the + end; the concurrency limit can reduce achieved RPS. +- `observe`: health URLs, host PIDs, PID files, and Docker container names/IDs. + Set PIDs for disjoint process trees to avoid counting descendants twice. + `box_data_root` discovers Box containers mounted below that deployment's data + directory on every sample, including sandboxes created as new groups speak. + The complete Job sets it automatically; on an existing instance, set it to + the instance's Box data directory when sampling dynamic Docker sandboxes. +- `thresholds`: minimum achieved/target RPS ratio and maximum P95 seconds per + case, in addition to the existing business and health/resource checks. The + sample limits are starting acceptance settings, not established product SLOs. +- `cleanup`: `on-success` (default), `always`, or `never`. Only recorded resources + created by this run are deleted, in reverse order. Plugins and the existing + instance remain installed. Failed/unknown creations are not automatically + retried; inspect the instance if a creation outcome is unknown. + +Failed runs retain resources by default. After inspection, clean them with the +same profile and credentials: + +```sh +python3 tools/regression_suite.py cleanup \ + --profile tools/regression_profiles/groups.json --output .qa/regression/run-001 +``` + +Cleanup checks the Workspace against `owned.json` and records remaining items +after partial failures. Keep `owned.json` until cleanup completes. Each run also +contains `profile.json`, resolved `resources.json`, and one directory per phase +with the original request/resource records. The root `summary.json` includes +target/achieved RPS, phase results, threshold failures, and the final verdict. + +The `regression_suite.py` entry reuses an existing instance and backend processes. +Use the complete lifecycle Job below to own deployment, credentials and backends. +The independent `normal.json` cases use a new-session CLI entry. Group traffic +uses the signed HTTP Bot adapter instead, as described below. Real IM adapter +acceptance and resource-leak verdicts need separate scenarios. Test the suite +plumbing on Linux with: + +```sh +python3 -m unittest discover -s tools -p test_regression_suite.py -v +``` + +### Persistent group traffic + +The group profile creates three HTTP Bots routed to chat, Box, and RAG Pipelines. +Their signed `/bots//sync` ingress produces real `GroupMessage` events and +uses Core's session manager. Four groups follow a `2/1/1` case assignment; +members take turns within each group. The same member IDs appear in different +groups to exercise group isolation. Box session selection uses the group launcher ID. + +In each group phase, `groups` selects the number of stable sessions and +`members_per_group` selects the rotating member identities. `rps` remains the +aggregate target, so a group's target speaking interval is `groups / rps` +seconds. `workers` bounds total in-flight requests independently of group count. +For example, 8 groups at 4 RPS target one message every 2 seconds per group. +Both the Suite and Job accept `--groups` and `--members-per-group` overrides. +The group count must cover the full weighted case sequence (4 in the sample). + +Only one request is in flight per group. A failure or unknown outcome stops that +group without retry while other groups continue. Every selected group must +complete at least two turns in total for the phase to pass. The fake model checks +that the preceding user message and assistant acknowledgement survived in actual +model input, including when a different member speaks next. Cross-group markers +fail the oracle. The normal Box/tool-output and RAG-context checks still apply. + +`group-state.json` carries group IDs, turn numbers and acknowledgements across +phases; existing groups keep their conversations when later phases add groups. +It refuses changed cases or a state left in progress by interruption. Do not +clear that marker to blindly resume: inspect the run and use a fresh output +directory when the last turn's outcome is unknown. Each phase summary includes +group-level turn counts and member IDs alongside case latency and health data. + +The Suite generates the HTTP Bot signing secret and passes it through +`LANGBOT_QA_GROUP_SECRET`. Resource request bodies containing credentials are +stored under `.private/`; do not share that directory. Cases specify `http_group` +with `url`, `secret_env`, and `message` instead of CLI `args`. Core needs no code +changes. The `/sync` response collapses streaming parts: this tests multi-turn +group processing, not real IM delivery or callback streaming order. +Every selected group must also receive traffic in each phase. Group latency +measures the signed HTTP request through its completed reply; independent CLI +latency includes CLI startup. Compare results within the same traffic mode. + +`lbctl` v0.3 may report Bot verification failure after a successful creation +because its safe projection omits event bindings. The Suite records the returned +Bot UUID before reading the full Core Bot API with the endpoint, Workspace and +environment credential confirmed by `lbctl whoami`. It then +checks the masked signing secret and effective route with a signed warmup turn +before load. The creation is not retried. + +## Complete lifecycle Job + +[`regression_job.py`](regression_job.py) owns the deployment as well as the suite. +Use [`regression_profiles/job.json`](regression_profiles/job.json) to pin the Core +Release and select the suite and plugin input directory. The job runs this chain: + +1. `lbctl install --profile all` creates a fresh deployment inside the run directory. +2. A Compose override starts our mock backend on the deployment's private network + and injects a generated test-only API key. The Job binds its own CLI context. +3. The suite installs plugins, creates resources and ingests documents. Each case + runs once as warmup before measured traffic starts. +4. Configured load phases run with health, process-tree and container sampling. +5. Service-log tails and raw request/resource records are retained. Successful + Jobs stop the deployment, remove only sandboxes mounted under their own data + root, and use `lbctl uninstall --purge-data` to reclaim the deployment. + +Both generation and embedding use our mock backend, with no external model +service or real model credentials. Plugin archives, the pinned Core image, +`lbctl`, Docker and Compose v2 are Job inputs/prerequisites. This wrapper uses +PyYAML, already included in LangBot's dependencies, to verify the CLI's deployment +record and recover ownership after an interrupted install. + +Run on a Linux host with Docker access and permission to inspect the selected +container processes. Use an idle dedicated Docker daemon: current Box runtime +startup cleanup can remove sandboxes from another runtime. The Job refuses to +start when existing Box sandboxes are present. The current `lbctl` installer +also reserves host port 5401 and 2280–2285, independently of its HTTP port. + +Set the plugin directory and the `lbctl` executable in `job.json`, then run: + +```sh +uv run --no-sync python tools/regression_job.py validate --profile tools/regression_profiles/job.json +uv run --no-sync python tools/regression_job.py run \ + --profile tools/regression_profiles/job.json --output .qa/regression/job-001 +``` + +`--duration`, `--rps` and `--workers` override every load phase. Each output +directory must be new. Job cleanup policy is `on-success`, `always`, or `never`; +it governs the whole deployment, whereas suite cleanup governs API resources +when a suite runs on an existing deployment. `purge_data: false` preserves the +deployment data after removing containers. Failed Jobs retain their environment +by default; after inspection use: + +```sh +uv run --no-sync python tools/regression_job.py cleanup \ + --output .qa/regression/job-001 +``` + +The root `summary.json` is the final Job verdict; `suite/` contains the measured +results, `mock/requests.jsonl` contains backend recordings, and `services.log` +contains up to 10,000 log lines per service. Generated credentials and deployment +inputs stay in `.private/`; share the result files, not that directory. A Job has +its own CLI locator/config and records its deployment identity before cleanup. +Changing that identity makes cleanup fail closed. Cleanup uses the saved Job +state and does not require the original profile or plugin input files. + +The Job replaces the real embedding model with `mock-embedding`. This measures +LangBot/plugin/Box/RAG overhead, not real-model inference cost or retrieval +quality. Run the Job plumbing checks on Linux with: + +```sh +uv run --no-sync python -m unittest discover -s tools -p 'test_*.py' -v +``` + +## Finite core smoke + +Use [`regression_profiles/smoke.json`](regression_profiles/smoke.json) for a finite +functional check on an existing Core. It prepares 35 resources through `lbctl`, +ingests HTML and plain-text fixtures through GeneralParsers, then executes 28 +cases / 43 requests. Reports group the verdicts by stage: + +| Stage | Cases and independent checks | +| --- | --- | +| Configuration | Identity and plugins; provider, model, Pipeline and knowledge-base readback; document completed status and Workspace/KB ownership | +| Ingress | Missing and incorrect signatures return HTTP 401 / code 40101 without creating a group AgentRun; valid signatures are exercised by group conversations | +| Conversation | Ordinary and Unicode chat; chat, Box and RAG groups each use two interleaved groups with three turns and rotating members; model input must retain prior user/reply history without cross-group history | +| Model | Slow SSE reply; HTTP 401, 429 and 503 primary failures followed by successful fallback; recorded model/status/stream sequence must match | +| Tools | Docker write/read across rounds, two tool calls in one model response, and expected exit code 7 with separate stdout/stderr returned to the model | +| Delivery | ASCII and Unicode filenames/content delivered as one attachment with exact bytes | +| Retrieval | HTML and text parsing/ingestion, RAG model input, retrieval from two knowledge bases with distinct sentinels and KB ownership, and an empty knowledge base returning an empty list | + +Cleanup is checked separately: delete journaled API resources and confirm their +UUIDs are absent from CLI list readback. Evidence goes to `cleanup-checks.json`; +cleanup failures affect the overall verdict. + +Start a dedicated `mock_llm.py` with the bundled scenarios and `--record` as +shown above. Copy the smoke profile and configure the CLI executable/config, +service URLs, plugin archives, `vars.mock_records` to that mock recording file, +and `vars.core_db` to the target Core's existing SQLite `langbot.db`. The suite +must read both local files. Keep fixture paths valid relative to the copied +profile. Do not share the mock with other traffic during this run: fallback +checks compare requests appended during each case. + +Set `plugins: []` to reuse installed plugins; LocalAgent, LangRAG and +GeneralParsers must still be enabled and initialized. The embedding endpoint +must serve the configured model (`all-MiniLM-L6-v2` by default). Generation is +deterministic mock traffic; Core, plugins, Box, document parsing and retrieval +execute normally. + +```sh +uv run --no-sync python tools/regression_suite.py validate --profile /path/to/smoke.json +uv run --no-sync python tools/regression_suite.py run \ + --profile /path/to/smoke.json --output .qa/smoke-001 +``` + +A correct reply alone does not pass. Each Pipeline/group request must match one +new AgentRun in the authenticated workspace with `completed` status and a persisted +`run.completed` event. The ledger is opened read-only. This version requires +local SQLite access; remote-only Core and PostgreSQL ledgers are not supported. +The observation timeout does not extend the Runner deadline or retry requests. +Unknown outcomes fail the suite; later turns of that group are skipped while +independent cases continue. An expected tool exit code of 7 passes only when +its output is correct and the agent handles the result and completes normally. + +`summary.json` contains the overall and per-stage verdicts; exit code 0 means all +checks and required cleanup passed. `cases.json` records per-request evidence, +`environment.json` records identity/capabilities/plugins, and `owned.json` tracks +created resources. Successful runs delete those API resources. Failures retain +them by default; after inspection use: + +```sh +uv run --no-sync python tools/regression_suite.py cleanup \ + --profile /path/to/smoke.json --output .qa/smoke-001 +``` + +Box sessions and AgentRun records follow the reused service's normal retention +policy. Load override flags are rejected for smoke. `stage` labels cases; +`fields` compares exact response values by dotted path (including array indices), +`forbid` rejects response substrings, and `mock_requests` checks the actual +model/status/stream sequence. `box_exec` defaults to exit code 0; error-handling +cases can specify `exit_code` and `stderr` explicitly. + +To use a fresh deployment, select this suite in `job.json`; the Job supplies its +SQLite/recording paths and uses mock embedding. Smoke skips load warmup and timed +phases. The dedicated Docker daemon requirement above still applies. + +## PR smoke gate + +`Core Smoke` runs regression-tool tests and a real Core smoke in parallel on +GitHub-hosted Linux runners. The smoke starts Core from the checked-out candidate, +uses the SDK from `uv.lock`, the existing official-plugin source pin, `lbctl` +v0.3.0, a Docker Box and mock generation/embedding. It needs no model credentials +or access to a developer's running environment. + +Dependency installation, service startup, sandbox image download and plugin +installation are preparation. After readiness, the gate allows **60 seconds total** +for API resource creation, 28 cases / 37 requests (two turns per group), verdict +checks and API cleanup. A missing verdict, failed check or timeout fails the gate. +The ordinary expanded profile keeps three turns / 43 requests. The entire cold +workflow takes longer than a minute; it has a separate 15-minute timeout. + +On an existing prepared stack, use: + +```sh +uv run --no-sync python tools/regression_gate.py \ + --profile /path/to/smoke.json --output .qa/gate-001 +``` + +`gate.json` records the budget, measured duration, verdict and process resource +peaks. `resources.jsonl` samples configured process trees every second (RSS, +cumulative CPU time, threads and file descriptors); suite artifacts contain +per-case timings and business evidence. These are diagnostics, not calibrated +performance-regression thresholds. Docker sandbox resource sampling remains in +the longer load runner. A timeout kills the local Suite/CLI process group without +retrying requests; a reused remote service can still have an in-flight run, so +inspect its retained ownership journal before cleanup. CI tears down its own +service processes and owned sandboxes afterward. + +The workflow always creates its two checks on PRs and cancels superseded runs. +After the workflow is merged, configure `Regression tools` and `Core smoke (60s)` +as required status checks in the repository ruleset. A green workflow alone does +not enforce merge protection. Longer load and soak tests use the existing Suite +and Job entry points with separately calibrated latency, throughput and resource +budgets; the one-minute smoke does not replace them. diff --git a/tools/fixtures/smoke-isolation.txt b/tools/fixtures/smoke-isolation.txt new file mode 100644 index 000000000..70ec425a5 --- /dev/null +++ b/tools/fixtures/smoke-isolation.txt @@ -0,0 +1,2 @@ +The separate knowledge base sentinel is cedar-isolation-4826. +This document belongs only to the second smoke knowledge base. diff --git a/tools/mock_llm.py b/tools/mock_llm.py new file mode 100644 index 000000000..6a97cb3fa --- /dev/null +++ b/tools/mock_llm.py @@ -0,0 +1,401 @@ +#!/usr/bin/env python3 +"""A deterministic OpenAI Chat Completions server for local regression tests.""" + +import argparse +import base64 +import hashlib +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import math +from pathlib import Path +import re +import struct +import threading +import time +import uuid + + +MAX_BODY_BYTES = 1024 * 1024 +USAGE = {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15} +GROUP_TURN = re.compile(r'LB_GROUP=(\S+) LB_TURN=(\d+) LB_TOKEN=([a-f0-9]+) LB_PREVIOUS=(none|[a-f0-9]+)') + + +def group_ack(messages, start): + """Require a real previous user turn and reply, with no other group's history.""" + turns = [] + for index, message in enumerate(messages): + if message.get('role') == 'user': + for match in GROUP_TURN.finditer(str(message.get('content', ''))): + turns.append((index, *match.groups())) + if not turns: + return '' + position, group, turn, token, previous = turns[-1] + if position != start or any(item[1] != group for item in turns): + raise ValueError('group history contains another group or lacks the current turn') + if int(turn) == 1: + if len(turns) != 1 or previous != 'none': + raise ValueError('first group turn must have an empty history') + else: + if len(turns) < 2: + raise ValueError('previous group turn is missing from model history') + prior_position, _, prior_turn, prior_token, _ = turns[-2] + replies = messages[prior_position + 1 : start] + if ( + int(prior_turn) != int(turn) - 1 + or prior_token != previous + or not any( + message.get('role') == 'assistant' and f'GROUP_ACK={previous}' in str(message.get('content', '')) + for message in replies + ) + ): + raise ValueError('previous group turn or assistant acknowledgement does not match') + return f' GROUP_ACK={token}' + + +def mock_embeddings(body): + """Return deterministic normalized token hashes, not semantic model inference.""" + inputs = body.get('input') + inputs = [inputs] if isinstance(inputs, str) else inputs + if ( + not isinstance(inputs, list) + or not 1 <= len(inputs) <= 128 + or any(not isinstance(text, str) or not text for text in inputs) + ): + raise ValueError('embedding input must contain 1..128 non-empty strings') + dimensions = body.get('dimensions', 384) + if isinstance(dimensions, bool) or not isinstance(dimensions, int) or not 1 <= dimensions <= 4096: + raise ValueError('embedding dimensions must be an integer in 1..4096') + encoding = body.get('encoding_format', 'float') + if encoding not in ('float', 'base64'): + raise ValueError('embedding encoding_format must be float or base64') + data = [] + for index, text in enumerate(inputs): + vector = [0.0] * dimensions + for token in re.findall(r'\w+', text.casefold()) or [text]: + digest = hashlib.sha256(token.encode()).digest() + vector[int.from_bytes(digest[:4], 'little') % dimensions] += 1 if digest[4] % 2 else -1 + norm = math.sqrt(sum(value * value for value in vector)) + if not norm: + vector[0], norm = 1.0, 1.0 + vector = [value / norm for value in vector] + embedding = ( + base64.b64encode(struct.pack(f'<{dimensions}f', *vector)).decode() if encoding == 'base64' else vector + ) + data.append({'object': 'embedding', 'index': index, 'embedding': embedding}) + return {'object': 'list', 'model': body['model'], 'data': data, 'usage': {'prompt_tokens': 10, 'total_tokens': 10}} + + +def select_message(body, scenario): + messages = body.get('messages') + if not isinstance(messages, list) or not messages or any(not isinstance(m, dict) for m in messages): + raise ValueError('messages must be a non-empty array of objects') + if not isinstance(body.get('stream', False), bool): + raise ValueError('stream must be a boolean') + if body.get('stream_options') is not None and not isinstance(body['stream_options'], dict): + raise ValueError('stream_options must be an object') + # Select from the current turn's history, never a shared request counter. + start = next( + (i for i in range(len(messages) - 1, -1, -1) if messages[i].get('role') == 'user'), + None, + ) + if start is None: + raise ValueError('a user message is required') + acknowledgement = group_ack(messages, start) + turn = messages[start:] + calls = [i for i, m in enumerate(turn) if m.get('role') == 'assistant' and m.get('tool_calls')] + for index, position in enumerate(calls): + end = calls[index + 1] if index + 1 < len(calls) else len(turn) + requested = {call['id'] for call in turn[position]['tool_calls']} + returned = [m.get('tool_call_id') for m in turn[position + 1 : end] if m.get('role') == 'tool'] + if len(returned) != len(requested) or set(returned) != requested: + raise ValueError('each tool call must have exactly one matching tool result') + steps = scenario['steps'] + if len(calls) >= len(steps): + raise ValueError('tool history exceeds the configured scenario') + step = steps[len(calls)] + if expected_prompt := step.get('expect_prompt'): + if expected_prompt not in json.dumps(messages, ensure_ascii=False): + raise ValueError(f'model input must contain {expected_prompt!r}') + expected = step.get('expect_tool_output') + if expected is not None: + results = turn[calls[-1] + 1 :] if calls else [] + result_text = '\n'.join(str(m.get('content', '')) for m in results if m.get('role') == 'tool') + if expected not in result_text: + raise ValueError(f'tool result must contain {expected!r}') + content = step.get('content') + message = {'role': 'assistant', 'content': content + acknowledgement if content is not None else None} + if tool_calls := step.get('tool_calls'): + tools = body.get('tools', []) + if not isinstance(tools, list) or any( + not isinstance(t, dict) or not isinstance(t.get('function'), dict) for t in tools + ): + raise ValueError('tools must be an array of objects') + available = {t.get('function', {}).get('name') for t in tools if t.get('type') == 'function'} + if body.get('tool_choice') == 'none' or any(call['name'] not in available for call in tool_calls): + raise ValueError('scenario tools must be advertised and enabled by the caller') + message['tool_calls'] = [ + { + 'id': f'call_mock_{len(calls)}_{i}', + 'type': 'function', + 'function': { + 'name': call['name'], + 'arguments': json.dumps(call['arguments']), + }, + } + for i, call in enumerate(tool_calls) + ] + return message + + +class MockServer(ThreadingHTTPServer): + daemon_threads = True + + def __init__(self, address, scenarios, record_path=None): + super().__init__(address, Handler) + self.scenarios = scenarios + self.record_path = record_path + self.record_lock = threading.Lock() + + def record(self, entry): + if self.record_path: + with ( + self.record_lock, + self.record_path.open('a', encoding='utf-8') as stream, + ): + stream.write(json.dumps(entry, ensure_ascii=False) + '\n') + + +class Handler(BaseHTTPRequestHandler): + def log_message(self, *_args): + pass + + def json_response(self, status, payload): + data = json.dumps(payload, ensure_ascii=False).encode() + self.send_response(status) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(data))) + self.end_headers() + self.wfile.write(data) + + def do_GET(self): + if self.path == '/healthz': + self.json_response(200, {'status': 'ok'}) + elif self.path == '/v1/models': + self.json_response( + 200, + { + 'object': 'list', + 'data': [ + { + 'id': name, + 'object': 'model', + 'created': 0, + 'owned_by': 'mock', + } + for name in [*self.server.scenarios, 'mock-embedding'] + ], + }, + ) + else: + self.json_response( + 404, + { + 'error': { + 'message': 'unknown endpoint', + 'type': 'invalid_request_error', + } + }, + ) + + def do_POST(self): + entry = {'request_id': uuid.uuid4().hex, 'time': time.time(), 'path': self.path, 'status': 400} + try: + if self.path not in ('/v1/chat/completions', '/v1/embeddings'): + entry['status'] = 404 + self.json_response( + 404, + { + 'error': { + 'message': 'unknown endpoint', + 'type': 'invalid_request_error', + } + }, + ) + return + self.connection.settimeout(10) + length = int(self.headers.get('Content-Length', 0)) + if self.headers.get('Transfer-Encoding') or not 0 < length <= MAX_BODY_BYTES: + raise ValueError('a Content-Length between 1 and 1048576 is required') + body = json.loads(self.rfile.read(length)) + if not isinstance(body, dict) or not isinstance(body.get('model'), str): + raise ValueError('request must be an object with a model name') + entry['request'] = body + if self.path == '/v1/embeddings': + if body['model'] != 'mock-embedding': + raise ValueError('embedding model must be mock-embedding') + payload = mock_embeddings(body) + entry.update(status=200, response=payload) + self.json_response(200, payload) + return + scenario = self.server.scenarios.get(body['model']) + if scenario is None: + raise ValueError('unknown model; use GET /v1/models') + message = select_message(body, scenario) + time.sleep(scenario.get('delay_seconds', 0)) + status = scenario.get('status', 200) + entry['status'] = status + if status != 200: + self.json_response( + status, + { + 'error': { + 'message': 'injected mock failure', + 'type': 'api_error', + 'code': str(status), + } + }, + ) + return + entry['response'] = message + base = { + 'id': 'chatcmpl-' + entry['request_id'], + 'created': int(time.time()), + 'model': body['model'], + } + finish = 'tool_calls' if message.get('tool_calls') else 'stop' + if body.get('stream'): + self.stream_response(body, scenario, message, base, finish) + else: + self.json_response( + 200, + { + **base, + 'object': 'chat.completion', + 'choices': [{'index': 0, 'message': message, 'finish_reason': finish}], + 'usage': USAGE, + }, + ) + except (ValueError, KeyError, TypeError, TimeoutError) as exc: + entry.update(status=400, error=str(exc)) + self.json_response(400, {'error': {'message': str(exc), 'type': 'invalid_request_error'}}) + except (BrokenPipeError, ConnectionResetError): + entry['disconnected'] = True + finally: + self.server.record(entry) + + def stream_response(self, body, scenario, message, base, finish): + self.send_response(200) + self.send_header('Content-Type', 'text/event-stream') + self.send_header('Cache-Control', 'no-cache') + self.end_headers() + + def send(delta, reason=None): + payload = { + **base, + 'object': 'chat.completion.chunk', + 'choices': [{'index': 0, 'delta': delta, 'finish_reason': reason}], + } + self.wfile.write(('data: ' + json.dumps(payload, ensure_ascii=False) + '\n\n').encode()) + self.wfile.flush() + time.sleep(scenario.get('chunk_delay_seconds', 0)) + + send({'role': 'assistant'}) + content = message.get('content') or '' + for offset in range(0, len(content), 8): + send({'content': content[offset : offset + 8]}) + for index, call in enumerate(message.get('tool_calls', [])): + send( + { + 'tool_calls': [ + { + 'index': index, + 'id': call['id'], + 'type': 'function', + 'function': { + 'name': call['function']['name'], + 'arguments': '', + }, + } + ] + } + ) + arguments = call['function']['arguments'] + for offset in range(0, len(arguments), 16): + send( + { + 'tool_calls': [ + { + 'index': index, + 'function': {'arguments': arguments[offset : offset + 16]}, + } + ] + } + ) + send({}, finish) + if (body.get('stream_options') or {}).get('include_usage'): + payload = { + **base, + 'object': 'chat.completion.chunk', + 'choices': [], + 'usage': USAGE, + } + self.wfile.write(('data: ' + json.dumps(payload) + '\n\n').encode()) + self.wfile.write(b'data: [DONE]\n\n') + self.wfile.flush() + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--host', default='127.0.0.1') + parser.add_argument('--port', type=int, default=18080) + parser.add_argument( + '--scenarios', + type=Path, + default=Path(__file__).with_name('mock_llm_scenarios.json'), + ) + parser.add_argument( + '--record', + type=Path, + help='Append request/response JSONL without HTTP headers; use test data only', + ) + args = parser.parse_args() + scenarios = json.loads(args.scenarios.read_text(encoding='utf-8')) + if not isinstance(scenarios, dict) or not scenarios: + parser.error('scenarios must be a non-empty object keyed by model name') + for name, scenario in scenarios.items(): + if not isinstance(scenario, dict) or not isinstance(scenario.get('steps'), list) or not scenario['steps']: + parser.error(f'{name}: steps must be a non-empty array') + if any(not isinstance(step, dict) for step in scenario['steps']): + parser.error(f'{name}: each step must be an object') + for step in scenario['steps']: + if not isinstance(step.get('content'), str) and not step.get('tool_calls'): + parser.error(f'{name}: each step needs content or tool_calls') + if 'expect_tool_output' in step and not isinstance(step['expect_tool_output'], str): + parser.error(f'{name}: expect_tool_output must be a string') + if 'expect_prompt' in step and not isinstance(step['expect_prompt'], str): + parser.error(f'{name}: expect_prompt must be a string') + if 'tool_calls' in step: + calls = step['tool_calls'] + if not isinstance(calls, list) or any( + not isinstance(call, dict) + or not isinstance(call.get('name'), str) + or not isinstance(call.get('arguments'), dict) + for call in calls + ): + parser.error(f'{name}: tool_calls need a name and arguments object') + if scenario.get('status', 200) not in (200, 401, 429, 500, 503): + parser.error(f'{name}: unsupported injected HTTP status') + for key in ('delay_seconds', 'chunk_delay_seconds'): + value = scenario.get(key, 0) + if not isinstance(value, (int, float)) or not 0 <= value <= 60: + parser.error(f'{name}: {key} must be between 0 and 60') + with MockServer((args.host, args.port), scenarios, args.record) as server: + print(f'Mock LLM: http://{args.host}:{server.server_port}/v1', flush=True) + try: + server.serve_forever() + except KeyboardInterrupt: + pass + + +if __name__ == '__main__': + main() diff --git a/tools/mock_llm_scenarios.json b/tools/mock_llm_scenarios.json new file mode 100644 index 000000000..0753c5b6a --- /dev/null +++ b/tools/mock_llm_scenarios.json @@ -0,0 +1,169 @@ +{ + "mock-chat": { + "steps": [{"content": "LANGBOT_CHAT_OK"}] + }, + "mock-rag": { + "steps": [{"expect_prompt": "aurora-parser-rag-9137", "content": "LANGBOT_RAG_OK"}] + }, + "mock-box": { + "steps": [ + {"tool_calls": [{"name": "exec", "arguments": {"command": "printf LANGBOT_BOX_SENTINEL > /workspace/mock-llm.txt; cat /workspace/mock-llm.txt"}}]}, + {"expect_tool_output": "LANGBOT_BOX_SENTINEL", "tool_calls": [{"name": "exec", "arguments": {"command": "cat /workspace/mock-llm.txt"}}]}, + {"expect_tool_output": "LANGBOT_BOX_SENTINEL", "content": "LANGBOT_BOX_OK"} + ] + }, + "mock-401": {"status": 401, "steps": [{"content": "unused"}]}, + "mock-429": {"status": 429, "steps": [{"content": "unused"}]}, + "mock-503": {"status": 503, "steps": [{"content": "unused"}]}, + "mock-slow": {"delay_seconds": 2, "steps": [{"content": "LANGBOT_SLOW_OK"}]}, + "mock-slow-stream": {"chunk_delay_seconds": 0.2, "steps": [{"content": "LANGBOT_SLOW_STREAM_OK"}]}, + "mock-smoke-box": { + "steps": [ + { + "tool_calls": [ + { + "name": "exec", + "arguments": { + "command": "printf LANGBOT_BOX_SENTINEL > /tmp/langbot-smoke.txt; cat /tmp/langbot-smoke.txt" + } + } + ] + }, + { + "expect_tool_output": "LANGBOT_BOX_SENTINEL", + "tool_calls": [ + { + "name": "exec", + "arguments": { + "command": "cat /tmp/langbot-smoke.txt" + } + } + ] + }, + { + "expect_tool_output": "LANGBOT_BOX_SENTINEL", + "content": "LANGBOT_BOX_OK" + } + ] + }, + "mock-smoke-export": { + "steps": [ + { + "tool_calls": [ + { + "name": "exec", + "arguments": { + "command": "printf LANGBOT_EXPORT_SENTINEL > /tmp/langbot-smoke-output.txt; cat /tmp/langbot-smoke-output.txt" + } + } + ] + }, + { + "expect_tool_output": "LANGBOT_EXPORT_SENTINEL", + "tool_calls": [ + { + "name": "export_box_files", + "arguments": { + "paths": [ + "/tmp/langbot-smoke-output.txt" + ] + } + } + ] + }, + { + "expect_tool_output": "accepted", + "content": "LANGBOT_EXPORT_OK" + } + ] + }, + "mock-smoke-unicode": { + "steps": [ + { + "expect_prompt": "请原样处理：你好，世界 🌍", + "content": "你好，世界 🌍 LANGBOT_UNICODE_OK" + } + ] + }, + "mock-smoke-exec-error": { + "steps": [ + { + "tool_calls": [ + { + "name": "exec", + "arguments": { + "command": "printf EXPECTED_STDOUT; printf EXPECTED_STDERR >&2; exit 7" + } + } + ] + }, + { + "expect_tool_output": "EXPECTED_STDERR", + "content": "LANGBOT_EXEC_ERROR_HANDLED" + } + ] + }, + "mock-smoke-parallel-tools": { + "steps": [ + { + "tool_calls": [ + { + "name": "exec", + "arguments": { + "command": "printf PARALLEL_OK_A" + } + }, + { + "name": "exec", + "arguments": { + "command": "printf PARALLEL_OK_B" + } + } + ] + }, + { + "expect_tool_output": "PARALLEL_OK_B", + "content": "LANGBOT_PARALLEL_TOOLS_OK" + } + ] + }, + "mock-smoke-export-unicode": { + "steps": [ + { + "tool_calls": [ + { + "name": "exec", + "arguments": { + "command": "printf '你好，文件 🌍' > /tmp/输出.txt; cat /tmp/输出.txt" + } + } + ] + }, + { + "expect_tool_output": "你好，文件 🌍", + "tool_calls": [ + { + "name": "export_box_files", + "arguments": { + "paths": [ + "/tmp/输出.txt" + ] + } + } + ] + }, + { + "expect_tool_output": "accepted", + "content": "LANGBOT_UNICODE_EXPORT_OK" + } + ] + }, + "mock-smoke-rag-other": { + "steps": [ + { + "expect_prompt": "cedar-isolation-4826", + "content": "LANGBOT_OTHER_RAG_OK" + } + ] + } +} diff --git a/tools/regression_ci.py b/tools/regression_ci.py new file mode 100644 index 000000000..0d57d4585 --- /dev/null +++ b/tools/regression_ci.py @@ -0,0 +1,235 @@ +#!/usr/bin/env python3 +"""Prepare a disposable Linux CI stack from this checkout, then run the 60s gate.""" + +import argparse +import json +import os +from pathlib import Path +import secrets +import shutil +import signal +import subprocess +import sys +import time + +import yaml + +from regression_gate import supervise +from regression_suite import Suite +from regression_traffic import owned_box_containers, read_endpoint_snapshot + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402 + + +def ready(url, processes, timeout=120): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if any(p.poll() is not None for p in processes): + raise RuntimeError('A smoke dependency exited; inspect service logs') + try: + read_endpoint_snapshot(url, timeout_seconds=1) + return + except (RuntimeError, OSError): + time.sleep(0.2) + raise RuntimeError(f'Service did not become ready: {url}') + + +def run(args): + if subprocess.check_output( + ['docker', 'ps', '-a', '--filter', 'label=langbot.box=true', '--format', '{{.ID}}'], text=True, timeout=10 + ).strip(): + raise RuntimeError('CI needs a dedicated Docker daemon with no existing Box sandboxes') + root = args.output.resolve() + root.mkdir(parents=True, exist_ok=False, mode=0o700) + private = root / '.private' + private.mkdir(mode=0o700) + processes, logs = [], [] + key, runtime_token, box_token = (secrets.token_urlsafe(32) for _ in range(3)) + env = dict( + os.environ, + LANGBOT_QA_API_KEY=key, + LANGBOT_DATA_ROOT=str(private / 'data'), + PYTHONPATH=str(ROOT / 'src'), + LITELLM_LOCAL_MODEL_COST_MAP='True', + PYTHONUNBUFFERED='1', + LANGBOT_PLUGIN_RUNTIME_CONTROL_TOKEN=runtime_token, + LANGBOT_BOX_CONTROL_TOKEN=box_token, + ) + + def start(name, command): + log = (root / f'{name}.log').open('w') + logs.append(log) + process = subprocess.Popen( + command, cwd=private, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True + ) + processes.append(process) + return process.pid + + try: + config_path = create_minimal_config(private, port=15301) + create_test_directories(private) + config = yaml.safe_load(config_path.read_text()) + config['api'].update(global_api_key=key, host='127.0.0.1') + config['plugin'].update(enable=True, runtime_ws_url='ws://127.0.0.1:15340/control/ws') + config['box'].update(enabled=True, backend='local', runtime={'endpoint': 'ws://127.0.0.1:15310/control/ws'}) + config['box']['local'].update( + host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine' + ) + config_path.write_text(yaml.safe_dump(config)) + profile_path = ROOT / 'tools/regression_profiles/smoke.json' + profile = json.loads(profile_path.read_text()) + cli_config = private / 'lbctl.yaml' + cli_config.write_text( + yaml.safe_dump( + { + 'version': 1, + 'current_context': 'smoke', + 'contexts': { + 'smoke': { + 'endpoint': 'http://127.0.0.1:15301', + 'credential': {'type': 'env', 'name': 'LANGBOT_QA_API_KEY'}, + } + }, + } + ) + ) + profile['cli'] = {'executable': str(args.lbctl.resolve()), 'config': str(cli_config)} + profile['vars'].update( + mock_base_url='http://127.0.0.1:18080/v1', + embedding_base_url='http://127.0.0.1:18080/v1', + core_base_url='http://127.0.0.1:15301', + core_health='http://127.0.0.1:15301/healthz', + core_db=str(private / 'data/langbot.db'), + mock_records=str(root / 'mock-requests.jsonl'), + ) + profile['plugins'] = [] + # Two turns retain the history/isolation oracle while leaving room for setup and cleanup. + profile['smoke']['group_turns'] = 2 + for resource in profile['resources']: + if resource['kind'] == 'model' and resource.get('type') == 'embedding': + resource['body']['name'] = 'mock-embedding' + for document in resource.get('documents', []): + document['file'] = str((profile_path.parent / document['file']).resolve()) + pids = {} + pids['mock'] = start( + 'mock', + [ + sys.executable, + str(ROOT / 'tools/mock_llm.py'), + '--port', + '18080', + '--record', + str(root / 'mock-requests.jsonl'), + ], + ) + pids['runtime'] = start( + 'runtime', + [ + sys.executable, + '-m', + 'langbot_plugin.cli.__init__', + 'rt', + '--ws-control-port', + '15340', + '--ws-debug-port', + '15341', + ], + ) + pids['box'] = start( + 'box', + [ + sys.executable, + '-m', + 'langbot_plugin.cli.__init__', + 'box', + '--host', + '127.0.0.1', + '--ws-control-port', + '15310', + ], + ) + pids['core'] = start( + 'core', [sys.executable, str(ROOT / 'main.py'), '--standalone-runtime', '--standalone-box'] + ) + profile['observe'] = { + 'health': { + 'core': 'http://127.0.0.1:15301/healthz', + 'box': 'http://127.0.0.1:15310/readyz', + 'mock': 'http://127.0.0.1:18080/healthz', + }, + 'pids': pids, + } + for url in profile['observe']['health'].values(): + ready(url, processes) + preparation = root / 'preparation' + preparation.mkdir() + suite = Suite(profile, profile_path, preparation, env) + identity = suite.call(['whoami']) + # Bind the CLI to the newly bootstrapped Workspace before installing plugins. + cli_data = yaml.safe_load(cli_config.read_text()) + cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid'] + cli_config.write_text(yaml.safe_dump(cli_data)) + for directory in ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers'): + source = args.plugins / directory + package_source = private / source.name + shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__')) + archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source) + suite.call(['plugin', 'install', 'local', '--file', archive, '--wait']) + generated = private / 'smoke.json' + generated.write_text(json.dumps(profile, indent=2)) + return supervise( + [ + sys.executable, + str(ROOT / 'tools/regression_suite.py'), + 'run', + '--profile', + str(generated), + '--output', + str(root / 'gate/suite'), + ], + root / 'gate', + 60, + pids, + env, + ) + finally: + for process in reversed(processes): + if process.poll() is None: + os.killpg(process.pid, signal.SIGTERM) + try: + process.wait(timeout=5) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + for log in logs: + log.close() + # This is a disposable CI daemon; still delete only this run's mounted sandboxes. + for container in owned_box_containers(private / 'box'): + subprocess.run(['docker', 'rm', '-f', container], check=True, capture_output=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--lbctl', type=Path, required=True) + parser.add_argument('--plugins', type=Path, required=True) + args = parser.parse_args() + if args.output.exists(): + parser.error('--output must be a new directory') + try: + result = run(args) + except Exception as exc: + verdict = args.output / 'gate' / 'gate.json' + result = json.loads(verdict.read_text()) if verdict.exists() else {} + result.update(passed=False, infrastructure_error=f'{type(exc).__name__}: {exc}') + verdict.parent.mkdir(parents=True, exist_ok=True) + verdict.write_text(json.dumps(result, indent=2) + '\n') + raise + print(json.dumps(result, indent=2)) + raise SystemExit(0 if result['passed'] else 1) + + +if __name__ == '__main__': + main() diff --git a/tools/regression_gate.py b/tools/regression_gate.py new file mode 100644 index 000000000..2e3217b8b --- /dev/null +++ b/tools/regression_gate.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +"""Bound a prepared-environment smoke run and record process resource samples.""" + +import argparse +import json +import os +from pathlib import Path +import signal +import subprocess +import sys +import time + +from regression_suite import Suite, validate +from regression_traffic import read_process_snapshot + + +def supervise(command, output, budget, pids, env=None): + output.mkdir(parents=True, exist_ok=False, mode=0o700) + started = time.monotonic() + samples, errors = [], [] + timed_out = False + with (output / 'suite.log').open('w') as log: + process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True) + try: + while process.poll() is None: + remaining = budget - (time.monotonic() - started) + if remaining <= 0: + timed_out = True + os.killpg(process.pid, signal.SIGKILL) + process.wait() + break + sample = {'elapsed_seconds': time.monotonic() - started, 'processes': {}} + for name, pid in pids.items(): + try: + sample['processes'][name] = read_process_snapshot(pid) + except (RuntimeError, OSError) as exc: + errors.append(f'{name}: {exc}') + samples.append(sample) + remaining = budget - (time.monotonic() - started) + try: + process.wait(timeout=max(0, min(1, remaining))) + except subprocess.TimeoutExpired: + pass + finally: + if process.poll() is None: + os.killpg(process.pid, signal.SIGKILL) + process.wait() + elapsed = time.monotonic() - started + (output / 'resources.jsonl').write_text(''.join(json.dumps(sample) + '\n' for sample in samples)) + summary_path = output / 'suite' / 'summary.json' + summary = json.loads(summary_path.read_text()) if summary_path.exists() else {} + result = { + 'passed': process.returncode == 0 + and summary.get('passed') is True + and elapsed <= budget + and not timed_out + and not errors, + 'budget_seconds': budget, + 'elapsed_seconds': elapsed, + 'timed_out': timed_out, + 'exit_code': process.returncode, + 'resource_errors': errors, + 'requests': summary.get('requests'), + 'status': summary.get('status'), + 'stages': summary.get('stages'), + 'resource_peaks': { + name: { + metric: max((s['processes'].get(name, {}).get(metric, 0) for s in samples), default=0) + for metric in ('rss_bytes', 'threads', 'open_fds') + } + for name in pids + }, + } + (output / 'gate.json').write_text(json.dumps(result, indent=2) + '\n') + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--profile', type=Path, required=True) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + profile = json.loads(args.profile.read_text()) + validate(profile) + if 'smoke' not in profile: + parser.error('a smoke profile is required') + suite = Suite(profile, args.profile.resolve(), args.output / 'suite') + pids = dict(profile.get('observe', {}).get('pids', {})) + for name, path in profile.get('observe', {}).get('pid_files', {}).items(): + pids[name] = int(Path(suite.path(path)).read_text().strip()) + result = supervise( + [ + sys.executable, + str(Path(__file__).with_name('regression_suite.py')), + 'run', + '--profile', + str(args.profile.resolve()), + '--output', + str(args.output.resolve() / 'suite'), + ], + args.output, + 60, + pids, + ) + print(json.dumps(result, indent=2)) + raise SystemExit(0 if result['passed'] else 1) + + +if __name__ == '__main__': + main() diff --git a/tools/regression_job.py b/tools/regression_job.py new file mode 100644 index 000000000..099c85ab7 --- /dev/null +++ b/tools/regression_job.py @@ -0,0 +1,480 @@ +#!/usr/bin/env python3 +"""Own one complete lbctl deployment, mock backend, regression, and teardown.""" + +import argparse +import copy +import json +import os +from pathlib import Path +import re +import secrets +import shutil +import subprocess +import time +import urllib.error +import urllib.request + +import yaml + +from regression_suite import Suite, number, resolve, validate +from regression_traffic import owned_box_containers + + +KEY_ENV = 'LANGBOT_QA_API_KEY' + + +def write_json(path, data): + temporary = path.with_suffix(path.suffix + '.tmp') + temporary.write_text(json.dumps(data, ensure_ascii=False, indent=2)) + temporary.chmod(0o600) + temporary.replace(path) + + +def load_profile(path): + profile = json.loads(path.read_text()) + if not isinstance(profile, dict) or profile.get('version') != 1: + raise ValueError('job profile version must be 1') + if profile.keys() - {'version', 'lbctl', 'suite', 'deployment', 'vars', 'cleanup', 'purge_data', 'mock_image'}: + raise ValueError('unknown job profile field') + deployment = profile.get('deployment', {}) + if not isinstance(deployment, dict) or deployment.keys() - {'version', 'port'}: + raise ValueError('deployment accepts version and port') + version = deployment.get('version', '') + if not isinstance(version, str) or not re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', version): + raise ValueError('deployment.version must pin a stable vX.Y.Z Release') + number(deployment.get('port', 5300), 'deployment port', minimum=1, integer=True) + if deployment.get('port', 5300) > 65535: + raise ValueError('deployment port must be <= 65535') + if profile.get('cleanup', 'on-success') not in ('always', 'on-success', 'never'): + raise ValueError('cleanup must be always/on-success/never') + if not isinstance(profile.get('purge_data', True), bool): + raise ValueError('purge_data must be boolean') + suite_path = Path(profile['suite']).expanduser() + suite_path = suite_path if suite_path.is_absolute() else path.parent / suite_path + suite = json.loads(suite_path.read_text()) + suite.setdefault('vars', {}).update(profile.get('vars', {})) + suite['vars'].update( + mock_base_url='http://regression_mock:18080/v1', embedding_base_url='http://regression_mock:18080/v1' + ) + suite['cli'] = {'config': 'job-generated.yaml'} + for item in suite.get('resources', []): + if item['kind'] == 'model' and item.get('type') == 'embedding': + item['body']['name'] = 'mock-embedding' + validate(suite) + return profile, suite, suite_path.resolve() + + +class Job: + def __init__(self, profile, suite_profile, suite_path, output): + self.profile = profile + self.suite_profile = copy.deepcopy(suite_profile) + self.suite_path = suite_path + self.output = output.resolve() + self.private = self.output / '.private' + self.deploy = self.private / 'deployment' + self.state_path = self.output / 'state.json' + self.cli_config = self.private / 'lbctl.yaml' + self.env = {**os.environ, 'XDG_CONFIG_HOME': str(self.private / 'config')} + self.cli = [profile.get('lbctl', 'lbctl'), '--config', str(self.cli_config), '-o', 'json'] + self.redactions = [] + self.state = {'job_id': secrets.token_hex(8), 'record': None, 'bound': False, 'phase': 'created'} + + def redact(self, value): + for secret in self.redactions: + value = value.replace(secret, '[REDACTED]') + return value + + def save(self): + write_json(self.state_path, self.state) + + def recover_record(self): + if (self.private / 'job-id').read_text().strip() != self.state['job_id']: + raise RuntimeError('Job ownership marker differs from state.json') + path = self.deploy / 'deployment.yaml' + if not path.exists(): + if self.state.get('record'): + raise RuntimeError('deployment record disappeared; refusing cleanup') + return + if path.is_symlink(): + raise RuntimeError('deployment record must not be a symlink') + try: + record = yaml.safe_load(path.read_text()) + except yaml.YAMLError as exc: + raise ValueError('invalid deployment YAML') from exc + if not isinstance(record, dict) or not record.get('deployment_id'): + raise RuntimeError('invalid deployment record') + if ( + Path(record['compose_file']).resolve().parent != self.deploy + or Path(record['data_dir']).resolve() != self.deploy / 'data' + or Path(record['env_file']).resolve().parent != self.deploy + ): + raise RuntimeError('deployment record is outside this Job') + if self.state.get('record') and record['deployment_id'] != self.state['record']['deployment_id']: + raise RuntimeError('deployment identity changed; refusing cleanup') + self.state['record'] = record + self.save() + + def load_redactions(self): + if self.state.get('record'): + path = Path(self.state['record']['env_file']) + if path.exists(): + for line in path.read_text().splitlines(): + name, separator, value = line.partition('=') + if separator and any(word in name for word in ('TOKEN', 'KEY')) and len(value) >= 16: + self.redactions.append(value) + + def command(self, args, timeout=900): + result = subprocess.run(args, capture_output=True, text=True, env=self.env, timeout=timeout) + return result.returncode, self.redact(result.stdout), self.redact(result.stderr) + + def checked(self, args, timeout=900): + code, stdout, stderr = self.command(args, timeout) + if code: + raise RuntimeError((stderr + stdout)[-4000:]) + return stdout + + def cli_call(self, args): + code, stdout, stderr = self.command([*self.cli, *args]) + result = json.loads(stdout) if stdout.strip() else {} + data = result.get('data', {}) + if args[0] == 'install' and isinstance(data, dict) and data.get('record'): + record = data['record'] + if ( + Path(record['compose_file']).resolve().parent != self.deploy + or Path(record['data_dir']).resolve() != self.deploy / 'data' + ): + raise RuntimeError('install returned a deployment outside this Job') + self.state['record'] = record + self.save() # Installation may have changed state even when its final health check failed. + if code: + raise RuntimeError((stderr + stdout)[-4000:]) + return data + + def compose(self, *args, override=False): + record = self.state['record'] + command = ['docker', 'compose', '--project-name', record['compose_project'], '--file', record['compose_file']] + if override: + command += ['--file', str(self.private / 'compose.override.json')] + command += ['--env-file', record['env_file'], '--profile', 'all', *args] + return self.checked(command) + + def project_containers(self): + project = self.state['record']['compose_project'] + output = self.checked( + [ + 'docker', + 'ps', + '-a', + '--filter', + f'label=com.docker.compose.project={project}', + '--format', + '{{.ID}} {{.Label "com.docker.compose.service"}}', + ] + ) + return {service: container for container, service in (line.split() for line in output.splitlines() if line)} + + def address(self, container): + networks = json.loads( + self.checked(['docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', container]) + ) + addresses = [network['IPAddress'] for network in networks.values() if network.get('IPAddress')] + if len(addresses) != 1: + raise RuntimeError('expected one private container network') + return addresses[0] + + @staticmethod + def ready(url): + deadline = time.monotonic() + 120 + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + while time.monotonic() < deadline: + try: + with opener.open(url, timeout=2) as response: + payload = json.load(response) + if response.status == 200 and payload.get('ready') is not False and payload.get('code', 0) == 0: + return + except (urllib.error.URLError, TimeoutError, OSError, ValueError): + pass + time.sleep(0.5) + raise RuntimeError(f'readiness deadline exceeded: {url}') + + def prepare(self): + variables = { + **self.suite_profile.get('vars', {}), + 'profile_dir': str(self.suite_path.parent), + 'run_dir': str(self.output), + } + inputs = [plugin['file'] for plugin in self.suite_profile.get('plugins', [])] + inputs += [ + document['file'] + for item in self.suite_profile.get('resources', []) + for document in item.get('documents', []) + ] + for name in inputs: + path = Path(resolve(name, variables)).expanduser() + path = path if path.is_absolute() else self.suite_path.parent / path + if not path.is_file(): + raise ValueError(f'Job input file is missing: {path}') + self.state['phase'] = 'install' + self.save() + # Current Box runtimes reap sandboxes from previous runtime IDs on startup. + # Do not start one alongside another user's live sandbox on this daemon. + boxes = self.checked(['docker', 'ps', '-a', '--filter', 'label=langbot.box=true', '--format', '{{.ID}}']) + if boxes.strip(): + raise RuntimeError('Docker daemon has existing Box sandboxes; use an idle dedicated test daemon') + key = secrets.token_urlsafe(32) + self.env[KEY_ENV] = key + self.env['LANGBOT_API_KEY'] = key + self.redactions.append(key) + (self.private / 'api-key').write_text(key) + (self.private / 'api-key').chmod(0o600) + deployment = self.profile['deployment'] + self.cli_call( + [ + 'install', + '--dir', + str(self.deploy), + '--version', + deployment['version'], + '--profile', + 'all', + '--port', + str(deployment.get('port', 5300)), + ] + ) + self.recover_record() + self.load_redactions() + mock_results = self.output / 'mock' + mock_results.mkdir(mode=0o700) + override = { + 'services': { + 'langbot': { + 'environment': {'API__GLOBAL_API_KEY': key}, + 'volumes': [f'{self.deploy / "data/box"}:{self.deploy / "data/box"}'], + }, + 'regression_mock': { + 'image': self.profile.get('mock_image', 'python:3.12-alpine'), + 'volumes': [f'{Path(__file__).parent.resolve()}:/regression:ro', f'{mock_results}:/results'], + 'command': [ + 'python', + '/regression/mock_llm.py', + '--host', + '0.0.0.0', + '--port', + '18080', + '--record', + '/results/requests.jsonl', + ], + 'networks': ['langbot_network'], + 'labels': {'langbot.regression_job': self.state['job_id']}, + }, + }, + } + write_json(self.private / 'compose.override.json', override) + self.compose('up', '-d', override=True) + containers = self.project_containers() + for service in ('langbot', 'langbot_plugin_runtime', 'langbot_box', 'regression_mock'): + if service not in containers: + raise RuntimeError(f'missing deployment service: {service}') + endpoint = self.state['record']['endpoint'] + health = { + 'core': endpoint + '/healthz', + 'mock': f'http://{self.address(containers["regression_mock"])}:18080/healthz', + 'box': f'http://{self.address(containers["langbot_box"])}:5410/readyz', + } + for url in health.values(): + self.ready(url) + identity = self.cli_call(['--endpoint', endpoint, 'whoami']) + self.cli_call( + [ + 'context', + 'add', + 'regression', + '--endpoint', + endpoint, + '--api-key-env', + KEY_ENV, + '--expect-workspace', + identity['workspace_uuid'], + ] + ) + self.cli_call(['context', 'use', 'regression']) + self.state.update( + bound=True, workspace_uuid=identity['workspace_uuid'], containers=containers, phase='resources' + ) + self.save() + profile = self.suite_profile + profile.setdefault('vars', {})['core_base_url'] = endpoint + if 'smoke' in profile: + if 'mock_records' in profile['smoke']: + profile['smoke']['mock_records'] = str(self.output / 'mock' / 'requests.jsonl') + profile['smoke']['ledger_db'] = str(Path(self.state['record']['data_dir']) / 'langbot.db') + profile['cli'] = {'executable': self.cli[0], 'config': str(self.cli_config)} + profile['observe'] = {'health': health, 'pids': {}, 'containers': list(containers.values())} + for service, container in containers.items(): + pid = int(self.checked(['docker', 'inspect', '--format', '{{.State.Pid}}', container]).strip()) + profile['observe']['pids'][service] = pid + destination = self.output / 'suite' + destination.mkdir(mode=0o700) + suite = Suite(profile, self.suite_path, destination, env=self.env) + self.redactions.extend(suite.secrets) + suite.write('profile.json', profile) + suite.prepare() + if 'smoke' in profile: + return suite + # Prime every configured business case before measuring steady load. + for case in profile['cases']: + case = resolve(case, suite.variables) + if 'http_group' in case: + continue # Suite preparation already verifies signed group ingress and the masked secret. + data = suite.call(case['args'], case.get('body')) + if case['expect'] not in json.dumps(data, ensure_ascii=False): + raise RuntimeError(f'warmup oracle failed: {case["name"]}') + profile['observe']['box_data_root'] = str(Path(self.state['record']['data_dir']) / 'box') + suite.write('profile.json', profile) + return suite + + def owned_sandboxes(self): + record = self.state.get('record') + if not record: + return [] + return owned_box_containers(Path(record['data_dir']) / 'box', self.checked) + + def collect(self): + self.recover_record() + self.load_redactions() + if self.state.get('record'): + logs = self.compose('logs', '--no-color', '--tail', '10000') + (self.output / 'services.log').write_text(logs) + + def cleanup(self): + self.recover_record() + self.load_redactions() + record = self.state.get('record') + if self.state['phase'] == 'cleaned': + if record and self.project_containers(): + raise RuntimeError('containers appeared after Job cleanup; inspect deployment before acting') + return + if record: + if ( + Path(record['compose_file']).resolve().parent != self.deploy + or Path(record['data_dir']).resolve() != self.deploy / 'data' + ): + raise RuntimeError('cleanup refused: deployment is outside this Job') + if self.state['bound']: + self.cli_call(['stop']) + else: + self.compose('stop', override=(self.private / 'compose.override.json').exists()) + sandboxes = self.owned_sandboxes() + if sandboxes: + self.checked(['docker', 'rm', '-f', *sandboxes]) + if self.state['bound']: + self.cli_call( + ['uninstall', '--yes', *(['--purge-data'] if self.profile.get('purge_data', True) else [])] + ) + else: + self.compose('down', '--remove-orphans', override=(self.private / 'compose.override.json').exists()) + if self.profile.get('purge_data', True) and (self.deploy / 'data').exists(): + shutil.rmtree(self.deploy / 'data') + for path in (self.private / 'api-key', self.private / 'compose.override.json'): + path.unlink(missing_ok=True) + self.state['phase'] = 'cleaned' + self.save() + + def run(self): + self.output.mkdir(parents=True, exist_ok=False, mode=0o700) + self.private.mkdir(mode=0o700) + (self.private / 'job-id').write_text(self.state['job_id']) + write_json(self.output / 'job-profile.json', self.profile) + self.save() + result = {'errors': [], 'completed': False} + try: + suite = self.prepare() + self.state['phase'] = 'traffic' + self.save() + summary = suite.run() + summary['passed'] = not summary['errors'] + suite.write('summary.json', summary) + result.update(completed=True, suite=summary, errors=list(summary['errors'])) + self.state['phase'] = 'finished' + self.save() + except KeyboardInterrupt: + result['errors'].append('Job interrupted; in-flight outcomes may be unknown') + except (RuntimeError, ValueError, OSError, KeyError, subprocess.TimeoutExpired) as exc: + result['errors'].append(self.redact(str(exc))) + finally: + if not result['completed'] and not result['errors']: + result['errors'].append('Job aborted before completion') + try: + self.collect() + except (RuntimeError, ValueError, OSError, subprocess.TimeoutExpired) as exc: + result['errors'].append('artifact collection: ' + self.redact(str(exc))) + policy = self.profile.get('cleanup', 'on-success') + if policy == 'always' or (policy == 'on-success' and not result['errors']): + try: + self.cleanup() + except (RuntimeError, ValueError, OSError, KeyError, subprocess.TimeoutExpired) as exc: + result['errors'].append('cleanup: ' + self.redact(str(exc))) + result.update(passed=not result['errors'], phase=self.state['phase']) + write_json(self.output / 'summary.json', result) + return result + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('validate', 'run', 'cleanup')) + parser.add_argument('--profile', type=Path) + parser.add_argument('--output', type=Path) + parser.add_argument('--duration', type=float) + parser.add_argument('--rps', type=float) + parser.add_argument('--workers', type=int) + parser.add_argument('--groups', type=int) + parser.add_argument('--members-per-group', type=int) + args = parser.parse_args() + try: + if args.action == 'cleanup': + if args.output is None: + raise ValueError('--output is required for cleanup') + profile = json.loads((args.output / 'job-profile.json').read_text()) + job = Job(profile, {}, args.output / 'unused-profile.json', args.output) + job.state = json.loads(job.state_path.read_text()) + if (job.private / 'api-key').exists(): + key = (job.private / 'api-key').read_text() + job.env[KEY_ENV] = key + job.env['LANGBOT_API_KEY'] = key + job.redactions.append(key) + job.cleanup() + return + if args.profile is None: + raise ValueError('--profile is required for run/validate') + profile, suite, path = load_profile(args.profile.resolve()) + if 'smoke' in suite and any( + getattr(args, flag) is not None for flag in ('duration', 'rps', 'workers', 'groups', 'members_per_group') + ): + raise ValueError('Load overrides are not applicable to smoke mode') + for phase in suite.get('phases', []): + for flag, field in [ + ('duration', 'duration_seconds'), + ('rps', 'rps'), + ('workers', 'workers'), + ('groups', 'groups'), + ('members_per_group', 'members_per_group'), + ]: + if getattr(args, flag) is not None: + phase[field] = getattr(args, flag) + validate(suite) + if args.action == 'validate': + print('Job profile valid; no deployment or backend changed.') + return + if args.output is None: + raise ValueError('--output is required for run') + job = Job(profile, suite, path, args.output) + result = job.run() + print(json.dumps(result, ensure_ascii=False, indent=2)) + if not result['passed']: + raise SystemExit(1) + except (RuntimeError, ValueError, OSError, KeyError, subprocess.TimeoutExpired) as exc: + parser.exit(1, f'{type(exc).__name__}: {exc}\n') + + +if __name__ == '__main__': + main() diff --git a/tools/regression_profiles/groups.json b/tools/regression_profiles/groups.json new file mode 100644 index 000000000..dd6d57e87 --- /dev/null +++ b/tools/regression_profiles/groups.json @@ -0,0 +1,464 @@ +{ + "version": 1, + "cli": { + "executable": "lbctl", + "config": "./lbctl.yaml" + }, + "vars": { + "plugin_dir": "/path/to/plugin-zips", + "mock_base_url": "http://127.0.0.1:18080/v1", + "embedding_base_url": "http://127.0.0.1:15410/v1", + "core_health": "http://127.0.0.1:5300/healthz", + "core_base_url": "http://127.0.0.1:5300" + }, + "plugins": [ + { + "source": "local", + "file": "${plugin_dir}/LocalAgent.zip" + }, + { + "source": "local", + "file": "${plugin_dir}/LangRAG.zip" + }, + { + "source": "local", + "file": "${plugin_dir}/GeneralParsers.zip" + } + ], + "resources": [ + { + "id": "mock_provider", + "kind": "provider", + "body": { + "name": "regression mock", + "requester": "openai-chat-completions", + "base_url": "${mock_base_url}", + "api_keys": [ + "test-only" + ] + } + }, + { + "id": "embedding_provider", + "kind": "provider", + "body": { + "name": "regression embedding", + "requester": "openai-chat-completions", + "base_url": "${embedding_base_url}", + "api_keys": [ + "test-only" + ] + } + }, + { + "id": "chat_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-chat", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "box_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-box", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "rag_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-rag", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "embedding_model", + "kind": "model", + "type": "embedding", + "body": { + "name": "all-MiniLM-L6-v2", + "provider_uuid": "${embedding_provider}" + } + }, + { + "id": "knowledge_base", + "kind": "knowledge-base", + "body": { + "name": "regression parser golden", + "knowledge_engine_plugin_id": "langbot-team/LangRAG", + "creation_settings": { + "embedding_model_uuid": "${embedding_model}", + "index_type": "chunk", + "chunk_size": 512, + "overlap": 50 + }, + "retrieval_settings": { + "search_type": "vector", + "top_k": 5, + "query_rewrite": "off", + "rerank": "off" + } + }, + "documents": [ + { + "file": "../../skills/skills/langbot-testing/fixtures/rag/parser-golden.html", + "parser_plugin_id": "langbot-team/GeneralParsers" + } + ] + }, + { + "id": "chat_pipeline", + "kind": "pipeline", + "body": { + "name": "regression chat", + "description": "Normal deterministic chat", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${chat_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": false, + "timeout": 30, + "max-round": 20, + "box-session-id-template": "{launcher_type}_{launcher_id}" + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "box_pipeline", + "kind": "pipeline", + "body": { + "name": "regression Box", + "description": "Real Box write and read", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${box_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": true, + "box-session-id-template": "{launcher_type}_{launcher_id}", + "timeout": 30, + "max-tool-iterations": 5, + "max-round": 20 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "rag_pipeline", + "kind": "pipeline", + "body": { + "name": "regression RAG", + "description": "Retrieved model input oracle", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${rag_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [ + "${knowledge_base}" + ], + "box-enabled": false, + "timeout": 30, + "retrieval-top-k": 5, + "max-round": 20, + "box-session-id-template": "{launcher_type}_{launcher_id}" + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "chat_bot", + "kind": "bot", + "body": { + "name": "regression group chat", + "description": "Synthetic signed group ingress", + "adapter": "http_bot", + "enable": true, + "adapter_config": { + "inbound_secret": "${group_secret}", + "signature_required": true, + "callback_url": "http://127.0.0.1:9/unused-sync-only", + "default_session_type": "group", + "callback_timeout": 20, + "callback_max_retries": 0 + }, + "event_bindings": [ + { + "event_pattern": "message.received", + "target_type": "pipeline", + "target_uuid": "${chat_pipeline}", + "enabled": true, + "filters": [] + } + ], + "plugin_processors": [] + } + }, + { + "id": "box_bot", + "kind": "bot", + "body": { + "name": "regression group box", + "description": "Synthetic signed group ingress", + "adapter": "http_bot", + "enable": true, + "adapter_config": { + "inbound_secret": "${group_secret}", + "signature_required": true, + "callback_url": "http://127.0.0.1:9/unused-sync-only", + "default_session_type": "group", + "callback_timeout": 20, + "callback_max_retries": 0 + }, + "event_bindings": [ + { + "event_pattern": "message.received", + "target_type": "pipeline", + "target_uuid": "${box_pipeline}", + "enabled": true, + "filters": [] + } + ], + "plugin_processors": [] + } + }, + { + "id": "rag_bot", + "kind": "bot", + "body": { + "name": "regression group rag", + "description": "Synthetic signed group ingress", + "adapter": "http_bot", + "enable": true, + "adapter_config": { + "inbound_secret": "${group_secret}", + "signature_required": true, + "callback_url": "http://127.0.0.1:9/unused-sync-only", + "default_session_type": "group", + "callback_timeout": 20, + "callback_max_retries": 0 + }, + "event_bindings": [ + { + "event_pattern": "message.received", + "target_type": "pipeline", + "target_uuid": "${rag_pipeline}", + "enabled": true, + "filters": [] + } + ], + "plugin_processors": [] + } + } + ], + "cases": [ + { + "name": "chat", + "weight": 2, + "http_group": { + "url": "${core_base_url}/bots/${chat_bot}/sync", + "secret_env": "LANGBOT_QA_GROUP_SECRET", + "message": "ai What is the documented parser-rag sentinel?" + }, + "expect": "LANGBOT_CHAT_OK" + }, + { + "name": "box", + "weight": 1, + "http_group": { + "url": "${core_base_url}/bots/${box_bot}/sync", + "secret_env": "LANGBOT_QA_GROUP_SECRET", + "message": "ai What is the documented parser-rag sentinel?" + }, + "expect": "LANGBOT_BOX_OK" + }, + { + "name": "rag_chat", + "weight": 1, + "http_group": { + "url": "${core_base_url}/bots/${rag_bot}/sync", + "secret_env": "LANGBOT_QA_GROUP_SECRET", + "message": "ai What is the documented parser-rag sentinel?" + }, + "expect": "LANGBOT_RAG_OK" + } + ], + "phases": [ + { + "name": "normal", + "duration_seconds": 600, + "rps": 2, + "workers": 4, + "cooldown_seconds": 30, + "groups": 4, + "members_per_group": 3 + } + ], + "observe": { + "health": { + "core": "${core_health}", + "mock": "http://127.0.0.1:18080/healthz", + "box": "http://127.0.0.1:5410/readyz" + }, + "pids": {}, + "pid_files": {}, + "containers": [] + }, + "cleanup": "on-success", + "thresholds": { + "min_rps_ratio": 0, + "max_p95_seconds": { + "chat": 5, + "box": 10, + "rag_chat": 6 + } + } +} diff --git a/tools/regression_profiles/job.json b/tools/regression_profiles/job.json new file mode 100644 index 000000000..70ca15c4d --- /dev/null +++ b/tools/regression_profiles/job.json @@ -0,0 +1,15 @@ +{ + "version": 1, + "lbctl": "lbctl", + "suite": "groups.json", + "deployment": { + "version": "v4.11.0", + "port": 15351 + }, + "vars": { + "plugin_dir": "/path/to/plugin-zips" + }, + "mock_image": "python:3.12-alpine", + "cleanup": "on-success", + "purge_data": true +} diff --git a/tools/regression_profiles/normal.json b/tools/regression_profiles/normal.json new file mode 100644 index 000000000..6c5bd78e8 --- /dev/null +++ b/tools/regression_profiles/normal.json @@ -0,0 +1,111 @@ +{ + "version": 1, + "cli": {"executable": "lbctl", "config": "./lbctl.yaml"}, + "vars": { + "plugin_dir": "/path/to/plugin-zips", + "mock_base_url": "http://127.0.0.1:18080/v1", + "embedding_base_url": "http://127.0.0.1:15410/v1", + "core_health": "http://127.0.0.1:5300/healthz" + }, + "plugins": [ + {"source": "local", "file": "${plugin_dir}/LocalAgent.zip"}, + {"source": "local", "file": "${plugin_dir}/LangRAG.zip"}, + {"source": "local", "file": "${plugin_dir}/GeneralParsers.zip"} + ], + "resources": [ + {"id": "mock_provider", "kind": "provider", "body": { + "name": "regression mock", "requester": "openai-chat-completions", + "base_url": "${mock_base_url}", "api_keys": ["test-only"] + }}, + {"id": "embedding_provider", "kind": "provider", "body": { + "name": "regression embedding", "requester": "openai-chat-completions", + "base_url": "${embedding_base_url}", "api_keys": ["test-only"] + }}, + {"id": "chat_model", "kind": "model", "type": "llm", "body": { + "name": "mock-chat", "provider_uuid": "${mock_provider}", + "abilities": ["func_call"], "context_length": 8192, + "extra_args": {"request_policy": {"retries": 0}} + }}, + {"id": "box_model", "kind": "model", "type": "llm", "body": { + "name": "mock-box", "provider_uuid": "${mock_provider}", + "abilities": ["func_call"], "context_length": 8192, + "extra_args": {"request_policy": {"retries": 0}} + }}, + {"id": "rag_model", "kind": "model", "type": "llm", "body": { + "name": "mock-rag", "provider_uuid": "${mock_provider}", + "abilities": ["func_call"], "context_length": 8192, + "extra_args": {"request_policy": {"retries": 0}} + }}, + {"id": "embedding_model", "kind": "model", "type": "embedding", "body": { + "name": "all-MiniLM-L6-v2", "provider_uuid": "${embedding_provider}" + }}, + {"id": "knowledge_base", "kind": "knowledge-base", "body": { + "name": "regression parser golden", "knowledge_engine_plugin_id": "langbot-team/LangRAG", + "creation_settings": {"embedding_model_uuid": "${embedding_model}", "index_type": "chunk", "chunk_size": 512, "overlap": 50}, + "retrieval_settings": {"search_type": "vector", "top_k": 5, "query_rewrite": "off", "rerank": "off"} + }, "documents": [{ + "file": "../../skills/skills/langbot-testing/fixtures/rag/parser-golden.html", + "parser_plugin_id": "langbot-team/GeneralParsers" + }]}, + {"id": "chat_pipeline", "kind": "pipeline", "body": { + "name": "regression chat", "description": "Normal deterministic chat", + "config": { + "ai": { + "runner": {"id": "plugin:langbot-team/LocalAgent/default", "expire-time": 0}, + "runner_config": {"plugin:langbot-team/LocalAgent/default": { + "model": {"primary": "${chat_model}", "fallbacks": []}, + "prompt": [{"role": "system", "content": "You are a QA regression assistant."}], + "knowledge-bases": [], "box-enabled": false, "timeout": 30 + }} + }, + "safety": {"rate-limit": {"limitation": 10000}}, + "output": {"force-delay": {"min": 0, "max": 0}} + } + }}, + {"id": "box_pipeline", "kind": "pipeline", "body": { + "name": "regression Box", "description": "Real Box write and read", + "config": { + "ai": { + "runner": {"id": "plugin:langbot-team/LocalAgent/default", "expire-time": 0}, + "runner_config": {"plugin:langbot-team/LocalAgent/default": { + "model": {"primary": "${box_model}", "fallbacks": []}, + "prompt": [{"role": "system", "content": "You are a QA regression assistant."}], + "knowledge-bases": [], "box-enabled": true, "box-session-id-template": "{global}", + "timeout": 30, "max-tool-iterations": 5 + }} + }, + "safety": {"rate-limit": {"limitation": 10000}}, + "output": {"force-delay": {"min": 0, "max": 0}} + } + }}, + {"id": "rag_pipeline", "kind": "pipeline", "body": { + "name": "regression RAG", "description": "Retrieved model input oracle", + "config": { + "ai": { + "runner": {"id": "plugin:langbot-team/LocalAgent/default", "expire-time": 0}, + "runner_config": {"plugin:langbot-team/LocalAgent/default": { + "model": {"primary": "${rag_model}", "fallbacks": []}, + "prompt": [{"role": "system", "content": "You are a QA regression assistant."}], + "knowledge-bases": ["${knowledge_base}"], "box-enabled": false, + "timeout": 30, "retrieval-top-k": 5 + }} + }, + "safety": {"rate-limit": {"limitation": 10000}}, + "output": {"force-delay": {"min": 0, "max": 0}} + } + }} + ], + "cases": [ + {"name": "chat", "weight": 2, "args": ["pipeline", "run", "${chat_pipeline}", "--message", "Run the normal QA scenario", "--timeout", "90s"], "expect": "LANGBOT_CHAT_OK"}, + {"name": "box", "weight": 1, "args": ["pipeline", "run", "${box_pipeline}", "--message", "Run the normal QA scenario", "--timeout", "90s"], "expect": "LANGBOT_BOX_OK"}, + {"name": "rag_chat", "weight": 1, "args": ["pipeline", "run", "${rag_pipeline}", "--message", "What is the documented parser-rag sentinel?", "--timeout", "90s"], "expect": "LANGBOT_RAG_OK"}, + {"name": "rag_retrieve", "weight": 1, "args": ["knowledge-base", "retrieve", "${knowledge_base}"], "body": {"query": "What is the parser-rag golden sentinel and parser-engine pair?", "retrieval_settings": {"search_type": "vector", "top_k": 5}}, "expect": "aurora-parser-rag-9137"} + ], + "phases": [{"name": "normal", "duration_seconds": 600, "rps": 2, "workers": 4, "cooldown_seconds": 30}], + "observe": { + "health": {"core": "${core_health}", "mock": "http://127.0.0.1:18080/healthz", "box": "http://127.0.0.1:5410/readyz"}, + "pids": {}, "pid_files": {}, "containers": [] + }, + "cleanup": "on-success", + "thresholds": {"min_rps_ratio": 0.95, "max_p95_seconds": {"chat": 3, "box": 6, "rag_chat": 4, "rag_retrieve": 1}} +} diff --git a/tools/regression_profiles/smoke.json b/tools/regression_profiles/smoke.json new file mode 100644 index 000000000..123dfa331 --- /dev/null +++ b/tools/regression_profiles/smoke.json @@ -0,0 +1,1644 @@ +{ + "version": 1, + "cli": { + "executable": "lbctl", + "config": "./lbctl.yaml" + }, + "vars": { + "plugin_dir": "/path/to/plugin-zips", + "mock_base_url": "http://127.0.0.1:18080/v1", + "embedding_base_url": "http://127.0.0.1:15410/v1", + "core_health": "http://127.0.0.1:5300/healthz", + "core_base_url": "http://127.0.0.1:5300", + "core_db": "/path/to/core/data/langbot.db", + "mock_records": "/path/to/dedicated-mock-requests.jsonl" + }, + "plugins": [ + { + "source": "local", + "file": "${plugin_dir}/LocalAgent.zip" + }, + { + "source": "local", + "file": "${plugin_dir}/LangRAG.zip" + }, + { + "source": "local", + "file": "${plugin_dir}/GeneralParsers.zip" + } + ], + "resources": [ + { + "id": "mock_provider", + "kind": "provider", + "body": { + "name": "regression mock", + "requester": "openai-chat-completions", + "base_url": "${mock_base_url}", + "api_keys": [ + "test-only" + ] + } + }, + { + "id": "embedding_provider", + "kind": "provider", + "body": { + "name": "regression embedding", + "requester": "openai-chat-completions", + "base_url": "${embedding_base_url}", + "api_keys": [ + "test-only" + ] + } + }, + { + "id": "unicode_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-smoke-unicode", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "slow_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-slow-stream", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "error401_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-401", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "error429_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-429", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "error503_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-503", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "exec_error_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-smoke-exec-error", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "parallel_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-smoke-parallel-tools", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "unicode_export_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-smoke-export-unicode", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "other_rag_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-smoke-rag-other", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "chat_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-chat", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "box_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-smoke-box", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "export_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-smoke-export", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "rag_model", + "kind": "model", + "type": "llm", + "body": { + "name": "mock-rag", + "provider_uuid": "${mock_provider}", + "abilities": [ + "func_call" + ], + "context_length": 8192, + "extra_args": { + "request_policy": { + "retries": 0 + } + } + } + }, + { + "id": "embedding_model", + "kind": "model", + "type": "embedding", + "body": { + "name": "all-MiniLM-L6-v2", + "provider_uuid": "${embedding_provider}" + } + }, + { + "id": "knowledge_base", + "kind": "knowledge-base", + "body": { + "name": "regression parser golden", + "knowledge_engine_plugin_id": "langbot-team/LangRAG", + "creation_settings": { + "embedding_model_uuid": "${embedding_model}", + "index_type": "chunk", + "chunk_size": 512, + "overlap": 50 + }, + "retrieval_settings": { + "search_type": "vector", + "top_k": 5, + "query_rewrite": "off", + "rerank": "off" + } + }, + "documents": [ + { + "file": "../../skills/skills/langbot-testing/fixtures/rag/parser-golden.html", + "parser_plugin_id": "langbot-team/GeneralParsers" + } + ] + }, + { + "id": "other_knowledge_base", + "kind": "knowledge-base", + "body": { + "name": "smoke other knowledge base", + "knowledge_engine_plugin_id": "langbot-team/LangRAG", + "creation_settings": { + "embedding_model_uuid": "${embedding_model}", + "index_type": "chunk", + "chunk_size": 512, + "overlap": 50 + }, + "retrieval_settings": { + "search_type": "vector", + "top_k": 5, + "query_rewrite": "off", + "rerank": "off" + } + }, + "documents": [ + { + "file": "../fixtures/smoke-isolation.txt", + "parser_plugin_id": "langbot-team/GeneralParsers" + } + ] + }, + { + "id": "empty_knowledge_base", + "kind": "knowledge-base", + "body": { + "name": "smoke empty knowledge base", + "knowledge_engine_plugin_id": "langbot-team/LangRAG", + "creation_settings": { + "embedding_model_uuid": "${embedding_model}", + "index_type": "chunk", + "chunk_size": 512, + "overlap": 50 + }, + "retrieval_settings": { + "search_type": "vector", + "top_k": 5, + "query_rewrite": "off", + "rerank": "off" + } + } + }, + { + "id": "chat_pipeline", + "kind": "pipeline", + "body": { + "name": "regression chat", + "description": "Normal deterministic chat", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${chat_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": false, + "timeout": 30, + "max-round": 20 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "box_pipeline", + "kind": "pipeline", + "body": { + "name": "regression Box", + "description": "Real Box write and read", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${box_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": true, + "box-session-id-template": "{launcher_type}_{launcher_id}", + "timeout": 30, + "max-tool-iterations": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + }, + { + "id": "rag_pipeline", + "kind": "pipeline", + "body": { + "name": "regression RAG", + "description": "Retrieved model input oracle", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${rag_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [ + "${knowledge_base}" + ], + "box-enabled": false, + "timeout": 30, + "retrieval-top-k": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + }, + { + "id": "chat_bot", + "kind": "bot", + "body": { + "name": "regression group chat", + "description": "Synthetic signed group ingress", + "adapter": "http_bot", + "enable": true, + "adapter_config": { + "inbound_secret": "${group_secret}", + "signature_required": true, + "callback_url": "http://127.0.0.1:9/unused-sync-only", + "default_session_type": "group", + "callback_timeout": 20, + "callback_max_retries": 0 + }, + "event_bindings": [ + { + "event_pattern": "message.received", + "target_type": "pipeline", + "target_uuid": "${chat_pipeline}", + "enabled": true, + "filters": [] + } + ], + "plugin_processors": [] + } + }, + { + "id": "export_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke file export", + "description": "Explicit Box file delivery", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${export_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": true, + "box-session-id-template": "{launcher_type}_{launcher_id}", + "timeout": 30, + "max-tool-iterations": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + }, + { + "id": "unicode_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke unicode", + "description": "Smoke unicode", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${unicode_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": false, + "timeout": 30, + "max-round": 20 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "slow_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke slow", + "description": "Smoke slow", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${slow_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": false, + "timeout": 30, + "max-round": 20 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "fallback401_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke fallback401", + "description": "Smoke fallback401", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${error401_model}", + "fallbacks": [ + "${chat_model}" + ] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": false, + "timeout": 30, + "max-round": 20 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "fallback429_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke fallback429", + "description": "Smoke fallback429", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${error429_model}", + "fallbacks": [ + "${chat_model}" + ] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": false, + "timeout": 30, + "max-round": 20 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "fallback503_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke fallback503", + "description": "Smoke fallback503", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${error503_model}", + "fallbacks": [ + "${chat_model}" + ] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": false, + "timeout": 30, + "max-round": 20 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + }, + "trigger": { + "group-respond-rules": { + "prefix": [ + "ai " + ] + } + } + } + } + }, + { + "id": "exec_error_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke exec_error", + "description": "Smoke exec_error", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${exec_error_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": true, + "box-session-id-template": "{launcher_type}_{launcher_id}", + "timeout": 30, + "max-tool-iterations": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + }, + { + "id": "parallel_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke parallel", + "description": "Smoke parallel", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${parallel_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": true, + "box-session-id-template": "{launcher_type}_{launcher_id}", + "timeout": 30, + "max-tool-iterations": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + }, + { + "id": "unicode_export_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke unicode_export", + "description": "Smoke unicode_export", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${unicode_export_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [], + "box-enabled": true, + "box-session-id-template": "{launcher_type}_{launcher_id}", + "timeout": 30, + "max-tool-iterations": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + }, + { + "id": "other_rag_pipeline", + "kind": "pipeline", + "body": { + "name": "smoke other_rag", + "description": "Smoke other_rag", + "config": { + "ai": { + "runner": { + "id": "plugin:langbot-team/LocalAgent/default", + "expire-time": 0 + }, + "runner_config": { + "plugin:langbot-team/LocalAgent/default": { + "model": { + "primary": "${other_rag_model}", + "fallbacks": [] + }, + "prompt": [ + { + "role": "system", + "content": "You are a QA regression assistant." + } + ], + "knowledge-bases": [ + "${other_knowledge_base}" + ], + "box-enabled": false, + "timeout": 30, + "retrieval-top-k": 5 + } + } + }, + "safety": { + "rate-limit": { + "limitation": 10000 + } + }, + "output": { + "force-delay": { + "min": 0, + "max": 0 + }, + "misc": { + "at-sender": false, + "quote-origin": false + } + } + } + } + }, + { + "id": "box_bot", + "kind": "bot", + "body": { + "name": "smoke group box", + "description": "Synthetic signed group ingress", + "adapter": "http_bot", + "enable": true, + "adapter_config": { + "inbound_secret": "${group_secret}", + "signature_required": true, + "callback_url": "http://127.0.0.1:9/unused-sync-only", + "default_session_type": "group", + "callback_timeout": 20, + "callback_max_retries": 0 + }, + "event_bindings": [ + { + "event_pattern": "message.received", + "target_type": "pipeline", + "target_uuid": "${box_pipeline}", + "enabled": true, + "filters": [] + } + ], + "plugin_processors": [] + } + }, + { + "id": "rag_bot", + "kind": "bot", + "body": { + "name": "smoke group rag", + "description": "Synthetic signed group ingress", + "adapter": "http_bot", + "enable": true, + "adapter_config": { + "inbound_secret": "${group_secret}", + "signature_required": true, + "callback_url": "http://127.0.0.1:9/unused-sync-only", + "default_session_type": "group", + "callback_timeout": 20, + "callback_max_retries": 0 + }, + "event_bindings": [ + { + "event_pattern": "message.received", + "target_type": "pipeline", + "target_uuid": "${rag_pipeline}", + "enabled": true, + "filters": [] + } + ], + "plugin_processors": [] + } + } + ], + "cases": [ + { + "name": "identity", + "stage": "configuration", + "args": [ + "whoami" + ], + "expect": "${workspace_uuid}" + }, + { + "name": "plugins", + "stage": "configuration", + "args": [ + "plugin", + "list" + ], + "expect": "LocalAgent" + }, + { + "name": "provider_readback", + "stage": "configuration", + "args": [ + "provider", + "get", + "${mock_provider}" + ], + "expect": "${mock_provider}" + }, + { + "name": "model_readback", + "stage": "configuration", + "args": [ + "model", + "get", + "${chat_model}", + "--type", + "llm" + ], + "expect": "mock-chat" + }, + { + "name": "pipeline_readback", + "stage": "configuration", + "args": [ + "pipeline", + "get", + "${box_pipeline}" + ], + "expect": "${box_model}" + }, + { + "name": "kb_readback", + "stage": "configuration", + "args": [ + "knowledge-base", + "get", + "${knowledge_base}" + ], + "expect": "${knowledge_base}", + "fields": { + "base.uuid": "${knowledge_base}", + "base.knowledge_engine_plugin_id": "langbot-team/LangRAG" + } + }, + { + "name": "document_readback", + "stage": "configuration", + "args": [ + "knowledge-base", + "file", + "list", + "${knowledge_base}" + ], + "expect": "html", + "fields": { + "files.0.status": "completed", + "files.0.kb_id": "${knowledge_base}", + "files.0.workspace_uuid": "${workspace_uuid}" + } + }, + { + "name": "missing_signature", + "stage": "ingress", + "http_group": { + "url": "${core_base_url}/bots/${chat_bot}/sync", + "secret_env": "LANGBOT_QA_GROUP_SECRET", + "message": "ai What is the documented parser-rag sentinel?" + }, + "reject_group": "missing_signature", + "expect": "40101", + "fields": { + "http_status": 401, + "code": 40101 + } + }, + { + "name": "bad_signature", + "stage": "ingress", + "http_group": { + "url": "${core_base_url}/bots/${chat_bot}/sync", + "secret_env": "LANGBOT_QA_GROUP_SECRET", + "message": "ai What is the documented parser-rag sentinel?" + }, + "reject_group": "bad_signature", + "expect": "40101", + "fields": { + "http_status": 401, + "code": 40101 + } + }, + { + "name": "chat", + "args": [ + "pipeline", + "run", + "${chat_pipeline}", + "--message", + "Run the normal QA scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_CHAT_OK", + "stage": "conversation" + }, + { + "name": "unicode", + "stage": "conversation", + "args": [ + "pipeline", + "run", + "${unicode_pipeline}", + "--message", + "请原样处理：你好，世界 🌍", + "--timeout", + "90s" + ], + "expect": "你好，世界 🌍 LANGBOT_UNICODE_OK" + }, + { + "name": "slow", + "stage": "model", + "args": [ + "pipeline", + "run", + "${slow_pipeline}", + "--message", + "Run the smoke scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_SLOW_STREAM_OK", + "mock_requests": [ + { + "model": "mock-slow-stream", + "status": 200, + "stream": true + } + ] + }, + { + "name": "fallback401", + "stage": "model", + "args": [ + "pipeline", + "run", + "${fallback401_pipeline}", + "--message", + "Run the smoke scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_CHAT_OK", + "mock_requests": [ + { + "model": "mock-401", + "status": 401, + "stream": true + }, + { + "model": "mock-chat", + "status": 200, + "stream": true + } + ] + }, + { + "name": "fallback429", + "stage": "model", + "args": [ + "pipeline", + "run", + "${fallback429_pipeline}", + "--message", + "Run the smoke scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_CHAT_OK", + "mock_requests": [ + { + "model": "mock-429", + "status": 429, + "stream": true + }, + { + "model": "mock-chat", + "status": 200, + "stream": true + } + ] + }, + { + "name": "fallback503", + "stage": "model", + "args": [ + "pipeline", + "run", + "${fallback503_pipeline}", + "--message", + "Run the smoke scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_CHAT_OK", + "mock_requests": [ + { + "model": "mock-503", + "status": 503, + "stream": true + }, + { + "model": "mock-chat", + "status": 200, + "stream": true + } + ] + }, + { + "name": "exec_error", + "stage": "tools", + "args": [ + "pipeline", + "run", + "${exec_error_pipeline}", + "--message", + "Run the smoke scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_EXEC_ERROR_HANDLED", + "box_exec": { + "count": 1, + "backend": "docker", + "stdout": "EXPECTED_STDOUT", + "stderr": "EXPECTED_STDERR", + "exit_code": 7 + } + }, + { + "name": "parallel", + "stage": "tools", + "args": [ + "pipeline", + "run", + "${parallel_pipeline}", + "--message", + "Run the smoke scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_PARALLEL_TOOLS_OK", + "box_exec": { + "count": 2, + "backend": "docker", + "stdout": "PARALLEL_OK_" + } + }, + { + "name": "unicode_export", + "stage": "delivery", + "args": [ + "pipeline", + "run", + "${unicode_export_pipeline}", + "--message", + "Run the smoke scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_UNICODE_EXPORT_OK", + "box_exec": { + "count": 1, + "backend": "docker", + "stdout": "你好，文件 🌍" + }, + "attachment": { + "name": "输出.txt", + "text": "你好，文件 🌍" + } + }, + { + "name": "other_rag", + "stage": "retrieval", + "args": [ + "pipeline", + "run", + "${other_rag_pipeline}", + "--message", + "Run the smoke scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_OTHER_RAG_OK" + }, + { + "name": "box", + "args": [ + "pipeline", + "run", + "${box_pipeline}", + "--message", + "Run the normal QA scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_BOX_OK", + "box_exec": { + "count": 2, + "backend": "docker", + "stdout": "LANGBOT_BOX_SENTINEL" + }, + "stage": "tools" + }, + { + "name": "box_export", + "args": [ + "pipeline", + "run", + "${export_pipeline}", + "--message", + "Run the file export QA scenario", + "--timeout", + "90s" + ], + "expect": "LANGBOT_EXPORT_OK", + "box_exec": { + "count": 1, + "backend": "docker", + "stdout": "LANGBOT_EXPORT_SENTINEL" + }, + "attachment": { + "name": "langbot-smoke-output.txt", + "text": "LANGBOT_EXPORT_SENTINEL" + }, + "stage": "delivery" + }, + { + "name": "rag_chat", + "args": [ + "pipeline", + "run", + "${rag_pipeline}", + "--message", + "What is the documented parser-rag sentinel?", + "--timeout", + "90s" + ], + "expect": "LANGBOT_RAG_OK", + "stage": "retrieval" + }, + { + "name": "rag_retrieve", + "args": [ + "knowledge-base", + "retrieve", + "${knowledge_base}" + ], + "body": { + "query": "What is the parser-rag golden sentinel and parser-engine pair?", + "retrieval_settings": { + "search_type": "vector", + "top_k": 5 + } + }, + "expect": "aurora-parser-rag-9137", + "stage": "retrieval", + "fields": { + "results.0.metadata._langbot_knowledge_base_uuid": "${knowledge_base}" + }, + "forbid": [ + "cedar-isolation-4826" + ] + }, + { + "name": "retrieve_other", + "args": [ + "knowledge-base", + "retrieve", + "${other_knowledge_base}" + ], + "body": { + "query": "What is the parser-rag golden sentinel and parser-engine pair?", + "retrieval_settings": { + "search_type": "vector", + "top_k": 5 + } + }, + "expect": "cedar-isolation-4826", + "stage": "retrieval", + "fields": { + "results.0.metadata._langbot_knowledge_base_uuid": "${other_knowledge_base}" + }, + "forbid": [ + "aurora-parser-rag-9137" + ] + }, + { + "name": "retrieve_empty", + "args": [ + "knowledge-base", + "retrieve", + "${empty_knowledge_base}" + ], + "body": { + "query": "What is the parser-rag golden sentinel and parser-engine pair?", + "retrieval_settings": { + "search_type": "vector", + "top_k": 5 + } + }, + "expect": "results", + "stage": "retrieval", + "fields": { + "results": [] + } + }, + { + "name": "group_chat_history", + "http_group": { + "url": "${core_base_url}/bots/${chat_bot}/sync", + "secret_env": "LANGBOT_QA_GROUP_SECRET", + "message": "ai What is the documented parser-rag sentinel?" + }, + "expect": "LANGBOT_CHAT_OK", + "stage": "conversation" + }, + { + "name": "group_box_history", + "http_group": { + "url": "${core_base_url}/bots/${box_bot}/sync", + "secret_env": "LANGBOT_QA_GROUP_SECRET", + "message": "ai What is the documented parser-rag sentinel?" + }, + "expect": "LANGBOT_BOX_OK", + "stage": "conversation", + "box_exec": { + "count": 2, + "backend": "docker", + "stdout": "LANGBOT_BOX_SENTINEL" + } + }, + { + "name": "group_rag_history", + "http_group": { + "url": "${core_base_url}/bots/${rag_bot}/sync", + "secret_env": "LANGBOT_QA_GROUP_SECRET", + "message": "ai What is the documented parser-rag sentinel?" + }, + "expect": "LANGBOT_RAG_OK", + "stage": "conversation" + } + ], + "observe": { + "health": { + "core": "${core_health}", + "mock": "http://127.0.0.1:18080/healthz", + "box": "http://127.0.0.1:5410/readyz" + }, + "pids": {}, + "pid_files": {}, + "containers": [] + }, + "cleanup": "on-success", + "smoke": { + "ledger_db": "${core_db}", + "timeout_seconds": 30, + "group_turns": 3, + "groups_per_case": 2, + "required_plugins": [ + "langbot-team/LocalAgent", + "langbot-team/LangRAG", + "langbot-team/GeneralParsers" + ], + "mock_records": "${mock_records}" + } +} diff --git a/tools/regression_smoke.py b/tools/regression_smoke.py new file mode 100644 index 000000000..e462844ec --- /dev/null +++ b/tools/regression_smoke.py @@ -0,0 +1,328 @@ +"""Run finite CLI/group smoke cases with an independent read-only AgentRun oracle.""" + +from collections import Counter +from contextlib import closing +import base64 +import json +from pathlib import Path +import sqlite3 +import time +import urllib.error +import urllib.request +from uuid import uuid4 + +from regression_traffic import group_request, read_endpoint_snapshot + + +class UnresolvedRun(RuntimeError): + """The request cannot yet be matched to one authoritative terminal run.""" + + +def completed_run(connection, workspace, conversation, after_id, timeout): + deadline = time.monotonic() + timeout + while True: + rows = connection.execute( + 'SELECT run_id, status, status_reason, finished_at FROM agent_run ' + 'WHERE workspace_id=? AND conversation_id=? AND id>? ORDER BY id', + (workspace, conversation, after_id), + ).fetchall() + if len(rows) > 1: + raise UnresolvedRun('More than one AgentRun matches this request') + if rows and (rows[0]['finished_at'] or rows[0]['status'] in {'completed', 'failed', 'timeout', 'cancelled'}): + return dict(rows[0]) + if time.monotonic() >= deadline: + raise UnresolvedRun('No unique terminal AgentRun before the observation deadline; not retried') + time.sleep(0.1) + + +def check_run(connection, run, case): + if run['status'] != 'completed': + raise ValueError(f'AgentRun {run["run_id"]} ended as {run["status"]}: {run["status_reason"]}') + events = [ + (kind, json.loads(data or '{}')) + for kind, data in connection.execute( + 'SELECT type, data_json FROM agent_run_event WHERE run_id=? ORDER BY sequence', (run['run_id'],) + ) + ] + if any(not isinstance(data, dict) for _kind, data in events): + raise ValueError('AgentRun event payload must be an object') + if not any(kind == 'run.completed' for kind, _data in events): + raise ValueError('Completed AgentRun is missing its persisted run.completed event') + checks = {'terminal_event': True} + if expected := case.get('box_exec'): + calls = [ + data.get('result', {}) + for kind, data in events + if kind == 'tool.call.completed' and data.get('tool_name') == 'exec' + ] + if len(calls) != expected['count'] or any( + not isinstance(item, dict) + or item.get('backend') != expected['backend'] + or item.get('exit_code') != expected.get('exit_code', 0) + or not isinstance(item.get('stdout'), str) + or expected['stdout'] not in item['stdout'] + or ('stderr' in expected and expected['stderr'] not in str(item.get('stderr', ''))) + for item in calls + ): + raise ValueError('Box execution count/backend/exit code/output did not match the oracle') + checks['box_exec'] = {'calls': len(calls), 'backend': expected['backend'], 'output_matched': True} + return checks + + +def check_attachment(data, expected): + replies = data.get('replies') + if not isinstance(replies, list) or any( + not isinstance(reply, dict) + or not isinstance(reply.get('message_chain'), list) + or any(not isinstance(part, dict) for part in reply['message_chain']) + for reply in replies + ): + raise ValueError('Malformed file delivery response') + attachments = [part for reply in replies for part in reply.get('message_chain', []) if part.get('type') == 'File'] + if len(attachments) != 1 or attachments[0].get('name') != expected['name']: + raise ValueError('Expected exactly one file attachment with the configured name') + encoded = attachments[0].get('base64') + if not isinstance(encoded, str): + raise ValueError('Attachment must contain base64 bytes') + content = base64.b64decode(encoded, validate=True) + if content != expected['text'].encode('utf-8'): + raise ValueError('Exported attachment bytes differ from the expected fixture') + return {'name': expected['name'], 'bytes': len(content), 'content_matched': True} + + +def check_fields(data, expected): + for path, value in expected.items(): + actual = data + try: + for key in path.split('.'): + actual = actual[int(key)] if isinstance(actual, list) else actual[key] + except (KeyError, IndexError, TypeError, ValueError) as exc: + raise ValueError(f'Response field {path} is missing or malformed') from exc + if actual != value: + raise ValueError(f'Response field {path} differs from the expected value') + return expected + + +def check_mock_requests(path, offset, expected): + deadline = time.monotonic() + 3 + while True: + with path.open('rb') as stream: + stream.seek(offset) + lines = stream.read().splitlines() + records = [json.loads(line) for line in lines] + actual = [ + {'model': row['request']['model'], 'status': row['status'], 'stream': row['request'].get('stream', False)} + for row in records + if row['path'] == '/v1/chat/completions' + ] + if len(actual) >= len(expected) or time.monotonic() >= deadline: + if actual != expected: + raise ValueError(f'Model request sequence differs: expected {expected}, got {actual}') + return actual + time.sleep(0.05) + + +def rejected_group(case, group): + settings = case['http_group'] + headers = {'Content-Type': 'application/json'} + if case['reject_group'] == 'bad_signature': + headers.update({'X-LB-Timestamp': str(int(time.time())), 'X-LB-Signature': 'sha256=' + '0' * 64}) + request = urllib.request.Request( + settings['url'], + headers=headers, + data=json.dumps( + { + 'session_id': group, + 'session_type': 'group', + 'sender': {'id': 'qa-rejected'}, + 'message': [{'type': 'Plain', 'text': settings['message']}], + } + ).encode(), + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(request, timeout=10): + raise ValueError('Unsigned/tampered group message was accepted') + except urllib.error.HTTPError as exc: + payload = json.load(exc) + if exc.code != 401 or not isinstance(payload, dict) or payload.get('code') != 40101: + raise ValueError('Expected HTTP 401 with bad_signature code 40101') from exc + return {'http_status': exc.code, 'code': payload['code']} + + +def verify_cleanup(suite, deleted): + checks = [] + for kind, model_type in dict.fromkeys((item['kind'], item.get('type')) for item in deleted): + args = [kind, 'list'] + (['--type', model_type] if model_type else []) + data = suite.call(args) + key = { + 'provider': 'providers', + 'model': 'models', + 'pipeline': 'pipelines', + 'bot': 'bots', + 'knowledge-base': 'bases', + }[kind] + items = data.get(key) if isinstance(data, dict) else None + if not isinstance(items, list) or any( + not isinstance(item, dict) or not isinstance(item.get('uuid'), str) for item in items + ): + raise ValueError(f'Malformed {kind} list response during cleanup verification') + visible = {item['uuid'] for item in items} + expected_absent = {item['uuid'] for item in deleted if item['kind'] == kind and item.get('type') == model_type} + remaining = sorted(expected_absent & visible) + checks.append({'kind': kind, 'type': model_type, 'deleted_count': len(expected_absent), 'remaining': remaining}) + suite.write('cleanup-checks.json', checks) + if any(row['remaining'] for row in checks): + raise ValueError('Deleted resources remain visible in CLI list readback') + + +def run_smoke(suite): + # Imported here because Suite dispatches to this module after profile validation. + from regression_suite import resolve + + settings = suite.profile['smoke'] + path = Path(suite.path(settings['ledger_db'])).resolve() + if not path.is_file(): + raise ValueError('smoke.ledger_db must be an existing target Core SQLite database') + mock_path = Path(suite.path(settings['mock_records'])) if settings.get('mock_records') else None + rows, errors, health = [], [], [] + started = time.monotonic() + + def redact(value): + for secret in suite.secrets: + value = value.replace(secret, '[REDACTED]') + return value + + def sample_health(phase): + for name, url in resolve(suite.profile['observe']['health'], suite.variables).items(): + row = {'phase': phase, 'endpoint': name} + try: + row['result'] = read_endpoint_snapshot(url, timeout_seconds=3) + except (RuntimeError, OSError, ValueError) as exc: + row['error'] = redact(str(exc)) + errors.append(f'{phase}/{name}: health check failed') + health.append(row) + suite.write('health.json', health) + + sample_health('before') + # shortcut: local SQLite only; use a terminal-run API when lbctl exposes one. + with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True, timeout=5)) as connection: + connection.row_factory = sqlite3.Row + connection.execute('PRAGMA query_only=ON') + for original in suite.profile['cases']: + case = resolve(original, suite.variables) + is_group = 'http_group' in case + rejected = 'reject_group' in case + runner = not rejected and (is_group or case.get('args', [])[:2] == ['pipeline', 'run']) + groups = ( + [f'qa-smoke-{uuid4().hex}' for _ in range(settings.get('groups_per_case', 2))] + if is_group and not rejected + else [f'qa-smoke-{uuid4().hex}' if rejected else None] + ) + previous = {group: 'none' for group in groups} + retired = set() + for turn in range(1, (settings.get('group_turns', 2) if is_group and not rejected else 1) + 1): + for group in groups: + row = { + 'case': case['name'], + 'stage': case.get('stage', 'execution'), + 'status': 'failed', + 'requires_runner': runner, + } + if is_group: + row.update(group_id=group, turn=turn, member_id=f'qa-member-{turn % 2:03d}') + if group in retired: + row.update(status='skipped', error='Earlier turn failed or is unknown; group not retried') + rows.append(row) + continue + before = time.monotonic() + after_id = connection.execute('SELECT COALESCE(MAX(id),0) FROM agent_run').fetchone()[0] + token = uuid4().hex + offset = mock_path.stat().st_size if case.get('mock_requests') and mock_path.exists() else 0 + try: + if rejected: + data = rejected_group(case, group) + count = connection.execute( + 'SELECT COUNT(*) FROM agent_run WHERE workspace_id=? AND conversation_id=? AND id>?', + (suite.variables['workspace_uuid'], 'group_' + group, after_id), + ).fetchone()[0] + if count: + raise ValueError('Rejected ingress created an AgentRun') + elif is_group: + data = group_request( + case, group, row['member_id'], turn, token, previous[group], suite.env + )['data'] + else: + data = suite.call(case['args'], case.get('body')) + if case['expect'] not in json.dumps(data, ensure_ascii=False): + raise ValueError('Expected business response was not returned') + row['output'] = data + if any(value in json.dumps(data, ensure_ascii=False) for value in case.get('forbid', [])): + raise ValueError('Response contains forbidden content from another knowledge base') + row['checks'] = {} + if case.get('fields'): + row['checks']['fields'] = check_fields(data, case['fields']) + if runner: + if not isinstance(data, dict): + raise ValueError('Runner response must be an object') + session = data.get('session_id') + if not isinstance(session, str) or not session: + raise UnresolvedRun('Reply has no session ID for AgentRun reconciliation') + run = completed_run( + connection, + suite.variables['workspace_uuid'], + 'group_' + session if is_group else session, + after_id, + settings.get('timeout_seconds', 30), + ) + row['run'] = run + row['checks'].update(check_run(connection, run, case)) + if case.get('attachment'): + row['checks']['attachment'] = check_attachment(data, case['attachment']) + if case.get('mock_requests'): + row['checks']['mock_requests'] = check_mock_requests( + mock_path, offset, case['mock_requests'] + ) + row['status'] = 'passed' + previous[group] = token + except urllib.error.HTTPError as exc: + row.update( + status='unknown' if exc.code >= 500 else 'failed', error=f'HTTP {exc.code}; not retried' + ) + except (UnresolvedRun, urllib.error.URLError, TimeoutError) as exc: + row.update(status='unknown', error=redact(str(exc))) + except (RuntimeError, ValueError, OSError, KeyError, IndexError, sqlite3.Error) as exc: + message = redact(str(exc)) + row.update( + status='unknown' + if 'outcome unknown' in message or 'result_unknown' in message + else 'failed', + error=message, + ) + row['seconds'] = time.monotonic() - before + if row['status'] != 'passed': + errors.append(f'{case["name"]}: {row["status"]}: {row["error"]}') + if is_group: + retired.add(group) + rows.append(row) + suite.write('cases.json', rows) + suite.write( + 'progress.json', + {'completed_requests': len(rows), 'status': dict(Counter(x['status'] for x in rows))}, + ) + suite.write('cases.json', rows) + sample_health('after') + return { + 'mode': 'smoke', + 'phases': [], + 'cases': rows, + 'errors': errors, + 'elapsed_seconds': time.monotonic() - started, + 'status': dict(Counter(row['status'] for row in rows)), + 'stages': { + stage: dict(Counter(row['status'] for row in rows if row['stage'] == stage)) + for stage in dict.fromkeys(row['stage'] for row in rows) + }, + 'case_count': len(suite.profile['cases']), + 'requests': len(rows), + } diff --git a/tools/regression_suite.py b/tools/regression_suite.py new file mode 100644 index 000000000..4397d3905 --- /dev/null +++ b/tools/regression_suite.py @@ -0,0 +1,717 @@ +#!/usr/bin/env python3 +"""Prepare an lbctl regression from JSON, run load phases, and clean owned resources.""" + +import argparse +import copy +import json +import math +import os +from pathlib import Path +import re +import secrets +import subprocess +import sys +import urllib.error +import urllib.request +from uuid import uuid4 + +from regression_traffic import group_request + + +KINDS = {'provider', 'model', 'pipeline', 'knowledge-base', 'bot'} +REFERENCE = re.compile(r'\$\{([a-zA-Z][a-zA-Z0-9_-]*)\}') + + +def resolve(value, variables): + if isinstance(value, dict): + return {key: resolve(item, variables) for key, item in value.items()} + if isinstance(value, list): + return [resolve(item, variables) for item in value] + if isinstance(value, str): + return REFERENCE.sub(lambda match: str(variables[match[1]]), value) + return value + + +def merge(base, patch): + result = copy.deepcopy(base) + for key, value in patch.items(): + result[key] = ( + merge(result[key], value) if isinstance(value, dict) and isinstance(result.get(key), dict) else value + ) + return result + + +def contains_fields(actual, expected): + """Check submitted fields while allowing server-generated defaults.""" + if isinstance(expected, dict): + return isinstance(actual, dict) and all( + key in actual and contains_fields(actual[key], value) for key, value in expected.items() + ) + if isinstance(expected, list): + return ( + isinstance(actual, list) + and len(actual) == len(expected) + and all(contains_fields(left, right) for left, right in zip(actual, expected)) + ) + return actual == expected + + +def number(value, label, *, minimum=0, integer=False): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise ValueError(f'{label} must be a finite number') + if value < minimum or (integer and not isinstance(value, int)): + raise ValueError(f'{label} must be {"an integer " if integer else ""}>= {minimum}') + + +def validate(profile): + if not isinstance(profile, dict) or profile.get('version') != 1: + raise ValueError('profile version must be 1') + allowed = { + 'version', + 'cli', + 'vars', + 'plugins', + 'resources', + 'cases', + 'phases', + 'observe', + 'cleanup', + 'thresholds', + 'smoke', + } + if extra := profile.keys() - allowed: + raise ValueError(f'unknown profile fields: {sorted(extra)}') + for field in ('cli', 'vars', 'observe', 'thresholds'): + if field in profile and not isinstance(profile[field], dict): + raise ValueError(f'{field} must be an object') + for field in ('plugins', 'resources', 'cases', 'phases'): + if field in profile and ( + not isinstance(profile[field], list) or any(not isinstance(item, dict) for item in profile[field]) + ): + raise ValueError(f'{field} must be an array of objects') + cli = profile.get('cli', {}) + if not isinstance(cli, dict) or cli.keys() - {'config', 'executable'}: + raise ValueError('cli accepts config and executable') + if not isinstance(cli.get('config'), str) or not cli['config']: + raise ValueError('cli.config must name a Workspace-bound lbctl config') + variables = {'profile_dir': '.', 'run_dir': '.', 'workspace_uuid': 'workspace', 'group_secret': 'private'} + supplied = profile.get('vars', {}) + if not isinstance(supplied, dict) or variables.keys() & supplied.keys(): + raise ValueError('vars must be an object without reserved names') + if any(not isinstance(value, (str, int, float)) or '${' in str(value) for value in supplied.values()): + raise ValueError('vars values must be scalar literals, without nested references') + variables.update(supplied) + for plugin in profile.get('plugins', []): + if plugin.keys() - {'source', 'file'}: + raise ValueError('plugins accept source and file') + if plugin.get('source') not in {'local', 'github', 'marketplace'} or not plugin.get('file'): + raise ValueError('plugins need source local/github/marketplace and file') + resolve(plugin, variables) + for item in profile.get('resources', []): + if item.keys() - {'id', 'kind', 'type', 'body', 'documents'}: + raise ValueError('unknown resource field') + name = item.get('id', '') + if not re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]*', name) or name in variables: + raise ValueError(f'invalid or duplicate resource id: {name}') + if item.get('kind') not in KINDS or not isinstance(item.get('body'), dict): + raise ValueError(f'{name}: resource needs kind and body') + if 'uuid' in item['body']: + raise ValueError(f'{name}: resources must create new objects; body.uuid is forbidden') + if item['kind'] == 'bot' and ( + item['body'].get('adapter') != 'http_bot' + or not isinstance(item['body'].get('adapter_config'), dict) + or item['body'].get('adapter_config', {}).get('inbound_secret') != '${group_secret}' + or not any( + isinstance(case.get('http_group'), dict) and '${' + name + '}' in case['http_group'].get('url', '') + for case in profile.get('cases', []) + ) + ): + raise ValueError('Bot resources require http_bot, ${group_secret}, and a matching http_group case') + if item['kind'] == 'model' and item.get('type') not in {'llm', 'embedding', 'rerank'}: + raise ValueError(f'{name}: model type must be llm/embedding/rerank') + resolve(item['body'], variables) # References must point backwards; never guess missing UUIDs. + variables[name] = name + documents = item.get('documents', []) + if not isinstance(documents, list) or any(not isinstance(doc, dict) for doc in documents): + raise ValueError(f'{name}: documents must be an array of objects') + for document in documents: + if item['kind'] != 'knowledge-base' or not document.get('file') or not document.get('parser_plugin_id'): + raise ValueError(f'{name}: documents need file and parser_plugin_id on a knowledge-base') + resolve(document, variables) + cases = profile.get('cases') + if not isinstance(cases, list) or not cases: + raise ValueError('cases must be a non-empty array') + names = set() + for case in cases: + if case.keys() - { + 'name', + 'weight', + 'args', + 'expect', + 'body', + 'http_group', + 'box_exec', + 'attachment', + 'stage', + 'mock_requests', + 'fields', + 'forbid', + 'reject_group', + }: + raise ValueError('unknown case field') + if not isinstance(case.get('name'), str) or not case['name'] or case['name'] in names: + raise ValueError('case names must be non-empty and unique') + names.add(case['name']) + if 'stage' in case and (not isinstance(case['stage'], str) or not case['stage']): + raise ValueError('case stage must be a non-empty string') + if any(key in case for key in ('mock_requests', 'fields', 'forbid', 'reject_group')) and 'smoke' not in profile: + raise ValueError('extended case checks require smoke mode') + if 'fields' in case and (not isinstance(case['fields'], dict) or not case['fields']): + raise ValueError('fields must be a non-empty mapping of dotted paths to exact values') + if 'fields' in case and any(not isinstance(key, str) or not key for key in case['fields']): + raise ValueError('field paths must be non-empty strings') + if 'forbid' in case and ( + not isinstance(case['forbid'], list) + or not case['forbid'] + or any(not isinstance(item, str) or not item for item in case['forbid']) + ): + raise ValueError('forbid must contain non-empty response substrings') + if 'mock_requests' in case: + requests = case['mock_requests'] + if not isinstance(requests, list) or not requests or not profile.get('smoke', {}).get('mock_records'): + raise ValueError('mock_requests requires expected requests and smoke.mock_records') + for request in requests: + if not isinstance(request, dict) or set(request) != {'model', 'status', 'stream'}: + raise ValueError('mock_requests entries require model, status and stream') + if ( + not isinstance(request['model'], str) + or not request['model'] + or not isinstance(request['stream'], bool) + ): + raise ValueError('invalid mock request model/stream') + number(request['status'], 'mock request status', minimum=100, integer=True) + if 'reject_group' in case and ( + 'http_group' not in case + or case['reject_group'] not in ('missing_signature', 'bad_signature') + or any(key in case for key in ('box_exec', 'attachment', 'mock_requests')) + ): + raise ValueError('reject_group must name a signature rejection on an HTTP group case') + if 'http_group' in case: + settings = case['http_group'] + if ( + 'args' in case + or 'body' in case + or not isinstance(settings, dict) + or set(settings) != {'url', 'secret_env', 'message'} + or any(not isinstance(value, str) or not value for value in settings.values()) + or not resolve(settings['url'], variables).startswith(('http://', 'https://')) + or settings['secret_env'] != 'LANGBOT_QA_GROUP_SECRET' + ): + raise ValueError( + 'http_group needs url, message and secret_env=LANGBOT_QA_GROUP_SECRET; no CLI args/body' + ) + elif ( + not isinstance(case.get('args'), list) + or not case['args'] + or not all(isinstance(x, str) for x in case['args']) + ): + raise ValueError('case args must be a non-empty string array') + if not isinstance(case.get('expect'), str) or not case['expect']: + raise ValueError('case expect must be a non-empty string') + number(case.get('weight', 1), 'case weight', minimum=1, integer=True) + resolve(case, variables) + if 'body' in case and (not isinstance(case['body'], dict) or '--file' in case['args']): + raise ValueError('case body must be an object and cannot coexist with --file') + if 'box_exec' in case: + check = case['box_exec'] + if ( + not isinstance(check, dict) + or not {'count', 'backend', 'stdout'} <= set(check) + or check.keys() - {'count', 'backend', 'stdout', 'exit_code', 'stderr'} + ): + raise ValueError('box_exec requires count, backend and stdout') + if 'exit_code' in check: + number(check['exit_code'], 'box_exec.exit_code', minimum=0, integer=True) + if 'stderr' in check and not isinstance(check['stderr'], str): + raise ValueError('box_exec.stderr must be a string') + number(check['count'], 'box_exec.count', minimum=1, integer=True) + if any(not isinstance(check[k], str) or not check[k] for k in ('backend', 'stdout')): + raise ValueError('box_exec backend/stdout must be non-empty strings') + if 'smoke' not in profile: + raise ValueError('box_exec checks require smoke mode') + if 'http_group' not in case and case.get('args', [])[:2] != ['pipeline', 'run']: + raise ValueError('box_exec requires a Pipeline or group case') + if 'attachment' in case: + check = case['attachment'] + if ( + 'smoke' not in profile + or case.get('args', [])[:2] != ['pipeline', 'run'] + or not isinstance(check, dict) + or set(check) != {'name', 'text'} + or any(not isinstance(value, str) or not value for value in check.values()) + ): + raise ValueError('attachment requires name/text on a smoke Pipeline case') + if sum(case.get('weight', 1) for case in cases) > 1000: + raise ValueError('total case weight must be <= 1000') + smoke = profile.get('smoke') + if 'smoke' in profile: + if not isinstance(smoke, dict) or smoke.keys() - { + 'ledger_db', + 'mock_records', + 'timeout_seconds', + 'group_turns', + 'groups_per_case', + 'required_plugins', + }: + raise ValueError('invalid smoke settings') + if not isinstance(smoke.get('ledger_db'), str) or not smoke['ledger_db']: + raise ValueError('smoke.ledger_db must name the target Core SQLite database') + if 'mock_records' in smoke and (not isinstance(smoke['mock_records'], str) or not smoke['mock_records']): + raise ValueError('smoke.mock_records must name the dedicated mock recording file') + resolve(smoke, variables) + number(smoke.get('timeout_seconds', 30), 'smoke timeout_seconds', minimum=0.001) + number(smoke.get('group_turns', 2), 'smoke group_turns', minimum=2, integer=True) + number(smoke.get('groups_per_case', 2), 'smoke groups_per_case', minimum=2, integer=True) + if smoke.get('group_turns', 2) > 10 or smoke.get('groups_per_case', 2) > 10: + raise ValueError('smoke group_turns/groups_per_case must be <= 10') + required = smoke.get('required_plugins', []) + if not isinstance(required, list) or any(not isinstance(p, str) or not p for p in required): + raise ValueError('smoke.required_plugins must be an array of plugin IDs') + if profile.get('phases') or profile.get('thresholds'): + raise ValueError('smoke uses fixed case counts; load phases/thresholds are not applicable') + phases = profile.get('phases', []) + if not isinstance(phases, list) or (smoke is None and not phases): + raise ValueError('phases must be a non-empty array') + names = set() + for phase in phases: + name = phase.get('name', '') + if not re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]*', name) or name in names: + raise ValueError('phase names must be unique simple names') + names.add(name) + if phase.keys() - { + 'name', + 'duration_seconds', + 'rps', + 'workers', + 'cooldown_seconds', + 'groups', + 'members_per_group', + }: + raise ValueError(f'{name}: unknown phase field') + number(phase.get('duration_seconds'), 'duration_seconds', minimum=0.001) + number(phase.get('rps'), 'rps', minimum=0.001) + number(phase.get('workers'), 'workers', minimum=1, integer=True) + number(phase.get('cooldown_seconds', 0), 'cooldown_seconds') + if phase['workers'] > 32: + raise ValueError('workers must be <= 32') + number(phase.get('groups', 0), 'groups', integer=True) + number(phase.get('members_per_group', 3), 'members_per_group', minimum=1, integer=True) + if phase.get('groups', 0) > 4096 or phase.get('members_per_group', 3) > 1000: + raise ValueError('groups must be <= 4096 and members_per_group <= 1000') + group_cases = ['http_group' in case for case in cases] + if any(group_cases) and ( + not all(group_cases) or phase.get('groups', 0) < sum(case.get('weight', 1) for case in cases) + ): + raise ValueError('group phases need only http_group cases and enough groups for all weighted cases') + if not any(group_cases) and phase.get('groups', 0): + raise ValueError('groups require http_group cases') + if profile.get('cleanup', 'on-success') not in {'always', 'on-success', 'never'}: + raise ValueError('cleanup must be always/on-success/never') + thresholds = profile.get('thresholds', {}) + if extra := thresholds.keys() - {'min_rps_ratio', 'max_p95_seconds'}: + raise ValueError(f'unknown thresholds: {sorted(extra)}') + number(thresholds.get('min_rps_ratio', 0), 'min_rps_ratio') + if thresholds.get('min_rps_ratio', 0) > 1: + raise ValueError('min_rps_ratio must be <= 1') + if not isinstance(thresholds.get('max_p95_seconds', {}), dict): + raise ValueError('max_p95_seconds must map case names to seconds') + for name, value in thresholds.get('max_p95_seconds', {}).items(): + if name not in {case['name'] for case in cases}: + raise ValueError(f'unknown case in max_p95_seconds: {name}') + number(value, 'max_p95_seconds', minimum=0.001) + observed = resolve(profile.get('observe', {}), variables) + if not isinstance(observed, dict) or observed.keys() - { + 'health', + 'pids', + 'pid_files', + 'containers', + 'box_data_root', + }: + raise ValueError('observe accepts health, pids, pid_files, containers, box_data_root') + if 'box_data_root' in observed and ( + not isinstance(observed['box_data_root'], str) or not observed['box_data_root'] + ): + raise ValueError('observe.box_data_root must name the deployment Box data directory') + if not isinstance(observed.get('health'), dict) or not observed['health']: + raise ValueError('observe.health must contain at least one health endpoint') + for name, url in observed['health'].items(): + if not isinstance(name, str) or not isinstance(url, str) or not url.startswith(('http://', 'https://')): + raise ValueError('health endpoints must map names to HTTP(S) URLs') + for field in ('pids', 'pid_files'): + if not isinstance(observed.get(field, {}), dict): + raise ValueError(f'observe.{field} must be an object') + for pid in observed.get('pids', {}).values(): + number(pid, 'PID', minimum=1, integer=True) + if not isinstance(observed.get('containers', []), list) or any( + not isinstance(item, str) or not item for item in observed.get('containers', []) + ): + raise ValueError('observe.containers must be an array of container names/IDs') + + +class Suite: + def __init__(self, profile, profile_path, output, env=None): + self.profile = profile + self.output = output.resolve() + self.variables = { + **profile.get('vars', {}), + 'profile_dir': str(profile_path.parent.resolve()), + 'run_dir': str(self.output), + } + self.cli = [ + profile.get('cli', {}).get('executable', 'lbctl'), + '--config', + self.path(profile['cli']['config']), + '-o', + 'json', + ] + self.owned = [] + self.connection = {} + self.env = dict(os.environ if env is None else env) + if any('http_group' in case for case in profile['cases']): + self.variables['group_secret'] = secrets.token_urlsafe(32) + self.env['LANGBOT_QA_GROUP_SECRET'] = self.variables['group_secret'] + self.secrets = [ + value for key, value in self.env.items() if ('API_KEY' in key or key == 'LANGBOT_QA_GROUP_SECRET') and value + ] + + def path(self, value): + path = Path(resolve(value, self.variables)).expanduser() + return str(path if path.is_absolute() else Path(self.variables['profile_dir']) / path) + + def write(self, name, value): + path = self.output / name + if name == 'request.json': + private = self.output / '.private' + private.mkdir(exist_ok=True, mode=0o700) + path = private / name + path.write_text(json.dumps(value, ensure_ascii=False, indent=2)) + path.chmod(0o600) + return str(path) + + def call(self, args, body=None, *, defer_bot_readback=False): + command = [*self.cli, *args] + if body is not None: + command += ['--file', self.write('request.json', body)] + try: + result = subprocess.run(command, capture_output=True, text=True, timeout=360, env=self.env) + except subprocess.TimeoutExpired as exc: + raise RuntimeError('lbctl timed out; outcome unknown, no automatic retry') from exc + if result.returncode: + if defer_bot_readback and args == ['bot', 'create']: + response = json.loads(result.stdout) + data = response.get('data', {}) + if ( + response.get('error', {}).get('type') == 'verification_failed' + and data.get('operation') == 'bot.create' + and data.get('uuid') + and data.get('bot', {}).get('uuid') == data['uuid'] + ): + return data + error = result.stdout + result.stderr + for secret in self.secrets: + error = error.replace(secret, '[REDACTED]') + raise RuntimeError(error[:3000]) + response = json.loads(result.stdout) + if args == ['whoami']: + self.connection = response.get('meta') or {} + return response['data'] + + def read_bot(self, bot_uuid): + """Read full Bot fields using the endpoint and environment credential confirmed by lbctl.""" + source = self.connection.get('credential_source', '') + key = self.env.get(source.removeprefix('env:')) if source.startswith('env:') else None + if not key or not self.connection.get('endpoint'): + raise RuntimeError('Group Bot readback requires environment-backed lbctl credentials') + self.secrets.append(key) + request = urllib.request.Request( + self.connection['endpoint'].rstrip('/') + '/api/v1/platform/bots/' + bot_uuid, + headers={'Authorization': 'Bearer ' + key, 'X-Workspace-Id': self.variables['workspace_uuid']}, + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + try: + with opener.open(request, timeout=30) as response: + payload = json.load(response) + except (urllib.error.URLError, OSError) as exc: + raise RuntimeError('Bot readback request failed; the creation was not retried') from exc + if ( + not isinstance(payload, dict) + or payload.get('code') != 0 + or not isinstance(payload.get('data'), dict) + or not isinstance(payload['data'].get('bot'), dict) + ): + raise RuntimeError('Bot readback did not return a successful Bot resource') + return payload['data']['bot'] + + def journal(self): + self.write('owned.json', {'workspace_uuid': self.variables['workspace_uuid'], 'resources': self.owned}) + + def prepare(self): + if 'smoke' in self.profile and not Path(self.path(self.profile['smoke']['ledger_db'])).is_file(): + raise ValueError('smoke.ledger_db must be an existing target Core SQLite database') + identity = self.call(['whoami']) + self.variables['workspace_uuid'] = identity['workspace_uuid'] + self.journal() + for plugin in self.profile.get('plugins', []): + self.call(['plugin', 'install', plugin['source'], '--file', self.path(plugin['file']), '--wait']) + if 'smoke' in self.profile: + capabilities = self.call(['capabilities']) + plugins = self.call(['plugin', 'list']) + ready = { + f'{item["author"]}/{item["name"]}' + for item in plugins['plugins'] + if item.get('enabled') and item.get('status') == 'initialized' + } + missing = set(self.profile['smoke'].get('required_plugins', [])) - ready + self.write('environment.json', {'identity': identity, 'capabilities': capabilities, 'plugins': plugins}) + if missing: + raise RuntimeError(f'Required plugins are not initialized: {sorted(missing)}') + for item in self.profile.get('resources', []): + kind = item['kind'] + body = resolve(item['body'], self.variables) + config = body.pop('config', None) if kind == 'pipeline' else None + args = [kind, 'apply' if kind == 'pipeline' else 'create'] + if kind == 'model': + args += ['--type', item['type']] + created = self.call(args, body, defer_bot_readback=True) if kind == 'bot' else self.call(args, body) + created = created.get('pipeline', created) if kind == 'pipeline' else created + uuid = created['uuid'] + self.variables[item['id']] = uuid + self.owned.append({'kind': kind, 'uuid': uuid, **({'type': item['type']} if kind == 'model' else {})}) + self.journal() # Record ownership before any later configuration or ingestion can fail. + if kind == 'bot': + # lbctl 0.3's Bot projection omits event bindings; verify the existing Core API instead. + actual = self.read_bot(uuid) + expected = copy.deepcopy(body) + expected['adapter_config'].pop('inbound_secret') + if not contains_fields(actual, expected): + raise RuntimeError(f'Bot readback differs from submitted non-secret fields: {uuid}') + if config is not None: + current = self.call(['pipeline', 'get', uuid]) + current = current.get('pipeline', current) + self.call( + ['pipeline', 'apply'], + { + 'uuid': uuid, + 'name': current['name'], + 'description': current['description'], + 'config': merge(current['config'], config), + }, + ) + for document in item.get('documents', []): + self.call( + [ + 'knowledge-base', + 'ingest', + uuid, + '--file', + self.path(document['file']), + '--parser-plugin-id', + document['parser_plugin_id'], + '--wait', + ] + ) + self.write( + 'resources.json', + { + key: value + for key, value in self.variables.items() + if key not in self.profile.get('vars', {}) and key != 'group_secret' + }, + ) + if 'smoke' in self.profile: + return + warmups = [] + for case in self.profile['cases']: + if 'http_group' in case: + case = resolve(case, self.variables) + group = f'qa-warmup-{uuid4().hex[:12]}' + response = group_request(case, group, 'qa-member-000', 1, uuid4().hex, 'none', self.env) + warmups.append({'case': case['name'], 'output': response}) + self.write('warmup.json', warmups) + + def cleanup(self): + saved = json.loads((self.output / 'owned.json').read_text()) + identity = self.call(['whoami']) + if saved['workspace_uuid'] != identity['workspace_uuid']: + raise RuntimeError('cleanup refused: Workspace differs from owned.json') + failures = [] + original = saved['resources'].copy() + for resource in reversed(original): + args = [resource['kind'], 'delete', resource['uuid'], '--yes'] + if resource['kind'] == 'model': + args += ['--type', resource['type']] + try: + self.call(args) + except (RuntimeError, ValueError, OSError) as exc: + failures.append(str(exc)) + else: + saved['resources'].remove(resource) + self.write('owned.json', saved) + if 'smoke' in self.profile: + from regression_smoke import verify_cleanup + + try: + verify_cleanup(self, [item for item in original if item not in saved['resources']]) + except (RuntimeError, ValueError, OSError, KeyError) as exc: + failures.append(f'cleanup readback failed: {exc}') + return failures + + def run(self): + if 'smoke' in self.profile: + from regression_smoke import run_smoke + + return run_smoke(self) + cases = [] + for original in self.profile['cases']: + case = resolve(original, self.variables) + if 'body' in case: + case['args'] += ['--file', self.write(f'case-{len(cases)}.json', case.pop('body'))] + cases.extend( + [{key: case[key] for key in ('name', 'args', 'expect', 'http_group') if key in case}] + * case.get('weight', 1) + ) + cases_path = self.write('cases.json', cases) + observed = resolve(self.profile.get('observe', {}), self.variables) + observe_args = [] + for name, pid in observed.get('pids', {}).items(): + observe_args += ['--pid', f'{name}={int(pid)}'] + for name, path in observed.get('pid_files', {}).items(): + observe_args += ['--pid', f'{name}={int(Path(self.path(path)).read_text().strip())}'] + for name, url in observed.get('health', {}).items(): + observe_args += ['--health', f'{name}={url}'] + for container in observed.get('containers', []): + observe_args += ['--container', container] + if observed.get('box_data_root'): + observe_args += ['--box-data-root', self.path(observed['box_data_root'])] + results = [] + errors = [] + for phase in self.profile['phases']: + destination = self.output / phase['name'] + command = [ + sys.executable, + str(Path(__file__).with_name('regression_traffic.py')), + '--lbctl', + self.cli[0], + '--config', + self.cli[2], + '--cases', + cases_path, + '--output', + str(destination), + '--duration', + str(phase['duration_seconds']), + '--rps', + str(phase['rps']), + '--workers', + str(phase['workers']), + '--cooldown', + str(phase.get('cooldown_seconds', 0)), + '--shared-case-sequence', + '--groups', + str(phase.get('groups', 0)), + '--members-per-group', + str(phase.get('members_per_group', 3)), + '--group-state', + str(self.output / 'group-state.json'), + *observe_args, + ] + with (self.output / f'{phase["name"]}.log').open('w') as log: + result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, env=self.env) + summary = json.loads((destination / 'summary.json').read_text()) + summary['name'] = phase['name'] + summary['achieved_rps'] = summary['requests'] / summary['load_elapsed_seconds'] + results.append(summary) + if result.returncode: + errors.append(f'{phase["name"]}: business or health/resource checks failed') + thresholds = self.profile.get('thresholds', {}) + if summary['achieved_rps'] < phase['rps'] * thresholds.get('min_rps_ratio', 0): + errors.append(f'{phase["name"]}: achieved RPS below configured threshold') + for name, maximum in thresholds.get('max_p95_seconds', {}).items(): + p95 = summary['cases'][name]['p95_seconds'] + if p95 is None or p95 > maximum: + errors.append(f'{phase["name"]}/{name}: P95 missing or above configured threshold') + return {'phases': results, 'errors': errors} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('action', choices=('validate', 'run', 'cleanup')) + parser.add_argument('--profile', required=True, type=Path) + parser.add_argument('--output', type=Path, help='New run directory; existing run directory for cleanup') + parser.add_argument('--duration', type=float, help='Override duration_seconds in every phase') + parser.add_argument('--rps', type=float, help='Override RPS in every phase') + parser.add_argument('--workers', type=int, help='Override workers in every phase') + parser.add_argument('--groups', type=int, help='Override group count in every group phase') + parser.add_argument('--members-per-group', type=int) + args = parser.parse_args() + try: + profile = json.loads(args.profile.read_text()) + if 'smoke' in profile and any( + getattr(args, flag) is not None for flag in ('duration', 'rps', 'workers', 'groups', 'members_per_group') + ): + raise ValueError('Load overrides are not applicable to smoke mode') + for phase in profile.get('phases', []): + for flag, field in [ + ('duration', 'duration_seconds'), + ('rps', 'rps'), + ('workers', 'workers'), + ('groups', 'groups'), + ('members_per_group', 'members_per_group'), + ]: + if getattr(args, flag) is not None: + phase[field] = getattr(args, flag) + validate(profile) + if args.action == 'validate': + print('Profile valid (no services or resources changed).') + return + if args.output is None: + raise ValueError('--output is required for run/cleanup') + suite = Suite(profile, args.profile.resolve(), args.output) + if args.action == 'cleanup': + errors = suite.cleanup() + if errors: + raise RuntimeError('\n'.join(errors)) + return + args.output.mkdir(parents=True, exist_ok=False, mode=0o700) + suite.write('profile.json', profile) + result = {'phases': [], 'errors': []} + completed = False + try: + suite.prepare() + result = suite.run() + completed = True + except KeyboardInterrupt: + result['errors'].append('Interrupted; in-flight outcomes may be unknown. Inspect retained resources.') + except (RuntimeError, ValueError, OSError, KeyError) as exc: + result['errors'].append(str(exc)) + finally: + if not completed and not result['errors']: + result['errors'].append('Run aborted before completion; inspect logs and retained resources.') + policy = profile.get('cleanup', 'on-success') + if policy == 'always' or (policy == 'on-success' and not result['errors']): + if (args.output / 'owned.json').exists(): + try: + result['errors'].extend(suite.cleanup()) + except (RuntimeError, ValueError, OSError, KeyError) as exc: + result['errors'].append(f'cleanup failed: {exc}') + result['passed'] = not result['errors'] + suite.write('summary.json', result) + print(json.dumps(result, ensure_ascii=False, indent=2)) + if not result['passed']: + raise SystemExit(1) + except (RuntimeError, ValueError, OSError, KeyError) as exc: + parser.exit(1, f'{type(exc).__name__}: {exc}\n') + + +if __name__ == '__main__': + main() diff --git a/tools/regression_traffic.py b/tools/regression_traffic.py new file mode 100644 index 000000000..c147190cf --- /dev/null +++ b/tools/regression_traffic.py @@ -0,0 +1,449 @@ +#!/usr/bin/env python3 +"""Drive independent CLI requests or persistent group conversations with Linux resource sampling.""" + +import argparse +from collections import Counter +from concurrent.futures import ThreadPoolExecutor +import json +import hashlib +import hmac +import math +import os +from pathlib import Path +import subprocess +import sys +import threading +import time +import urllib.error +import urllib.request +import uuid + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) +from cloud_runtime_soak import read_endpoint_snapshot, read_process_snapshot + + +def percentile(values, fraction): + return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] if values else None + + +def owned_box_containers(data_root, checked=None): + """Discover sandboxes mounted below one run's data root, including newly created groups.""" + if checked is None: + + def checked(command): + return subprocess.check_output(command, text=True, timeout=10) + + candidates = checked(['docker', 'ps', '-a', '--filter', 'label=langbot.box=true', '--format', '{{.ID}}']).split() + if not candidates: + return [] + root = Path(data_root).resolve() + descriptions = checked(['docker', 'inspect', '--format', '{{json .}}', *candidates]) + owned = [] + for line in descriptions.splitlines(): + if not line: + continue + entry = json.loads(line) + if any( + Path(mount['Source']).resolve().is_relative_to(root) for mount in entry['Mounts'] if mount.get('Source') + ): + owned.append(entry['Id']) + return owned + + +def group_request(case, group_id, member_id, turn, token, previous, env=None): + """Send one signed GroupMessage through the existing HTTP Bot adapter.""" + settings = case['http_group'] + secret = (os.environ if env is None else env)[settings['secret_env']] + marker = f'LB_GROUP={group_id} LB_TURN={turn} LB_TOKEN={token} LB_PREVIOUS={previous}' + body = json.dumps( + { + 'session_id': group_id, + 'session_type': 'group', + 'sender': {'id': member_id, 'name': member_id, 'group_name': group_id}, + 'message': [{'type': 'Plain', 'text': settings['message'] + ' ' + marker}], + } + ).encode() + stamp = str(int(time.time())) + signature = hmac.new(secret.encode(), stamp.encode() + b'.' + body, hashlib.sha256).hexdigest() + request = urllib.request.Request( + settings['url'], + data=body, + headers={ + 'Content-Type': 'application/json', + 'X-LB-Timestamp': stamp, + 'X-LB-Signature': 'sha256=' + signature, + 'X-LB-Idempotency-Key': token, + }, + ) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({})) + with opener.open(request, timeout=90) as response: + payload = json.load(response) + if response.status != 200 or not isinstance(payload, dict) or payload.get('code') != 0: + raise ValueError('group ingress did not return a completed response') + data = payload.get('data', {}) + if not isinstance(data, dict): + raise ValueError('group ingress response data must be an object') + parts = data.get('message') + if not isinstance(parts, list) or any(not isinstance(part, dict) for part in parts): + raise ValueError('group reply must contain message components') + text = ''.join(str(part.get('text', '')) for part in parts if part.get('type') == 'Plain') + if data.get('session_id') != group_id or not data.get('reply_to'): + raise ValueError('group reply session or message correlation is missing') + if case['expect'] not in text or f'GROUP_ACK={token}' not in text: + raise ValueError('group reply failed its business or conversation-history oracle') + return payload + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--lbctl', required=True) + parser.add_argument('--config', required=True) + parser.add_argument( + '--cases', type=Path, required=True, help='JSON cases with name, expect, and args or http_group' + ) + parser.add_argument('--output', type=Path, required=True) + parser.add_argument('--duration', type=float, default=600) + parser.add_argument('--rps', type=float, default=2) + parser.add_argument('--workers', type=int, default=4) + parser.add_argument( + '--groups', type=int, default=0, help='Stable group sessions; zero uses independent CLI requests' + ) + parser.add_argument('--members-per-group', type=int, default=3) + parser.add_argument( + '--group-state', type=Path, help='Carry group histories across load phases; refuses interrupted runs' + ) + parser.add_argument('--cooldown', type=float, default=30) + parser.add_argument('--pid', action='append', default=[], metavar='NAME=PID') + parser.add_argument('--health', action='append', default=[], metavar='NAME=URL') + parser.add_argument('--container', action='append', default=[]) + parser.add_argument('--box-data-root', type=Path, help='Dynamically sample Box containers mounted under this root') + parser.add_argument( + '--shared-case-sequence', action='store_true', help='Share the weighted case sequence across workers' + ) + args = parser.parse_args() + if ( + not all(math.isfinite(value) for value in (args.duration, args.rps, args.cooldown)) + or args.duration <= 0 + or args.rps <= 0 + or not 1 <= args.workers <= 32 + or args.cooldown < 0 + ): + parser.error('duration/rps must be positive, workers 1..32, cooldown nonnegative') + if not 0 <= args.groups <= 4096 or not 1 <= args.members_per_group <= 1000: + parser.error('groups must be 0..4096 (0 selects CLI mode), members-per-group 1..1000') + cases = json.loads(args.cases.read_text()) + if ( + not isinstance(cases, list) + or not cases + or any( + not isinstance(case, dict) + or not isinstance(case.get('name'), str) + or ( + not args.groups + and (not isinstance(case.get('args'), list) or not all(isinstance(item, str) for item in case['args'])) + ) + or not isinstance(case.get('expect'), str) + or not case['expect'] + for case in cases + ) + ): + parser.error('cases must contain name, string args, and expected response text') + if args.groups: + if args.groups < len(cases): + parser.error('groups must cover the complete weighted case sequence') + for case in cases: + settings = case.get('http_group', {}) + if ( + not isinstance(settings, dict) + or set(settings) != {'url', 'secret_env', 'message'} + or not all(isinstance(value, str) and value for value in settings.values()) + or not settings['url'].startswith(('http://', 'https://')) + or not os.environ.get(settings['secret_env']) + ): + parser.error('group cases need HTTP(S) url, message and a populated secret_env') + args.output.mkdir(parents=True, exist_ok=False) + pids = {name: int(pid) for name, pid in (value.split('=', 1) for value in args.pid)} + endpoints = dict(value.split('=', 1) for value in args.health) + rows = [] + sampling_errors = [] + lock = threading.Lock() + group_condition = threading.Condition(lock) + stop = threading.Event() + phase = ['load'] + next_case = 0 + started = time.monotonic() + deadline = started + args.duration + redactions = [value for name, value in os.environ.items() if 'API_KEY' in name and value] + group_state = None + groups = [] + if args.groups: + redactions += [os.environ[case['http_group']['secret_env']] for case in cases] + state_path = args.group_state or args.output / 'group-state.json' + fingerprint = [(case['name'], case['http_group']) for case in cases] + group_state = ( + json.loads(state_path.read_text()) + if state_path.exists() + else { + 'run_id': uuid.uuid4().hex[:12], + 'cases': fingerprint, + 'groups': [], + 'in_progress': False, + } + ) + if group_state.get('in_progress') or json.dumps(group_state['cases']) != json.dumps(fingerprint): + parser.error('group state is interrupted or belongs to different cases; inspect it before reuse') + for index in range(len(group_state['groups']), args.groups): + group_state['groups'].append( + { + 'id': f'qa-{group_state["run_id"]}-g{index:04d}', + 'case': index % len(cases), + 'turn': 1, + 'previous': 'none', + 'failed': False, + } + ) + groups = [ + dict(item, due=started + index / args.rps, busy=False) + for index, item in enumerate(group_state['groups'][: args.groups]) + ] + group_state['in_progress'] = True + state_path.write_text(json.dumps(group_state, indent=2)) + state_path.chmod(0o600) + + def redact(text): + for secret in redactions: + text = text.replace(secret, '[REDACTED]') + return text + + def sample(): + with (args.output / 'resources.jsonl').open('w') as stream: + while not stop.is_set(): + row = { + 'time': time.time(), + 'elapsed': time.monotonic() - started, + 'phase': phase[0], + 'targets': {}, + 'health': {}, + } + for name, pid in pids.items(): + try: + row['targets'][name] = read_process_snapshot(pid) + except (OSError, RuntimeError) as exc: + row['targets'][name] = {'error': str(exc)} + for name, url in endpoints.items(): + try: + row['health'][name] = read_endpoint_snapshot(url, timeout_seconds=3) + except RuntimeError as exc: + row['health'][name] = {'error': str(exc)} + containers = set(args.container) + if args.box_data_root: + containers.update(owned_box_containers(args.box_data_root)) + if containers: + result = subprocess.run( + ['docker', 'stats', '--no-stream', '--format', '{{json .}}', *sorted(containers)], + capture_output=True, + text=True, + timeout=10, + ) + row['containers'] = [json.loads(line) for line in result.stdout.splitlines() if line] + if result.returncode: + row['container_error'] = result.stderr.strip() + stream.write(json.dumps(row) + '\n') + stream.flush() + with lock: + progress = { + 'elapsed': row['elapsed'], + 'phase': phase[0], + 'requests': len(rows), + 'status': dict(Counter(item['status'] for item in rows)), + } + (args.output / 'progress.json').write_text(json.dumps(progress)) + stop.wait(3) + + def worker(index): + nonlocal next_case + iteration = 0 + due = started + index / args.rps + while due < deadline: + if stop.wait(max(0, due - time.monotonic())) or time.monotonic() >= deadline: + break + if args.shared_case_sequence: + with lock: + case = cases[next_case % len(cases)] + next_case += 1 + else: + case = cases[(iteration * args.workers + index) % len(cases)] + before = time.monotonic() + row = {'case': case['name'], 'time': time.time(), 'elapsed': before - started, 'status': 'failed'} + try: + result = subprocess.run( + [args.lbctl, '--config', args.config, '-o', 'json', *case['args']], + capture_output=True, + text=True, + timeout=90, + ) + row['exit_code'] = result.returncode + row['output'] = json.loads(redact(result.stdout)) + if result.returncode == 0 and case['expect'] in json.dumps(row['output'], ensure_ascii=False): + row['status'] = 'ok' + elif 'result_unknown' in result.stdout: + row['status'] = 'unknown' + else: + row['error'] = redact(result.stderr)[:2000] or 'exit code or response oracle failed' + except subprocess.TimeoutExpired: + row.update(status='unknown', error='CLI timed out; server outcome unknown; request was not retried') + except (OSError, ValueError) as exc: + row['error'] = str(exc) + row['seconds'] = time.monotonic() - before + with lock: + rows.append(row) + with (args.output / 'requests.jsonl').open('a') as stream: + stream.write(json.dumps(row, ensure_ascii=False) + '\n') + iteration += 1 + due = max(due + args.workers / args.rps, time.monotonic()) + + def group_worker(_index): + while not stop.is_set(): + with group_condition: + while True: + now = time.monotonic() + if stop.is_set() or now >= deadline: + return + available = [group for group in groups if not group['busy'] and not group['failed']] + if not available: + if not any(group['busy'] for group in groups): + return + group_condition.wait(0.1) + continue + group = min(available, key=lambda item: item['due']) + if group['due'] > now: + group_condition.wait(min(1, group['due'] - now)) + continue + group['busy'] = True + break + case = cases[group['case']] + before = time.monotonic() + token = uuid.uuid4().hex + member = f'qa-member-{(group["turn"] - 1) % args.members_per_group:03d}' + row = { + 'case': case['name'], + 'group_id': group['id'], + 'member_id': member, + 'turn': group['turn'], + 'time': time.time(), + 'elapsed': before - started, + 'status': 'failed', + } + try: + row['output'] = group_request(case, group['id'], member, group['turn'], token, group['previous']) + row['status'] = 'ok' + except urllib.error.HTTPError as exc: + row.update(http_status=exc.code, error=redact(exc.read(2000).decode(errors='replace'))) + if exc.code >= 500: + row['status'] = 'unknown' + except (urllib.error.URLError, TimeoutError, OSError) as exc: + row.update(status='unknown', error=redact(str(exc)) + '; not retried') + except (ValueError, KeyError) as exc: + row['error'] = redact(str(exc)) + row['seconds'] = time.monotonic() - before + with group_condition: + rows.append(row) + with (args.output / 'requests.jsonl').open('a') as stream: + stream.write(json.dumps(row, ensure_ascii=False) + '\n') + group.update(busy=False, failed=row['status'] != 'ok') + if row['status'] == 'ok': + group.update(turn=group['turn'] + 1, previous=token) + group['due'] = max(group['due'] + args.groups / args.rps, time.monotonic()) + group_condition.notify_all() + + def sampling_guard(): + try: + sample() + except Exception as exc: + sampling_errors.append(f'{type(exc).__name__}: {exc}') + stop.set() + + sampler = threading.Thread(target=sampling_guard) + sampler.start() + try: + with ThreadPoolExecutor(max_workers=args.workers) as pool: + list(pool.map(group_worker if args.groups else worker, range(args.workers))) + stop.wait(max(0, deadline - time.monotonic())) + load_elapsed = time.monotonic() - started + phase[0] = 'cooldown' + stop.wait(args.cooldown) + finally: + stop.set() + sampler.join() + summary = { + 'duration_requested_seconds': args.duration, + 'load_elapsed_seconds': load_elapsed, + 'target_rps': args.rps, + 'workers': args.workers, + 'cooldown_seconds': args.cooldown, + 'requests': len(rows), + 'status': dict(Counter(row['status'] for row in rows)), + 'cases': {}, + } + samples = [json.loads(line) for line in (args.output / 'resources.jsonl').read_text().splitlines()] + summary['health_samples'] = len(samples) + summary['health_failures'] = sum( + any('error' in target for target in [*sample['targets'].values(), *sample['health'].values()]) + or bool(sample.get('container_error')) + for sample in samples + ) + summary['sampling_errors'] = sampling_errors + if args.groups: + for index, group in enumerate(groups): + group_state['groups'][index] = {key: value for key, value in group.items() if key not in {'due', 'busy'}} + group_state['in_progress'] = False + temporary = state_path.with_suffix('.tmp') + temporary.write_text(json.dumps(group_state, indent=2)) + temporary.chmod(0o600) + temporary.replace(state_path) + summary['groups'] = { + 'configured': args.groups, + 'members_per_group': args.members_per_group, + 'target_rps_per_group': args.rps / args.groups, + 'sessions': { + group['id']: { + 'case': cases[group['case']]['name'], + 'successful_turns_total': group['turn'] - 1, + 'failed': group['failed'], + 'requests_this_phase': sum(row.get('group_id') == group['id'] for row in rows), + 'members_this_phase': sorted( + {row['member_id'] for row in rows if row.get('group_id') == group['id']} + ), + } + for group in groups + }, + } + summary['group_failures'] = [ + group_id + for group_id, result in summary['groups']['sessions'].items() + if result['failed'] or result['successful_turns_total'] < 2 or not result['requests_this_phase'] + ] + for name in dict.fromkeys(case['name'] for case in cases): + selected = [row for row in rows if row['case'] == name] + times = [row['seconds'] for row in selected if row['status'] == 'ok'] + summary['cases'][name] = { + 'requests': len(selected), + 'status': dict(Counter(row['status'] for row in selected)), + 'p50_seconds': percentile(times, 0.5), + 'p95_seconds': percentile(times, 0.95), + 'p99_seconds': percentile(times, 0.99), + } + (args.output / 'summary.json').write_text(json.dumps(summary, indent=2)) + print(json.dumps(summary, indent=2), flush=True) + if ( + not rows + or any(row['status'] != 'ok' for row in rows) + or summary['health_failures'] + or sampling_errors + or summary.get('group_failures') + ): + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/tools/test_mock_llm.py b/tools/test_mock_llm.py new file mode 100644 index 000000000..8c8d74577 --- /dev/null +++ b/tools/test_mock_llm.py @@ -0,0 +1,209 @@ +"""Exercise the real HTTP server without LangBot, credentials or dependencies.""" + +from concurrent.futures import ThreadPoolExecutor +import base64 +import json +from pathlib import Path +import tempfile +import struct +import threading +import unittest +from urllib.error import HTTPError +from urllib.request import Request, urlopen + +from mock_llm import MockServer + + +class MockLLMTest(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmp = tempfile.TemporaryDirectory() + cls.record = Path(cls.tmp.name) / 'requests.jsonl' + scenarios = json.loads(Path(__file__).with_name('mock_llm_scenarios.json').read_text()) + cls.server = MockServer(('127.0.0.1', 0), scenarios, cls.record) + cls.thread = threading.Thread(target=cls.server.serve_forever) + cls.thread.start() + cls.url = f'http://127.0.0.1:{cls.server.server_port}' + + @classmethod + def tearDownClass(cls): + cls.server.shutdown() + cls.server.server_close() + cls.thread.join() + cls.tmp.cleanup() + + def post(self, model='mock-chat', **kwargs): + body = { + 'model': model, + 'messages': [{'role': 'user', 'content': 'test'}], + **kwargs, + } + request = Request( + self.url + '/v1/chat/completions', + data=json.dumps(body).encode(), + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'Bearer must-not-be-recorded', + }, + ) + with urlopen(request, timeout=5) as response: + return response.read().decode() + + def test_chat_and_model_discovery(self): + with urlopen(self.url + '/v1/models') as response: + models = json.load(response) + self.assertIn('mock-box', [m['id'] for m in models['data']]) + self.assertEqual( + json.loads(self.post())['choices'][0]['message']['content'], + 'LANGBOT_CHAT_OK', + ) + + def test_stream_and_usage(self): + events = self.post(stream=True, stream_options={'include_usage': True}).splitlines() + self.assertEqual(events[-2], 'data: [DONE]') + chunks = [json.loads(line[6:]) for line in events if line.startswith('data: {')] + content = ''.join(c['choices'][0]['delta'].get('content', '') for c in chunks if c['choices']) + self.assertEqual(content, 'LANGBOT_CHAT_OK') + self.assertEqual(chunks[-1]['usage']['total_tokens'], 15) + + def test_group_history_requires_previous_user_reply_and_isolation(self): + first = {'role': 'user', 'content': 'LB_GROUP=group-a LB_TURN=1 LB_TOKEN=aaa LB_PREVIOUS=none'} + second = {'role': 'user', 'content': 'LB_GROUP=group-a LB_TURN=2 LB_TOKEN=bbb LB_PREVIOUS=aaa'} + reply = json.loads(self.post(messages=[first]))['choices'][0]['message'] + self.assertIn('GROUP_ACK=aaa', reply['content']) + result = json.loads(self.post(messages=[first, reply, second]))['choices'][0]['message'] + self.assertIn('GROUP_ACK=bbb', result['content']) + other = {'role': 'user', 'content': 'LB_GROUP=group-b LB_TURN=1 LB_TOKEN=ccc LB_PREVIOUS=none'} + for messages in ([second], [first, second], [other, reply, second]): + with self.subTest(messages=messages), self.assertRaises(HTTPError) as failure: + self.post(messages=messages) + self.assertEqual(failure.exception.code, 400) + + def test_embedding_http_float_base64_and_validation(self): + def embed(**patch): + body = {'model': 'mock-embedding', 'input': ['repeatable fixture', 'another fixture'], **patch} + request = Request( + self.url + '/v1/embeddings', + data=json.dumps(body).encode(), + headers={'Content-Type': 'application/json'}, + ) + with urlopen(request, timeout=5) as response: + return json.load(response) + + first = embed()['data'] + self.assertEqual(first, embed()['data']) + self.assertEqual([item['index'] for item in first], [0, 1]) + self.assertEqual(len(first[0]['embedding']), 384) + self.assertAlmostEqual(sum(value * value for value in first[0]['embedding']), 1) + encoded = embed(encoding_format='base64')['data'][0]['embedding'] + decoded = struct.unpack('<384f', base64.b64decode(encoded)) + for actual, expected in zip(decoded, first[0]['embedding']): + self.assertAlmostEqual(actual, expected, places=6) + for patch in ({'input': []}, {'dimensions': 0}, {'encoding_format': 'bad'}, {'model': 'unknown'}): + with self.subTest(patch=patch), self.assertRaises(HTTPError) as caught: + embed(**patch) + self.assertEqual(caught.exception.code, 400) + + def box_turn(self, streaming=False): + messages = [{'role': 'user', 'content': 'run box checks'}] + tools = [ + { + 'type': 'function', + 'function': {'name': 'exec', 'parameters': {'type': 'object'}}, + } + ] + commands = [] + for _ in range(2): + wire = self.post('mock-box', messages=messages, tools=tools, stream=streaming) + if streaming: + deltas = [ + json.loads(line[6:])['choices'][0]['delta'] + for line in wire.splitlines() + if line.startswith('data: {') + ] + call = next(d['tool_calls'][0] for d in deltas if d.get('tool_calls') and 'id' in d['tool_calls'][0]) + arguments = ''.join(d['tool_calls'][0]['function']['arguments'] for d in deltas if d.get('tool_calls')) + call = { + **call, + 'function': {**call['function'], 'arguments': arguments}, + } + call.pop('index') + message = {'role': 'assistant', 'content': None, 'tool_calls': [call]} + else: + message = json.loads(wire)['choices'][0]['message'] + call = message['tool_calls'][0] + commands.append(json.loads(call['function']['arguments'])['command']) + messages.extend( + [ + message, + { + 'role': 'tool', + 'tool_call_id': call['id'], + 'content': 'LANGBOT_BOX_SENTINEL', + }, + ] + ) + final = json.loads(self.post('mock-box', messages=messages, tools=tools)) + self.assertEqual(final['choices'][0]['message']['content'], 'LANGBOT_BOX_OK') + self.assertIn('printf LANGBOT_BOX_SENTINEL', commands[0]) + self.assertEqual(commands[1], 'cat /workspace/mock-llm.txt') + + def test_tool_rounds_and_fragmented_stream(self): + self.box_turn() + self.box_turn(streaming=True) + + def test_concurrent_turns_do_not_share_progress(self): + with ThreadPoolExecutor(max_workers=4) as pool: + list(pool.map(lambda _: self.box_turn(), range(8))) + + def test_rag_requires_expected_model_input(self): + with self.assertRaises(HTTPError) as caught: + self.post('mock-rag') + self.assertEqual(caught.exception.code, 400) + response = json.loads( + self.post( + 'mock-rag', + messages=[ + {'role': 'system', 'content': 'Retrieved document: aurora-parser-rag-9137'}, + {'role': 'user', 'content': 'What is the documented sentinel?'}, + ], + ) + ) + self.assertEqual(response['choices'][0]['message']['content'], 'LANGBOT_RAG_OK') + + def test_errors_and_invalid_requests(self): + for status in (401, 429, 503): + with self.subTest(status=status), self.assertRaises(HTTPError) as caught: + self.post(f'mock-{status}') + self.assertEqual(caught.exception.code, status) + for kwargs in ( + {'model': 'unknown'}, + {'messages': []}, + {'stream_options': []}, + {'model': 'mock-box', 'tools': []}, + ): + with self.subTest(kwargs=kwargs), self.assertRaises(HTTPError) as caught: + self.post(**kwargs) + self.assertEqual(caught.exception.code, 400) + messages = [ + {'role': 'user', 'content': 'test'}, + {'role': 'assistant', 'tool_calls': [{'id': 'call1'}]}, + ] + with self.assertRaises(HTTPError) as caught: + self.post('mock-box', messages=messages) + self.assertEqual(caught.exception.code, 400) + messages.append({'role': 'tool', 'tool_call_id': 'call1', 'content': 'wrong output'}) + with self.assertRaises(HTTPError) as caught: + self.post('mock-box', messages=messages) + self.assertEqual(caught.exception.code, 400) + + def test_record_omits_authorization(self): + self.post() + with self.server.record_lock: + recorded = self.record.read_text() + self.assertNotIn('must-not-be-recorded', recorded) + self.assertIn('"request_id"', recorded) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_regression_gate.py b/tools/test_regression_gate.py new file mode 100644 index 000000000..360796f8d --- /dev/null +++ b/tools/test_regression_gate.py @@ -0,0 +1,66 @@ +"""A gate must fail closed on timeout, missing verdict, and inner failure.""" + +import os +import json +from pathlib import Path +import sys +import tempfile +import unittest +from unittest.mock import patch + +from regression_gate import supervise + + +class GateTest(unittest.TestCase): + def test_success_records_elapsed_time_and_resources(self): + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'gate' + code = ( + 'from pathlib import Path; import json; ' + f'p=Path({str(output / "suite")!r}); p.mkdir(); ' + '(p/"summary.json").write_text(json.dumps({"passed":True,"requests":1}))' + ) + result = supervise([sys.executable, '-c', code], output, 5, {'test': os.getpid()}) + self.assertTrue(result['passed']) + self.assertLess(result['elapsed_seconds'], 5) + self.assertGreater(result['resource_peaks']['test']['rss_bytes'], 0) + + def test_timeout_is_failed_without_retry(self): + with tempfile.TemporaryDirectory() as directory: + result = supervise([sys.executable, '-c', 'import time; time.sleep(30)'], Path(directory) / 'gate', 0.1, {}) + self.assertFalse(result['passed']) + self.assertTrue(result['timed_out']) + self.assertLess(result['elapsed_seconds'], 2) + + def test_teardown_failure_overrides_successful_smoke_verdict(self): + from regression_ci import main + + with tempfile.TemporaryDirectory() as directory: + output = Path(directory) / 'ci' + + def fail_teardown(args): + verdict = args.output / 'gate' / 'gate.json' + verdict.parent.mkdir(parents=True) + verdict.write_text(json.dumps({'passed': True})) + raise RuntimeError('sandbox teardown failed') + + with ( + patch('regression_ci.run', side_effect=fail_teardown), + patch.object( + sys, 'argv', ['regression_ci', '--output', str(output), '--lbctl', 'unused', '--plugins', 'unused'] + ), + self.assertRaises(RuntimeError), + ): + main() + result = json.loads((output / 'gate/gate.json').read_text()) + self.assertFalse(result['passed']) + self.assertIn('sandbox teardown failed', result['infrastructure_error']) + + def test_zero_exit_without_business_verdict_is_failed(self): + with tempfile.TemporaryDirectory() as directory: + result = supervise([sys.executable, '-c', 'pass'], Path(directory) / 'gate', 5, {}) + self.assertFalse(result['passed']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_regression_groups.py b/tools/test_regression_groups.py new file mode 100644 index 000000000..3d7bd5d0c --- /dev/null +++ b/tools/test_regression_groups.py @@ -0,0 +1,240 @@ +"""Exercise signed group traffic, member rotation, phase continuity and fail-closed history.""" + +import hashlib +import hmac +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import re +import subprocess +import sys +import tempfile +import threading +import time +import unittest + +from regression_suite import Suite, validate + + +class GroupTests(unittest.TestCase): + def test_bot_readback_uses_the_cli_confirmed_endpoint_and_workspace(self): + received = [] + + class Readback(BaseHTTPRequestHandler): + def do_GET(self): + received.append((self.path, self.headers.get('Authorization'), self.headers.get('X-Workspace-Id'))) + body = json.dumps({'code': 0, 'data': {'bot': {'uuid': 'owned-bot', 'event_bindings': []}}}).encode() + self.send_response(200) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *_args): + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Readback) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + with tempfile.TemporaryDirectory() as folder: + current = json.loads((Path(__file__).with_name('regression_profiles') / 'groups.json').read_text()) + suite = Suite(current, Path(folder) / 'profile.json', Path(folder), env={'TEST_API_KEY': 'test-key'}) + suite.variables['workspace_uuid'] = 'bound-workspace' + suite.connection = { + 'endpoint': f'http://127.0.0.1:{server.server_port}', + 'credential_source': 'env:TEST_API_KEY', + } + self.assertEqual(suite.read_bot('owned-bot')['uuid'], 'owned-bot') + self.assertEqual(received, [('/api/v1/platform/bots/owned-bot', 'Bearer test-key', 'bound-workspace')]) + finally: + server.shutdown() + server.server_close() + thread.join() + + def test_groups_profile_and_invalid_modes(self): + path = Path(__file__).with_name('regression_profiles') / 'groups.json' + profile = json.loads(path.read_text()) + validate(profile) + for field, value in [('groups', 3), ('members_per_group', 0), ('groups', 4097)]: + current = json.loads(path.read_text()) + current['phases'][0][field] = value + with self.subTest(field=field), self.assertRaises(ValueError): + validate(current) + + def test_signed_groups_continue_across_phases_and_stop_after_bad_reply(self): + secret = 'test-group-signing-key' + histories = {} + failures = [] + active = set() + lock = threading.Lock() + bad_reply = False + + class Ingress(BaseHTTPRequestHandler): + def do_GET(self): + self.respond({'code': 0}) + + def respond(self, payload, status=200): + body = json.dumps(payload).encode() + self.send_response(status) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + def do_POST(self): + raw = self.rfile.read(int(self.headers['Content-Length'])) + stamp = self.headers['X-LB-Timestamp'] + expected = ( + 'sha256=' + hmac.new(secret.encode(), stamp.encode() + b'.' + raw, hashlib.sha256).hexdigest() + ) + payload = json.loads(raw) + group = payload['session_id'] + fields = dict(re.findall(r'LB_(GROUP|TURN|TOKEN|PREVIOUS)=(\S+)', payload['message'][0]['text'])) + with lock: + prior = histories.get(group, []) + if ( + self.headers['X-LB-Signature'] != expected + or group in active + or payload['session_type'] != 'group' + or fields['GROUP'] != group + or int(fields['TURN']) != len(prior) + 1 + or fields['PREVIOUS'] != (prior[-1]['token'] if prior else 'none') + ): + failures.append(payload) + active.add(group) + time.sleep(0.01) + with lock: + histories.setdefault(group, []).append( + {'token': fields['TOKEN'], 'member': payload['sender']['id']} + ) + active.remove(group) + ack = 'missing' if bad_reply and group.endswith('g0000') else fields['TOKEN'] + self.respond( + { + 'code': 0, + 'data': { + 'session_id': group, + 'reply_to': 'incoming-' + fields['TOKEN'], + 'message': [ + {'type': 'Plain', 'text': 'TEST_OK GROUP_'}, + {'type': 'Plain', 'text': 'ACK=' + ack}, + ], + }, + }, + status=500 if bad_reply == 'unknown' and group.endswith('g0000') else 200, + ) + + def log_message(self, *_args): + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Ingress) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + cases = root / 'cases.json' + cases.write_text( + json.dumps( + [ + { + 'name': 'group', + 'expect': 'TEST_OK', + 'http_group': { + 'url': f'http://127.0.0.1:{server.server_port}/bots/test/sync', + 'secret_env': 'LANGBOT_QA_GROUP_SECRET', + 'message': 'ai Test group history', + }, + } + ] + ) + ) + state = root / 'group-state.json' + command = [ + sys.executable, + str(Path(__file__).with_name('regression_traffic.py')), + '--lbctl', + 'unused', + '--config', + 'unused', + '--cases', + str(cases), + '--duration', + '0.8', + '--rps', + '40', + '--workers', + '3', + '--cooldown', + '0', + '--members-per-group', + '2', + '--health', + f'core=http://127.0.0.1:{server.server_port}/healthz', + ] + env = {**os.environ, 'LANGBOT_QA_GROUP_SECRET': secret} + summaries = [] + for groups in (4, 8): + output = root / f'phase-{groups}' + result = subprocess.run( + [*command, '--groups', str(groups), '--group-state', str(state), '--output', str(output)], + env=env, + capture_output=True, + text=True, + timeout=15, + ) + self.assertEqual(result.returncode, 0, result.stdout + result.stderr) + summaries.append(json.loads((output / 'summary.json').read_text())) + first = summaries[0]['groups']['sessions'] + second = summaries[1]['groups']['sessions'] + self.assertEqual(len(first), 4) + self.assertEqual(len(second), 8) + for group in first: + self.assertGreater(second[group]['successful_turns_total'], first[group]['successful_turns_total']) + self.assertEqual(len(second[group]['members_this_phase']), 2) + self.assertFalse(failures) + self.assertNotIn(secret, state.read_text()) + bad_reply = True + output = root / 'failed' + result = subprocess.run( + [*command, '--groups', '4', '--output', str(output)], + env=env, + capture_output=True, + text=True, + timeout=15, + ) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + summary = json.loads((output / 'summary.json').read_text()) + failed = next(item for group, item in summary['groups']['sessions'].items() if group.endswith('g0000')) + self.assertTrue(failed['failed']) + self.assertEqual(failed['requests_this_phase'], 1) + bad_reply = 'unknown' + result = subprocess.run( + [*command, '--groups', '4', '--output', str(root / 'unknown')], + env=env, + capture_output=True, + text=True, + timeout=15, + ) + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + unknown = json.loads((root / 'unknown/summary.json').read_text()) + self.assertEqual(unknown['status'].get('unknown'), 1) + interrupted = json.loads(state.read_text()) + interrupted['in_progress'] = True + state.write_text(json.dumps(interrupted)) + result = subprocess.run( + [*command, '--groups', '4', '--group-state', str(state), '--output', str(root / 'interrupted')], + env=env, + capture_output=True, + text=True, + timeout=15, + ) + self.assertNotEqual(result.returncode, 0) + self.assertIn('interrupted', result.stderr) + finally: + server.shutdown() + server.server_close() + thread.join() + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_regression_job.py b/tools/test_regression_job.py new file mode 100644 index 000000000..774eb5964 --- /dev/null +++ b/tools/test_regression_job.py @@ -0,0 +1,195 @@ +"""Exercise Job ownership and lifecycle with controlled CLI/Docker responses.""" + +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import json +import os +from pathlib import Path +import shutil +import sys +import tempfile +import threading +import unittest +from unittest.mock import patch + +import yaml + +from regression_job import Job, main +from test_regression_suite import FAKE_CLI, profile + + +class ControlledJob(Job): + def __init__(self, manifest, suite, path, output, health, fail_install=False): + super().__init__(manifest, suite, path, output) + self.health = health + self.fail_install = fail_install + self.calls = [] + self.env['LBCTL_TEST_ROOT'] = str(output.parent) + + def command(self, args, timeout=900): + self.calls.append(args) + if args[: len(self.cli)] == self.cli: + operation = args[len(self.cli) :] + if operation[0] == 'install': + self.deploy.mkdir() + (self.deploy / 'data').mkdir() + (self.deploy / 'compose.yaml').write_text('services: {}') + (self.deploy / '.env').write_text( + 'LANGBOT_PLUGIN_RUNTIME_CONTROL_TOKEN=' + 't' * 32 + '\nLANGBOT_BOX_ENABLED=true\n' + ) + record = { + 'deployment_id': 'owned-deployment', + 'compose_project': 'test-project', + 'compose_file': str(self.deploy / 'compose.yaml'), + 'data_dir': str(self.deploy / 'data'), + 'env_file': str(self.deploy / '.env'), + 'endpoint': self.health, + } + (self.deploy / 'deployment.yaml').write_text(yaml.safe_dump(record)) + # Simulate a timeout/failed final probe with no returned record. + return ( + (1, json.dumps({'data': {}}), 'install failed') + if self.fail_install + else (0, json.dumps({'data': {'record': record}}), '') + ) + if operation[0] == '--endpoint': + return 0, json.dumps({'data': {'workspace_uuid': 'test-workspace'}}), '' + if operation[0] == 'uninstall': + shutil.rmtree(self.deploy / 'data') + return 0, json.dumps({'data': {}}), '' + if args[:2] == ['docker', 'ps']: + if 'label=langbot.box=true' in args: + return 0, '', '' + return ( + 0, + '\n'.join( + f'id-{name} {name}' + for name in ('langbot', 'langbot_plugin_runtime', 'langbot_box', 'regression_mock') + ), + '', + ) + if args[:2] == ['docker', 'inspect']: + if '{{.State.Pid}}' in args: + return 0, str(os.getpid()), '' + return 0, json.dumps({'network': {'IPAddress': '127.0.0.1'}}), '' + if args[:2] == ['docker', 'compose']: + if 'logs' in args: + return 0, self.redact(self.env.get('LANGBOT_QA_API_KEY', '') + '\n' + 't' * 32), '' + return 0, '', '' + raise AssertionError(args) + + @staticmethod + def ready(url): + pass + + def prepare(self): + suite = super().prepare() + # Real traffic driver and HTTP health probe; infrastructure calls are controlled above. + suite.profile['observe'] = {'health': {'core': self.health}} + return suite + + +class JobTests(unittest.TestCase): + def test_lifecycle_success_and_partial_install_recovery(self): + class Health(BaseHTTPRequestHandler): + def do_GET(self): + self.send_response(200) + self.end_headers() + self.wfile.write(b'{"code":0}') + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Health) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + cli = root / 'lbctl-test' + cli.write_text(f'#!{sys.executable}\n' + FAKE_CLI) + cli.chmod(0o700) + manifest = {'lbctl': str(cli), 'deployment': {'version': 'v4.11.0', 'port': 15351}} + suite = profile() + suite['resources'] = [{'id': 'chat', 'kind': 'pipeline', 'body': {'name': 'QA', 'config': {'ai': {}}}}] + suite['cases'][0].update(args=['pipeline', 'run', '${chat}'], expect='TEST_OK') + suite['phases'][0].update(duration_seconds=0.5, rps=4, workers=2) + health = f'http://127.0.0.1:{server.server_port}' + job = ControlledJob(manifest, suite, root / 'profile.json', root / 'success', health) + result = job.run() + self.assertTrue(result['passed'], result) + self.assertEqual(result['phase'], 'cleaned') + self.assertFalse((job.deploy / 'data').exists()) + self.assertFalse((job.private / 'api-key').exists()) + self.assertIn('--purge-data', job.calls[-1]) + self.assertNotIn('t' * 32, (job.output / 'services.log').read_text()) + + job = ControlledJob(manifest, suite, root / 'profile.json', root / 'failed', health, fail_install=True) + result = job.run() + self.assertFalse(result['passed']) + self.assertEqual(job.state['record']['deployment_id'], 'owned-deployment') + self.assertTrue((job.deploy / 'data').exists()) + self.assertFalse(any('uninstall' in call for call in job.calls)) + with ( + patch.object(sys, 'argv', ['regression_job.py', 'cleanup', '--output', str(job.output)]), + patch('regression_job.Job', return_value=job), + ): + main() + self.assertFalse((job.deploy / 'data').exists()) + self.assertEqual(job.state['phase'], 'cleaned') + finally: + server.shutdown() + server.server_close() + thread.join() + + def test_cleanup_refuses_changed_deployment(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + job = Job({}, profile(), root / 'profile.json', root) + job.private.mkdir() + job.deploy.mkdir() + (job.private / 'job-id').write_text(job.state['job_id']) + record = { + 'deployment_id': 'old', + 'compose_file': str(job.deploy / 'compose.yaml'), + 'data_dir': str(job.deploy / 'data'), + 'env_file': str(job.deploy / '.env'), + } + job.state['record'] = record + (job.deploy / 'deployment.yaml').write_text(yaml.safe_dump({**record, 'deployment_id': 'new'})) + with self.assertRaisesRegex(RuntimeError, 'identity changed'): + job.cleanup() + + def test_sandbox_cleanup_uses_owned_mount_roots(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + job = Job({}, profile(), root / 'profile.json', root) + job.state['record'] = {'data_dir': str(job.deploy / 'data')} + containers = ['ours', 'other'] + + def checked(args, timeout=900): + if args[1] == 'ps': + return '\n'.join(containers) + return '\n'.join( + json.dumps( + { + 'Id': container, + 'Mounts': [ + { + 'Source': str(job.deploy / 'data/box/tenant') + if container != 'other' + else '/unrelated/box/tenant' + } + ], + } + ) + for container in args[4:] + ) + + job.checked = checked + self.assertEqual(job.owned_sandboxes(), ['ours']) + containers.append('later') + self.assertEqual(job.owned_sandboxes(), ['ours', 'later']) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_regression_smoke.py b/tools/test_regression_smoke.py new file mode 100644 index 000000000..b87ae26ae --- /dev/null +++ b/tools/test_regression_smoke.py @@ -0,0 +1,205 @@ +"""Challenge the smoke verdict independently of a successful model reply.""" + +import base64 +import copy +import json +from pathlib import Path +import sqlite3 +import tempfile +from types import SimpleNamespace +import unittest + +from regression_smoke import ( + UnresolvedRun, + check_attachment, + check_fields, + check_mock_requests, + check_run, + completed_run, + verify_cleanup, +) +from regression_suite import validate + + +class SmokeOracleTest(unittest.TestCase): + def setUp(self): + self.db = sqlite3.connect(':memory:') + self.db.row_factory = sqlite3.Row + self.addCleanup(self.db.close) + self.db.executescript(""" + CREATE TABLE agent_run (id INTEGER, run_id TEXT, workspace_id TEXT, + conversation_id TEXT, status TEXT, status_reason TEXT, finished_at TEXT); + CREATE TABLE agent_run_event (run_id TEXT, sequence INTEGER, type TEXT, data_json TEXT); + INSERT INTO agent_run VALUES (1, 'run', 'workspace', 'conversation', 'completed', '', 'now'); + INSERT INTO agent_run_event VALUES ('run', 1, 'run.completed', '{}'); + """) + + def run_row(self, workspace='workspace', after=0): + return completed_run(self.db, workspace, 'conversation', after, 0) + + def test_terminal_status_and_event_are_both_required(self): + self.assertTrue(check_run(self.db, self.run_row(), {})['terminal_event']) + self.db.execute("UPDATE agent_run SET status='failed'") + with self.assertRaises(ValueError): + check_run(self.db, self.run_row(), {}) + self.db.execute("UPDATE agent_run SET status='completed'") + self.db.execute('DELETE FROM agent_run_event') + with self.assertRaises(ValueError): + check_run(self.db, self.run_row(), {}) + + def test_missing_stale_wrong_workspace_and_ambiguous_runs_are_unknown(self): + for workspace, after in [('other', 0), ('workspace', 1)]: + with self.subTest(workspace=workspace, after=after), self.assertRaises(UnresolvedRun): + self.run_row(workspace, after) + self.db.execute("UPDATE agent_run SET status='running', finished_at=NULL") + with self.assertRaises(UnresolvedRun): + self.run_row() + self.db.execute('INSERT INTO agent_run SELECT * FROM agent_run') + with self.assertRaises(UnresolvedRun): + self.run_row() + + def test_box_requires_actual_backend_success_and_output(self): + case = {'box_exec': {'count': 1, 'backend': 'docker', 'stdout': 'sentinel'}} + good = {'backend': 'docker', 'exit_code': 0, 'stdout': 'sentinel'} + for result, passes in [ + (good, True), + ({**good, 'backend': 'host'}, False), + ({**good, 'exit_code': 1}, False), + ({**good, 'stdout': ''}, False), + (None, False), + ]: + self.db.execute('DELETE FROM agent_run_event WHERE sequence=2') + self.db.execute( + 'INSERT INTO agent_run_event VALUES (?, ?, ?, ?)', + ('run', 2, 'tool.call.completed', json.dumps({'tool_name': 'exec', 'result': result})), + ) + with self.subTest(result=result): + if passes: + check_run(self.db, self.run_row(), case) + else: + with self.assertRaises(ValueError): + check_run(self.db, self.run_row(), case) + with self.assertRaises(ValueError): + check_run(self.db, self.run_row(), {'box_exec': {**case['box_exec'], 'count': 2}}) + + def test_attachment_requires_exact_bytes_and_single_file(self): + expected = {'name': 'out.txt', 'text': 'sentinel'} + part = {'type': 'File', 'name': 'out.txt', 'base64': base64.b64encode(b'sentinel').decode()} + data = {'replies': [{'message_chain': [part]}]} + self.assertTrue(check_attachment(data, expected)['content_matched']) + for parts in [ + [], + [part, part], + [{**part, 'base64': '!!!'}], + [{**part, 'base64': 'd3Jvbmc='}], + [{**part, 'name': 'other'}], + [None], + ]: + with self.subTest(parts=parts), self.assertRaises(ValueError): + check_attachment({'replies': [{'message_chain': parts}]}, expected) + + def test_fields_check_empty_results_and_missing_paths(self): + check_fields({'results': []}, {'results': []}) + check_fields({'results': [{'metadata': {'kb': 'owned'}}]}, {'results.0.metadata.kb': 'owned'}) + for data in [{'results': [1]}, {}, {'results': None}]: + with self.subTest(data=data), self.assertRaises(ValueError): + check_fields(data, {'results': []}) + with self.assertRaises(ValueError): + check_fields({'results': None}, {'results.0.metadata.kb': 'owned'}) + + def test_fallback_must_call_failed_primary_then_streaming_backup(self): + expected = [ + {'model': 'primary', 'status': 429, 'stream': True}, + {'model': 'backup', 'status': 200, 'stream': True}, + ] + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'requests.jsonl' + prefix = b'earlier request ignored\n' + for rows in [ + expected, + list(reversed(expected)), + [expected[0], {**expected[1], 'stream': False}], + [expected[0], {**expected[1], 'status': 503}], + ]: + path.write_bytes( + prefix + + b''.join( + ( + json.dumps( + { + 'path': '/v1/chat/completions', + 'status': row['status'], + 'request': {'model': row['model'], 'stream': row['stream']}, + } + ) + + '\n' + ).encode() + for row in rows + ) + ) + if rows == expected: + self.assertEqual(check_mock_requests(path, len(prefix), expected), expected) + else: + with self.assertRaises(ValueError): + check_mock_requests(path, len(prefix), expected) + + def test_expected_tool_failure_requires_exact_exit_and_stderr(self): + case = {'box_exec': {'count': 1, 'backend': 'docker', 'stdout': 'out', 'exit_code': 7, 'stderr': 'err'}} + for exit_code, stderr in [(7, 'err'), (0, 'err'), (7, '')]: + self.db.execute('DELETE FROM agent_run_event WHERE sequence=2') + self.db.execute( + 'INSERT INTO agent_run_event VALUES (?, ?, ?, ?)', + ( + 'run', + 2, + 'tool.call.completed', + json.dumps( + { + 'tool_name': 'exec', + 'result': {'backend': 'docker', 'stdout': 'out', 'exit_code': exit_code, 'stderr': stderr}, + } + ), + ), + ) + if exit_code == 7 and stderr == 'err': + check_run(self.db, self.run_row(), case) + else: + with self.assertRaises(ValueError): + check_run(self.db, self.run_row(), case) + + def test_cleanup_readback_rejects_resources_still_visible(self): + deleted = [{'kind': 'pipeline', 'uuid': 'owned'}] + evidence = {} + for visible in ['unrelated', 'owned']: + suite = SimpleNamespace( + call=lambda args: {'pipelines': [{'uuid': visible}]}, + write=lambda name, value: evidence.update({name: value}), + ) + if visible == 'owned': + with self.assertRaises(ValueError): + verify_cleanup(suite, deleted) + else: + verify_cleanup(suite, deleted) + self.assertEqual(evidence['cleanup-checks.json'][0]['remaining'], []) + suite.call = lambda args: {'unexpected': []} + with self.assertRaises(ValueError): + verify_cleanup(suite, deleted) + + def test_profile_supports_mixed_cases_and_rejects_ignored_oracles(self): + profile = json.loads(Path(__file__).with_name('regression_profiles').joinpath('smoke.json').read_text()) + validate(profile) + for changes in [{'thresholds': {'min_rps_ratio': 1}}, {'smoke': {'ledger_db': ''}}, {'phases': None}]: + with self.subTest(changes=changes), self.assertRaises(ValueError): + validate({**profile, **changes}) + wrong = copy.deepcopy(profile) + next(case for case in wrong['cases'] if case['name'] == 'rag_retrieve')['box_exec'] = { + 'count': 1, + 'backend': 'docker', + 'stdout': 'x', + } + with self.assertRaises(ValueError): + validate(wrong) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_regression_suite.py b/tools/test_regression_suite.py new file mode 100644 index 000000000..ae295588a --- /dev/null +++ b/tools/test_regression_suite.py @@ -0,0 +1,309 @@ +"""Focused checks for profile validation, ownership and the CLI preparation sequence.""" + +import copy +from contextlib import redirect_stdout +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import threading +import unittest +from unittest.mock import patch + +from regression_suite import Suite, main, merge, resolve, validate + + +FAKE_CLI = r""" +import json, os, sys +from pathlib import Path +args = sys.argv[5:] +root = Path(os.environ['LBCTL_TEST_ROOT']) +state_path = root / 'cli-state.json' +state = json.loads(state_path.read_text()) if state_path.exists() else {} +body = json.loads(Path(args[args.index('--file') + 1]).read_text()) if '--file' in args else {} +with (root / 'cli-calls.jsonl').open('a') as log: + log.write(json.dumps({'args': args, 'body': body}) + '\n') +if args == ['whoami']: + data = {'workspace_uuid': 'test-workspace'} +elif args[:2] == ['pipeline', 'run']: + assert args[2] in state + data = {'status': 'completed', 'reply': 'TEST_OK'} +elif args[1] == 'delete': + del state[args[2]] + state_path.write_text(json.dumps(state)) + data = {} +elif args[:2] == ['pipeline', 'get']: + data = state[args[2]] +elif args[1] in ['create', 'apply']: + uuid = body.get('uuid', f'created-{len(state)}') + data = {'uuid': uuid, 'name': body.get('name', 'test'), 'description': '', 'config': body.get('config', {'ai': {'default': True}})} + state[uuid] = data + state_path.write_text(json.dumps(state)) +else: + raise RuntimeError(args) +print(json.dumps({'ok': True, 'data': data})) +""" + + +def profile(): + return { + 'version': 1, + 'cli': {'config': 'lbctl.yaml'}, + 'cases': [{'name': 'chat', 'args': ['pipeline', 'run', 'existing'], 'expect': 'OK'}], + 'phases': [{'name': 'normal', 'duration_seconds': 1, 'rps': 2, 'workers': 4}], + 'observe': {'health': {'core': 'http://127.0.0.1:5300/healthz'}}, + } + + +class ProfileTests(unittest.TestCase): + def test_created_bot_verification_failure_can_only_defer_bot_readback(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + suite = Suite(profile(), root / 'profile.json', root) + response = { + 'data': {'operation': 'bot.create', 'uuid': 'owned-bot', 'bot': {'uuid': 'owned-bot'}}, + 'error': {'type': 'verification_failed'}, + } + result = subprocess.CompletedProcess([], 1, json.dumps(response), '') + with patch('regression_suite.subprocess.run', return_value=result): + with self.assertRaises(RuntimeError): + suite.call(['bot', 'create'], {'name': 'QA'}) + self.assertEqual( + suite.call(['bot', 'create'], {'name': 'QA'}, defer_bot_readback=True)['uuid'], 'owned-bot' + ) + with self.assertRaises(RuntimeError): + suite.call(['pipeline', 'apply'], {}, defer_bot_readback=True) + + def test_group_bot_is_owned_before_raw_readback_and_warmed_before_load(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + current = profile() + current['resources'] = [ + { + 'id': 'chat_bot', + 'kind': 'bot', + 'body': { + 'name': 'QA', + 'adapter': 'http_bot', + 'enable': True, + 'adapter_config': {'inbound_secret': '${group_secret}'}, + 'event_bindings': [{'target_type': 'pipeline', 'target_uuid': 'pipeline-a'}], + }, + } + ] + current['cases'] = [ + { + 'name': 'chat', + 'expect': 'OK', + 'http_group': { + 'url': 'http://localhost/bots/${chat_bot}/sync', + 'message': 'ai Test', + 'secret_env': 'LANGBOT_QA_GROUP_SECRET', + }, + } + ] + current['phases'][0]['groups'] = 1 + validate(current) + for correct_route in (False, True): + suite = Suite(current, root / 'profile.json', root) + + def call(args, body=None, **kwargs): + if args == ['whoami']: + return {'workspace_uuid': 'workspace-a'} + if args == ['bot', 'create']: + self.assertTrue(kwargs['defer_bot_readback']) + return {'uuid': 'owned-bot'} + raise AssertionError(args) + + suite.call = call + actual = copy.deepcopy(current['resources'][0]['body']) + actual['adapter_config']['inbound_secret'] = '***' + if not correct_route: + actual['event_bindings'][0]['target_uuid'] = 'wrong-pipeline' + with ( + patch.object(suite, 'read_bot', return_value=actual), + patch('regression_suite.group_request', return_value={'code': 0}) as warmup, + ): + if correct_route: + suite.prepare() + warmup.assert_called_once() + self.assertNotIn(suite.variables['group_secret'], (root / 'resources.json').read_text()) + else: + with self.assertRaisesRegex(RuntimeError, 'Bot readback differs'): + suite.prepare() + warmup.assert_not_called() + self.assertEqual( + json.loads((root / 'owned.json').read_text())['resources'], + [{'kind': 'bot', 'uuid': 'owned-bot'}], + ) + + def test_normal_profile_and_overrides_contract(self): + validate(json.loads(Path(__file__).with_name('regression_profiles').joinpath('normal.json').read_text())) + bad = profile() + for field, value in [('rps', float('nan')), ('workers', 33), ('duration_seconds', -1)]: + with self.subTest(field=field): + current = copy.deepcopy(bad) + current['phases'][0][field] = value + with self.assertRaises(ValueError): + validate(current) + + def test_missing_refs_and_existing_resource_overwrite_are_rejected(self): + current = profile() + current['cases'][0]['args'][-1] = '${missing}' + with self.assertRaises(KeyError): + validate(current) + current = profile() + current['resources'] = [{'id': 'pipeline', 'kind': 'pipeline', 'body': {'uuid': 'existing'}}] + with self.assertRaisesRegex(ValueError, 'uuid is forbidden'): + validate(current) + + def test_merge_keeps_defaults_but_replaces_lists(self): + base = {'ai': {'enabled': True, 'models': ['old']}, 'output': {'x': 1}} + actual = merge(base, {'ai': {'models': ['new']}}) + self.assertEqual(actual, {'ai': {'enabled': True, 'models': ['new']}, 'output': {'x': 1}}) + self.assertEqual(base['ai']['models'], ['old']) + self.assertEqual(resolve(['${id}', {'id': '${id}'}], {'id': 'created'}), ['created', {'id': 'created'}]) + + def test_ownership_recorded_before_configuration_failure(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + current = profile() + current['resources'] = [{'id': 'chat', 'kind': 'pipeline', 'body': {'name': 'QA', 'config': {'ai': {}}}}] + suite = Suite(current, root / 'profile.json', root) + calls = [] + + def call(args, body=None): + calls.append((args, body)) + if args == ['whoami']: + return {'workspace_uuid': 'workspace-a'} + if args == ['pipeline', 'apply'] and 'uuid' not in body: + return {'uuid': 'owned-pipeline'} + if args[:2] == ['pipeline', 'get']: + return {'uuid': 'owned-pipeline', 'name': 'QA', 'description': '', 'config': {'ai': {'old': 1}}} + if args == ['pipeline', 'apply']: + raise RuntimeError('configuration failed') + return {} + + suite.call = call + with self.assertRaisesRegex(RuntimeError, 'configuration failed'): + suite.prepare() + self.assertEqual( + json.loads((root / 'owned.json').read_text())['resources'], + [{'kind': 'pipeline', 'uuid': 'owned-pipeline'}], + ) + self.assertEqual(suite.cleanup(), []) + self.assertEqual(calls[-1][0], ['pipeline', 'delete', 'owned-pipeline', '--yes']) + self.assertEqual(json.loads((root / 'owned.json').read_text())['resources'], []) + + def test_cleanup_refuses_another_workspace(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + (root / 'owned.json').write_text(json.dumps({'workspace_uuid': 'other', 'resources': []})) + suite = Suite(profile(), root / 'profile.json', root) + suite.call = lambda args: {'workspace_uuid': 'current'} + with self.assertRaisesRegex(RuntimeError, 'Workspace differs'): + suite.cleanup() + + def test_aborted_run_is_not_treated_as_successful_cleanup(self): + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + config = root / 'profile.json' + config.write_text(json.dumps(profile())) + for error_type in (KeyboardInterrupt, TypeError): + output = root / error_type.__name__ + + def interrupt(suite): + suite.variables['workspace_uuid'] = 'test-workspace' + suite.journal() + raise error_type + + argv = ['regression_suite.py', 'run', '--profile', str(config), '--output', str(output)] + with ( + patch.object(sys, 'argv', argv), + patch.object(Suite, 'prepare', interrupt), + patch.object(Suite, 'cleanup') as cleanup, + redirect_stdout(io.StringIO()), + ): + with self.assertRaises(SystemExit if error_type is KeyboardInterrupt else TypeError): + main() + cleanup.assert_not_called() + self.assertFalse(json.loads((output / 'summary.json').read_text())['passed']) + + def test_subprocess_phases_and_cleanup_with_controlled_cli(self): + class Health(BaseHTTPRequestHandler): + def do_GET(self): + payload = b'{"code":0}' + self.send_response(200) + self.send_header('Content-Length', str(len(payload))) + self.end_headers() + self.wfile.write(payload) + + def log_message(self, *args): + pass + + server = ThreadingHTTPServer(('127.0.0.1', 0), Health) + thread = threading.Thread(target=server.serve_forever) + thread.start() + try: + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + cli = root / 'lbctl-test' + cli.write_text(f'#!{sys.executable}\n' + FAKE_CLI) + cli.chmod(0o700) + current = profile() + current['cli']['executable'] = str(cli) + current['resources'] = [ + {'id': 'provider', 'kind': 'provider', 'body': {'name': 'test'}}, + { + 'id': 'pipeline', + 'kind': 'pipeline', + 'body': {'name': 'test', 'config': {'ai': {'model': '${provider}'}}}, + }, + ] + current['cases'] = [ + {'name': 'chat', 'args': ['pipeline', 'run', '${pipeline}'], 'expect': 'TEST_OK', 'weight': 2}, + {'name': 'box', 'args': ['pipeline', 'run', '${pipeline}'], 'expect': 'TEST_OK', 'weight': 1}, + ] + current['phases'] = [ + {'name': name, 'duration_seconds': 0.5, 'rps': 12, 'workers': 3} for name in ('low', 'high') + ] + current['observe'] = {'health': {'core': f'http://127.0.0.1:{server.server_port}/healthz'}} + current['cleanup'] = 'always' + config = root / 'profile.json' + command = [ + sys.executable, + str(Path(__file__).with_name('regression_suite.py')), + 'run', + '--profile', + str(config), + ] + environment = {**os.environ, 'LBCTL_TEST_ROOT': folder} + for enforce_latency in (False, True): + current['thresholds'] = {'max_p95_seconds': {'chat': 0.001}} if enforce_latency else {} + config.write_text(json.dumps(current)) + output = root / ('failure' if enforce_latency else 'success') + result = subprocess.run( + [*command, '--output', str(output)], env=environment, capture_output=True, text=True, timeout=30 + ) + summary = json.loads((output / 'summary.json').read_text()) + self.assertEqual(result.returncode, int(enforce_latency), result.stdout + result.stderr) + self.assertEqual(summary['passed'], not enforce_latency) + self.assertEqual(len(summary['phases']), 2) + for phase in summary['phases']: + self.assertGreater(phase['achieved_rps'], 0) + self.assertEqual(phase['health_failures'], 0) + self.assertGreaterEqual(phase['cases']['chat']['requests'], phase['cases']['box']['requests']) + self.assertEqual(json.loads((output / 'owned.json').read_text())['resources'], []) + self.assertEqual(json.loads((root / 'cli-state.json').read_text()), {}) + finally: + server.shutdown() + server.server_close() + thread.join() + + +if __name__ == '__main__': + unittest.main()