mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-25 09:55:25 +08:00
test(ci): add a 60-second core smoke gate and regression jobs (#2711)
* test(tools): add deterministic lbctl regression job * test(tools): simulate persistent multi-group conversations * test(tools): add finite core smoke with terminal run and file oracles * test(tools): expand smoke coverage across execution stages * ci: add a timed real core smoke and lightweight tool checks * fix(ci): keep infrastructure failures visible in smoke verdicts
This commit is contained in:
huanghuoguoguo
@@ -0,0 +1,113 @@
|
||||
name: Core Smoke
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
push:
|
||||
branches: [master, develop]
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
concurrency:
|
||||
group: core-smoke-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
tools:
|
||||
name: Regression tools
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 5
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- run: python -m pip install PyYAML==6.0.3
|
||||
- name: Check regression tools
|
||||
run: timeout 60s python -m unittest discover -s tools -p 'test_*.py' -v
|
||||
|
||||
smoke:
|
||||
name: Core smoke (60s)
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- uses: astral-sh/setup-uv@v4
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Install locked dependencies
|
||||
run: uv sync --dev --frozen
|
||||
- name: Read official plugin source pin
|
||||
id: plugins
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
revision = json.loads(Path('tests/fixtures/legacy_plugin_identity_sources.json').read_text())['revision']
|
||||
with open(os.environ['GITHUB_OUTPUT'], 'a') as stream:
|
||||
stream.write(f'revision={revision}\n')
|
||||
PY
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
repository: langbot-app/langbot-plugins
|
||||
ref: ${{ steps.plugins.outputs.revision }}
|
||||
path: .smoke-plugins
|
||||
persist-credentials: false
|
||||
- name: Prepare CLI and sandbox image
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
mkdir -p .smoke-bin
|
||||
gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .smoke-bin \
|
||||
--pattern lbctl_linux_amd64 --pattern checksums.txt
|
||||
cd .smoke-bin
|
||||
sha256sum --check --ignore-missing checksums.txt
|
||||
chmod +x lbctl_linux_amd64
|
||||
docker pull python:3.12-alpine
|
||||
- name: Prepare real services and run the 60-second smoke
|
||||
run: |
|
||||
uv run --no-sync python tools/regression_ci.py \
|
||||
--lbctl .smoke-bin/lbctl_linux_amd64 --plugins .smoke-plugins --output .qa/core-smoke
|
||||
- name: Publish gate result
|
||||
if: always()
|
||||
run: |
|
||||
python - <<'PY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
path = Path('.qa/core-smoke/gate/gate.json')
|
||||
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
|
||||
if path.exists():
|
||||
data = json.loads(path.read_text())
|
||||
out.write(f"Core smoke: {'PASS' if data['passed'] else 'FAIL'}; "
|
||||
f"{data.get('elapsed_seconds', 0):.2f}/60 seconds; {data.get('requests', 0)} requests.\n")
|
||||
if data.get('infrastructure_error'):
|
||||
out.write('Preparation or teardown failed; inspect service logs.\n')
|
||||
out.write('Service/dependency preparation is outside the 60-second execution budget.\n')
|
||||
else:
|
||||
out.write('Core smoke did not produce a verdict; inspect preparation failure.\n')
|
||||
PY
|
||||
- name: Preserve verdicts and resource samples
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: core-smoke-${{ github.run_id }}
|
||||
path: |
|
||||
.qa/core-smoke/gate/gate.json
|
||||
.qa/core-smoke/gate/resources.jsonl
|
||||
.qa/core-smoke/gate/suite.log
|
||||
.qa/core-smoke/gate/suite/cases.json
|
||||
.qa/core-smoke/gate/suite/summary.json
|
||||
.qa/core-smoke/gate/suite/health.json
|
||||
.qa/core-smoke/gate/suite/cleanup-checks.json
|
||||
.qa/core-smoke/*.log
|
||||
if-no-files-found: warn
|
||||
retention-days: 7
|
||||
@@ -9,6 +9,10 @@ on:
|
||||
pull_request:
|
||||
types: [opened, synchronize, reopened, ready_for_review]
|
||||
|
||||
concurrency:
|
||||
group: lint-${{ github.event.pull_request.number || github.ref }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
ruff:
|
||||
name: Ruff Lint & Format
|
||||
@@ -25,14 +29,11 @@ jobs:
|
||||
- name: Install uv
|
||||
uses: astral-sh/setup-uv@v4
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv sync --dev
|
||||
|
||||
- name: Run ruff check
|
||||
run: uv run ruff check src/langbot/ tests/ --output-format=concise
|
||||
run: uvx ruff@0.14.14 check src/langbot/ tests/ tools/ --output-format=concise
|
||||
|
||||
- name: Run ruff format
|
||||
run: uv run ruff format src --check
|
||||
run: uvx ruff@0.14.14 format src tools --check
|
||||
|
||||
frontend:
|
||||
name: Frontend Lint
|
||||
|
||||
+403
@@ -0,0 +1,403 @@
|
||||
# LangBot regression tools
|
||||
|
||||
Run the deterministic OpenAI Chat Completions endpoint with Python 3.11+, without
|
||||
API credentials or third-party dependencies:
|
||||
|
||||
```sh
|
||||
python3 tools/mock_llm.py --port 18080 --record /tmp/langbot-mock-requests.jsonl
|
||||
```
|
||||
|
||||
In LangBot, create an `openai-chat-completions` Provider whose base URL is
|
||||
`http://127.0.0.1:18080/v1` and whose API key is any test-only value. Use a model
|
||||
from `GET /v1/models`. The loopback address works when Core and this tool run on
|
||||
the same host; containers must use an address reachable from their own network.
|
||||
|
||||
| Model | Behavior |
|
||||
| --- | --- |
|
||||
| `mock-chat` | Returns `LANGBOT_CHAT_OK`, including SSE streaming |
|
||||
| `mock-rag` | Requires the golden document sentinel in the model input, then returns `LANGBOT_RAG_OK` |
|
||||
| `mock-embedding` | Returns deterministic normalized token-hash vectors from `/v1/embeddings` |
|
||||
| `mock-box` | Calls `exec` twice to write/read `/workspace/mock-llm.txt`, then returns `LANGBOT_BOX_OK` |
|
||||
| `mock-401`, `mock-429`, `mock-503` | Returns the selected HTTP failure |
|
||||
| `mock-slow` | Delays the response by two seconds |
|
||||
| `mock-slow-stream` | Delays each SSE chunk by 0.2 seconds |
|
||||
|
||||
`mock-rag` requires `aurora-parser-rag-9137` in the submitted messages. The
|
||||
[golden HTML fixture](../skills/skills/langbot-testing/fixtures/rag/parser-golden.html)
|
||||
contains this marker; retrieve it into the model context without including it
|
||||
in the user's question.
|
||||
|
||||
For `mock-box`, enable Box in the Runner and expose the native `exec` tool. Each
|
||||
tool result must contain `LANGBOT_BOX_SENTINEL` before the next step is returned.
|
||||
Tool-call IDs and fragmented streaming arguments use the normal OpenAI protocol.
|
||||
Scenario progress comes from each request's current conversation turn, so
|
||||
concurrent sessions and retries do not consume a shared script counter.
|
||||
Tagged group traffic additionally requires the previous user turn and assistant
|
||||
acknowledgement in the model input, and rejects another group's history. Final
|
||||
responses carry a per-turn acknowledgement checked by the traffic driver.
|
||||
|
||||
Add fixed response/tool sequences with `--scenarios path/to/scenarios.json`, using
|
||||
`mock_llm_scenarios.json` as the example. `expect_tool_output` checks a substring
|
||||
in the previous tool results; `expect_prompt` checks the submitted messages.
|
||||
The server returns tool calls; it never executes
|
||||
their commands. Request recordings contain prompts, tool results, responses,
|
||||
status, and disconnects, but no HTTP headers. Use test data only.
|
||||
|
||||
Run the focused HTTP checks on the Linux test host:
|
||||
|
||||
```sh
|
||||
python3 -m unittest discover -s tools -p test_mock_llm.py -v
|
||||
```
|
||||
|
||||
These checks validate the mock server. A LangBot regression must additionally
|
||||
verify real tool outputs and retrieved document IDs/content. Token usage is
|
||||
synthetic, and this endpoint does not evaluate answer quality or embedding models.
|
||||
|
||||
`mock-embedding` accepts string/batch inputs, float/base64 encoding, and optional
|
||||
dimensions (default 384). It tests the embedding protocol and RAG plumbing;
|
||||
token hashes do not establish semantic retrieval quality.
|
||||
|
||||
## Normal traffic and resource collection
|
||||
|
||||
`regression_traffic.py` executes a JSON list of normal `lbctl` cases at a selected
|
||||
rate. Each case has a name, CLI arguments, and an expected response substring:
|
||||
|
||||
```json
|
||||
[{"name":"chat","args":["pipeline","run","PIPELINE_UUID","--message","Hello"],"expect":"LANGBOT_CHAT_OK"}]
|
||||
```
|
||||
|
||||
Prepare a Workspace-bound CLI context and its API key environment variable, then
|
||||
run on the Linux test host:
|
||||
|
||||
```sh
|
||||
python3 tools/regression_traffic.py \
|
||||
--lbctl /path/to/lbctl --config /path/to/lbctl.yaml \
|
||||
--cases /path/to/cases.json --output /path/to/new-results-directory \
|
||||
--duration 600 --rps 2 --workers 4 --cooldown 30 \
|
||||
--pid core=CORE_PID --pid mock=MOCK_PID \
|
||||
--health core=http://127.0.0.1:5300/healthz \
|
||||
--health mock=http://127.0.0.1:18080/healthz
|
||||
```
|
||||
|
||||
Add `--container CONTAINER_ID` for Docker CPU/memory/PID statistics. The executing
|
||||
account must be able to inspect the selected process trees and Docker daemon.
|
||||
Resource collection reuses `scripts/cloud_runtime_soak.py`; it does not require
|
||||
cgroup v2. PID resource totals include descendants, so select disjoint trees.
|
||||
|
||||
The result directory contains `requests.jsonl`, `resources.jsonl`, `progress.json`,
|
||||
and `summary.json`. The requested rate is a target: four workers bound concurrency,
|
||||
and slow requests lower the achieved rate. Requests with unknown outcomes are
|
||||
recorded without retry. A command fails its business check if it returns an error
|
||||
or omits the expected response. Health/resource collection failures also make the
|
||||
run fail. API key values are redacted from recorded CLI output.
|
||||
|
||||
Latency percentiles include the entire CLI invocation and use successful
|
||||
requests only. Achieved request rate is `requests / load_elapsed_seconds`.
|
||||
`health_samples` counts sampling rounds; `health_failures` counts rounds with
|
||||
any failed health probe or process/container resource read.
|
||||
|
||||
## Configured regression suites
|
||||
|
||||
[`regression_suite.py`](regression_suite.py) turns a JSON profile into a repeatable
|
||||
resource preparation, traffic, observation, verdict, and cleanup sequence. Start
|
||||
with [`regression_profiles/groups.json`](regression_profiles/groups.json) for
|
||||
continuous group conversations. The earlier
|
||||
[`normal.json`](regression_profiles/normal.json) profile retains independent CLI
|
||||
requests; parallel CLI workers alone do not simulate persistent groups.
|
||||
|
||||
Use a dedicated QA Workspace. First provision the instance with `lbctl install`
|
||||
or your source deployment, enable Docker Box, start the mock and embedding
|
||||
backends, and prepare a Workspace-bound `lbctl` config. Set the profile's CLI
|
||||
config path, backend URLs, and three plugin archive paths. The sample embedding
|
||||
endpoint must serve `all-MiniLM-L6-v2`; the mock serves generation only. Relative
|
||||
paths resolve against the profile directory, so adjust them if you move it.
|
||||
|
||||
Run on the Linux test host:
|
||||
|
||||
```sh
|
||||
python3 tools/regression_suite.py validate --profile tools/regression_profiles/groups.json
|
||||
python3 tools/regression_suite.py run \
|
||||
--profile tools/regression_profiles/groups.json --output .qa/regression/run-001
|
||||
```
|
||||
|
||||
`validate` checks profile syntax and references without contacting the instance.
|
||||
The output directory for `run` must be new. CLI options `--duration`, `--rps`, and
|
||||
`--workers` override those values in every phase. No additional Python dependency
|
||||
is needed.
|
||||
|
||||
The profile controls:
|
||||
|
||||
- `plugins`: local archives or GitHub/marketplace request files, installed with
|
||||
`lbctl plugin install ... --wait`.
|
||||
- `resources`: Providers, models, knowledge bases, document ingestion, Pipelines,
|
||||
and Bots, created in dependency order. `${resource_id}` refers to a UUID
|
||||
returned by an earlier creation; `${workspace_uuid}` is the current Workspace.
|
||||
Pipeline config patches merge with actual server defaults. Independent CLI
|
||||
cases can use existing resources supplied through `vars`; group mode creates
|
||||
its own signed HTTP Bots.
|
||||
- `cases`: CLI argument arrays or `http_group` settings and expected business text. Integer `weight`
|
||||
values set the mix; `2/1/1/1` means `40/20/20/20`. An optional `body` object
|
||||
becomes the case's request file. All workers share the weighted sequence.
|
||||
- `phases`: each phase has a name, duration, target RPS, worker limit, and optional
|
||||
cooldown. Multiple phases provide a step load. In-flight requests drain at the
|
||||
end; the concurrency limit can reduce achieved RPS.
|
||||
- `observe`: health URLs, host PIDs, PID files, and Docker container names/IDs.
|
||||
Set PIDs for disjoint process trees to avoid counting descendants twice.
|
||||
`box_data_root` discovers Box containers mounted below that deployment's data
|
||||
directory on every sample, including sandboxes created as new groups speak.
|
||||
The complete Job sets it automatically; on an existing instance, set it to
|
||||
the instance's Box data directory when sampling dynamic Docker sandboxes.
|
||||
- `thresholds`: minimum achieved/target RPS ratio and maximum P95 seconds per
|
||||
case, in addition to the existing business and health/resource checks. The
|
||||
sample limits are starting acceptance settings, not established product SLOs.
|
||||
- `cleanup`: `on-success` (default), `always`, or `never`. Only recorded resources
|
||||
created by this run are deleted, in reverse order. Plugins and the existing
|
||||
instance remain installed. Failed/unknown creations are not automatically
|
||||
retried; inspect the instance if a creation outcome is unknown.
|
||||
|
||||
Failed runs retain resources by default. After inspection, clean them with the
|
||||
same profile and credentials:
|
||||
|
||||
```sh
|
||||
python3 tools/regression_suite.py cleanup \
|
||||
--profile tools/regression_profiles/groups.json --output .qa/regression/run-001
|
||||
```
|
||||
|
||||
Cleanup checks the Workspace against `owned.json` and records remaining items
|
||||
after partial failures. Keep `owned.json` until cleanup completes. Each run also
|
||||
contains `profile.json`, resolved `resources.json`, and one directory per phase
|
||||
with the original request/resource records. The root `summary.json` includes
|
||||
target/achieved RPS, phase results, threshold failures, and the final verdict.
|
||||
|
||||
The `regression_suite.py` entry reuses an existing instance and backend processes.
|
||||
Use the complete lifecycle Job below to own deployment, credentials and backends.
|
||||
The independent `normal.json` cases use a new-session CLI entry. Group traffic
|
||||
uses the signed HTTP Bot adapter instead, as described below. Real IM adapter
|
||||
acceptance and resource-leak verdicts need separate scenarios. Test the suite
|
||||
plumbing on Linux with:
|
||||
|
||||
```sh
|
||||
python3 -m unittest discover -s tools -p test_regression_suite.py -v
|
||||
```
|
||||
|
||||
### Persistent group traffic
|
||||
|
||||
The group profile creates three HTTP Bots routed to chat, Box, and RAG Pipelines.
|
||||
Their signed `/bots/<uuid>/sync` ingress produces real `GroupMessage` events and
|
||||
uses Core's session manager. Four groups follow a `2/1/1` case assignment;
|
||||
members take turns within each group. The same member IDs appear in different
|
||||
groups to exercise group isolation. Box session selection uses the group launcher ID.
|
||||
|
||||
In each group phase, `groups` selects the number of stable sessions and
|
||||
`members_per_group` selects the rotating member identities. `rps` remains the
|
||||
aggregate target, so a group's target speaking interval is `groups / rps`
|
||||
seconds. `workers` bounds total in-flight requests independently of group count.
|
||||
For example, 8 groups at 4 RPS target one message every 2 seconds per group.
|
||||
Both the Suite and Job accept `--groups` and `--members-per-group` overrides.
|
||||
The group count must cover the full weighted case sequence (4 in the sample).
|
||||
|
||||
Only one request is in flight per group. A failure or unknown outcome stops that
|
||||
group without retry while other groups continue. Every selected group must
|
||||
complete at least two turns in total for the phase to pass. The fake model checks
|
||||
that the preceding user message and assistant acknowledgement survived in actual
|
||||
model input, including when a different member speaks next. Cross-group markers
|
||||
fail the oracle. The normal Box/tool-output and RAG-context checks still apply.
|
||||
|
||||
`group-state.json` carries group IDs, turn numbers and acknowledgements across
|
||||
phases; existing groups keep their conversations when later phases add groups.
|
||||
It refuses changed cases or a state left in progress by interruption. Do not
|
||||
clear that marker to blindly resume: inspect the run and use a fresh output
|
||||
directory when the last turn's outcome is unknown. Each phase summary includes
|
||||
group-level turn counts and member IDs alongside case latency and health data.
|
||||
|
||||
The Suite generates the HTTP Bot signing secret and passes it through
|
||||
`LANGBOT_QA_GROUP_SECRET`. Resource request bodies containing credentials are
|
||||
stored under `.private/`; do not share that directory. Cases specify `http_group`
|
||||
with `url`, `secret_env`, and `message` instead of CLI `args`. Core needs no code
|
||||
changes. The `/sync` response collapses streaming parts: this tests multi-turn
|
||||
group processing, not real IM delivery or callback streaming order.
|
||||
Every selected group must also receive traffic in each phase. Group latency
|
||||
measures the signed HTTP request through its completed reply; independent CLI
|
||||
latency includes CLI startup. Compare results within the same traffic mode.
|
||||
|
||||
`lbctl` v0.3 may report Bot verification failure after a successful creation
|
||||
because its safe projection omits event bindings. The Suite records the returned
|
||||
Bot UUID before reading the full Core Bot API with the endpoint, Workspace and
|
||||
environment credential confirmed by `lbctl whoami`. It then
|
||||
checks the masked signing secret and effective route with a signed warmup turn
|
||||
before load. The creation is not retried.
|
||||
|
||||
## Complete lifecycle Job
|
||||
|
||||
[`regression_job.py`](regression_job.py) owns the deployment as well as the suite.
|
||||
Use [`regression_profiles/job.json`](regression_profiles/job.json) to pin the Core
|
||||
Release and select the suite and plugin input directory. The job runs this chain:
|
||||
|
||||
1. `lbctl install --profile all` creates a fresh deployment inside the run directory.
|
||||
2. A Compose override starts our mock backend on the deployment's private network
|
||||
and injects a generated test-only API key. The Job binds its own CLI context.
|
||||
3. The suite installs plugins, creates resources and ingests documents. Each case
|
||||
runs once as warmup before measured traffic starts.
|
||||
4. Configured load phases run with health, process-tree and container sampling.
|
||||
5. Service-log tails and raw request/resource records are retained. Successful
|
||||
Jobs stop the deployment, remove only sandboxes mounted under their own data
|
||||
root, and use `lbctl uninstall --purge-data` to reclaim the deployment.
|
||||
|
||||
Both generation and embedding use our mock backend, with no external model
|
||||
service or real model credentials. Plugin archives, the pinned Core image,
|
||||
`lbctl`, Docker and Compose v2 are Job inputs/prerequisites. This wrapper uses
|
||||
PyYAML, already included in LangBot's dependencies, to verify the CLI's deployment
|
||||
record and recover ownership after an interrupted install.
|
||||
|
||||
Run on a Linux host with Docker access and permission to inspect the selected
|
||||
container processes. Use an idle dedicated Docker daemon: current Box runtime
|
||||
startup cleanup can remove sandboxes from another runtime. The Job refuses to
|
||||
start when existing Box sandboxes are present. The current `lbctl` installer
|
||||
also reserves host port 5401 and 2280–2285, independently of its HTTP port.
|
||||
|
||||
Set the plugin directory and the `lbctl` executable in `job.json`, then run:
|
||||
|
||||
```sh
|
||||
uv run --no-sync python tools/regression_job.py validate --profile tools/regression_profiles/job.json
|
||||
uv run --no-sync python tools/regression_job.py run \
|
||||
--profile tools/regression_profiles/job.json --output .qa/regression/job-001
|
||||
```
|
||||
|
||||
`--duration`, `--rps` and `--workers` override every load phase. Each output
|
||||
directory must be new. Job cleanup policy is `on-success`, `always`, or `never`;
|
||||
it governs the whole deployment, whereas suite cleanup governs API resources
|
||||
when a suite runs on an existing deployment. `purge_data: false` preserves the
|
||||
deployment data after removing containers. Failed Jobs retain their environment
|
||||
by default; after inspection use:
|
||||
|
||||
```sh
|
||||
uv run --no-sync python tools/regression_job.py cleanup \
|
||||
--output .qa/regression/job-001
|
||||
```
|
||||
|
||||
The root `summary.json` is the final Job verdict; `suite/` contains the measured
|
||||
results, `mock/requests.jsonl` contains backend recordings, and `services.log`
|
||||
contains up to 10,000 log lines per service. Generated credentials and deployment
|
||||
inputs stay in `.private/`; share the result files, not that directory. A Job has
|
||||
its own CLI locator/config and records its deployment identity before cleanup.
|
||||
Changing that identity makes cleanup fail closed. Cleanup uses the saved Job
|
||||
state and does not require the original profile or plugin input files.
|
||||
|
||||
The Job replaces the real embedding model with `mock-embedding`. This measures
|
||||
LangBot/plugin/Box/RAG overhead, not real-model inference cost or retrieval
|
||||
quality. Run the Job plumbing checks on Linux with:
|
||||
|
||||
```sh
|
||||
uv run --no-sync python -m unittest discover -s tools -p 'test_*.py' -v
|
||||
```
|
||||
|
||||
## Finite core smoke
|
||||
|
||||
Use [`regression_profiles/smoke.json`](regression_profiles/smoke.json) for a finite
|
||||
functional check on an existing Core. It prepares 35 resources through `lbctl`,
|
||||
ingests HTML and plain-text fixtures through GeneralParsers, then executes 28
|
||||
cases / 43 requests. Reports group the verdicts by stage:
|
||||
|
||||
| Stage | Cases and independent checks |
|
||||
| --- | --- |
|
||||
| Configuration | Identity and plugins; provider, model, Pipeline and knowledge-base readback; document completed status and Workspace/KB ownership |
|
||||
| Ingress | Missing and incorrect signatures return HTTP 401 / code 40101 without creating a group AgentRun; valid signatures are exercised by group conversations |
|
||||
| Conversation | Ordinary and Unicode chat; chat, Box and RAG groups each use two interleaved groups with three turns and rotating members; model input must retain prior user/reply history without cross-group history |
|
||||
| Model | Slow SSE reply; HTTP 401, 429 and 503 primary failures followed by successful fallback; recorded model/status/stream sequence must match |
|
||||
| Tools | Docker write/read across rounds, two tool calls in one model response, and expected exit code 7 with separate stdout/stderr returned to the model |
|
||||
| Delivery | ASCII and Unicode filenames/content delivered as one attachment with exact bytes |
|
||||
| Retrieval | HTML and text parsing/ingestion, RAG model input, retrieval from two knowledge bases with distinct sentinels and KB ownership, and an empty knowledge base returning an empty list |
|
||||
|
||||
Cleanup is checked separately: delete journaled API resources and confirm their
|
||||
UUIDs are absent from CLI list readback. Evidence goes to `cleanup-checks.json`;
|
||||
cleanup failures affect the overall verdict.
|
||||
|
||||
Start a dedicated `mock_llm.py` with the bundled scenarios and `--record` as
|
||||
shown above. Copy the smoke profile and configure the CLI executable/config,
|
||||
service URLs, plugin archives, `vars.mock_records` to that mock recording file,
|
||||
and `vars.core_db` to the target Core's existing SQLite `langbot.db`. The suite
|
||||
must read both local files. Keep fixture paths valid relative to the copied
|
||||
profile. Do not share the mock with other traffic during this run: fallback
|
||||
checks compare requests appended during each case.
|
||||
|
||||
Set `plugins: []` to reuse installed plugins; LocalAgent, LangRAG and
|
||||
GeneralParsers must still be enabled and initialized. The embedding endpoint
|
||||
must serve the configured model (`all-MiniLM-L6-v2` by default). Generation is
|
||||
deterministic mock traffic; Core, plugins, Box, document parsing and retrieval
|
||||
execute normally.
|
||||
|
||||
```sh
|
||||
uv run --no-sync python tools/regression_suite.py validate --profile /path/to/smoke.json
|
||||
uv run --no-sync python tools/regression_suite.py run \
|
||||
--profile /path/to/smoke.json --output .qa/smoke-001
|
||||
```
|
||||
|
||||
A correct reply alone does not pass. Each Pipeline/group request must match one
|
||||
new AgentRun in the authenticated workspace with `completed` status and a persisted
|
||||
`run.completed` event. The ledger is opened read-only. This version requires
|
||||
local SQLite access; remote-only Core and PostgreSQL ledgers are not supported.
|
||||
The observation timeout does not extend the Runner deadline or retry requests.
|
||||
Unknown outcomes fail the suite; later turns of that group are skipped while
|
||||
independent cases continue. An expected tool exit code of 7 passes only when
|
||||
its output is correct and the agent handles the result and completes normally.
|
||||
|
||||
`summary.json` contains the overall and per-stage verdicts; exit code 0 means all
|
||||
checks and required cleanup passed. `cases.json` records per-request evidence,
|
||||
`environment.json` records identity/capabilities/plugins, and `owned.json` tracks
|
||||
created resources. Successful runs delete those API resources. Failures retain
|
||||
them by default; after inspection use:
|
||||
|
||||
```sh
|
||||
uv run --no-sync python tools/regression_suite.py cleanup \
|
||||
--profile /path/to/smoke.json --output .qa/smoke-001
|
||||
```
|
||||
|
||||
Box sessions and AgentRun records follow the reused service's normal retention
|
||||
policy. Load override flags are rejected for smoke. `stage` labels cases;
|
||||
`fields` compares exact response values by dotted path (including array indices),
|
||||
`forbid` rejects response substrings, and `mock_requests` checks the actual
|
||||
model/status/stream sequence. `box_exec` defaults to exit code 0; error-handling
|
||||
cases can specify `exit_code` and `stderr` explicitly.
|
||||
|
||||
To use a fresh deployment, select this suite in `job.json`; the Job supplies its
|
||||
SQLite/recording paths and uses mock embedding. Smoke skips load warmup and timed
|
||||
phases. The dedicated Docker daemon requirement above still applies.
|
||||
|
||||
## PR smoke gate
|
||||
|
||||
`Core Smoke` runs regression-tool tests and a real Core smoke in parallel on
|
||||
GitHub-hosted Linux runners. The smoke starts Core from the checked-out candidate,
|
||||
uses the SDK from `uv.lock`, the existing official-plugin source pin, `lbctl`
|
||||
v0.3.0, a Docker Box and mock generation/embedding. It needs no model credentials
|
||||
or access to a developer's running environment.
|
||||
|
||||
Dependency installation, service startup, sandbox image download and plugin
|
||||
installation are preparation. After readiness, the gate allows **60 seconds total**
|
||||
for API resource creation, 28 cases / 37 requests (two turns per group), verdict
|
||||
checks and API cleanup. A missing verdict, failed check or timeout fails the gate.
|
||||
The ordinary expanded profile keeps three turns / 43 requests. The entire cold
|
||||
workflow takes longer than a minute; it has a separate 15-minute timeout.
|
||||
|
||||
On an existing prepared stack, use:
|
||||
|
||||
```sh
|
||||
uv run --no-sync python tools/regression_gate.py \
|
||||
--profile /path/to/smoke.json --output .qa/gate-001
|
||||
```
|
||||
|
||||
`gate.json` records the budget, measured duration, verdict and process resource
|
||||
peaks. `resources.jsonl` samples configured process trees every second (RSS,
|
||||
cumulative CPU time, threads and file descriptors); suite artifacts contain
|
||||
per-case timings and business evidence. These are diagnostics, not calibrated
|
||||
performance-regression thresholds. Docker sandbox resource sampling remains in
|
||||
the longer load runner. A timeout kills the local Suite/CLI process group without
|
||||
retrying requests; a reused remote service can still have an in-flight run, so
|
||||
inspect its retained ownership journal before cleanup. CI tears down its own
|
||||
service processes and owned sandboxes afterward.
|
||||
|
||||
The workflow always creates its two checks on PRs and cancels superseded runs.
|
||||
After the workflow is merged, configure `Regression tools` and `Core smoke (60s)`
|
||||
as required status checks in the repository ruleset. A green workflow alone does
|
||||
not enforce merge protection. Longer load and soak tests use the existing Suite
|
||||
and Job entry points with separately calibrated latency, throughput and resource
|
||||
budgets; the one-minute smoke does not replace them.
|
||||
@@ -0,0 +1,2 @@
|
||||
The separate knowledge base sentinel is cedar-isolation-4826.
|
||||
This document belongs only to the second smoke knowledge base.
|
||||
@@ -0,0 +1,401 @@
|
||||
#!/usr/bin/env python3
|
||||
"""A deterministic OpenAI Chat Completions server for local regression tests."""
|
||||
|
||||
import argparse
|
||||
import base64
|
||||
import hashlib
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import math
|
||||
from pathlib import Path
|
||||
import re
|
||||
import struct
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
|
||||
|
||||
MAX_BODY_BYTES = 1024 * 1024
|
||||
USAGE = {'prompt_tokens': 10, 'completion_tokens': 5, 'total_tokens': 15}
|
||||
GROUP_TURN = re.compile(r'LB_GROUP=(\S+) LB_TURN=(\d+) LB_TOKEN=([a-f0-9]+) LB_PREVIOUS=(none|[a-f0-9]+)')
|
||||
|
||||
|
||||
def group_ack(messages, start):
|
||||
"""Require a real previous user turn and reply, with no other group's history."""
|
||||
turns = []
|
||||
for index, message in enumerate(messages):
|
||||
if message.get('role') == 'user':
|
||||
for match in GROUP_TURN.finditer(str(message.get('content', ''))):
|
||||
turns.append((index, *match.groups()))
|
||||
if not turns:
|
||||
return ''
|
||||
position, group, turn, token, previous = turns[-1]
|
||||
if position != start or any(item[1] != group for item in turns):
|
||||
raise ValueError('group history contains another group or lacks the current turn')
|
||||
if int(turn) == 1:
|
||||
if len(turns) != 1 or previous != 'none':
|
||||
raise ValueError('first group turn must have an empty history')
|
||||
else:
|
||||
if len(turns) < 2:
|
||||
raise ValueError('previous group turn is missing from model history')
|
||||
prior_position, _, prior_turn, prior_token, _ = turns[-2]
|
||||
replies = messages[prior_position + 1 : start]
|
||||
if (
|
||||
int(prior_turn) != int(turn) - 1
|
||||
or prior_token != previous
|
||||
or not any(
|
||||
message.get('role') == 'assistant' and f'GROUP_ACK={previous}' in str(message.get('content', ''))
|
||||
for message in replies
|
||||
)
|
||||
):
|
||||
raise ValueError('previous group turn or assistant acknowledgement does not match')
|
||||
return f' GROUP_ACK={token}'
|
||||
|
||||
|
||||
def mock_embeddings(body):
|
||||
"""Return deterministic normalized token hashes, not semantic model inference."""
|
||||
inputs = body.get('input')
|
||||
inputs = [inputs] if isinstance(inputs, str) else inputs
|
||||
if (
|
||||
not isinstance(inputs, list)
|
||||
or not 1 <= len(inputs) <= 128
|
||||
or any(not isinstance(text, str) or not text for text in inputs)
|
||||
):
|
||||
raise ValueError('embedding input must contain 1..128 non-empty strings')
|
||||
dimensions = body.get('dimensions', 384)
|
||||
if isinstance(dimensions, bool) or not isinstance(dimensions, int) or not 1 <= dimensions <= 4096:
|
||||
raise ValueError('embedding dimensions must be an integer in 1..4096')
|
||||
encoding = body.get('encoding_format', 'float')
|
||||
if encoding not in ('float', 'base64'):
|
||||
raise ValueError('embedding encoding_format must be float or base64')
|
||||
data = []
|
||||
for index, text in enumerate(inputs):
|
||||
vector = [0.0] * dimensions
|
||||
for token in re.findall(r'\w+', text.casefold()) or [text]:
|
||||
digest = hashlib.sha256(token.encode()).digest()
|
||||
vector[int.from_bytes(digest[:4], 'little') % dimensions] += 1 if digest[4] % 2 else -1
|
||||
norm = math.sqrt(sum(value * value for value in vector))
|
||||
if not norm:
|
||||
vector[0], norm = 1.0, 1.0
|
||||
vector = [value / norm for value in vector]
|
||||
embedding = (
|
||||
base64.b64encode(struct.pack(f'<{dimensions}f', *vector)).decode() if encoding == 'base64' else vector
|
||||
)
|
||||
data.append({'object': 'embedding', 'index': index, 'embedding': embedding})
|
||||
return {'object': 'list', 'model': body['model'], 'data': data, 'usage': {'prompt_tokens': 10, 'total_tokens': 10}}
|
||||
|
||||
|
||||
def select_message(body, scenario):
|
||||
messages = body.get('messages')
|
||||
if not isinstance(messages, list) or not messages or any(not isinstance(m, dict) for m in messages):
|
||||
raise ValueError('messages must be a non-empty array of objects')
|
||||
if not isinstance(body.get('stream', False), bool):
|
||||
raise ValueError('stream must be a boolean')
|
||||
if body.get('stream_options') is not None and not isinstance(body['stream_options'], dict):
|
||||
raise ValueError('stream_options must be an object')
|
||||
# Select from the current turn's history, never a shared request counter.
|
||||
start = next(
|
||||
(i for i in range(len(messages) - 1, -1, -1) if messages[i].get('role') == 'user'),
|
||||
None,
|
||||
)
|
||||
if start is None:
|
||||
raise ValueError('a user message is required')
|
||||
acknowledgement = group_ack(messages, start)
|
||||
turn = messages[start:]
|
||||
calls = [i for i, m in enumerate(turn) if m.get('role') == 'assistant' and m.get('tool_calls')]
|
||||
for index, position in enumerate(calls):
|
||||
end = calls[index + 1] if index + 1 < len(calls) else len(turn)
|
||||
requested = {call['id'] for call in turn[position]['tool_calls']}
|
||||
returned = [m.get('tool_call_id') for m in turn[position + 1 : end] if m.get('role') == 'tool']
|
||||
if len(returned) != len(requested) or set(returned) != requested:
|
||||
raise ValueError('each tool call must have exactly one matching tool result')
|
||||
steps = scenario['steps']
|
||||
if len(calls) >= len(steps):
|
||||
raise ValueError('tool history exceeds the configured scenario')
|
||||
step = steps[len(calls)]
|
||||
if expected_prompt := step.get('expect_prompt'):
|
||||
if expected_prompt not in json.dumps(messages, ensure_ascii=False):
|
||||
raise ValueError(f'model input must contain {expected_prompt!r}')
|
||||
expected = step.get('expect_tool_output')
|
||||
if expected is not None:
|
||||
results = turn[calls[-1] + 1 :] if calls else []
|
||||
result_text = '\n'.join(str(m.get('content', '')) for m in results if m.get('role') == 'tool')
|
||||
if expected not in result_text:
|
||||
raise ValueError(f'tool result must contain {expected!r}')
|
||||
content = step.get('content')
|
||||
message = {'role': 'assistant', 'content': content + acknowledgement if content is not None else None}
|
||||
if tool_calls := step.get('tool_calls'):
|
||||
tools = body.get('tools', [])
|
||||
if not isinstance(tools, list) or any(
|
||||
not isinstance(t, dict) or not isinstance(t.get('function'), dict) for t in tools
|
||||
):
|
||||
raise ValueError('tools must be an array of objects')
|
||||
available = {t.get('function', {}).get('name') for t in tools if t.get('type') == 'function'}
|
||||
if body.get('tool_choice') == 'none' or any(call['name'] not in available for call in tool_calls):
|
||||
raise ValueError('scenario tools must be advertised and enabled by the caller')
|
||||
message['tool_calls'] = [
|
||||
{
|
||||
'id': f'call_mock_{len(calls)}_{i}',
|
||||
'type': 'function',
|
||||
'function': {
|
||||
'name': call['name'],
|
||||
'arguments': json.dumps(call['arguments']),
|
||||
},
|
||||
}
|
||||
for i, call in enumerate(tool_calls)
|
||||
]
|
||||
return message
|
||||
|
||||
|
||||
class MockServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, address, scenarios, record_path=None):
|
||||
super().__init__(address, Handler)
|
||||
self.scenarios = scenarios
|
||||
self.record_path = record_path
|
||||
self.record_lock = threading.Lock()
|
||||
|
||||
def record(self, entry):
|
||||
if self.record_path:
|
||||
with (
|
||||
self.record_lock,
|
||||
self.record_path.open('a', encoding='utf-8') as stream,
|
||||
):
|
||||
stream.write(json.dumps(entry, ensure_ascii=False) + '\n')
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *_args):
|
||||
pass
|
||||
|
||||
def json_response(self, status, payload):
|
||||
data = json.dumps(payload, ensure_ascii=False).encode()
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Type', 'application/json')
|
||||
self.send_header('Content-Length', str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def do_GET(self):
|
||||
if self.path == '/healthz':
|
||||
self.json_response(200, {'status': 'ok'})
|
||||
elif self.path == '/v1/models':
|
||||
self.json_response(
|
||||
200,
|
||||
{
|
||||
'object': 'list',
|
||||
'data': [
|
||||
{
|
||||
'id': name,
|
||||
'object': 'model',
|
||||
'created': 0,
|
||||
'owned_by': 'mock',
|
||||
}
|
||||
for name in [*self.server.scenarios, 'mock-embedding']
|
||||
],
|
||||
},
|
||||
)
|
||||
else:
|
||||
self.json_response(
|
||||
404,
|
||||
{
|
||||
'error': {
|
||||
'message': 'unknown endpoint',
|
||||
'type': 'invalid_request_error',
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
def do_POST(self):
|
||||
entry = {'request_id': uuid.uuid4().hex, 'time': time.time(), 'path': self.path, 'status': 400}
|
||||
try:
|
||||
if self.path not in ('/v1/chat/completions', '/v1/embeddings'):
|
||||
entry['status'] = 404
|
||||
self.json_response(
|
||||
404,
|
||||
{
|
||||
'error': {
|
||||
'message': 'unknown endpoint',
|
||||
'type': 'invalid_request_error',
|
||||
}
|
||||
},
|
||||
)
|
||||
return
|
||||
self.connection.settimeout(10)
|
||||
length = int(self.headers.get('Content-Length', 0))
|
||||
if self.headers.get('Transfer-Encoding') or not 0 < length <= MAX_BODY_BYTES:
|
||||
raise ValueError('a Content-Length between 1 and 1048576 is required')
|
||||
body = json.loads(self.rfile.read(length))
|
||||
if not isinstance(body, dict) or not isinstance(body.get('model'), str):
|
||||
raise ValueError('request must be an object with a model name')
|
||||
entry['request'] = body
|
||||
if self.path == '/v1/embeddings':
|
||||
if body['model'] != 'mock-embedding':
|
||||
raise ValueError('embedding model must be mock-embedding')
|
||||
payload = mock_embeddings(body)
|
||||
entry.update(status=200, response=payload)
|
||||
self.json_response(200, payload)
|
||||
return
|
||||
scenario = self.server.scenarios.get(body['model'])
|
||||
if scenario is None:
|
||||
raise ValueError('unknown model; use GET /v1/models')
|
||||
message = select_message(body, scenario)
|
||||
time.sleep(scenario.get('delay_seconds', 0))
|
||||
status = scenario.get('status', 200)
|
||||
entry['status'] = status
|
||||
if status != 200:
|
||||
self.json_response(
|
||||
status,
|
||||
{
|
||||
'error': {
|
||||
'message': 'injected mock failure',
|
||||
'type': 'api_error',
|
||||
'code': str(status),
|
||||
}
|
||||
},
|
||||
)
|
||||
return
|
||||
entry['response'] = message
|
||||
base = {
|
||||
'id': 'chatcmpl-' + entry['request_id'],
|
||||
'created': int(time.time()),
|
||||
'model': body['model'],
|
||||
}
|
||||
finish = 'tool_calls' if message.get('tool_calls') else 'stop'
|
||||
if body.get('stream'):
|
||||
self.stream_response(body, scenario, message, base, finish)
|
||||
else:
|
||||
self.json_response(
|
||||
200,
|
||||
{
|
||||
**base,
|
||||
'object': 'chat.completion',
|
||||
'choices': [{'index': 0, 'message': message, 'finish_reason': finish}],
|
||||
'usage': USAGE,
|
||||
},
|
||||
)
|
||||
except (ValueError, KeyError, TypeError, TimeoutError) as exc:
|
||||
entry.update(status=400, error=str(exc))
|
||||
self.json_response(400, {'error': {'message': str(exc), 'type': 'invalid_request_error'}})
|
||||
except (BrokenPipeError, ConnectionResetError):
|
||||
entry['disconnected'] = True
|
||||
finally:
|
||||
self.server.record(entry)
|
||||
|
||||
def stream_response(self, body, scenario, message, base, finish):
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Type', 'text/event-stream')
|
||||
self.send_header('Cache-Control', 'no-cache')
|
||||
self.end_headers()
|
||||
|
||||
def send(delta, reason=None):
|
||||
payload = {
|
||||
**base,
|
||||
'object': 'chat.completion.chunk',
|
||||
'choices': [{'index': 0, 'delta': delta, 'finish_reason': reason}],
|
||||
}
|
||||
self.wfile.write(('data: ' + json.dumps(payload, ensure_ascii=False) + '\n\n').encode())
|
||||
self.wfile.flush()
|
||||
time.sleep(scenario.get('chunk_delay_seconds', 0))
|
||||
|
||||
send({'role': 'assistant'})
|
||||
content = message.get('content') or ''
|
||||
for offset in range(0, len(content), 8):
|
||||
send({'content': content[offset : offset + 8]})
|
||||
for index, call in enumerate(message.get('tool_calls', [])):
|
||||
send(
|
||||
{
|
||||
'tool_calls': [
|
||||
{
|
||||
'index': index,
|
||||
'id': call['id'],
|
||||
'type': 'function',
|
||||
'function': {
|
||||
'name': call['function']['name'],
|
||||
'arguments': '',
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
arguments = call['function']['arguments']
|
||||
for offset in range(0, len(arguments), 16):
|
||||
send(
|
||||
{
|
||||
'tool_calls': [
|
||||
{
|
||||
'index': index,
|
||||
'function': {'arguments': arguments[offset : offset + 16]},
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
send({}, finish)
|
||||
if (body.get('stream_options') or {}).get('include_usage'):
|
||||
payload = {
|
||||
**base,
|
||||
'object': 'chat.completion.chunk',
|
||||
'choices': [],
|
||||
'usage': USAGE,
|
||||
}
|
||||
self.wfile.write(('data: ' + json.dumps(payload) + '\n\n').encode())
|
||||
self.wfile.write(b'data: [DONE]\n\n')
|
||||
self.wfile.flush()
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--host', default='127.0.0.1')
|
||||
parser.add_argument('--port', type=int, default=18080)
|
||||
parser.add_argument(
|
||||
'--scenarios',
|
||||
type=Path,
|
||||
default=Path(__file__).with_name('mock_llm_scenarios.json'),
|
||||
)
|
||||
parser.add_argument(
|
||||
'--record',
|
||||
type=Path,
|
||||
help='Append request/response JSONL without HTTP headers; use test data only',
|
||||
)
|
||||
args = parser.parse_args()
|
||||
scenarios = json.loads(args.scenarios.read_text(encoding='utf-8'))
|
||||
if not isinstance(scenarios, dict) or not scenarios:
|
||||
parser.error('scenarios must be a non-empty object keyed by model name')
|
||||
for name, scenario in scenarios.items():
|
||||
if not isinstance(scenario, dict) or not isinstance(scenario.get('steps'), list) or not scenario['steps']:
|
||||
parser.error(f'{name}: steps must be a non-empty array')
|
||||
if any(not isinstance(step, dict) for step in scenario['steps']):
|
||||
parser.error(f'{name}: each step must be an object')
|
||||
for step in scenario['steps']:
|
||||
if not isinstance(step.get('content'), str) and not step.get('tool_calls'):
|
||||
parser.error(f'{name}: each step needs content or tool_calls')
|
||||
if 'expect_tool_output' in step and not isinstance(step['expect_tool_output'], str):
|
||||
parser.error(f'{name}: expect_tool_output must be a string')
|
||||
if 'expect_prompt' in step and not isinstance(step['expect_prompt'], str):
|
||||
parser.error(f'{name}: expect_prompt must be a string')
|
||||
if 'tool_calls' in step:
|
||||
calls = step['tool_calls']
|
||||
if not isinstance(calls, list) or any(
|
||||
not isinstance(call, dict)
|
||||
or not isinstance(call.get('name'), str)
|
||||
or not isinstance(call.get('arguments'), dict)
|
||||
for call in calls
|
||||
):
|
||||
parser.error(f'{name}: tool_calls need a name and arguments object')
|
||||
if scenario.get('status', 200) not in (200, 401, 429, 500, 503):
|
||||
parser.error(f'{name}: unsupported injected HTTP status')
|
||||
for key in ('delay_seconds', 'chunk_delay_seconds'):
|
||||
value = scenario.get(key, 0)
|
||||
if not isinstance(value, (int, float)) or not 0 <= value <= 60:
|
||||
parser.error(f'{name}: {key} must be between 0 and 60')
|
||||
with MockServer((args.host, args.port), scenarios, args.record) as server:
|
||||
print(f'Mock LLM: http://{args.host}:{server.server_port}/v1', flush=True)
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
pass
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,169 @@
|
||||
{
|
||||
"mock-chat": {
|
||||
"steps": [{"content": "LANGBOT_CHAT_OK"}]
|
||||
},
|
||||
"mock-rag": {
|
||||
"steps": [{"expect_prompt": "aurora-parser-rag-9137", "content": "LANGBOT_RAG_OK"}]
|
||||
},
|
||||
"mock-box": {
|
||||
"steps": [
|
||||
{"tool_calls": [{"name": "exec", "arguments": {"command": "printf LANGBOT_BOX_SENTINEL > /workspace/mock-llm.txt; cat /workspace/mock-llm.txt"}}]},
|
||||
{"expect_tool_output": "LANGBOT_BOX_SENTINEL", "tool_calls": [{"name": "exec", "arguments": {"command": "cat /workspace/mock-llm.txt"}}]},
|
||||
{"expect_tool_output": "LANGBOT_BOX_SENTINEL", "content": "LANGBOT_BOX_OK"}
|
||||
]
|
||||
},
|
||||
"mock-401": {"status": 401, "steps": [{"content": "unused"}]},
|
||||
"mock-429": {"status": 429, "steps": [{"content": "unused"}]},
|
||||
"mock-503": {"status": 503, "steps": [{"content": "unused"}]},
|
||||
"mock-slow": {"delay_seconds": 2, "steps": [{"content": "LANGBOT_SLOW_OK"}]},
|
||||
"mock-slow-stream": {"chunk_delay_seconds": 0.2, "steps": [{"content": "LANGBOT_SLOW_STREAM_OK"}]},
|
||||
"mock-smoke-box": {
|
||||
"steps": [
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "printf LANGBOT_BOX_SENTINEL > /tmp/langbot-smoke.txt; cat /tmp/langbot-smoke.txt"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "LANGBOT_BOX_SENTINEL",
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "cat /tmp/langbot-smoke.txt"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "LANGBOT_BOX_SENTINEL",
|
||||
"content": "LANGBOT_BOX_OK"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mock-smoke-export": {
|
||||
"steps": [
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "printf LANGBOT_EXPORT_SENTINEL > /tmp/langbot-smoke-output.txt; cat /tmp/langbot-smoke-output.txt"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "LANGBOT_EXPORT_SENTINEL",
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "export_box_files",
|
||||
"arguments": {
|
||||
"paths": [
|
||||
"/tmp/langbot-smoke-output.txt"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "accepted",
|
||||
"content": "LANGBOT_EXPORT_OK"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mock-smoke-unicode": {
|
||||
"steps": [
|
||||
{
|
||||
"expect_prompt": "请原样处理：你好，世界 🌍",
|
||||
"content": "你好，世界 🌍 LANGBOT_UNICODE_OK"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mock-smoke-exec-error": {
|
||||
"steps": [
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "printf EXPECTED_STDOUT; printf EXPECTED_STDERR >&2; exit 7"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "EXPECTED_STDERR",
|
||||
"content": "LANGBOT_EXEC_ERROR_HANDLED"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mock-smoke-parallel-tools": {
|
||||
"steps": [
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "printf PARALLEL_OK_A"
|
||||
}
|
||||
},
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "printf PARALLEL_OK_B"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "PARALLEL_OK_B",
|
||||
"content": "LANGBOT_PARALLEL_TOOLS_OK"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mock-smoke-export-unicode": {
|
||||
"steps": [
|
||||
{
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "exec",
|
||||
"arguments": {
|
||||
"command": "printf '你好，文件 🌍' > /tmp/输出.txt; cat /tmp/输出.txt"
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "你好，文件 🌍",
|
||||
"tool_calls": [
|
||||
{
|
||||
"name": "export_box_files",
|
||||
"arguments": {
|
||||
"paths": [
|
||||
"/tmp/输出.txt"
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"expect_tool_output": "accepted",
|
||||
"content": "LANGBOT_UNICODE_EXPORT_OK"
|
||||
}
|
||||
]
|
||||
},
|
||||
"mock-smoke-rag-other": {
|
||||
"steps": [
|
||||
{
|
||||
"expect_prompt": "cedar-isolation-4826",
|
||||
"content": "LANGBOT_OTHER_RAG_OK"
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,235 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare a disposable Linux CI stack from this checkout, then run the 60s gate."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import secrets
|
||||
import shutil
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
import yaml
|
||||
|
||||
from regression_gate import supervise
|
||||
from regression_suite import Suite
|
||||
from regression_traffic import owned_box_containers, read_endpoint_snapshot
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402
|
||||
|
||||
|
||||
def ready(url, processes, timeout=120):
|
||||
deadline = time.monotonic() + timeout
|
||||
while time.monotonic() < deadline:
|
||||
if any(p.poll() is not None for p in processes):
|
||||
raise RuntimeError('A smoke dependency exited; inspect service logs')
|
||||
try:
|
||||
read_endpoint_snapshot(url, timeout_seconds=1)
|
||||
return
|
||||
except (RuntimeError, OSError):
|
||||
time.sleep(0.2)
|
||||
raise RuntimeError(f'Service did not become ready: {url}')
|
||||
|
||||
|
||||
def run(args):
|
||||
if subprocess.check_output(
|
||||
['docker', 'ps', '-a', '--filter', 'label=langbot.box=true', '--format', '{{.ID}}'], text=True, timeout=10
|
||||
).strip():
|
||||
raise RuntimeError('CI needs a dedicated Docker daemon with no existing Box sandboxes')
|
||||
root = args.output.resolve()
|
||||
root.mkdir(parents=True, exist_ok=False, mode=0o700)
|
||||
private = root / '.private'
|
||||
private.mkdir(mode=0o700)
|
||||
processes, logs = [], []
|
||||
key, runtime_token, box_token = (secrets.token_urlsafe(32) for _ in range(3))
|
||||
env = dict(
|
||||
os.environ,
|
||||
LANGBOT_QA_API_KEY=key,
|
||||
LANGBOT_DATA_ROOT=str(private / 'data'),
|
||||
PYTHONPATH=str(ROOT / 'src'),
|
||||
LITELLM_LOCAL_MODEL_COST_MAP='True',
|
||||
PYTHONUNBUFFERED='1',
|
||||
LANGBOT_PLUGIN_RUNTIME_CONTROL_TOKEN=runtime_token,
|
||||
LANGBOT_BOX_CONTROL_TOKEN=box_token,
|
||||
)
|
||||
|
||||
def start(name, command):
|
||||
log = (root / f'{name}.log').open('w')
|
||||
logs.append(log)
|
||||
process = subprocess.Popen(
|
||||
command, cwd=private, env=env, stdout=log, stderr=subprocess.STDOUT, start_new_session=True
|
||||
)
|
||||
processes.append(process)
|
||||
return process.pid
|
||||
|
||||
try:
|
||||
config_path = create_minimal_config(private, port=15301)
|
||||
create_test_directories(private)
|
||||
config = yaml.safe_load(config_path.read_text())
|
||||
config['api'].update(global_api_key=key, host='127.0.0.1')
|
||||
config['plugin'].update(enable=True, runtime_ws_url='ws://127.0.0.1:15340/control/ws')
|
||||
config['box'].update(enabled=True, backend='local', runtime={'endpoint': 'ws://127.0.0.1:15310/control/ws'})
|
||||
config['box']['local'].update(
|
||||
host_root=str(private / 'box'), allowed_mount_roots=[str(private / 'box')], image='python:3.12-alpine'
|
||||
)
|
||||
config_path.write_text(yaml.safe_dump(config))
|
||||
profile_path = ROOT / 'tools/regression_profiles/smoke.json'
|
||||
profile = json.loads(profile_path.read_text())
|
||||
cli_config = private / 'lbctl.yaml'
|
||||
cli_config.write_text(
|
||||
yaml.safe_dump(
|
||||
{
|
||||
'version': 1,
|
||||
'current_context': 'smoke',
|
||||
'contexts': {
|
||||
'smoke': {
|
||||
'endpoint': 'http://127.0.0.1:15301',
|
||||
'credential': {'type': 'env', 'name': 'LANGBOT_QA_API_KEY'},
|
||||
}
|
||||
},
|
||||
}
|
||||
)
|
||||
)
|
||||
profile['cli'] = {'executable': str(args.lbctl.resolve()), 'config': str(cli_config)}
|
||||
profile['vars'].update(
|
||||
mock_base_url='http://127.0.0.1:18080/v1',
|
||||
embedding_base_url='http://127.0.0.1:18080/v1',
|
||||
core_base_url='http://127.0.0.1:15301',
|
||||
core_health='http://127.0.0.1:15301/healthz',
|
||||
core_db=str(private / 'data/langbot.db'),
|
||||
mock_records=str(root / 'mock-requests.jsonl'),
|
||||
)
|
||||
profile['plugins'] = []
|
||||
# Two turns retain the history/isolation oracle while leaving room for setup and cleanup.
|
||||
profile['smoke']['group_turns'] = 2
|
||||
for resource in profile['resources']:
|
||||
if resource['kind'] == 'model' and resource.get('type') == 'embedding':
|
||||
resource['body']['name'] = 'mock-embedding'
|
||||
for document in resource.get('documents', []):
|
||||
document['file'] = str((profile_path.parent / document['file']).resolve())
|
||||
pids = {}
|
||||
pids['mock'] = start(
|
||||
'mock',
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / 'tools/mock_llm.py'),
|
||||
'--port',
|
||||
'18080',
|
||||
'--record',
|
||||
str(root / 'mock-requests.jsonl'),
|
||||
],
|
||||
)
|
||||
pids['runtime'] = start(
|
||||
'runtime',
|
||||
[
|
||||
sys.executable,
|
||||
'-m',
|
||||
'langbot_plugin.cli.__init__',
|
||||
'rt',
|
||||
'--ws-control-port',
|
||||
'15340',
|
||||
'--ws-debug-port',
|
||||
'15341',
|
||||
],
|
||||
)
|
||||
pids['box'] = start(
|
||||
'box',
|
||||
[
|
||||
sys.executable,
|
||||
'-m',
|
||||
'langbot_plugin.cli.__init__',
|
||||
'box',
|
||||
'--host',
|
||||
'127.0.0.1',
|
||||
'--ws-control-port',
|
||||
'15310',
|
||||
],
|
||||
)
|
||||
pids['core'] = start(
|
||||
'core', [sys.executable, str(ROOT / 'main.py'), '--standalone-runtime', '--standalone-box']
|
||||
)
|
||||
profile['observe'] = {
|
||||
'health': {
|
||||
'core': 'http://127.0.0.1:15301/healthz',
|
||||
'box': 'http://127.0.0.1:15310/readyz',
|
||||
'mock': 'http://127.0.0.1:18080/healthz',
|
||||
},
|
||||
'pids': pids,
|
||||
}
|
||||
for url in profile['observe']['health'].values():
|
||||
ready(url, processes)
|
||||
preparation = root / 'preparation'
|
||||
preparation.mkdir()
|
||||
suite = Suite(profile, profile_path, preparation, env)
|
||||
identity = suite.call(['whoami'])
|
||||
# Bind the CLI to the newly bootstrapped Workspace before installing plugins.
|
||||
cli_data = yaml.safe_load(cli_config.read_text())
|
||||
cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid']
|
||||
cli_config.write_text(yaml.safe_dump(cli_data))
|
||||
for directory in ('Runner/LocalAgent', 'KnowledgeEngine/LangRAG', 'misc/GeneralParsers'):
|
||||
source = args.plugins / directory
|
||||
package_source = private / source.name
|
||||
shutil.copytree(source, package_source, ignore=shutil.ignore_patterns('.git', '.venv*', '__pycache__'))
|
||||
archive = shutil.make_archive(str(private / source.name), 'zip', root_dir=package_source)
|
||||
suite.call(['plugin', 'install', 'local', '--file', archive, '--wait'])
|
||||
generated = private / 'smoke.json'
|
||||
generated.write_text(json.dumps(profile, indent=2))
|
||||
return supervise(
|
||||
[
|
||||
sys.executable,
|
||||
str(ROOT / 'tools/regression_suite.py'),
|
||||
'run',
|
||||
'--profile',
|
||||
str(generated),
|
||||
'--output',
|
||||
str(root / 'gate/suite'),
|
||||
],
|
||||
root / 'gate',
|
||||
60,
|
||||
pids,
|
||||
env,
|
||||
)
|
||||
finally:
|
||||
for process in reversed(processes):
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
for log in logs:
|
||||
log.close()
|
||||
# This is a disposable CI daemon; still delete only this run's mounted sandboxes.
|
||||
for container in owned_box_containers(private / 'box'):
|
||||
subprocess.run(['docker', 'rm', '-f', container], check=True, capture_output=True)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--lbctl', type=Path, required=True)
|
||||
parser.add_argument('--plugins', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
if args.output.exists():
|
||||
parser.error('--output must be a new directory')
|
||||
try:
|
||||
result = run(args)
|
||||
except Exception as exc:
|
||||
verdict = args.output / 'gate' / 'gate.json'
|
||||
result = json.loads(verdict.read_text()) if verdict.exists() else {}
|
||||
result.update(passed=False, infrastructure_error=f'{type(exc).__name__}: {exc}')
|
||||
verdict.parent.mkdir(parents=True, exist_ok=True)
|
||||
verdict.write_text(json.dumps(result, indent=2) + '\n')
|
||||
raise
|
||||
print(json.dumps(result, indent=2))
|
||||
raise SystemExit(0 if result['passed'] else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Bound a prepared-environment smoke run and record process resource samples."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
from regression_suite import Suite, validate
|
||||
from regression_traffic import read_process_snapshot
|
||||
|
||||
|
||||
def supervise(command, output, budget, pids, env=None):
|
||||
output.mkdir(parents=True, exist_ok=False, mode=0o700)
|
||||
started = time.monotonic()
|
||||
samples, errors = [], []
|
||||
timed_out = False
|
||||
with (output / 'suite.log').open('w') as log:
|
||||
process = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, env=env, start_new_session=True)
|
||||
try:
|
||||
while process.poll() is None:
|
||||
remaining = budget - (time.monotonic() - started)
|
||||
if remaining <= 0:
|
||||
timed_out = True
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
break
|
||||
sample = {'elapsed_seconds': time.monotonic() - started, 'processes': {}}
|
||||
for name, pid in pids.items():
|
||||
try:
|
||||
sample['processes'][name] = read_process_snapshot(pid)
|
||||
except (RuntimeError, OSError) as exc:
|
||||
errors.append(f'{name}: {exc}')
|
||||
samples.append(sample)
|
||||
remaining = budget - (time.monotonic() - started)
|
||||
try:
|
||||
process.wait(timeout=max(0, min(1, remaining)))
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.wait()
|
||||
elapsed = time.monotonic() - started
|
||||
(output / 'resources.jsonl').write_text(''.join(json.dumps(sample) + '\n' for sample in samples))
|
||||
summary_path = output / 'suite' / 'summary.json'
|
||||
summary = json.loads(summary_path.read_text()) if summary_path.exists() else {}
|
||||
result = {
|
||||
'passed': process.returncode == 0
|
||||
and summary.get('passed') is True
|
||||
and elapsed <= budget
|
||||
and not timed_out
|
||||
and not errors,
|
||||
'budget_seconds': budget,
|
||||
'elapsed_seconds': elapsed,
|
||||
'timed_out': timed_out,
|
||||
'exit_code': process.returncode,
|
||||
'resource_errors': errors,
|
||||
'requests': summary.get('requests'),
|
||||
'status': summary.get('status'),
|
||||
'stages': summary.get('stages'),
|
||||
'resource_peaks': {
|
||||
name: {
|
||||
metric: max((s['processes'].get(name, {}).get(metric, 0) for s in samples), default=0)
|
||||
for metric in ('rss_bytes', 'threads', 'open_fds')
|
||||
}
|
||||
for name in pids
|
||||
},
|
||||
}
|
||||
(output / 'gate.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--profile', type=Path, required=True)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
profile = json.loads(args.profile.read_text())
|
||||
validate(profile)
|
||||
if 'smoke' not in profile:
|
||||
parser.error('a smoke profile is required')
|
||||
suite = Suite(profile, args.profile.resolve(), args.output / 'suite')
|
||||
pids = dict(profile.get('observe', {}).get('pids', {}))
|
||||
for name, path in profile.get('observe', {}).get('pid_files', {}).items():
|
||||
pids[name] = int(Path(suite.path(path)).read_text().strip())
|
||||
result = supervise(
|
||||
[
|
||||
sys.executable,
|
||||
str(Path(__file__).with_name('regression_suite.py')),
|
||||
'run',
|
||||
'--profile',
|
||||
str(args.profile.resolve()),
|
||||
'--output',
|
||||
str(args.output.resolve() / 'suite'),
|
||||
],
|
||||
args.output,
|
||||
60,
|
||||
pids,
|
||||
)
|
||||
print(json.dumps(result, indent=2))
|
||||
raise SystemExit(0 if result['passed'] else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,480 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Own one complete lbctl deployment, mock backend, regression, and teardown."""
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import shutil
|
||||
import subprocess
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
|
||||
import yaml
|
||||
|
||||
from regression_suite import Suite, number, resolve, validate
|
||||
from regression_traffic import owned_box_containers
|
||||
|
||||
|
||||
KEY_ENV = 'LANGBOT_QA_API_KEY'
|
||||
|
||||
|
||||
def write_json(path, data):
|
||||
temporary = path.with_suffix(path.suffix + '.tmp')
|
||||
temporary.write_text(json.dumps(data, ensure_ascii=False, indent=2))
|
||||
temporary.chmod(0o600)
|
||||
temporary.replace(path)
|
||||
|
||||
|
||||
def load_profile(path):
|
||||
profile = json.loads(path.read_text())
|
||||
if not isinstance(profile, dict) or profile.get('version') != 1:
|
||||
raise ValueError('job profile version must be 1')
|
||||
if profile.keys() - {'version', 'lbctl', 'suite', 'deployment', 'vars', 'cleanup', 'purge_data', 'mock_image'}:
|
||||
raise ValueError('unknown job profile field')
|
||||
deployment = profile.get('deployment', {})
|
||||
if not isinstance(deployment, dict) or deployment.keys() - {'version', 'port'}:
|
||||
raise ValueError('deployment accepts version and port')
|
||||
version = deployment.get('version', '')
|
||||
if not isinstance(version, str) or not re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', version):
|
||||
raise ValueError('deployment.version must pin a stable vX.Y.Z Release')
|
||||
number(deployment.get('port', 5300), 'deployment port', minimum=1, integer=True)
|
||||
if deployment.get('port', 5300) > 65535:
|
||||
raise ValueError('deployment port must be <= 65535')
|
||||
if profile.get('cleanup', 'on-success') not in ('always', 'on-success', 'never'):
|
||||
raise ValueError('cleanup must be always/on-success/never')
|
||||
if not isinstance(profile.get('purge_data', True), bool):
|
||||
raise ValueError('purge_data must be boolean')
|
||||
suite_path = Path(profile['suite']).expanduser()
|
||||
suite_path = suite_path if suite_path.is_absolute() else path.parent / suite_path
|
||||
suite = json.loads(suite_path.read_text())
|
||||
suite.setdefault('vars', {}).update(profile.get('vars', {}))
|
||||
suite['vars'].update(
|
||||
mock_base_url='http://regression_mock:18080/v1', embedding_base_url='http://regression_mock:18080/v1'
|
||||
)
|
||||
suite['cli'] = {'config': 'job-generated.yaml'}
|
||||
for item in suite.get('resources', []):
|
||||
if item['kind'] == 'model' and item.get('type') == 'embedding':
|
||||
item['body']['name'] = 'mock-embedding'
|
||||
validate(suite)
|
||||
return profile, suite, suite_path.resolve()
|
||||
|
||||
|
||||
class Job:
|
||||
def __init__(self, profile, suite_profile, suite_path, output):
|
||||
self.profile = profile
|
||||
self.suite_profile = copy.deepcopy(suite_profile)
|
||||
self.suite_path = suite_path
|
||||
self.output = output.resolve()
|
||||
self.private = self.output / '.private'
|
||||
self.deploy = self.private / 'deployment'
|
||||
self.state_path = self.output / 'state.json'
|
||||
self.cli_config = self.private / 'lbctl.yaml'
|
||||
self.env = {**os.environ, 'XDG_CONFIG_HOME': str(self.private / 'config')}
|
||||
self.cli = [profile.get('lbctl', 'lbctl'), '--config', str(self.cli_config), '-o', 'json']
|
||||
self.redactions = []
|
||||
self.state = {'job_id': secrets.token_hex(8), 'record': None, 'bound': False, 'phase': 'created'}
|
||||
|
||||
def redact(self, value):
|
||||
for secret in self.redactions:
|
||||
value = value.replace(secret, '[REDACTED]')
|
||||
return value
|
||||
|
||||
def save(self):
|
||||
write_json(self.state_path, self.state)
|
||||
|
||||
def recover_record(self):
|
||||
if (self.private / 'job-id').read_text().strip() != self.state['job_id']:
|
||||
raise RuntimeError('Job ownership marker differs from state.json')
|
||||
path = self.deploy / 'deployment.yaml'
|
||||
if not path.exists():
|
||||
if self.state.get('record'):
|
||||
raise RuntimeError('deployment record disappeared; refusing cleanup')
|
||||
return
|
||||
if path.is_symlink():
|
||||
raise RuntimeError('deployment record must not be a symlink')
|
||||
try:
|
||||
record = yaml.safe_load(path.read_text())
|
||||
except yaml.YAMLError as exc:
|
||||
raise ValueError('invalid deployment YAML') from exc
|
||||
if not isinstance(record, dict) or not record.get('deployment_id'):
|
||||
raise RuntimeError('invalid deployment record')
|
||||
if (
|
||||
Path(record['compose_file']).resolve().parent != self.deploy
|
||||
or Path(record['data_dir']).resolve() != self.deploy / 'data'
|
||||
or Path(record['env_file']).resolve().parent != self.deploy
|
||||
):
|
||||
raise RuntimeError('deployment record is outside this Job')
|
||||
if self.state.get('record') and record['deployment_id'] != self.state['record']['deployment_id']:
|
||||
raise RuntimeError('deployment identity changed; refusing cleanup')
|
||||
self.state['record'] = record
|
||||
self.save()
|
||||
|
||||
def load_redactions(self):
|
||||
if self.state.get('record'):
|
||||
path = Path(self.state['record']['env_file'])
|
||||
if path.exists():
|
||||
for line in path.read_text().splitlines():
|
||||
name, separator, value = line.partition('=')
|
||||
if separator and any(word in name for word in ('TOKEN', 'KEY')) and len(value) >= 16:
|
||||
self.redactions.append(value)
|
||||
|
||||
def command(self, args, timeout=900):
|
||||
result = subprocess.run(args, capture_output=True, text=True, env=self.env, timeout=timeout)
|
||||
return result.returncode, self.redact(result.stdout), self.redact(result.stderr)
|
||||
|
||||
def checked(self, args, timeout=900):
|
||||
code, stdout, stderr = self.command(args, timeout)
|
||||
if code:
|
||||
raise RuntimeError((stderr + stdout)[-4000:])
|
||||
return stdout
|
||||
|
||||
def cli_call(self, args):
|
||||
code, stdout, stderr = self.command([*self.cli, *args])
|
||||
result = json.loads(stdout) if stdout.strip() else {}
|
||||
data = result.get('data', {})
|
||||
if args[0] == 'install' and isinstance(data, dict) and data.get('record'):
|
||||
record = data['record']
|
||||
if (
|
||||
Path(record['compose_file']).resolve().parent != self.deploy
|
||||
or Path(record['data_dir']).resolve() != self.deploy / 'data'
|
||||
):
|
||||
raise RuntimeError('install returned a deployment outside this Job')
|
||||
self.state['record'] = record
|
||||
self.save() # Installation may have changed state even when its final health check failed.
|
||||
if code:
|
||||
raise RuntimeError((stderr + stdout)[-4000:])
|
||||
return data
|
||||
|
||||
def compose(self, *args, override=False):
|
||||
record = self.state['record']
|
||||
command = ['docker', 'compose', '--project-name', record['compose_project'], '--file', record['compose_file']]
|
||||
if override:
|
||||
command += ['--file', str(self.private / 'compose.override.json')]
|
||||
command += ['--env-file', record['env_file'], '--profile', 'all', *args]
|
||||
return self.checked(command)
|
||||
|
||||
def project_containers(self):
|
||||
project = self.state['record']['compose_project']
|
||||
output = self.checked(
|
||||
[
|
||||
'docker',
|
||||
'ps',
|
||||
'-a',
|
||||
'--filter',
|
||||
f'label=com.docker.compose.project={project}',
|
||||
'--format',
|
||||
'{{.ID}} {{.Label "com.docker.compose.service"}}',
|
||||
]
|
||||
)
|
||||
return {service: container for container, service in (line.split() for line in output.splitlines() if line)}
|
||||
|
||||
def address(self, container):
|
||||
networks = json.loads(
|
||||
self.checked(['docker', 'inspect', '--format', '{{json .NetworkSettings.Networks}}', container])
|
||||
)
|
||||
addresses = [network['IPAddress'] for network in networks.values() if network.get('IPAddress')]
|
||||
if len(addresses) != 1:
|
||||
raise RuntimeError('expected one private container network')
|
||||
return addresses[0]
|
||||
|
||||
@staticmethod
|
||||
def ready(url):
|
||||
deadline = time.monotonic() + 120
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
while time.monotonic() < deadline:
|
||||
try:
|
||||
with opener.open(url, timeout=2) as response:
|
||||
payload = json.load(response)
|
||||
if response.status == 200 and payload.get('ready') is not False and payload.get('code', 0) == 0:
|
||||
return
|
||||
except (urllib.error.URLError, TimeoutError, OSError, ValueError):
|
||||
pass
|
||||
time.sleep(0.5)
|
||||
raise RuntimeError(f'readiness deadline exceeded: {url}')
|
||||
|
||||
def prepare(self):
|
||||
variables = {
|
||||
**self.suite_profile.get('vars', {}),
|
||||
'profile_dir': str(self.suite_path.parent),
|
||||
'run_dir': str(self.output),
|
||||
}
|
||||
inputs = [plugin['file'] for plugin in self.suite_profile.get('plugins', [])]
|
||||
inputs += [
|
||||
document['file']
|
||||
for item in self.suite_profile.get('resources', [])
|
||||
for document in item.get('documents', [])
|
||||
]
|
||||
for name in inputs:
|
||||
path = Path(resolve(name, variables)).expanduser()
|
||||
path = path if path.is_absolute() else self.suite_path.parent / path
|
||||
if not path.is_file():
|
||||
raise ValueError(f'Job input file is missing: {path}')
|
||||
self.state['phase'] = 'install'
|
||||
self.save()
|
||||
# Current Box runtimes reap sandboxes from previous runtime IDs on startup.
|
||||
# Do not start one alongside another user's live sandbox on this daemon.
|
||||
boxes = self.checked(['docker', 'ps', '-a', '--filter', 'label=langbot.box=true', '--format', '{{.ID}}'])
|
||||
if boxes.strip():
|
||||
raise RuntimeError('Docker daemon has existing Box sandboxes; use an idle dedicated test daemon')
|
||||
key = secrets.token_urlsafe(32)
|
||||
self.env[KEY_ENV] = key
|
||||
self.env['LANGBOT_API_KEY'] = key
|
||||
self.redactions.append(key)
|
||||
(self.private / 'api-key').write_text(key)
|
||||
(self.private / 'api-key').chmod(0o600)
|
||||
deployment = self.profile['deployment']
|
||||
self.cli_call(
|
||||
[
|
||||
'install',
|
||||
'--dir',
|
||||
str(self.deploy),
|
||||
'--version',
|
||||
deployment['version'],
|
||||
'--profile',
|
||||
'all',
|
||||
'--port',
|
||||
str(deployment.get('port', 5300)),
|
||||
]
|
||||
)
|
||||
self.recover_record()
|
||||
self.load_redactions()
|
||||
mock_results = self.output / 'mock'
|
||||
mock_results.mkdir(mode=0o700)
|
||||
override = {
|
||||
'services': {
|
||||
'langbot': {
|
||||
'environment': {'API__GLOBAL_API_KEY': key},
|
||||
'volumes': [f'{self.deploy / "data/box"}:{self.deploy / "data/box"}'],
|
||||
},
|
||||
'regression_mock': {
|
||||
'image': self.profile.get('mock_image', 'python:3.12-alpine'),
|
||||
'volumes': [f'{Path(__file__).parent.resolve()}:/regression:ro', f'{mock_results}:/results'],
|
||||
'command': [
|
||||
'python',
|
||||
'/regression/mock_llm.py',
|
||||
'--host',
|
||||
'0.0.0.0',
|
||||
'--port',
|
||||
'18080',
|
||||
'--record',
|
||||
'/results/requests.jsonl',
|
||||
],
|
||||
'networks': ['langbot_network'],
|
||||
'labels': {'langbot.regression_job': self.state['job_id']},
|
||||
},
|
||||
},
|
||||
}
|
||||
write_json(self.private / 'compose.override.json', override)
|
||||
self.compose('up', '-d', override=True)
|
||||
containers = self.project_containers()
|
||||
for service in ('langbot', 'langbot_plugin_runtime', 'langbot_box', 'regression_mock'):
|
||||
if service not in containers:
|
||||
raise RuntimeError(f'missing deployment service: {service}')
|
||||
endpoint = self.state['record']['endpoint']
|
||||
health = {
|
||||
'core': endpoint + '/healthz',
|
||||
'mock': f'http://{self.address(containers["regression_mock"])}:18080/healthz',
|
||||
'box': f'http://{self.address(containers["langbot_box"])}:5410/readyz',
|
||||
}
|
||||
for url in health.values():
|
||||
self.ready(url)
|
||||
identity = self.cli_call(['--endpoint', endpoint, 'whoami'])
|
||||
self.cli_call(
|
||||
[
|
||||
'context',
|
||||
'add',
|
||||
'regression',
|
||||
'--endpoint',
|
||||
endpoint,
|
||||
'--api-key-env',
|
||||
KEY_ENV,
|
||||
'--expect-workspace',
|
||||
identity['workspace_uuid'],
|
||||
]
|
||||
)
|
||||
self.cli_call(['context', 'use', 'regression'])
|
||||
self.state.update(
|
||||
bound=True, workspace_uuid=identity['workspace_uuid'], containers=containers, phase='resources'
|
||||
)
|
||||
self.save()
|
||||
profile = self.suite_profile
|
||||
profile.setdefault('vars', {})['core_base_url'] = endpoint
|
||||
if 'smoke' in profile:
|
||||
if 'mock_records' in profile['smoke']:
|
||||
profile['smoke']['mock_records'] = str(self.output / 'mock' / 'requests.jsonl')
|
||||
profile['smoke']['ledger_db'] = str(Path(self.state['record']['data_dir']) / 'langbot.db')
|
||||
profile['cli'] = {'executable': self.cli[0], 'config': str(self.cli_config)}
|
||||
profile['observe'] = {'health': health, 'pids': {}, 'containers': list(containers.values())}
|
||||
for service, container in containers.items():
|
||||
pid = int(self.checked(['docker', 'inspect', '--format', '{{.State.Pid}}', container]).strip())
|
||||
profile['observe']['pids'][service] = pid
|
||||
destination = self.output / 'suite'
|
||||
destination.mkdir(mode=0o700)
|
||||
suite = Suite(profile, self.suite_path, destination, env=self.env)
|
||||
self.redactions.extend(suite.secrets)
|
||||
suite.write('profile.json', profile)
|
||||
suite.prepare()
|
||||
if 'smoke' in profile:
|
||||
return suite
|
||||
# Prime every configured business case before measuring steady load.
|
||||
for case in profile['cases']:
|
||||
case = resolve(case, suite.variables)
|
||||
if 'http_group' in case:
|
||||
continue # Suite preparation already verifies signed group ingress and the masked secret.
|
||||
data = suite.call(case['args'], case.get('body'))
|
||||
if case['expect'] not in json.dumps(data, ensure_ascii=False):
|
||||
raise RuntimeError(f'warmup oracle failed: {case["name"]}')
|
||||
profile['observe']['box_data_root'] = str(Path(self.state['record']['data_dir']) / 'box')
|
||||
suite.write('profile.json', profile)
|
||||
return suite
|
||||
|
||||
def owned_sandboxes(self):
|
||||
record = self.state.get('record')
|
||||
if not record:
|
||||
return []
|
||||
return owned_box_containers(Path(record['data_dir']) / 'box', self.checked)
|
||||
|
||||
def collect(self):
|
||||
self.recover_record()
|
||||
self.load_redactions()
|
||||
if self.state.get('record'):
|
||||
logs = self.compose('logs', '--no-color', '--tail', '10000')
|
||||
(self.output / 'services.log').write_text(logs)
|
||||
|
||||
def cleanup(self):
|
||||
self.recover_record()
|
||||
self.load_redactions()
|
||||
record = self.state.get('record')
|
||||
if self.state['phase'] == 'cleaned':
|
||||
if record and self.project_containers():
|
||||
raise RuntimeError('containers appeared after Job cleanup; inspect deployment before acting')
|
||||
return
|
||||
if record:
|
||||
if (
|
||||
Path(record['compose_file']).resolve().parent != self.deploy
|
||||
or Path(record['data_dir']).resolve() != self.deploy / 'data'
|
||||
):
|
||||
raise RuntimeError('cleanup refused: deployment is outside this Job')
|
||||
if self.state['bound']:
|
||||
self.cli_call(['stop'])
|
||||
else:
|
||||
self.compose('stop', override=(self.private / 'compose.override.json').exists())
|
||||
sandboxes = self.owned_sandboxes()
|
||||
if sandboxes:
|
||||
self.checked(['docker', 'rm', '-f', *sandboxes])
|
||||
if self.state['bound']:
|
||||
self.cli_call(
|
||||
['uninstall', '--yes', *(['--purge-data'] if self.profile.get('purge_data', True) else [])]
|
||||
)
|
||||
else:
|
||||
self.compose('down', '--remove-orphans', override=(self.private / 'compose.override.json').exists())
|
||||
if self.profile.get('purge_data', True) and (self.deploy / 'data').exists():
|
||||
shutil.rmtree(self.deploy / 'data')
|
||||
for path in (self.private / 'api-key', self.private / 'compose.override.json'):
|
||||
path.unlink(missing_ok=True)
|
||||
self.state['phase'] = 'cleaned'
|
||||
self.save()
|
||||
|
||||
def run(self):
|
||||
self.output.mkdir(parents=True, exist_ok=False, mode=0o700)
|
||||
self.private.mkdir(mode=0o700)
|
||||
(self.private / 'job-id').write_text(self.state['job_id'])
|
||||
write_json(self.output / 'job-profile.json', self.profile)
|
||||
self.save()
|
||||
result = {'errors': [], 'completed': False}
|
||||
try:
|
||||
suite = self.prepare()
|
||||
self.state['phase'] = 'traffic'
|
||||
self.save()
|
||||
summary = suite.run()
|
||||
summary['passed'] = not summary['errors']
|
||||
suite.write('summary.json', summary)
|
||||
result.update(completed=True, suite=summary, errors=list(summary['errors']))
|
||||
self.state['phase'] = 'finished'
|
||||
self.save()
|
||||
except KeyboardInterrupt:
|
||||
result['errors'].append('Job interrupted; in-flight outcomes may be unknown')
|
||||
except (RuntimeError, ValueError, OSError, KeyError, subprocess.TimeoutExpired) as exc:
|
||||
result['errors'].append(self.redact(str(exc)))
|
||||
finally:
|
||||
if not result['completed'] and not result['errors']:
|
||||
result['errors'].append('Job aborted before completion')
|
||||
try:
|
||||
self.collect()
|
||||
except (RuntimeError, ValueError, OSError, subprocess.TimeoutExpired) as exc:
|
||||
result['errors'].append('artifact collection: ' + self.redact(str(exc)))
|
||||
policy = self.profile.get('cleanup', 'on-success')
|
||||
if policy == 'always' or (policy == 'on-success' and not result['errors']):
|
||||
try:
|
||||
self.cleanup()
|
||||
except (RuntimeError, ValueError, OSError, KeyError, subprocess.TimeoutExpired) as exc:
|
||||
result['errors'].append('cleanup: ' + self.redact(str(exc)))
|
||||
result.update(passed=not result['errors'], phase=self.state['phase'])
|
||||
write_json(self.output / 'summary.json', result)
|
||||
return result
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=('validate', 'run', 'cleanup'))
|
||||
parser.add_argument('--profile', type=Path)
|
||||
parser.add_argument('--output', type=Path)
|
||||
parser.add_argument('--duration', type=float)
|
||||
parser.add_argument('--rps', type=float)
|
||||
parser.add_argument('--workers', type=int)
|
||||
parser.add_argument('--groups', type=int)
|
||||
parser.add_argument('--members-per-group', type=int)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
if args.action == 'cleanup':
|
||||
if args.output is None:
|
||||
raise ValueError('--output is required for cleanup')
|
||||
profile = json.loads((args.output / 'job-profile.json').read_text())
|
||||
job = Job(profile, {}, args.output / 'unused-profile.json', args.output)
|
||||
job.state = json.loads(job.state_path.read_text())
|
||||
if (job.private / 'api-key').exists():
|
||||
key = (job.private / 'api-key').read_text()
|
||||
job.env[KEY_ENV] = key
|
||||
job.env['LANGBOT_API_KEY'] = key
|
||||
job.redactions.append(key)
|
||||
job.cleanup()
|
||||
return
|
||||
if args.profile is None:
|
||||
raise ValueError('--profile is required for run/validate')
|
||||
profile, suite, path = load_profile(args.profile.resolve())
|
||||
if 'smoke' in suite and any(
|
||||
getattr(args, flag) is not None for flag in ('duration', 'rps', 'workers', 'groups', 'members_per_group')
|
||||
):
|
||||
raise ValueError('Load overrides are not applicable to smoke mode')
|
||||
for phase in suite.get('phases', []):
|
||||
for flag, field in [
|
||||
('duration', 'duration_seconds'),
|
||||
('rps', 'rps'),
|
||||
('workers', 'workers'),
|
||||
('groups', 'groups'),
|
||||
('members_per_group', 'members_per_group'),
|
||||
]:
|
||||
if getattr(args, flag) is not None:
|
||||
phase[field] = getattr(args, flag)
|
||||
validate(suite)
|
||||
if args.action == 'validate':
|
||||
print('Job profile valid; no deployment or backend changed.')
|
||||
return
|
||||
if args.output is None:
|
||||
raise ValueError('--output is required for run')
|
||||
job = Job(profile, suite, path, args.output)
|
||||
result = job.run()
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
if not result['passed']:
|
||||
raise SystemExit(1)
|
||||
except (RuntimeError, ValueError, OSError, KeyError, subprocess.TimeoutExpired) as exc:
|
||||
parser.exit(1, f'{type(exc).__name__}: {exc}\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,464 @@
|
||||
{
|
||||
"version": 1,
|
||||
"cli": {
|
||||
"executable": "lbctl",
|
||||
"config": "./lbctl.yaml"
|
||||
},
|
||||
"vars": {
|
||||
"plugin_dir": "/path/to/plugin-zips",
|
||||
"mock_base_url": "http://127.0.0.1:18080/v1",
|
||||
"embedding_base_url": "http://127.0.0.1:15410/v1",
|
||||
"core_health": "http://127.0.0.1:5300/healthz",
|
||||
"core_base_url": "http://127.0.0.1:5300"
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"source": "local",
|
||||
"file": "${plugin_dir}/LocalAgent.zip"
|
||||
},
|
||||
{
|
||||
"source": "local",
|
||||
"file": "${plugin_dir}/LangRAG.zip"
|
||||
},
|
||||
{
|
||||
"source": "local",
|
||||
"file": "${plugin_dir}/GeneralParsers.zip"
|
||||
}
|
||||
],
|
||||
"resources": [
|
||||
{
|
||||
"id": "mock_provider",
|
||||
"kind": "provider",
|
||||
"body": {
|
||||
"name": "regression mock",
|
||||
"requester": "openai-chat-completions",
|
||||
"base_url": "${mock_base_url}",
|
||||
"api_keys": [
|
||||
"test-only"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "embedding_provider",
|
||||
"kind": "provider",
|
||||
"body": {
|
||||
"name": "regression embedding",
|
||||
"requester": "openai-chat-completions",
|
||||
"base_url": "${embedding_base_url}",
|
||||
"api_keys": [
|
||||
"test-only"
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chat_model",
|
||||
"kind": "model",
|
||||
"type": "llm",
|
||||
"body": {
|
||||
"name": "mock-chat",
|
||||
"provider_uuid": "${mock_provider}",
|
||||
"abilities": [
|
||||
"func_call"
|
||||
],
|
||||
"context_length": 8192,
|
||||
"extra_args": {
|
||||
"request_policy": {
|
||||
"retries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "box_model",
|
||||
"kind": "model",
|
||||
"type": "llm",
|
||||
"body": {
|
||||
"name": "mock-box",
|
||||
"provider_uuid": "${mock_provider}",
|
||||
"abilities": [
|
||||
"func_call"
|
||||
],
|
||||
"context_length": 8192,
|
||||
"extra_args": {
|
||||
"request_policy": {
|
||||
"retries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "rag_model",
|
||||
"kind": "model",
|
||||
"type": "llm",
|
||||
"body": {
|
||||
"name": "mock-rag",
|
||||
"provider_uuid": "${mock_provider}",
|
||||
"abilities": [
|
||||
"func_call"
|
||||
],
|
||||
"context_length": 8192,
|
||||
"extra_args": {
|
||||
"request_policy": {
|
||||
"retries": 0
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "embedding_model",
|
||||
"kind": "model",
|
||||
"type": "embedding",
|
||||
"body": {
|
||||
"name": "all-MiniLM-L6-v2",
|
||||
"provider_uuid": "${embedding_provider}"
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "knowledge_base",
|
||||
"kind": "knowledge-base",
|
||||
"body": {
|
||||
"name": "regression parser golden",
|
||||
"knowledge_engine_plugin_id": "langbot-team/LangRAG",
|
||||
"creation_settings": {
|
||||
"embedding_model_uuid": "${embedding_model}",
|
||||
"index_type": "chunk",
|
||||
"chunk_size": 512,
|
||||
"overlap": 50
|
||||
},
|
||||
"retrieval_settings": {
|
||||
"search_type": "vector",
|
||||
"top_k": 5,
|
||||
"query_rewrite": "off",
|
||||
"rerank": "off"
|
||||
}
|
||||
},
|
||||
"documents": [
|
||||
{
|
||||
"file": "../../skills/skills/langbot-testing/fixtures/rag/parser-golden.html",
|
||||
"parser_plugin_id": "langbot-team/GeneralParsers"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "chat_pipeline",
|
||||
"kind": "pipeline",
|
||||
"body": {
|
||||
"name": "regression chat",
|
||||
"description": "Normal deterministic chat",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {
|
||||
"id": "plugin:langbot-team/LocalAgent/default",
|
||||
"expire-time": 0
|
||||
},
|
||||
"runner_config": {
|
||||
"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {
|
||||
"primary": "${chat_model}",
|
||||
"fallbacks": []
|
||||
},
|
||||
"prompt": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a QA regression assistant."
|
||||
}
|
||||
],
|
||||
"knowledge-bases": [],
|
||||
"box-enabled": false,
|
||||
"timeout": 30,
|
||||
"max-round": 20,
|
||||
"box-session-id-template": "{launcher_type}_{launcher_id}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"safety": {
|
||||
"rate-limit": {
|
||||
"limitation": 10000
|
||||
}
|
||||
},
|
||||
"output": {
|
||||
"force-delay": {
|
||||
"min": 0,
|
||||
"max": 0
|
||||
},
|
||||
"misc": {
|
||||
"at-sender": false,
|
||||
"quote-origin": false
|
||||
}
|
||||
},
|
||||
"trigger": {
|
||||
"group-respond-rules": {
|
||||
"prefix": [
|
||||
"ai "
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "box_pipeline",
|
||||
"kind": "pipeline",
|
||||
"body": {
|
||||
"name": "regression Box",
|
||||
"description": "Real Box write and read",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {
|
||||
"id": "plugin:langbot-team/LocalAgent/default",
|
||||
"expire-time": 0
|
||||
},
|
||||
"runner_config": {
|
||||
"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {
|
||||
"primary": "${box_model}",
|
||||
"fallbacks": []
|
||||
},
|
||||
"prompt": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a QA regression assistant."
|
||||
}
|
||||
],
|
||||
"knowledge-bases": [],
|
||||
"box-enabled": true,
|
||||
"box-session-id-template": "{launcher_type}_{launcher_id}",
|
||||
"timeout": 30,
|
||||
"max-tool-iterations": 5,
|
||||
"max-round": 20
|
||||
}
|
||||
}
|
||||
},
|
||||
"safety": {
|
||||
"rate-limit": {
|
||||
"limitation": 10000
|
||||
}
|
||||
},
|
||||
"output": {
|
||||
"force-delay": {
|
||||
"min": 0,
|
||||
"max": 0
|
||||
},
|
||||
"misc": {
|
||||
"at-sender": false,
|
||||
"quote-origin": false
|
||||
}
|
||||
},
|
||||
"trigger": {
|
||||
"group-respond-rules": {
|
||||
"prefix": [
|
||||
"ai "
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "rag_pipeline",
|
||||
"kind": "pipeline",
|
||||
"body": {
|
||||
"name": "regression RAG",
|
||||
"description": "Retrieved model input oracle",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {
|
||||
"id": "plugin:langbot-team/LocalAgent/default",
|
||||
"expire-time": 0
|
||||
},
|
||||
"runner_config": {
|
||||
"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {
|
||||
"primary": "${rag_model}",
|
||||
"fallbacks": []
|
||||
},
|
||||
"prompt": [
|
||||
{
|
||||
"role": "system",
|
||||
"content": "You are a QA regression assistant."
|
||||
}
|
||||
],
|
||||
"knowledge-bases": [
|
||||
"${knowledge_base}"
|
||||
],
|
||||
"box-enabled": false,
|
||||
"timeout": 30,
|
||||
"retrieval-top-k": 5,
|
||||
"max-round": 20,
|
||||
"box-session-id-template": "{launcher_type}_{launcher_id}"
|
||||
}
|
||||
}
|
||||
},
|
||||
"safety": {
|
||||
"rate-limit": {
|
||||
"limitation": 10000
|
||||
}
|
||||
},
|
||||
"output": {
|
||||
"force-delay": {
|
||||
"min": 0,
|
||||
"max": 0
|
||||
},
|
||||
"misc": {
|
||||
"at-sender": false,
|
||||
"quote-origin": false
|
||||
}
|
||||
},
|
||||
"trigger": {
|
||||
"group-respond-rules": {
|
||||
"prefix": [
|
||||
"ai "
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "chat_bot",
|
||||
"kind": "bot",
|
||||
"body": {
|
||||
"name": "regression group chat",
|
||||
"description": "Synthetic signed group ingress",
|
||||
"adapter": "http_bot",
|
||||
"enable": true,
|
||||
"adapter_config": {
|
||||
"inbound_secret": "${group_secret}",
|
||||
"signature_required": true,
|
||||
"callback_url": "http://127.0.0.1:9/unused-sync-only",
|
||||
"default_session_type": "group",
|
||||
"callback_timeout": 20,
|
||||
"callback_max_retries": 0
|
||||
},
|
||||
"event_bindings": [
|
||||
{
|
||||
"event_pattern": "message.received",
|
||||
"target_type": "pipeline",
|
||||
"target_uuid": "${chat_pipeline}",
|
||||
"enabled": true,
|
||||
"filters": []
|
||||
}
|
||||
],
|
||||
"plugin_processors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "box_bot",
|
||||
"kind": "bot",
|
||||
"body": {
|
||||
"name": "regression group box",
|
||||
"description": "Synthetic signed group ingress",
|
||||
"adapter": "http_bot",
|
||||
"enable": true,
|
||||
"adapter_config": {
|
||||
"inbound_secret": "${group_secret}",
|
||||
"signature_required": true,
|
||||
"callback_url": "http://127.0.0.1:9/unused-sync-only",
|
||||
"default_session_type": "group",
|
||||
"callback_timeout": 20,
|
||||
"callback_max_retries": 0
|
||||
},
|
||||
"event_bindings": [
|
||||
{
|
||||
"event_pattern": "message.received",
|
||||
"target_type": "pipeline",
|
||||
"target_uuid": "${box_pipeline}",
|
||||
"enabled": true,
|
||||
"filters": []
|
||||
}
|
||||
],
|
||||
"plugin_processors": []
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "rag_bot",
|
||||
"kind": "bot",
|
||||
"body": {
|
||||
"name": "regression group rag",
|
||||
"description": "Synthetic signed group ingress",
|
||||
"adapter": "http_bot",
|
||||
"enable": true,
|
||||
"adapter_config": {
|
||||
"inbound_secret": "${group_secret}",
|
||||
"signature_required": true,
|
||||
"callback_url": "http://127.0.0.1:9/unused-sync-only",
|
||||
"default_session_type": "group",
|
||||
"callback_timeout": 20,
|
||||
"callback_max_retries": 0
|
||||
},
|
||||
"event_bindings": [
|
||||
{
|
||||
"event_pattern": "message.received",
|
||||
"target_type": "pipeline",
|
||||
"target_uuid": "${rag_pipeline}",
|
||||
"enabled": true,
|
||||
"filters": []
|
||||
}
|
||||
],
|
||||
"plugin_processors": []
|
||||
}
|
||||
}
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"name": "chat",
|
||||
"weight": 2,
|
||||
"http_group": {
|
||||
"url": "${core_base_url}/bots/${chat_bot}/sync",
|
||||
"secret_env": "LANGBOT_QA_GROUP_SECRET",
|
||||
"message": "ai What is the documented parser-rag sentinel?"
|
||||
},
|
||||
"expect": "LANGBOT_CHAT_OK"
|
||||
},
|
||||
{
|
||||
"name": "box",
|
||||
"weight": 1,
|
||||
"http_group": {
|
||||
"url": "${core_base_url}/bots/${box_bot}/sync",
|
||||
"secret_env": "LANGBOT_QA_GROUP_SECRET",
|
||||
"message": "ai What is the documented parser-rag sentinel?"
|
||||
},
|
||||
"expect": "LANGBOT_BOX_OK"
|
||||
},
|
||||
{
|
||||
"name": "rag_chat",
|
||||
"weight": 1,
|
||||
"http_group": {
|
||||
"url": "${core_base_url}/bots/${rag_bot}/sync",
|
||||
"secret_env": "LANGBOT_QA_GROUP_SECRET",
|
||||
"message": "ai What is the documented parser-rag sentinel?"
|
||||
},
|
||||
"expect": "LANGBOT_RAG_OK"
|
||||
}
|
||||
],
|
||||
"phases": [
|
||||
{
|
||||
"name": "normal",
|
||||
"duration_seconds": 600,
|
||||
"rps": 2,
|
||||
"workers": 4,
|
||||
"cooldown_seconds": 30,
|
||||
"groups": 4,
|
||||
"members_per_group": 3
|
||||
}
|
||||
],
|
||||
"observe": {
|
||||
"health": {
|
||||
"core": "${core_health}",
|
||||
"mock": "http://127.0.0.1:18080/healthz",
|
||||
"box": "http://127.0.0.1:5410/readyz"
|
||||
},
|
||||
"pids": {},
|
||||
"pid_files": {},
|
||||
"containers": []
|
||||
},
|
||||
"cleanup": "on-success",
|
||||
"thresholds": {
|
||||
"min_rps_ratio": 0,
|
||||
"max_p95_seconds": {
|
||||
"chat": 5,
|
||||
"box": 10,
|
||||
"rag_chat": 6
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"version": 1,
|
||||
"lbctl": "lbctl",
|
||||
"suite": "groups.json",
|
||||
"deployment": {
|
||||
"version": "v4.11.0",
|
||||
"port": 15351
|
||||
},
|
||||
"vars": {
|
||||
"plugin_dir": "/path/to/plugin-zips"
|
||||
},
|
||||
"mock_image": "python:3.12-alpine",
|
||||
"cleanup": "on-success",
|
||||
"purge_data": true
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
{
|
||||
"version": 1,
|
||||
"cli": {"executable": "lbctl", "config": "./lbctl.yaml"},
|
||||
"vars": {
|
||||
"plugin_dir": "/path/to/plugin-zips",
|
||||
"mock_base_url": "http://127.0.0.1:18080/v1",
|
||||
"embedding_base_url": "http://127.0.0.1:15410/v1",
|
||||
"core_health": "http://127.0.0.1:5300/healthz"
|
||||
},
|
||||
"plugins": [
|
||||
{"source": "local", "file": "${plugin_dir}/LocalAgent.zip"},
|
||||
{"source": "local", "file": "${plugin_dir}/LangRAG.zip"},
|
||||
{"source": "local", "file": "${plugin_dir}/GeneralParsers.zip"}
|
||||
],
|
||||
"resources": [
|
||||
{"id": "mock_provider", "kind": "provider", "body": {
|
||||
"name": "regression mock", "requester": "openai-chat-completions",
|
||||
"base_url": "${mock_base_url}", "api_keys": ["test-only"]
|
||||
}},
|
||||
{"id": "embedding_provider", "kind": "provider", "body": {
|
||||
"name": "regression embedding", "requester": "openai-chat-completions",
|
||||
"base_url": "${embedding_base_url}", "api_keys": ["test-only"]
|
||||
}},
|
||||
{"id": "chat_model", "kind": "model", "type": "llm", "body": {
|
||||
"name": "mock-chat", "provider_uuid": "${mock_provider}",
|
||||
"abilities": ["func_call"], "context_length": 8192,
|
||||
"extra_args": {"request_policy": {"retries": 0}}
|
||||
}},
|
||||
{"id": "box_model", "kind": "model", "type": "llm", "body": {
|
||||
"name": "mock-box", "provider_uuid": "${mock_provider}",
|
||||
"abilities": ["func_call"], "context_length": 8192,
|
||||
"extra_args": {"request_policy": {"retries": 0}}
|
||||
}},
|
||||
{"id": "rag_model", "kind": "model", "type": "llm", "body": {
|
||||
"name": "mock-rag", "provider_uuid": "${mock_provider}",
|
||||
"abilities": ["func_call"], "context_length": 8192,
|
||||
"extra_args": {"request_policy": {"retries": 0}}
|
||||
}},
|
||||
{"id": "embedding_model", "kind": "model", "type": "embedding", "body": {
|
||||
"name": "all-MiniLM-L6-v2", "provider_uuid": "${embedding_provider}"
|
||||
}},
|
||||
{"id": "knowledge_base", "kind": "knowledge-base", "body": {
|
||||
"name": "regression parser golden", "knowledge_engine_plugin_id": "langbot-team/LangRAG",
|
||||
"creation_settings": {"embedding_model_uuid": "${embedding_model}", "index_type": "chunk", "chunk_size": 512, "overlap": 50},
|
||||
"retrieval_settings": {"search_type": "vector", "top_k": 5, "query_rewrite": "off", "rerank": "off"}
|
||||
}, "documents": [{
|
||||
"file": "../../skills/skills/langbot-testing/fixtures/rag/parser-golden.html",
|
||||
"parser_plugin_id": "langbot-team/GeneralParsers"
|
||||
}]},
|
||||
{"id": "chat_pipeline", "kind": "pipeline", "body": {
|
||||
"name": "regression chat", "description": "Normal deterministic chat",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {"id": "plugin:langbot-team/LocalAgent/default", "expire-time": 0},
|
||||
"runner_config": {"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {"primary": "${chat_model}", "fallbacks": []},
|
||||
"prompt": [{"role": "system", "content": "You are a QA regression assistant."}],
|
||||
"knowledge-bases": [], "box-enabled": false, "timeout": 30
|
||||
}}
|
||||
},
|
||||
"safety": {"rate-limit": {"limitation": 10000}},
|
||||
"output": {"force-delay": {"min": 0, "max": 0}}
|
||||
}
|
||||
}},
|
||||
{"id": "box_pipeline", "kind": "pipeline", "body": {
|
||||
"name": "regression Box", "description": "Real Box write and read",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {"id": "plugin:langbot-team/LocalAgent/default", "expire-time": 0},
|
||||
"runner_config": {"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {"primary": "${box_model}", "fallbacks": []},
|
||||
"prompt": [{"role": "system", "content": "You are a QA regression assistant."}],
|
||||
"knowledge-bases": [], "box-enabled": true, "box-session-id-template": "{global}",
|
||||
"timeout": 30, "max-tool-iterations": 5
|
||||
}}
|
||||
},
|
||||
"safety": {"rate-limit": {"limitation": 10000}},
|
||||
"output": {"force-delay": {"min": 0, "max": 0}}
|
||||
}
|
||||
}},
|
||||
{"id": "rag_pipeline", "kind": "pipeline", "body": {
|
||||
"name": "regression RAG", "description": "Retrieved model input oracle",
|
||||
"config": {
|
||||
"ai": {
|
||||
"runner": {"id": "plugin:langbot-team/LocalAgent/default", "expire-time": 0},
|
||||
"runner_config": {"plugin:langbot-team/LocalAgent/default": {
|
||||
"model": {"primary": "${rag_model}", "fallbacks": []},
|
||||
"prompt": [{"role": "system", "content": "You are a QA regression assistant."}],
|
||||
"knowledge-bases": ["${knowledge_base}"], "box-enabled": false,
|
||||
"timeout": 30, "retrieval-top-k": 5
|
||||
}}
|
||||
},
|
||||
"safety": {"rate-limit": {"limitation": 10000}},
|
||||
"output": {"force-delay": {"min": 0, "max": 0}}
|
||||
}
|
||||
}}
|
||||
],
|
||||
"cases": [
|
||||
{"name": "chat", "weight": 2, "args": ["pipeline", "run", "${chat_pipeline}", "--message", "Run the normal QA scenario", "--timeout", "90s"], "expect": "LANGBOT_CHAT_OK"},
|
||||
{"name": "box", "weight": 1, "args": ["pipeline", "run", "${box_pipeline}", "--message", "Run the normal QA scenario", "--timeout", "90s"], "expect": "LANGBOT_BOX_OK"},
|
||||
{"name": "rag_chat", "weight": 1, "args": ["pipeline", "run", "${rag_pipeline}", "--message", "What is the documented parser-rag sentinel?", "--timeout", "90s"], "expect": "LANGBOT_RAG_OK"},
|
||||
{"name": "rag_retrieve", "weight": 1, "args": ["knowledge-base", "retrieve", "${knowledge_base}"], "body": {"query": "What is the parser-rag golden sentinel and parser-engine pair?", "retrieval_settings": {"search_type": "vector", "top_k": 5}}, "expect": "aurora-parser-rag-9137"}
|
||||
],
|
||||
"phases": [{"name": "normal", "duration_seconds": 600, "rps": 2, "workers": 4, "cooldown_seconds": 30}],
|
||||
"observe": {
|
||||
"health": {"core": "${core_health}", "mock": "http://127.0.0.1:18080/healthz", "box": "http://127.0.0.1:5410/readyz"},
|
||||
"pids": {}, "pid_files": {}, "containers": []
|
||||
},
|
||||
"cleanup": "on-success",
|
||||
"thresholds": {"min_rps_ratio": 0.95, "max_p95_seconds": {"chat": 3, "box": 6, "rag_chat": 4, "rag_retrieve": 1}}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,328 @@
|
||||
"""Run finite CLI/group smoke cases with an independent read-only AgentRun oracle."""
|
||||
|
||||
from collections import Counter
|
||||
from contextlib import closing
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
from regression_traffic import group_request, read_endpoint_snapshot
|
||||
|
||||
|
||||
class UnresolvedRun(RuntimeError):
|
||||
"""The request cannot yet be matched to one authoritative terminal run."""
|
||||
|
||||
|
||||
def completed_run(connection, workspace, conversation, after_id, timeout):
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
rows = connection.execute(
|
||||
'SELECT run_id, status, status_reason, finished_at FROM agent_run '
|
||||
'WHERE workspace_id=? AND conversation_id=? AND id>? ORDER BY id',
|
||||
(workspace, conversation, after_id),
|
||||
).fetchall()
|
||||
if len(rows) > 1:
|
||||
raise UnresolvedRun('More than one AgentRun matches this request')
|
||||
if rows and (rows[0]['finished_at'] or rows[0]['status'] in {'completed', 'failed', 'timeout', 'cancelled'}):
|
||||
return dict(rows[0])
|
||||
if time.monotonic() >= deadline:
|
||||
raise UnresolvedRun('No unique terminal AgentRun before the observation deadline; not retried')
|
||||
time.sleep(0.1)
|
||||
|
||||
|
||||
def check_run(connection, run, case):
|
||||
if run['status'] != 'completed':
|
||||
raise ValueError(f'AgentRun {run["run_id"]} ended as {run["status"]}: {run["status_reason"]}')
|
||||
events = [
|
||||
(kind, json.loads(data or '{}'))
|
||||
for kind, data in connection.execute(
|
||||
'SELECT type, data_json FROM agent_run_event WHERE run_id=? ORDER BY sequence', (run['run_id'],)
|
||||
)
|
||||
]
|
||||
if any(not isinstance(data, dict) for _kind, data in events):
|
||||
raise ValueError('AgentRun event payload must be an object')
|
||||
if not any(kind == 'run.completed' for kind, _data in events):
|
||||
raise ValueError('Completed AgentRun is missing its persisted run.completed event')
|
||||
checks = {'terminal_event': True}
|
||||
if expected := case.get('box_exec'):
|
||||
calls = [
|
||||
data.get('result', {})
|
||||
for kind, data in events
|
||||
if kind == 'tool.call.completed' and data.get('tool_name') == 'exec'
|
||||
]
|
||||
if len(calls) != expected['count'] or any(
|
||||
not isinstance(item, dict)
|
||||
or item.get('backend') != expected['backend']
|
||||
or item.get('exit_code') != expected.get('exit_code', 0)
|
||||
or not isinstance(item.get('stdout'), str)
|
||||
or expected['stdout'] not in item['stdout']
|
||||
or ('stderr' in expected and expected['stderr'] not in str(item.get('stderr', '')))
|
||||
for item in calls
|
||||
):
|
||||
raise ValueError('Box execution count/backend/exit code/output did not match the oracle')
|
||||
checks['box_exec'] = {'calls': len(calls), 'backend': expected['backend'], 'output_matched': True}
|
||||
return checks
|
||||
|
||||
|
||||
def check_attachment(data, expected):
|
||||
replies = data.get('replies')
|
||||
if not isinstance(replies, list) or any(
|
||||
not isinstance(reply, dict)
|
||||
or not isinstance(reply.get('message_chain'), list)
|
||||
or any(not isinstance(part, dict) for part in reply['message_chain'])
|
||||
for reply in replies
|
||||
):
|
||||
raise ValueError('Malformed file delivery response')
|
||||
attachments = [part for reply in replies for part in reply.get('message_chain', []) if part.get('type') == 'File']
|
||||
if len(attachments) != 1 or attachments[0].get('name') != expected['name']:
|
||||
raise ValueError('Expected exactly one file attachment with the configured name')
|
||||
encoded = attachments[0].get('base64')
|
||||
if not isinstance(encoded, str):
|
||||
raise ValueError('Attachment must contain base64 bytes')
|
||||
content = base64.b64decode(encoded, validate=True)
|
||||
if content != expected['text'].encode('utf-8'):
|
||||
raise ValueError('Exported attachment bytes differ from the expected fixture')
|
||||
return {'name': expected['name'], 'bytes': len(content), 'content_matched': True}
|
||||
|
||||
|
||||
def check_fields(data, expected):
|
||||
for path, value in expected.items():
|
||||
actual = data
|
||||
try:
|
||||
for key in path.split('.'):
|
||||
actual = actual[int(key)] if isinstance(actual, list) else actual[key]
|
||||
except (KeyError, IndexError, TypeError, ValueError) as exc:
|
||||
raise ValueError(f'Response field {path} is missing or malformed') from exc
|
||||
if actual != value:
|
||||
raise ValueError(f'Response field {path} differs from the expected value')
|
||||
return expected
|
||||
|
||||
|
||||
def check_mock_requests(path, offset, expected):
|
||||
deadline = time.monotonic() + 3
|
||||
while True:
|
||||
with path.open('rb') as stream:
|
||||
stream.seek(offset)
|
||||
lines = stream.read().splitlines()
|
||||
records = [json.loads(line) for line in lines]
|
||||
actual = [
|
||||
{'model': row['request']['model'], 'status': row['status'], 'stream': row['request'].get('stream', False)}
|
||||
for row in records
|
||||
if row['path'] == '/v1/chat/completions'
|
||||
]
|
||||
if len(actual) >= len(expected) or time.monotonic() >= deadline:
|
||||
if actual != expected:
|
||||
raise ValueError(f'Model request sequence differs: expected {expected}, got {actual}')
|
||||
return actual
|
||||
time.sleep(0.05)
|
||||
|
||||
|
||||
def rejected_group(case, group):
|
||||
settings = case['http_group']
|
||||
headers = {'Content-Type': 'application/json'}
|
||||
if case['reject_group'] == 'bad_signature':
|
||||
headers.update({'X-LB-Timestamp': str(int(time.time())), 'X-LB-Signature': 'sha256=' + '0' * 64})
|
||||
request = urllib.request.Request(
|
||||
settings['url'],
|
||||
headers=headers,
|
||||
data=json.dumps(
|
||||
{
|
||||
'session_id': group,
|
||||
'session_type': 'group',
|
||||
'sender': {'id': 'qa-rejected'},
|
||||
'message': [{'type': 'Plain', 'text': settings['message']}],
|
||||
}
|
||||
).encode(),
|
||||
)
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
try:
|
||||
with opener.open(request, timeout=10):
|
||||
raise ValueError('Unsigned/tampered group message was accepted')
|
||||
except urllib.error.HTTPError as exc:
|
||||
payload = json.load(exc)
|
||||
if exc.code != 401 or not isinstance(payload, dict) or payload.get('code') != 40101:
|
||||
raise ValueError('Expected HTTP 401 with bad_signature code 40101') from exc
|
||||
return {'http_status': exc.code, 'code': payload['code']}
|
||||
|
||||
|
||||
def verify_cleanup(suite, deleted):
|
||||
checks = []
|
||||
for kind, model_type in dict.fromkeys((item['kind'], item.get('type')) for item in deleted):
|
||||
args = [kind, 'list'] + (['--type', model_type] if model_type else [])
|
||||
data = suite.call(args)
|
||||
key = {
|
||||
'provider': 'providers',
|
||||
'model': 'models',
|
||||
'pipeline': 'pipelines',
|
||||
'bot': 'bots',
|
||||
'knowledge-base': 'bases',
|
||||
}[kind]
|
||||
items = data.get(key) if isinstance(data, dict) else None
|
||||
if not isinstance(items, list) or any(
|
||||
not isinstance(item, dict) or not isinstance(item.get('uuid'), str) for item in items
|
||||
):
|
||||
raise ValueError(f'Malformed {kind} list response during cleanup verification')
|
||||
visible = {item['uuid'] for item in items}
|
||||
expected_absent = {item['uuid'] for item in deleted if item['kind'] == kind and item.get('type') == model_type}
|
||||
remaining = sorted(expected_absent & visible)
|
||||
checks.append({'kind': kind, 'type': model_type, 'deleted_count': len(expected_absent), 'remaining': remaining})
|
||||
suite.write('cleanup-checks.json', checks)
|
||||
if any(row['remaining'] for row in checks):
|
||||
raise ValueError('Deleted resources remain visible in CLI list readback')
|
||||
|
||||
|
||||
def run_smoke(suite):
|
||||
# Imported here because Suite dispatches to this module after profile validation.
|
||||
from regression_suite import resolve
|
||||
|
||||
settings = suite.profile['smoke']
|
||||
path = Path(suite.path(settings['ledger_db'])).resolve()
|
||||
if not path.is_file():
|
||||
raise ValueError('smoke.ledger_db must be an existing target Core SQLite database')
|
||||
mock_path = Path(suite.path(settings['mock_records'])) if settings.get('mock_records') else None
|
||||
rows, errors, health = [], [], []
|
||||
started = time.monotonic()
|
||||
|
||||
def redact(value):
|
||||
for secret in suite.secrets:
|
||||
value = value.replace(secret, '[REDACTED]')
|
||||
return value
|
||||
|
||||
def sample_health(phase):
|
||||
for name, url in resolve(suite.profile['observe']['health'], suite.variables).items():
|
||||
row = {'phase': phase, 'endpoint': name}
|
||||
try:
|
||||
row['result'] = read_endpoint_snapshot(url, timeout_seconds=3)
|
||||
except (RuntimeError, OSError, ValueError) as exc:
|
||||
row['error'] = redact(str(exc))
|
||||
errors.append(f'{phase}/{name}: health check failed')
|
||||
health.append(row)
|
||||
suite.write('health.json', health)
|
||||
|
||||
sample_health('before')
|
||||
# shortcut: local SQLite only; use a terminal-run API when lbctl exposes one.
|
||||
with closing(sqlite3.connect(path.as_uri() + '?mode=ro', uri=True, timeout=5)) as connection:
|
||||
connection.row_factory = sqlite3.Row
|
||||
connection.execute('PRAGMA query_only=ON')
|
||||
for original in suite.profile['cases']:
|
||||
case = resolve(original, suite.variables)
|
||||
is_group = 'http_group' in case
|
||||
rejected = 'reject_group' in case
|
||||
runner = not rejected and (is_group or case.get('args', [])[:2] == ['pipeline', 'run'])
|
||||
groups = (
|
||||
[f'qa-smoke-{uuid4().hex}' for _ in range(settings.get('groups_per_case', 2))]
|
||||
if is_group and not rejected
|
||||
else [f'qa-smoke-{uuid4().hex}' if rejected else None]
|
||||
)
|
||||
previous = {group: 'none' for group in groups}
|
||||
retired = set()
|
||||
for turn in range(1, (settings.get('group_turns', 2) if is_group and not rejected else 1) + 1):
|
||||
for group in groups:
|
||||
row = {
|
||||
'case': case['name'],
|
||||
'stage': case.get('stage', 'execution'),
|
||||
'status': 'failed',
|
||||
'requires_runner': runner,
|
||||
}
|
||||
if is_group:
|
||||
row.update(group_id=group, turn=turn, member_id=f'qa-member-{turn % 2:03d}')
|
||||
if group in retired:
|
||||
row.update(status='skipped', error='Earlier turn failed or is unknown; group not retried')
|
||||
rows.append(row)
|
||||
continue
|
||||
before = time.monotonic()
|
||||
after_id = connection.execute('SELECT COALESCE(MAX(id),0) FROM agent_run').fetchone()[0]
|
||||
token = uuid4().hex
|
||||
offset = mock_path.stat().st_size if case.get('mock_requests') and mock_path.exists() else 0
|
||||
try:
|
||||
if rejected:
|
||||
data = rejected_group(case, group)
|
||||
count = connection.execute(
|
||||
'SELECT COUNT(*) FROM agent_run WHERE workspace_id=? AND conversation_id=? AND id>?',
|
||||
(suite.variables['workspace_uuid'], 'group_' + group, after_id),
|
||||
).fetchone()[0]
|
||||
if count:
|
||||
raise ValueError('Rejected ingress created an AgentRun')
|
||||
elif is_group:
|
||||
data = group_request(
|
||||
case, group, row['member_id'], turn, token, previous[group], suite.env
|
||||
)['data']
|
||||
else:
|
||||
data = suite.call(case['args'], case.get('body'))
|
||||
if case['expect'] not in json.dumps(data, ensure_ascii=False):
|
||||
raise ValueError('Expected business response was not returned')
|
||||
row['output'] = data
|
||||
if any(value in json.dumps(data, ensure_ascii=False) for value in case.get('forbid', [])):
|
||||
raise ValueError('Response contains forbidden content from another knowledge base')
|
||||
row['checks'] = {}
|
||||
if case.get('fields'):
|
||||
row['checks']['fields'] = check_fields(data, case['fields'])
|
||||
if runner:
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError('Runner response must be an object')
|
||||
session = data.get('session_id')
|
||||
if not isinstance(session, str) or not session:
|
||||
raise UnresolvedRun('Reply has no session ID for AgentRun reconciliation')
|
||||
run = completed_run(
|
||||
connection,
|
||||
suite.variables['workspace_uuid'],
|
||||
'group_' + session if is_group else session,
|
||||
after_id,
|
||||
settings.get('timeout_seconds', 30),
|
||||
)
|
||||
row['run'] = run
|
||||
row['checks'].update(check_run(connection, run, case))
|
||||
if case.get('attachment'):
|
||||
row['checks']['attachment'] = check_attachment(data, case['attachment'])
|
||||
if case.get('mock_requests'):
|
||||
row['checks']['mock_requests'] = check_mock_requests(
|
||||
mock_path, offset, case['mock_requests']
|
||||
)
|
||||
row['status'] = 'passed'
|
||||
previous[group] = token
|
||||
except urllib.error.HTTPError as exc:
|
||||
row.update(
|
||||
status='unknown' if exc.code >= 500 else 'failed', error=f'HTTP {exc.code}; not retried'
|
||||
)
|
||||
except (UnresolvedRun, urllib.error.URLError, TimeoutError) as exc:
|
||||
row.update(status='unknown', error=redact(str(exc)))
|
||||
except (RuntimeError, ValueError, OSError, KeyError, IndexError, sqlite3.Error) as exc:
|
||||
message = redact(str(exc))
|
||||
row.update(
|
||||
status='unknown'
|
||||
if 'outcome unknown' in message or 'result_unknown' in message
|
||||
else 'failed',
|
||||
error=message,
|
||||
)
|
||||
row['seconds'] = time.monotonic() - before
|
||||
if row['status'] != 'passed':
|
||||
errors.append(f'{case["name"]}: {row["status"]}: {row["error"]}')
|
||||
if is_group:
|
||||
retired.add(group)
|
||||
rows.append(row)
|
||||
suite.write('cases.json', rows)
|
||||
suite.write(
|
||||
'progress.json',
|
||||
{'completed_requests': len(rows), 'status': dict(Counter(x['status'] for x in rows))},
|
||||
)
|
||||
suite.write('cases.json', rows)
|
||||
sample_health('after')
|
||||
return {
|
||||
'mode': 'smoke',
|
||||
'phases': [],
|
||||
'cases': rows,
|
||||
'errors': errors,
|
||||
'elapsed_seconds': time.monotonic() - started,
|
||||
'status': dict(Counter(row['status'] for row in rows)),
|
||||
'stages': {
|
||||
stage: dict(Counter(row['status'] for row in rows if row['stage'] == stage))
|
||||
for stage in dict.fromkeys(row['stage'] for row in rows)
|
||||
},
|
||||
'case_count': len(suite.profile['cases']),
|
||||
'requests': len(rows),
|
||||
}
|
||||
@@ -0,0 +1,717 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prepare an lbctl regression from JSON, run load phases, and clean owned resources."""
|
||||
|
||||
import argparse
|
||||
import copy
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import secrets
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from uuid import uuid4
|
||||
|
||||
from regression_traffic import group_request
|
||||
|
||||
|
||||
KINDS = {'provider', 'model', 'pipeline', 'knowledge-base', 'bot'}
|
||||
REFERENCE = re.compile(r'\$\{([a-zA-Z][a-zA-Z0-9_-]*)\}')
|
||||
|
||||
|
||||
def resolve(value, variables):
|
||||
if isinstance(value, dict):
|
||||
return {key: resolve(item, variables) for key, item in value.items()}
|
||||
if isinstance(value, list):
|
||||
return [resolve(item, variables) for item in value]
|
||||
if isinstance(value, str):
|
||||
return REFERENCE.sub(lambda match: str(variables[match[1]]), value)
|
||||
return value
|
||||
|
||||
|
||||
def merge(base, patch):
|
||||
result = copy.deepcopy(base)
|
||||
for key, value in patch.items():
|
||||
result[key] = (
|
||||
merge(result[key], value) if isinstance(value, dict) and isinstance(result.get(key), dict) else value
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
def contains_fields(actual, expected):
|
||||
"""Check submitted fields while allowing server-generated defaults."""
|
||||
if isinstance(expected, dict):
|
||||
return isinstance(actual, dict) and all(
|
||||
key in actual and contains_fields(actual[key], value) for key, value in expected.items()
|
||||
)
|
||||
if isinstance(expected, list):
|
||||
return (
|
||||
isinstance(actual, list)
|
||||
and len(actual) == len(expected)
|
||||
and all(contains_fields(left, right) for left, right in zip(actual, expected))
|
||||
)
|
||||
return actual == expected
|
||||
|
||||
|
||||
def number(value, label, *, minimum=0, integer=False):
|
||||
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
|
||||
raise ValueError(f'{label} must be a finite number')
|
||||
if value < minimum or (integer and not isinstance(value, int)):
|
||||
raise ValueError(f'{label} must be {"an integer " if integer else ""}>= {minimum}')
|
||||
|
||||
|
||||
def validate(profile):
|
||||
if not isinstance(profile, dict) or profile.get('version') != 1:
|
||||
raise ValueError('profile version must be 1')
|
||||
allowed = {
|
||||
'version',
|
||||
'cli',
|
||||
'vars',
|
||||
'plugins',
|
||||
'resources',
|
||||
'cases',
|
||||
'phases',
|
||||
'observe',
|
||||
'cleanup',
|
||||
'thresholds',
|
||||
'smoke',
|
||||
}
|
||||
if extra := profile.keys() - allowed:
|
||||
raise ValueError(f'unknown profile fields: {sorted(extra)}')
|
||||
for field in ('cli', 'vars', 'observe', 'thresholds'):
|
||||
if field in profile and not isinstance(profile[field], dict):
|
||||
raise ValueError(f'{field} must be an object')
|
||||
for field in ('plugins', 'resources', 'cases', 'phases'):
|
||||
if field in profile and (
|
||||
not isinstance(profile[field], list) or any(not isinstance(item, dict) for item in profile[field])
|
||||
):
|
||||
raise ValueError(f'{field} must be an array of objects')
|
||||
cli = profile.get('cli', {})
|
||||
if not isinstance(cli, dict) or cli.keys() - {'config', 'executable'}:
|
||||
raise ValueError('cli accepts config and executable')
|
||||
if not isinstance(cli.get('config'), str) or not cli['config']:
|
||||
raise ValueError('cli.config must name a Workspace-bound lbctl config')
|
||||
variables = {'profile_dir': '.', 'run_dir': '.', 'workspace_uuid': 'workspace', 'group_secret': 'private'}
|
||||
supplied = profile.get('vars', {})
|
||||
if not isinstance(supplied, dict) or variables.keys() & supplied.keys():
|
||||
raise ValueError('vars must be an object without reserved names')
|
||||
if any(not isinstance(value, (str, int, float)) or '${' in str(value) for value in supplied.values()):
|
||||
raise ValueError('vars values must be scalar literals, without nested references')
|
||||
variables.update(supplied)
|
||||
for plugin in profile.get('plugins', []):
|
||||
if plugin.keys() - {'source', 'file'}:
|
||||
raise ValueError('plugins accept source and file')
|
||||
if plugin.get('source') not in {'local', 'github', 'marketplace'} or not plugin.get('file'):
|
||||
raise ValueError('plugins need source local/github/marketplace and file')
|
||||
resolve(plugin, variables)
|
||||
for item in profile.get('resources', []):
|
||||
if item.keys() - {'id', 'kind', 'type', 'body', 'documents'}:
|
||||
raise ValueError('unknown resource field')
|
||||
name = item.get('id', '')
|
||||
if not re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]*', name) or name in variables:
|
||||
raise ValueError(f'invalid or duplicate resource id: {name}')
|
||||
if item.get('kind') not in KINDS or not isinstance(item.get('body'), dict):
|
||||
raise ValueError(f'{name}: resource needs kind and body')
|
||||
if 'uuid' in item['body']:
|
||||
raise ValueError(f'{name}: resources must create new objects; body.uuid is forbidden')
|
||||
if item['kind'] == 'bot' and (
|
||||
item['body'].get('adapter') != 'http_bot'
|
||||
or not isinstance(item['body'].get('adapter_config'), dict)
|
||||
or item['body'].get('adapter_config', {}).get('inbound_secret') != '${group_secret}'
|
||||
or not any(
|
||||
isinstance(case.get('http_group'), dict) and '${' + name + '}' in case['http_group'].get('url', '')
|
||||
for case in profile.get('cases', [])
|
||||
)
|
||||
):
|
||||
raise ValueError('Bot resources require http_bot, ${group_secret}, and a matching http_group case')
|
||||
if item['kind'] == 'model' and item.get('type') not in {'llm', 'embedding', 'rerank'}:
|
||||
raise ValueError(f'{name}: model type must be llm/embedding/rerank')
|
||||
resolve(item['body'], variables) # References must point backwards; never guess missing UUIDs.
|
||||
variables[name] = name
|
||||
documents = item.get('documents', [])
|
||||
if not isinstance(documents, list) or any(not isinstance(doc, dict) for doc in documents):
|
||||
raise ValueError(f'{name}: documents must be an array of objects')
|
||||
for document in documents:
|
||||
if item['kind'] != 'knowledge-base' or not document.get('file') or not document.get('parser_plugin_id'):
|
||||
raise ValueError(f'{name}: documents need file and parser_plugin_id on a knowledge-base')
|
||||
resolve(document, variables)
|
||||
cases = profile.get('cases')
|
||||
if not isinstance(cases, list) or not cases:
|
||||
raise ValueError('cases must be a non-empty array')
|
||||
names = set()
|
||||
for case in cases:
|
||||
if case.keys() - {
|
||||
'name',
|
||||
'weight',
|
||||
'args',
|
||||
'expect',
|
||||
'body',
|
||||
'http_group',
|
||||
'box_exec',
|
||||
'attachment',
|
||||
'stage',
|
||||
'mock_requests',
|
||||
'fields',
|
||||
'forbid',
|
||||
'reject_group',
|
||||
}:
|
||||
raise ValueError('unknown case field')
|
||||
if not isinstance(case.get('name'), str) or not case['name'] or case['name'] in names:
|
||||
raise ValueError('case names must be non-empty and unique')
|
||||
names.add(case['name'])
|
||||
if 'stage' in case and (not isinstance(case['stage'], str) or not case['stage']):
|
||||
raise ValueError('case stage must be a non-empty string')
|
||||
if any(key in case for key in ('mock_requests', 'fields', 'forbid', 'reject_group')) and 'smoke' not in profile:
|
||||
raise ValueError('extended case checks require smoke mode')
|
||||
if 'fields' in case and (not isinstance(case['fields'], dict) or not case['fields']):
|
||||
raise ValueError('fields must be a non-empty mapping of dotted paths to exact values')
|
||||
if 'fields' in case and any(not isinstance(key, str) or not key for key in case['fields']):
|
||||
raise ValueError('field paths must be non-empty strings')
|
||||
if 'forbid' in case and (
|
||||
not isinstance(case['forbid'], list)
|
||||
or not case['forbid']
|
||||
or any(not isinstance(item, str) or not item for item in case['forbid'])
|
||||
):
|
||||
raise ValueError('forbid must contain non-empty response substrings')
|
||||
if 'mock_requests' in case:
|
||||
requests = case['mock_requests']
|
||||
if not isinstance(requests, list) or not requests or not profile.get('smoke', {}).get('mock_records'):
|
||||
raise ValueError('mock_requests requires expected requests and smoke.mock_records')
|
||||
for request in requests:
|
||||
if not isinstance(request, dict) or set(request) != {'model', 'status', 'stream'}:
|
||||
raise ValueError('mock_requests entries require model, status and stream')
|
||||
if (
|
||||
not isinstance(request['model'], str)
|
||||
or not request['model']
|
||||
or not isinstance(request['stream'], bool)
|
||||
):
|
||||
raise ValueError('invalid mock request model/stream')
|
||||
number(request['status'], 'mock request status', minimum=100, integer=True)
|
||||
if 'reject_group' in case and (
|
||||
'http_group' not in case
|
||||
or case['reject_group'] not in ('missing_signature', 'bad_signature')
|
||||
or any(key in case for key in ('box_exec', 'attachment', 'mock_requests'))
|
||||
):
|
||||
raise ValueError('reject_group must name a signature rejection on an HTTP group case')
|
||||
if 'http_group' in case:
|
||||
settings = case['http_group']
|
||||
if (
|
||||
'args' in case
|
||||
or 'body' in case
|
||||
or not isinstance(settings, dict)
|
||||
or set(settings) != {'url', 'secret_env', 'message'}
|
||||
or any(not isinstance(value, str) or not value for value in settings.values())
|
||||
or not resolve(settings['url'], variables).startswith(('http://', 'https://'))
|
||||
or settings['secret_env'] != 'LANGBOT_QA_GROUP_SECRET'
|
||||
):
|
||||
raise ValueError(
|
||||
'http_group needs url, message and secret_env=LANGBOT_QA_GROUP_SECRET; no CLI args/body'
|
||||
)
|
||||
elif (
|
||||
not isinstance(case.get('args'), list)
|
||||
or not case['args']
|
||||
or not all(isinstance(x, str) for x in case['args'])
|
||||
):
|
||||
raise ValueError('case args must be a non-empty string array')
|
||||
if not isinstance(case.get('expect'), str) or not case['expect']:
|
||||
raise ValueError('case expect must be a non-empty string')
|
||||
number(case.get('weight', 1), 'case weight', minimum=1, integer=True)
|
||||
resolve(case, variables)
|
||||
if 'body' in case and (not isinstance(case['body'], dict) or '--file' in case['args']):
|
||||
raise ValueError('case body must be an object and cannot coexist with --file')
|
||||
if 'box_exec' in case:
|
||||
check = case['box_exec']
|
||||
if (
|
||||
not isinstance(check, dict)
|
||||
or not {'count', 'backend', 'stdout'} <= set(check)
|
||||
or check.keys() - {'count', 'backend', 'stdout', 'exit_code', 'stderr'}
|
||||
):
|
||||
raise ValueError('box_exec requires count, backend and stdout')
|
||||
if 'exit_code' in check:
|
||||
number(check['exit_code'], 'box_exec.exit_code', minimum=0, integer=True)
|
||||
if 'stderr' in check and not isinstance(check['stderr'], str):
|
||||
raise ValueError('box_exec.stderr must be a string')
|
||||
number(check['count'], 'box_exec.count', minimum=1, integer=True)
|
||||
if any(not isinstance(check[k], str) or not check[k] for k in ('backend', 'stdout')):
|
||||
raise ValueError('box_exec backend/stdout must be non-empty strings')
|
||||
if 'smoke' not in profile:
|
||||
raise ValueError('box_exec checks require smoke mode')
|
||||
if 'http_group' not in case and case.get('args', [])[:2] != ['pipeline', 'run']:
|
||||
raise ValueError('box_exec requires a Pipeline or group case')
|
||||
if 'attachment' in case:
|
||||
check = case['attachment']
|
||||
if (
|
||||
'smoke' not in profile
|
||||
or case.get('args', [])[:2] != ['pipeline', 'run']
|
||||
or not isinstance(check, dict)
|
||||
or set(check) != {'name', 'text'}
|
||||
or any(not isinstance(value, str) or not value for value in check.values())
|
||||
):
|
||||
raise ValueError('attachment requires name/text on a smoke Pipeline case')
|
||||
if sum(case.get('weight', 1) for case in cases) > 1000:
|
||||
raise ValueError('total case weight must be <= 1000')
|
||||
smoke = profile.get('smoke')
|
||||
if 'smoke' in profile:
|
||||
if not isinstance(smoke, dict) or smoke.keys() - {
|
||||
'ledger_db',
|
||||
'mock_records',
|
||||
'timeout_seconds',
|
||||
'group_turns',
|
||||
'groups_per_case',
|
||||
'required_plugins',
|
||||
}:
|
||||
raise ValueError('invalid smoke settings')
|
||||
if not isinstance(smoke.get('ledger_db'), str) or not smoke['ledger_db']:
|
||||
raise ValueError('smoke.ledger_db must name the target Core SQLite database')
|
||||
if 'mock_records' in smoke and (not isinstance(smoke['mock_records'], str) or not smoke['mock_records']):
|
||||
raise ValueError('smoke.mock_records must name the dedicated mock recording file')
|
||||
resolve(smoke, variables)
|
||||
number(smoke.get('timeout_seconds', 30), 'smoke timeout_seconds', minimum=0.001)
|
||||
number(smoke.get('group_turns', 2), 'smoke group_turns', minimum=2, integer=True)
|
||||
number(smoke.get('groups_per_case', 2), 'smoke groups_per_case', minimum=2, integer=True)
|
||||
if smoke.get('group_turns', 2) > 10 or smoke.get('groups_per_case', 2) > 10:
|
||||
raise ValueError('smoke group_turns/groups_per_case must be <= 10')
|
||||
required = smoke.get('required_plugins', [])
|
||||
if not isinstance(required, list) or any(not isinstance(p, str) or not p for p in required):
|
||||
raise ValueError('smoke.required_plugins must be an array of plugin IDs')
|
||||
if profile.get('phases') or profile.get('thresholds'):
|
||||
raise ValueError('smoke uses fixed case counts; load phases/thresholds are not applicable')
|
||||
phases = profile.get('phases', [])
|
||||
if not isinstance(phases, list) or (smoke is None and not phases):
|
||||
raise ValueError('phases must be a non-empty array')
|
||||
names = set()
|
||||
for phase in phases:
|
||||
name = phase.get('name', '')
|
||||
if not re.fullmatch(r'[a-zA-Z][a-zA-Z0-9_-]*', name) or name in names:
|
||||
raise ValueError('phase names must be unique simple names')
|
||||
names.add(name)
|
||||
if phase.keys() - {
|
||||
'name',
|
||||
'duration_seconds',
|
||||
'rps',
|
||||
'workers',
|
||||
'cooldown_seconds',
|
||||
'groups',
|
||||
'members_per_group',
|
||||
}:
|
||||
raise ValueError(f'{name}: unknown phase field')
|
||||
number(phase.get('duration_seconds'), 'duration_seconds', minimum=0.001)
|
||||
number(phase.get('rps'), 'rps', minimum=0.001)
|
||||
number(phase.get('workers'), 'workers', minimum=1, integer=True)
|
||||
number(phase.get('cooldown_seconds', 0), 'cooldown_seconds')
|
||||
if phase['workers'] > 32:
|
||||
raise ValueError('workers must be <= 32')
|
||||
number(phase.get('groups', 0), 'groups', integer=True)
|
||||
number(phase.get('members_per_group', 3), 'members_per_group', minimum=1, integer=True)
|
||||
if phase.get('groups', 0) > 4096 or phase.get('members_per_group', 3) > 1000:
|
||||
raise ValueError('groups must be <= 4096 and members_per_group <= 1000')
|
||||
group_cases = ['http_group' in case for case in cases]
|
||||
if any(group_cases) and (
|
||||
not all(group_cases) or phase.get('groups', 0) < sum(case.get('weight', 1) for case in cases)
|
||||
):
|
||||
raise ValueError('group phases need only http_group cases and enough groups for all weighted cases')
|
||||
if not any(group_cases) and phase.get('groups', 0):
|
||||
raise ValueError('groups require http_group cases')
|
||||
if profile.get('cleanup', 'on-success') not in {'always', 'on-success', 'never'}:
|
||||
raise ValueError('cleanup must be always/on-success/never')
|
||||
thresholds = profile.get('thresholds', {})
|
||||
if extra := thresholds.keys() - {'min_rps_ratio', 'max_p95_seconds'}:
|
||||
raise ValueError(f'unknown thresholds: {sorted(extra)}')
|
||||
number(thresholds.get('min_rps_ratio', 0), 'min_rps_ratio')
|
||||
if thresholds.get('min_rps_ratio', 0) > 1:
|
||||
raise ValueError('min_rps_ratio must be <= 1')
|
||||
if not isinstance(thresholds.get('max_p95_seconds', {}), dict):
|
||||
raise ValueError('max_p95_seconds must map case names to seconds')
|
||||
for name, value in thresholds.get('max_p95_seconds', {}).items():
|
||||
if name not in {case['name'] for case in cases}:
|
||||
raise ValueError(f'unknown case in max_p95_seconds: {name}')
|
||||
number(value, 'max_p95_seconds', minimum=0.001)
|
||||
observed = resolve(profile.get('observe', {}), variables)
|
||||
if not isinstance(observed, dict) or observed.keys() - {
|
||||
'health',
|
||||
'pids',
|
||||
'pid_files',
|
||||
'containers',
|
||||
'box_data_root',
|
||||
}:
|
||||
raise ValueError('observe accepts health, pids, pid_files, containers, box_data_root')
|
||||
if 'box_data_root' in observed and (
|
||||
not isinstance(observed['box_data_root'], str) or not observed['box_data_root']
|
||||
):
|
||||
raise ValueError('observe.box_data_root must name the deployment Box data directory')
|
||||
if not isinstance(observed.get('health'), dict) or not observed['health']:
|
||||
raise ValueError('observe.health must contain at least one health endpoint')
|
||||
for name, url in observed['health'].items():
|
||||
if not isinstance(name, str) or not isinstance(url, str) or not url.startswith(('http://', 'https://')):
|
||||
raise ValueError('health endpoints must map names to HTTP(S) URLs')
|
||||
for field in ('pids', 'pid_files'):
|
||||
if not isinstance(observed.get(field, {}), dict):
|
||||
raise ValueError(f'observe.{field} must be an object')
|
||||
for pid in observed.get('pids', {}).values():
|
||||
number(pid, 'PID', minimum=1, integer=True)
|
||||
if not isinstance(observed.get('containers', []), list) or any(
|
||||
not isinstance(item, str) or not item for item in observed.get('containers', [])
|
||||
):
|
||||
raise ValueError('observe.containers must be an array of container names/IDs')
|
||||
|
||||
|
||||
class Suite:
|
||||
def __init__(self, profile, profile_path, output, env=None):
|
||||
self.profile = profile
|
||||
self.output = output.resolve()
|
||||
self.variables = {
|
||||
**profile.get('vars', {}),
|
||||
'profile_dir': str(profile_path.parent.resolve()),
|
||||
'run_dir': str(self.output),
|
||||
}
|
||||
self.cli = [
|
||||
profile.get('cli', {}).get('executable', 'lbctl'),
|
||||
'--config',
|
||||
self.path(profile['cli']['config']),
|
||||
'-o',
|
||||
'json',
|
||||
]
|
||||
self.owned = []
|
||||
self.connection = {}
|
||||
self.env = dict(os.environ if env is None else env)
|
||||
if any('http_group' in case for case in profile['cases']):
|
||||
self.variables['group_secret'] = secrets.token_urlsafe(32)
|
||||
self.env['LANGBOT_QA_GROUP_SECRET'] = self.variables['group_secret']
|
||||
self.secrets = [
|
||||
value for key, value in self.env.items() if ('API_KEY' in key or key == 'LANGBOT_QA_GROUP_SECRET') and value
|
||||
]
|
||||
|
||||
def path(self, value):
|
||||
path = Path(resolve(value, self.variables)).expanduser()
|
||||
return str(path if path.is_absolute() else Path(self.variables['profile_dir']) / path)
|
||||
|
||||
def write(self, name, value):
|
||||
path = self.output / name
|
||||
if name == 'request.json':
|
||||
private = self.output / '.private'
|
||||
private.mkdir(exist_ok=True, mode=0o700)
|
||||
path = private / name
|
||||
path.write_text(json.dumps(value, ensure_ascii=False, indent=2))
|
||||
path.chmod(0o600)
|
||||
return str(path)
|
||||
|
||||
def call(self, args, body=None, *, defer_bot_readback=False):
|
||||
command = [*self.cli, *args]
|
||||
if body is not None:
|
||||
command += ['--file', self.write('request.json', body)]
|
||||
try:
|
||||
result = subprocess.run(command, capture_output=True, text=True, timeout=360, env=self.env)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise RuntimeError('lbctl timed out; outcome unknown, no automatic retry') from exc
|
||||
if result.returncode:
|
||||
if defer_bot_readback and args == ['bot', 'create']:
|
||||
response = json.loads(result.stdout)
|
||||
data = response.get('data', {})
|
||||
if (
|
||||
response.get('error', {}).get('type') == 'verification_failed'
|
||||
and data.get('operation') == 'bot.create'
|
||||
and data.get('uuid')
|
||||
and data.get('bot', {}).get('uuid') == data['uuid']
|
||||
):
|
||||
return data
|
||||
error = result.stdout + result.stderr
|
||||
for secret in self.secrets:
|
||||
error = error.replace(secret, '[REDACTED]')
|
||||
raise RuntimeError(error[:3000])
|
||||
response = json.loads(result.stdout)
|
||||
if args == ['whoami']:
|
||||
self.connection = response.get('meta') or {}
|
||||
return response['data']
|
||||
|
||||
def read_bot(self, bot_uuid):
|
||||
"""Read full Bot fields using the endpoint and environment credential confirmed by lbctl."""
|
||||
source = self.connection.get('credential_source', '')
|
||||
key = self.env.get(source.removeprefix('env:')) if source.startswith('env:') else None
|
||||
if not key or not self.connection.get('endpoint'):
|
||||
raise RuntimeError('Group Bot readback requires environment-backed lbctl credentials')
|
||||
self.secrets.append(key)
|
||||
request = urllib.request.Request(
|
||||
self.connection['endpoint'].rstrip('/') + '/api/v1/platform/bots/' + bot_uuid,
|
||||
headers={'Authorization': 'Bearer ' + key, 'X-Workspace-Id': self.variables['workspace_uuid']},
|
||||
)
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
try:
|
||||
with opener.open(request, timeout=30) as response:
|
||||
payload = json.load(response)
|
||||
except (urllib.error.URLError, OSError) as exc:
|
||||
raise RuntimeError('Bot readback request failed; the creation was not retried') from exc
|
||||
if (
|
||||
not isinstance(payload, dict)
|
||||
or payload.get('code') != 0
|
||||
or not isinstance(payload.get('data'), dict)
|
||||
or not isinstance(payload['data'].get('bot'), dict)
|
||||
):
|
||||
raise RuntimeError('Bot readback did not return a successful Bot resource')
|
||||
return payload['data']['bot']
|
||||
|
||||
def journal(self):
|
||||
self.write('owned.json', {'workspace_uuid': self.variables['workspace_uuid'], 'resources': self.owned})
|
||||
|
||||
def prepare(self):
|
||||
if 'smoke' in self.profile and not Path(self.path(self.profile['smoke']['ledger_db'])).is_file():
|
||||
raise ValueError('smoke.ledger_db must be an existing target Core SQLite database')
|
||||
identity = self.call(['whoami'])
|
||||
self.variables['workspace_uuid'] = identity['workspace_uuid']
|
||||
self.journal()
|
||||
for plugin in self.profile.get('plugins', []):
|
||||
self.call(['plugin', 'install', plugin['source'], '--file', self.path(plugin['file']), '--wait'])
|
||||
if 'smoke' in self.profile:
|
||||
capabilities = self.call(['capabilities'])
|
||||
plugins = self.call(['plugin', 'list'])
|
||||
ready = {
|
||||
f'{item["author"]}/{item["name"]}'
|
||||
for item in plugins['plugins']
|
||||
if item.get('enabled') and item.get('status') == 'initialized'
|
||||
}
|
||||
missing = set(self.profile['smoke'].get('required_plugins', [])) - ready
|
||||
self.write('environment.json', {'identity': identity, 'capabilities': capabilities, 'plugins': plugins})
|
||||
if missing:
|
||||
raise RuntimeError(f'Required plugins are not initialized: {sorted(missing)}')
|
||||
for item in self.profile.get('resources', []):
|
||||
kind = item['kind']
|
||||
body = resolve(item['body'], self.variables)
|
||||
config = body.pop('config', None) if kind == 'pipeline' else None
|
||||
args = [kind, 'apply' if kind == 'pipeline' else 'create']
|
||||
if kind == 'model':
|
||||
args += ['--type', item['type']]
|
||||
created = self.call(args, body, defer_bot_readback=True) if kind == 'bot' else self.call(args, body)
|
||||
created = created.get('pipeline', created) if kind == 'pipeline' else created
|
||||
uuid = created['uuid']
|
||||
self.variables[item['id']] = uuid
|
||||
self.owned.append({'kind': kind, 'uuid': uuid, **({'type': item['type']} if kind == 'model' else {})})
|
||||
self.journal() # Record ownership before any later configuration or ingestion can fail.
|
||||
if kind == 'bot':
|
||||
# lbctl 0.3's Bot projection omits event bindings; verify the existing Core API instead.
|
||||
actual = self.read_bot(uuid)
|
||||
expected = copy.deepcopy(body)
|
||||
expected['adapter_config'].pop('inbound_secret')
|
||||
if not contains_fields(actual, expected):
|
||||
raise RuntimeError(f'Bot readback differs from submitted non-secret fields: {uuid}')
|
||||
if config is not None:
|
||||
current = self.call(['pipeline', 'get', uuid])
|
||||
current = current.get('pipeline', current)
|
||||
self.call(
|
||||
['pipeline', 'apply'],
|
||||
{
|
||||
'uuid': uuid,
|
||||
'name': current['name'],
|
||||
'description': current['description'],
|
||||
'config': merge(current['config'], config),
|
||||
},
|
||||
)
|
||||
for document in item.get('documents', []):
|
||||
self.call(
|
||||
[
|
||||
'knowledge-base',
|
||||
'ingest',
|
||||
uuid,
|
||||
'--file',
|
||||
self.path(document['file']),
|
||||
'--parser-plugin-id',
|
||||
document['parser_plugin_id'],
|
||||
'--wait',
|
||||
]
|
||||
)
|
||||
self.write(
|
||||
'resources.json',
|
||||
{
|
||||
key: value
|
||||
for key, value in self.variables.items()
|
||||
if key not in self.profile.get('vars', {}) and key != 'group_secret'
|
||||
},
|
||||
)
|
||||
if 'smoke' in self.profile:
|
||||
return
|
||||
warmups = []
|
||||
for case in self.profile['cases']:
|
||||
if 'http_group' in case:
|
||||
case = resolve(case, self.variables)
|
||||
group = f'qa-warmup-{uuid4().hex[:12]}'
|
||||
response = group_request(case, group, 'qa-member-000', 1, uuid4().hex, 'none', self.env)
|
||||
warmups.append({'case': case['name'], 'output': response})
|
||||
self.write('warmup.json', warmups)
|
||||
|
||||
def cleanup(self):
|
||||
saved = json.loads((self.output / 'owned.json').read_text())
|
||||
identity = self.call(['whoami'])
|
||||
if saved['workspace_uuid'] != identity['workspace_uuid']:
|
||||
raise RuntimeError('cleanup refused: Workspace differs from owned.json')
|
||||
failures = []
|
||||
original = saved['resources'].copy()
|
||||
for resource in reversed(original):
|
||||
args = [resource['kind'], 'delete', resource['uuid'], '--yes']
|
||||
if resource['kind'] == 'model':
|
||||
args += ['--type', resource['type']]
|
||||
try:
|
||||
self.call(args)
|
||||
except (RuntimeError, ValueError, OSError) as exc:
|
||||
failures.append(str(exc))
|
||||
else:
|
||||
saved['resources'].remove(resource)
|
||||
self.write('owned.json', saved)
|
||||
if 'smoke' in self.profile:
|
||||
from regression_smoke import verify_cleanup
|
||||
|
||||
try:
|
||||
verify_cleanup(self, [item for item in original if item not in saved['resources']])
|
||||
except (RuntimeError, ValueError, OSError, KeyError) as exc:
|
||||
failures.append(f'cleanup readback failed: {exc}')
|
||||
return failures
|
||||
|
||||
def run(self):
|
||||
if 'smoke' in self.profile:
|
||||
from regression_smoke import run_smoke
|
||||
|
||||
return run_smoke(self)
|
||||
cases = []
|
||||
for original in self.profile['cases']:
|
||||
case = resolve(original, self.variables)
|
||||
if 'body' in case:
|
||||
case['args'] += ['--file', self.write(f'case-{len(cases)}.json', case.pop('body'))]
|
||||
cases.extend(
|
||||
[{key: case[key] for key in ('name', 'args', 'expect', 'http_group') if key in case}]
|
||||
* case.get('weight', 1)
|
||||
)
|
||||
cases_path = self.write('cases.json', cases)
|
||||
observed = resolve(self.profile.get('observe', {}), self.variables)
|
||||
observe_args = []
|
||||
for name, pid in observed.get('pids', {}).items():
|
||||
observe_args += ['--pid', f'{name}={int(pid)}']
|
||||
for name, path in observed.get('pid_files', {}).items():
|
||||
observe_args += ['--pid', f'{name}={int(Path(self.path(path)).read_text().strip())}']
|
||||
for name, url in observed.get('health', {}).items():
|
||||
observe_args += ['--health', f'{name}={url}']
|
||||
for container in observed.get('containers', []):
|
||||
observe_args += ['--container', container]
|
||||
if observed.get('box_data_root'):
|
||||
observe_args += ['--box-data-root', self.path(observed['box_data_root'])]
|
||||
results = []
|
||||
errors = []
|
||||
for phase in self.profile['phases']:
|
||||
destination = self.output / phase['name']
|
||||
command = [
|
||||
sys.executable,
|
||||
str(Path(__file__).with_name('regression_traffic.py')),
|
||||
'--lbctl',
|
||||
self.cli[0],
|
||||
'--config',
|
||||
self.cli[2],
|
||||
'--cases',
|
||||
cases_path,
|
||||
'--output',
|
||||
str(destination),
|
||||
'--duration',
|
||||
str(phase['duration_seconds']),
|
||||
'--rps',
|
||||
str(phase['rps']),
|
||||
'--workers',
|
||||
str(phase['workers']),
|
||||
'--cooldown',
|
||||
str(phase.get('cooldown_seconds', 0)),
|
||||
'--shared-case-sequence',
|
||||
'--groups',
|
||||
str(phase.get('groups', 0)),
|
||||
'--members-per-group',
|
||||
str(phase.get('members_per_group', 3)),
|
||||
'--group-state',
|
||||
str(self.output / 'group-state.json'),
|
||||
*observe_args,
|
||||
]
|
||||
with (self.output / f'{phase["name"]}.log').open('w') as log:
|
||||
result = subprocess.run(command, stdout=log, stderr=subprocess.STDOUT, env=self.env)
|
||||
summary = json.loads((destination / 'summary.json').read_text())
|
||||
summary['name'] = phase['name']
|
||||
summary['achieved_rps'] = summary['requests'] / summary['load_elapsed_seconds']
|
||||
results.append(summary)
|
||||
if result.returncode:
|
||||
errors.append(f'{phase["name"]}: business or health/resource checks failed')
|
||||
thresholds = self.profile.get('thresholds', {})
|
||||
if summary['achieved_rps'] < phase['rps'] * thresholds.get('min_rps_ratio', 0):
|
||||
errors.append(f'{phase["name"]}: achieved RPS below configured threshold')
|
||||
for name, maximum in thresholds.get('max_p95_seconds', {}).items():
|
||||
p95 = summary['cases'][name]['p95_seconds']
|
||||
if p95 is None or p95 > maximum:
|
||||
errors.append(f'{phase["name"]}/{name}: P95 missing or above configured threshold')
|
||||
return {'phases': results, 'errors': errors}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('action', choices=('validate', 'run', 'cleanup'))
|
||||
parser.add_argument('--profile', required=True, type=Path)
|
||||
parser.add_argument('--output', type=Path, help='New run directory; existing run directory for cleanup')
|
||||
parser.add_argument('--duration', type=float, help='Override duration_seconds in every phase')
|
||||
parser.add_argument('--rps', type=float, help='Override RPS in every phase')
|
||||
parser.add_argument('--workers', type=int, help='Override workers in every phase')
|
||||
parser.add_argument('--groups', type=int, help='Override group count in every group phase')
|
||||
parser.add_argument('--members-per-group', type=int)
|
||||
args = parser.parse_args()
|
||||
try:
|
||||
profile = json.loads(args.profile.read_text())
|
||||
if 'smoke' in profile and any(
|
||||
getattr(args, flag) is not None for flag in ('duration', 'rps', 'workers', 'groups', 'members_per_group')
|
||||
):
|
||||
raise ValueError('Load overrides are not applicable to smoke mode')
|
||||
for phase in profile.get('phases', []):
|
||||
for flag, field in [
|
||||
('duration', 'duration_seconds'),
|
||||
('rps', 'rps'),
|
||||
('workers', 'workers'),
|
||||
('groups', 'groups'),
|
||||
('members_per_group', 'members_per_group'),
|
||||
]:
|
||||
if getattr(args, flag) is not None:
|
||||
phase[field] = getattr(args, flag)
|
||||
validate(profile)
|
||||
if args.action == 'validate':
|
||||
print('Profile valid (no services or resources changed).')
|
||||
return
|
||||
if args.output is None:
|
||||
raise ValueError('--output is required for run/cleanup')
|
||||
suite = Suite(profile, args.profile.resolve(), args.output)
|
||||
if args.action == 'cleanup':
|
||||
errors = suite.cleanup()
|
||||
if errors:
|
||||
raise RuntimeError('\n'.join(errors))
|
||||
return
|
||||
args.output.mkdir(parents=True, exist_ok=False, mode=0o700)
|
||||
suite.write('profile.json', profile)
|
||||
result = {'phases': [], 'errors': []}
|
||||
completed = False
|
||||
try:
|
||||
suite.prepare()
|
||||
result = suite.run()
|
||||
completed = True
|
||||
except KeyboardInterrupt:
|
||||
result['errors'].append('Interrupted; in-flight outcomes may be unknown. Inspect retained resources.')
|
||||
except (RuntimeError, ValueError, OSError, KeyError) as exc:
|
||||
result['errors'].append(str(exc))
|
||||
finally:
|
||||
if not completed and not result['errors']:
|
||||
result['errors'].append('Run aborted before completion; inspect logs and retained resources.')
|
||||
policy = profile.get('cleanup', 'on-success')
|
||||
if policy == 'always' or (policy == 'on-success' and not result['errors']):
|
||||
if (args.output / 'owned.json').exists():
|
||||
try:
|
||||
result['errors'].extend(suite.cleanup())
|
||||
except (RuntimeError, ValueError, OSError, KeyError) as exc:
|
||||
result['errors'].append(f'cleanup failed: {exc}')
|
||||
result['passed'] = not result['errors']
|
||||
suite.write('summary.json', result)
|
||||
print(json.dumps(result, ensure_ascii=False, indent=2))
|
||||
if not result['passed']:
|
||||
raise SystemExit(1)
|
||||
except (RuntimeError, ValueError, OSError, KeyError) as exc:
|
||||
parser.exit(1, f'{type(exc).__name__}: {exc}\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,449 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Drive independent CLI requests or persistent group conversations with Linux resource sampling."""
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import hashlib
|
||||
import hmac
|
||||
import math
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
import uuid
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts'))
|
||||
from cloud_runtime_soak import read_endpoint_snapshot, read_process_snapshot
|
||||
|
||||
|
||||
def percentile(values, fraction):
|
||||
return sorted(values)[max(0, math.ceil(len(values) * fraction) - 1)] if values else None
|
||||
|
||||
|
||||
def owned_box_containers(data_root, checked=None):
|
||||
"""Discover sandboxes mounted below one run's data root, including newly created groups."""
|
||||
if checked is None:
|
||||
|
||||
def checked(command):
|
||||
return subprocess.check_output(command, text=True, timeout=10)
|
||||
|
||||
candidates = checked(['docker', 'ps', '-a', '--filter', 'label=langbot.box=true', '--format', '{{.ID}}']).split()
|
||||
if not candidates:
|
||||
return []
|
||||
root = Path(data_root).resolve()
|
||||
descriptions = checked(['docker', 'inspect', '--format', '{{json .}}', *candidates])
|
||||
owned = []
|
||||
for line in descriptions.splitlines():
|
||||
if not line:
|
||||
continue
|
||||
entry = json.loads(line)
|
||||
if any(
|
||||
Path(mount['Source']).resolve().is_relative_to(root) for mount in entry['Mounts'] if mount.get('Source')
|
||||
):
|
||||
owned.append(entry['Id'])
|
||||
return owned
|
||||
|
||||
|
||||
def group_request(case, group_id, member_id, turn, token, previous, env=None):
|
||||
"""Send one signed GroupMessage through the existing HTTP Bot adapter."""
|
||||
settings = case['http_group']
|
||||
secret = (os.environ if env is None else env)[settings['secret_env']]
|
||||
marker = f'LB_GROUP={group_id} LB_TURN={turn} LB_TOKEN={token} LB_PREVIOUS={previous}'
|
||||
body = json.dumps(
|
||||
{
|
||||
'session_id': group_id,
|
||||
'session_type': 'group',
|
||||
'sender': {'id': member_id, 'name': member_id, 'group_name': group_id},
|
||||
'message': [{'type': 'Plain', 'text': settings['message'] + ' ' + marker}],
|
||||
}
|
||||
).encode()
|
||||
stamp = str(int(time.time()))
|
||||
signature = hmac.new(secret.encode(), stamp.encode() + b'.' + body, hashlib.sha256).hexdigest()
|
||||
request = urllib.request.Request(
|
||||
settings['url'],
|
||||
data=body,
|
||||
headers={
|
||||
'Content-Type': 'application/json',
|
||||
'X-LB-Timestamp': stamp,
|
||||
'X-LB-Signature': 'sha256=' + signature,
|
||||
'X-LB-Idempotency-Key': token,
|
||||
},
|
||||
)
|
||||
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}))
|
||||
with opener.open(request, timeout=90) as response:
|
||||
payload = json.load(response)
|
||||
if response.status != 200 or not isinstance(payload, dict) or payload.get('code') != 0:
|
||||
raise ValueError('group ingress did not return a completed response')
|
||||
data = payload.get('data', {})
|
||||
if not isinstance(data, dict):
|
||||
raise ValueError('group ingress response data must be an object')
|
||||
parts = data.get('message')
|
||||
if not isinstance(parts, list) or any(not isinstance(part, dict) for part in parts):
|
||||
raise ValueError('group reply must contain message components')
|
||||
text = ''.join(str(part.get('text', '')) for part in parts if part.get('type') == 'Plain')
|
||||
if data.get('session_id') != group_id or not data.get('reply_to'):
|
||||
raise ValueError('group reply session or message correlation is missing')
|
||||
if case['expect'] not in text or f'GROUP_ACK={token}' not in text:
|
||||
raise ValueError('group reply failed its business or conversation-history oracle')
|
||||
return payload
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--lbctl', required=True)
|
||||
parser.add_argument('--config', required=True)
|
||||
parser.add_argument(
|
||||
'--cases', type=Path, required=True, help='JSON cases with name, expect, and args or http_group'
|
||||
)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
parser.add_argument('--duration', type=float, default=600)
|
||||
parser.add_argument('--rps', type=float, default=2)
|
||||
parser.add_argument('--workers', type=int, default=4)
|
||||
parser.add_argument(
|
||||
'--groups', type=int, default=0, help='Stable group sessions; zero uses independent CLI requests'
|
||||
)
|
||||
parser.add_argument('--members-per-group', type=int, default=3)
|
||||
parser.add_argument(
|
||||
'--group-state', type=Path, help='Carry group histories across load phases; refuses interrupted runs'
|
||||
)
|
||||
parser.add_argument('--cooldown', type=float, default=30)
|
||||
parser.add_argument('--pid', action='append', default=[], metavar='NAME=PID')
|
||||
parser.add_argument('--health', action='append', default=[], metavar='NAME=URL')
|
||||
parser.add_argument('--container', action='append', default=[])
|
||||
parser.add_argument('--box-data-root', type=Path, help='Dynamically sample Box containers mounted under this root')
|
||||
parser.add_argument(
|
||||
'--shared-case-sequence', action='store_true', help='Share the weighted case sequence across workers'
|
||||
)
|
||||
args = parser.parse_args()
|
||||
if (
|
||||
not all(math.isfinite(value) for value in (args.duration, args.rps, args.cooldown))
|
||||
or args.duration <= 0
|
||||
or args.rps <= 0
|
||||
or not 1 <= args.workers <= 32
|
||||
or args.cooldown < 0
|
||||
):
|
||||
parser.error('duration/rps must be positive, workers 1..32, cooldown nonnegative')
|
||||
if not 0 <= args.groups <= 4096 or not 1 <= args.members_per_group <= 1000:
|
||||
parser.error('groups must be 0..4096 (0 selects CLI mode), members-per-group 1..1000')
|
||||
cases = json.loads(args.cases.read_text())
|
||||
if (
|
||||
not isinstance(cases, list)
|
||||
or not cases
|
||||
or any(
|
||||
not isinstance(case, dict)
|
||||
or not isinstance(case.get('name'), str)
|
||||
or (
|
||||
not args.groups
|
||||
and (not isinstance(case.get('args'), list) or not all(isinstance(item, str) for item in case['args']))
|
||||
)
|
||||
or not isinstance(case.get('expect'), str)
|
||||
or not case['expect']
|
||||
for case in cases
|
||||
)
|
||||
):
|
||||
parser.error('cases must contain name, string args, and expected response text')
|
||||
if args.groups:
|
||||
if args.groups < len(cases):
|
||||
parser.error('groups must cover the complete weighted case sequence')
|
||||
for case in cases:
|
||||
settings = case.get('http_group', {})
|
||||
if (
|
||||
not isinstance(settings, dict)
|
||||
or set(settings) != {'url', 'secret_env', 'message'}
|
||||
or not all(isinstance(value, str) and value for value in settings.values())
|
||||
or not settings['url'].startswith(('http://', 'https://'))
|
||||
or not os.environ.get(settings['secret_env'])
|
||||
):
|
||||
parser.error('group cases need HTTP(S) url, message and a populated secret_env')
|
||||
args.output.mkdir(parents=True, exist_ok=False)
|
||||
pids = {name: int(pid) for name, pid in (value.split('=', 1) for value in args.pid)}
|
||||
endpoints = dict(value.split('=', 1) for value in args.health)
|
||||
rows = []
|
||||
sampling_errors = []
|
||||
lock = threading.Lock()
|
||||
group_condition = threading.Condition(lock)
|
||||
stop = threading.Event()
|
||||
phase = ['load']
|
||||
next_case = 0
|
||||
started = time.monotonic()
|
||||
deadline = started + args.duration
|
||||
redactions = [value for name, value in os.environ.items() if 'API_KEY' in name and value]
|
||||
group_state = None
|
||||
groups = []
|
||||
if args.groups:
|
||||
redactions += [os.environ[case['http_group']['secret_env']] for case in cases]
|
||||
state_path = args.group_state or args.output / 'group-state.json'
|
||||
fingerprint = [(case['name'], case['http_group']) for case in cases]
|
||||
group_state = (
|
||||
json.loads(state_path.read_text())
|
||||
if state_path.exists()
|
||||
else {
|
||||
'run_id': uuid.uuid4().hex[:12],
|
||||
'cases': fingerprint,
|
||||
'groups': [],
|
||||
'in_progress': False,
|
||||
}
|
||||
)
|
||||
if group_state.get('in_progress') or json.dumps(group_state['cases']) != json.dumps(fingerprint):
|
||||
parser.error('group state is interrupted or belongs to different cases; inspect it before reuse')
|
||||
for index in range(len(group_state['groups']), args.groups):
|
||||
group_state['groups'].append(
|
||||
{
|
||||
'id': f'qa-{group_state["run_id"]}-g{index:04d}',
|
||||
'case': index % len(cases),
|
||||
'turn': 1,
|
||||
'previous': 'none',
|
||||
'failed': False,
|
||||
}
|
||||
)
|
||||
groups = [
|
||||
dict(item, due=started + index / args.rps, busy=False)
|
||||
for index, item in enumerate(group_state['groups'][: args.groups])
|
||||
]
|
||||
group_state['in_progress'] = True
|
||||
state_path.write_text(json.dumps(group_state, indent=2))
|
||||
state_path.chmod(0o600)
|
||||
|
||||
def redact(text):
|
||||
for secret in redactions:
|
||||
text = text.replace(secret, '[REDACTED]')
|
||||
return text
|
||||
|
||||
def sample():
|
||||
with (args.output / 'resources.jsonl').open('w') as stream:
|
||||
while not stop.is_set():
|
||||
row = {
|
||||
'time': time.time(),
|
||||
'elapsed': time.monotonic() - started,
|
||||
'phase': phase[0],
|
||||
'targets': {},
|
||||
'health': {},
|
||||
}
|
||||
for name, pid in pids.items():
|
||||
try:
|
||||
row['targets'][name] = read_process_snapshot(pid)
|
||||
except (OSError, RuntimeError) as exc:
|
||||
row['targets'][name] = {'error': str(exc)}
|
||||
for name, url in endpoints.items():
|
||||
try:
|
||||
row['health'][name] = read_endpoint_snapshot(url, timeout_seconds=3)
|
||||
except RuntimeError as exc:
|
||||
row['health'][name] = {'error': str(exc)}
|
||||
containers = set(args.container)
|
||||
if args.box_data_root:
|
||||
containers.update(owned_box_containers(args.box_data_root))
|
||||
if containers:
|
||||
result = subprocess.run(
|
||||
['docker', 'stats', '--no-stream', '--format', '{{json .}}', *sorted(containers)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=10,
|
||||
)
|
||||
row['containers'] = [json.loads(line) for line in result.stdout.splitlines() if line]
|
||||
if result.returncode:
|
||||
row['container_error'] = result.stderr.strip()
|
||||
stream.write(json.dumps(row) + '\n')
|
||||
stream.flush()
|
||||
with lock:
|
||||
progress = {
|
||||
'elapsed': row['elapsed'],
|
||||
'phase': phase[0],
|
||||
'requests': len(rows),
|
||||
'status': dict(Counter(item['status'] for item in rows)),
|
||||
}
|
||||
(args.output / 'progress.json').write_text(json.dumps(progress))
|
||||
stop.wait(3)
|
||||
|
||||
def worker(index):
|
||||
nonlocal next_case
|
||||
iteration = 0
|
||||
due = started + index / args.rps
|
||||
while due < deadline:
|
||||
if stop.wait(max(0, due - time.monotonic())) or time.monotonic() >= deadline:
|
||||
break
|
||||
if args.shared_case_sequence:
|
||||
with lock:
|
||||
case = cases[next_case % len(cases)]
|
||||
next_case += 1
|
||||
else:
|
||||
case = cases[(iteration * args.workers + index) % len(cases)]
|
||||
before = time.monotonic()
|
||||
row = {'case': case['name'], 'time': time.time(), 'elapsed': before - started, 'status': 'failed'}
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[args.lbctl, '--config', args.config, '-o', 'json', *case['args']],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=90,
|
||||
)
|
||||
row['exit_code'] = result.returncode
|
||||
row['output'] = json.loads(redact(result.stdout))
|
||||
if result.returncode == 0 and case['expect'] in json.dumps(row['output'], ensure_ascii=False):
|
||||
row['status'] = 'ok'
|
||||
elif 'result_unknown' in result.stdout:
|
||||
row['status'] = 'unknown'
|
||||
else:
|
||||
row['error'] = redact(result.stderr)[:2000] or 'exit code or response oracle failed'
|
||||
except subprocess.TimeoutExpired:
|
||||
row.update(status='unknown', error='CLI timed out; server outcome unknown; request was not retried')
|
||||
except (OSError, ValueError) as exc:
|
||||
row['error'] = str(exc)
|
||||
row['seconds'] = time.monotonic() - before
|
||||
with lock:
|
||||
rows.append(row)
|
||||
with (args.output / 'requests.jsonl').open('a') as stream:
|
||||
stream.write(json.dumps(row, ensure_ascii=False) + '\n')
|
||||
iteration += 1
|
||||
due = max(due + args.workers / args.rps, time.monotonic())
|
||||
|
||||
def group_worker(_index):
|
||||
while not stop.is_set():
|
||||
with group_condition:
|
||||
while True:
|
||||
now = time.monotonic()
|
||||
if stop.is_set() or now >= deadline:
|
||||
return
|
||||
available = [group for group in groups if not group['busy'] and not group['failed']]
|
||||
if not available:
|
||||
if not any(group['busy'] for group in groups):
|
||||
return
|
||||
group_condition.wait(0.1)
|
||||
continue
|
||||
group = min(available, key=lambda item: item['due'])
|
||||
if group['due'] > now:
|
||||
group_condition.wait(min(1, group['due'] - now))
|
||||
continue
|
||||
group['busy'] = True
|
||||
break
|
||||
case = cases[group['case']]
|
||||
before = time.monotonic()
|
||||
token = uuid.uuid4().hex
|
||||
member = f'qa-member-{(group["turn"] - 1) % args.members_per_group:03d}'
|
||||
row = {
|
||||
'case': case['name'],
|
||||
'group_id': group['id'],
|
||||
'member_id': member,
|
||||
'turn': group['turn'],
|
||||
'time': time.time(),
|
||||
'elapsed': before - started,
|
||||
'status': 'failed',
|
||||
}
|
||||
try:
|
||||
row['output'] = group_request(case, group['id'], member, group['turn'], token, group['previous'])
|
||||
row['status'] = 'ok'
|
||||
except urllib.error.HTTPError as exc:
|
||||
row.update(http_status=exc.code, error=redact(exc.read(2000).decode(errors='replace')))
|
||||
if exc.code >= 500:
|
||||
row['status'] = 'unknown'
|
||||
except (urllib.error.URLError, TimeoutError, OSError) as exc:
|
||||
row.update(status='unknown', error=redact(str(exc)) + '; not retried')
|
||||
except (ValueError, KeyError) as exc:
|
||||
row['error'] = redact(str(exc))
|
||||
row['seconds'] = time.monotonic() - before
|
||||
with group_condition:
|
||||
rows.append(row)
|
||||
with (args.output / 'requests.jsonl').open('a') as stream:
|
||||
stream.write(json.dumps(row, ensure_ascii=False) + '\n')
|
||||
group.update(busy=False, failed=row['status'] != 'ok')
|
||||
if row['status'] == 'ok':
|
||||
group.update(turn=group['turn'] + 1, previous=token)
|
||||
group['due'] = max(group['due'] + args.groups / args.rps, time.monotonic())
|
||||
group_condition.notify_all()
|
||||
|
||||
def sampling_guard():
|
||||
try:
|
||||
sample()
|
||||
except Exception as exc:
|
||||
sampling_errors.append(f'{type(exc).__name__}: {exc}')
|
||||
stop.set()
|
||||
|
||||
sampler = threading.Thread(target=sampling_guard)
|
||||
sampler.start()
|
||||
try:
|
||||
with ThreadPoolExecutor(max_workers=args.workers) as pool:
|
||||
list(pool.map(group_worker if args.groups else worker, range(args.workers)))
|
||||
stop.wait(max(0, deadline - time.monotonic()))
|
||||
load_elapsed = time.monotonic() - started
|
||||
phase[0] = 'cooldown'
|
||||
stop.wait(args.cooldown)
|
||||
finally:
|
||||
stop.set()
|
||||
sampler.join()
|
||||
summary = {
|
||||
'duration_requested_seconds': args.duration,
|
||||
'load_elapsed_seconds': load_elapsed,
|
||||
'target_rps': args.rps,
|
||||
'workers': args.workers,
|
||||
'cooldown_seconds': args.cooldown,
|
||||
'requests': len(rows),
|
||||
'status': dict(Counter(row['status'] for row in rows)),
|
||||
'cases': {},
|
||||
}
|
||||
samples = [json.loads(line) for line in (args.output / 'resources.jsonl').read_text().splitlines()]
|
||||
summary['health_samples'] = len(samples)
|
||||
summary['health_failures'] = sum(
|
||||
any('error' in target for target in [*sample['targets'].values(), *sample['health'].values()])
|
||||
or bool(sample.get('container_error'))
|
||||
for sample in samples
|
||||
)
|
||||
summary['sampling_errors'] = sampling_errors
|
||||
if args.groups:
|
||||
for index, group in enumerate(groups):
|
||||
group_state['groups'][index] = {key: value for key, value in group.items() if key not in {'due', 'busy'}}
|
||||
group_state['in_progress'] = False
|
||||
temporary = state_path.with_suffix('.tmp')
|
||||
temporary.write_text(json.dumps(group_state, indent=2))
|
||||
temporary.chmod(0o600)
|
||||
temporary.replace(state_path)
|
||||
summary['groups'] = {
|
||||
'configured': args.groups,
|
||||
'members_per_group': args.members_per_group,
|
||||
'target_rps_per_group': args.rps / args.groups,
|
||||
'sessions': {
|
||||
group['id']: {
|
||||
'case': cases[group['case']]['name'],
|
||||
'successful_turns_total': group['turn'] - 1,
|
||||
'failed': group['failed'],
|
||||
'requests_this_phase': sum(row.get('group_id') == group['id'] for row in rows),
|
||||
'members_this_phase': sorted(
|
||||
{row['member_id'] for row in rows if row.get('group_id') == group['id']}
|
||||
),
|
||||
}
|
||||
for group in groups
|
||||
},
|
||||
}
|
||||
summary['group_failures'] = [
|
||||
group_id
|
||||
for group_id, result in summary['groups']['sessions'].items()
|
||||
if result['failed'] or result['successful_turns_total'] < 2 or not result['requests_this_phase']
|
||||
]
|
||||
for name in dict.fromkeys(case['name'] for case in cases):
|
||||
selected = [row for row in rows if row['case'] == name]
|
||||
times = [row['seconds'] for row in selected if row['status'] == 'ok']
|
||||
summary['cases'][name] = {
|
||||
'requests': len(selected),
|
||||
'status': dict(Counter(row['status'] for row in selected)),
|
||||
'p50_seconds': percentile(times, 0.5),
|
||||
'p95_seconds': percentile(times, 0.95),
|
||||
'p99_seconds': percentile(times, 0.99),
|
||||
}
|
||||
(args.output / 'summary.json').write_text(json.dumps(summary, indent=2))
|
||||
print(json.dumps(summary, indent=2), flush=True)
|
||||
if (
|
||||
not rows
|
||||
or any(row['status'] != 'ok' for row in rows)
|
||||
or summary['health_failures']
|
||||
or sampling_errors
|
||||
or summary.get('group_failures')
|
||||
):
|
||||
raise SystemExit(1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,209 @@
|
||||
"""Exercise the real HTTP server without LangBot, credentials or dependencies."""
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import base64
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import struct
|
||||
import threading
|
||||
import unittest
|
||||
from urllib.error import HTTPError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from mock_llm import MockServer
|
||||
|
||||
|
||||
class MockLLMTest(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.tmp = tempfile.TemporaryDirectory()
|
||||
cls.record = Path(cls.tmp.name) / 'requests.jsonl'
|
||||
scenarios = json.loads(Path(__file__).with_name('mock_llm_scenarios.json').read_text())
|
||||
cls.server = MockServer(('127.0.0.1', 0), scenarios, cls.record)
|
||||
cls.thread = threading.Thread(target=cls.server.serve_forever)
|
||||
cls.thread.start()
|
||||
cls.url = f'http://127.0.0.1:{cls.server.server_port}'
|
||||
|
||||
@classmethod
|
||||
def tearDownClass(cls):
|
||||
cls.server.shutdown()
|
||||
cls.server.server_close()
|
||||
cls.thread.join()
|
||||
cls.tmp.cleanup()
|
||||
|
||||
def post(self, model='mock-chat', **kwargs):
|
||||
body = {
|
||||
'model': model,
|
||||
'messages': [{'role': 'user', 'content': 'test'}],
|
||||
**kwargs,
|
||||
}
|
||||
request = Request(
|
||||
self.url + '/v1/chat/completions',
|
||||
data=json.dumps(body).encode(),
|
||||
headers={
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': 'Bearer must-not-be-recorded',
|
||||
},
|
||||
)
|
||||
with urlopen(request, timeout=5) as response:
|
||||
return response.read().decode()
|
||||
|
||||
def test_chat_and_model_discovery(self):
|
||||
with urlopen(self.url + '/v1/models') as response:
|
||||
models = json.load(response)
|
||||
self.assertIn('mock-box', [m['id'] for m in models['data']])
|
||||
self.assertEqual(
|
||||
json.loads(self.post())['choices'][0]['message']['content'],
|
||||
'LANGBOT_CHAT_OK',
|
||||
)
|
||||
|
||||
def test_stream_and_usage(self):
|
||||
events = self.post(stream=True, stream_options={'include_usage': True}).splitlines()
|
||||
self.assertEqual(events[-2], 'data: [DONE]')
|
||||
chunks = [json.loads(line[6:]) for line in events if line.startswith('data: {')]
|
||||
content = ''.join(c['choices'][0]['delta'].get('content', '') for c in chunks if c['choices'])
|
||||
self.assertEqual(content, 'LANGBOT_CHAT_OK')
|
||||
self.assertEqual(chunks[-1]['usage']['total_tokens'], 15)
|
||||
|
||||
def test_group_history_requires_previous_user_reply_and_isolation(self):
|
||||
first = {'role': 'user', 'content': 'LB_GROUP=group-a LB_TURN=1 LB_TOKEN=aaa LB_PREVIOUS=none'}
|
||||
second = {'role': 'user', 'content': 'LB_GROUP=group-a LB_TURN=2 LB_TOKEN=bbb LB_PREVIOUS=aaa'}
|
||||
reply = json.loads(self.post(messages=[first]))['choices'][0]['message']
|
||||
self.assertIn('GROUP_ACK=aaa', reply['content'])
|
||||
result = json.loads(self.post(messages=[first, reply, second]))['choices'][0]['message']
|
||||
self.assertIn('GROUP_ACK=bbb', result['content'])
|
||||
other = {'role': 'user', 'content': 'LB_GROUP=group-b LB_TURN=1 LB_TOKEN=ccc LB_PREVIOUS=none'}
|
||||
for messages in ([second], [first, second], [other, reply, second]):
|
||||
with self.subTest(messages=messages), self.assertRaises(HTTPError) as failure:
|
||||
self.post(messages=messages)
|
||||
self.assertEqual(failure.exception.code, 400)
|
||||
|
||||
def test_embedding_http_float_base64_and_validation(self):
|
||||
def embed(**patch):
|
||||
body = {'model': 'mock-embedding', 'input': ['repeatable fixture', 'another fixture'], **patch}
|
||||
request = Request(
|
||||
self.url + '/v1/embeddings',
|
||||
data=json.dumps(body).encode(),
|
||||
headers={'Content-Type': 'application/json'},
|
||||
)
|
||||
with urlopen(request, timeout=5) as response:
|
||||
return json.load(response)
|
||||
|
||||
first = embed()['data']
|
||||
self.assertEqual(first, embed()['data'])
|
||||
self.assertEqual([item['index'] for item in first], [0, 1])
|
||||
self.assertEqual(len(first[0]['embedding']), 384)
|
||||
self.assertAlmostEqual(sum(value * value for value in first[0]['embedding']), 1)
|
||||
encoded = embed(encoding_format='base64')['data'][0]['embedding']
|
||||
decoded = struct.unpack('<384f', base64.b64decode(encoded))
|
||||
for actual, expected in zip(decoded, first[0]['embedding']):
|
||||
self.assertAlmostEqual(actual, expected, places=6)
|
||||
for patch in ({'input': []}, {'dimensions': 0}, {'encoding_format': 'bad'}, {'model': 'unknown'}):
|
||||
with self.subTest(patch=patch), self.assertRaises(HTTPError) as caught:
|
||||
embed(**patch)
|
||||
self.assertEqual(caught.exception.code, 400)
|
||||
|
||||
def box_turn(self, streaming=False):
|
||||
messages = [{'role': 'user', 'content': 'run box checks'}]
|
||||
tools = [
|
||||
{
|
||||
'type': 'function',
|
||||
'function': {'name': 'exec', 'parameters': {'type': 'object'}},
|
||||
}
|
||||
]
|
||||
commands = []
|
||||
for _ in range(2):
|
||||
wire = self.post('mock-box', messages=messages, tools=tools, stream=streaming)
|
||||
if streaming:
|
||||
deltas = [
|
||||
json.loads(line[6:])['choices'][0]['delta']
|
||||
for line in wire.splitlines()
|
||||
if line.startswith('data: {')
|
||||
]
|
||||
call = next(d['tool_calls'][0] for d in deltas if d.get('tool_calls') and 'id' in d['tool_calls'][0])
|
||||
arguments = ''.join(d['tool_calls'][0]['function']['arguments'] for d in deltas if d.get('tool_calls'))
|
||||
call = {
|
||||
**call,
|
||||
'function': {**call['function'], 'arguments': arguments},
|
||||
}
|
||||
call.pop('index')
|
||||
message = {'role': 'assistant', 'content': None, 'tool_calls': [call]}
|
||||
else:
|
||||
message = json.loads(wire)['choices'][0]['message']
|
||||
call = message['tool_calls'][0]
|
||||
commands.append(json.loads(call['function']['arguments'])['command'])
|
||||
messages.extend(
|
||||
[
|
||||
message,
|
||||
{
|
||||
'role': 'tool',
|
||||
'tool_call_id': call['id'],
|
||||
'content': 'LANGBOT_BOX_SENTINEL',
|
||||
},
|
||||
]
|
||||
)
|
||||
final = json.loads(self.post('mock-box', messages=messages, tools=tools))
|
||||
self.assertEqual(final['choices'][0]['message']['content'], 'LANGBOT_BOX_OK')
|
||||
self.assertIn('printf LANGBOT_BOX_SENTINEL', commands[0])
|
||||
self.assertEqual(commands[1], 'cat /workspace/mock-llm.txt')
|
||||
|
||||
def test_tool_rounds_and_fragmented_stream(self):
|
||||
self.box_turn()
|
||||
self.box_turn(streaming=True)
|
||||
|
||||
def test_concurrent_turns_do_not_share_progress(self):
|
||||
with ThreadPoolExecutor(max_workers=4) as pool:
|
||||
list(pool.map(lambda _: self.box_turn(), range(8)))
|
||||
|
||||
def test_rag_requires_expected_model_input(self):
|
||||
with self.assertRaises(HTTPError) as caught:
|
||||
self.post('mock-rag')
|
||||
self.assertEqual(caught.exception.code, 400)
|
||||
response = json.loads(
|
||||
self.post(
|
||||
'mock-rag',
|
||||
messages=[
|
||||
{'role': 'system', 'content': 'Retrieved document: aurora-parser-rag-9137'},
|
||||
{'role': 'user', 'content': 'What is the documented sentinel?'},
|
||||
],
|
||||
)
|
||||
)
|
||||
self.assertEqual(response['choices'][0]['message']['content'], 'LANGBOT_RAG_OK')
|
||||
|
||||
def test_errors_and_invalid_requests(self):
|
||||
for status in (401, 429, 503):
|
||||
with self.subTest(status=status), self.assertRaises(HTTPError) as caught:
|
||||
self.post(f'mock-{status}')
|
||||
self.assertEqual(caught.exception.code, status)
|
||||
for kwargs in (
|
||||
{'model': 'unknown'},
|
||||
{'messages': []},
|
||||
{'stream_options': []},
|
||||
{'model': 'mock-box', 'tools': []},
|
||||
):
|
||||
with self.subTest(kwargs=kwargs), self.assertRaises(HTTPError) as caught:
|
||||
self.post(**kwargs)
|
||||
self.assertEqual(caught.exception.code, 400)
|
||||
messages = [
|
||||
{'role': 'user', 'content': 'test'},
|
||||
{'role': 'assistant', 'tool_calls': [{'id': 'call1'}]},
|
||||
]
|
||||
with self.assertRaises(HTTPError) as caught:
|
||||
self.post('mock-box', messages=messages)
|
||||
self.assertEqual(caught.exception.code, 400)
|
||||
messages.append({'role': 'tool', 'tool_call_id': 'call1', 'content': 'wrong output'})
|
||||
with self.assertRaises(HTTPError) as caught:
|
||||
self.post('mock-box', messages=messages)
|
||||
self.assertEqual(caught.exception.code, 400)
|
||||
|
||||
def test_record_omits_authorization(self):
|
||||
self.post()
|
||||
with self.server.record_lock:
|
||||
recorded = self.record.read_text()
|
||||
self.assertNotIn('must-not-be-recorded', recorded)
|
||||
self.assertIn('"request_id"', recorded)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,66 @@
|
||||
"""A gate must fail closed on timeout, missing verdict, and inner failure."""
|
||||
|
||||
import os
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from regression_gate import supervise
|
||||
|
||||
|
||||
class GateTest(unittest.TestCase):
|
||||
def test_success_records_elapsed_time_and_resources(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory) / 'gate'
|
||||
code = (
|
||||
'from pathlib import Path; import json; '
|
||||
f'p=Path({str(output / "suite")!r}); p.mkdir(); '
|
||||
'(p/"summary.json").write_text(json.dumps({"passed":True,"requests":1}))'
|
||||
)
|
||||
result = supervise([sys.executable, '-c', code], output, 5, {'test': os.getpid()})
|
||||
self.assertTrue(result['passed'])
|
||||
self.assertLess(result['elapsed_seconds'], 5)
|
||||
self.assertGreater(result['resource_peaks']['test']['rss_bytes'], 0)
|
||||
|
||||
def test_timeout_is_failed_without_retry(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
result = supervise([sys.executable, '-c', 'import time; time.sleep(30)'], Path(directory) / 'gate', 0.1, {})
|
||||
self.assertFalse(result['passed'])
|
||||
self.assertTrue(result['timed_out'])
|
||||
self.assertLess(result['elapsed_seconds'], 2)
|
||||
|
||||
def test_teardown_failure_overrides_successful_smoke_verdict(self):
|
||||
from regression_ci import main
|
||||
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
output = Path(directory) / 'ci'
|
||||
|
||||
def fail_teardown(args):
|
||||
verdict = args.output / 'gate' / 'gate.json'
|
||||
verdict.parent.mkdir(parents=True)
|
||||
verdict.write_text(json.dumps({'passed': True}))
|
||||
raise RuntimeError('sandbox teardown failed')
|
||||
|
||||
with (
|
||||
patch('regression_ci.run', side_effect=fail_teardown),
|
||||
patch.object(
|
||||
sys, 'argv', ['regression_ci', '--output', str(output), '--lbctl', 'unused', '--plugins', 'unused']
|
||||
),
|
||||
self.assertRaises(RuntimeError),
|
||||
):
|
||||
main()
|
||||
result = json.loads((output / 'gate/gate.json').read_text())
|
||||
self.assertFalse(result['passed'])
|
||||
self.assertIn('sandbox teardown failed', result['infrastructure_error'])
|
||||
|
||||
def test_zero_exit_without_business_verdict_is_failed(self):
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
result = supervise([sys.executable, '-c', 'pass'], Path(directory) / 'gate', 5, {})
|
||||
self.assertFalse(result['passed'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,240 @@
|
||||
"""Exercise signed group traffic, member rotation, phase continuity and fail-closed history."""
|
||||
|
||||
import hashlib
|
||||
import hmac
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
|
||||
from regression_suite import Suite, validate
|
||||
|
||||
|
||||
class GroupTests(unittest.TestCase):
|
||||
def test_bot_readback_uses_the_cli_confirmed_endpoint_and_workspace(self):
|
||||
received = []
|
||||
|
||||
class Readback(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
received.append((self.path, self.headers.get('Authorization'), self.headers.get('X-Workspace-Id')))
|
||||
body = json.dumps({'code': 0, 'data': {'bot': {'uuid': 'owned-bot', 'event_bindings': []}}}).encode()
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *_args):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(('127.0.0.1', 0), Readback)
|
||||
thread = threading.Thread(target=server.serve_forever)
|
||||
thread.start()
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
current = json.loads((Path(__file__).with_name('regression_profiles') / 'groups.json').read_text())
|
||||
suite = Suite(current, Path(folder) / 'profile.json', Path(folder), env={'TEST_API_KEY': 'test-key'})
|
||||
suite.variables['workspace_uuid'] = 'bound-workspace'
|
||||
suite.connection = {
|
||||
'endpoint': f'http://127.0.0.1:{server.server_port}',
|
||||
'credential_source': 'env:TEST_API_KEY',
|
||||
}
|
||||
self.assertEqual(suite.read_bot('owned-bot')['uuid'], 'owned-bot')
|
||||
self.assertEqual(received, [('/api/v1/platform/bots/owned-bot', 'Bearer test-key', 'bound-workspace')])
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
def test_groups_profile_and_invalid_modes(self):
|
||||
path = Path(__file__).with_name('regression_profiles') / 'groups.json'
|
||||
profile = json.loads(path.read_text())
|
||||
validate(profile)
|
||||
for field, value in [('groups', 3), ('members_per_group', 0), ('groups', 4097)]:
|
||||
current = json.loads(path.read_text())
|
||||
current['phases'][0][field] = value
|
||||
with self.subTest(field=field), self.assertRaises(ValueError):
|
||||
validate(current)
|
||||
|
||||
def test_signed_groups_continue_across_phases_and_stop_after_bad_reply(self):
|
||||
secret = 'test-group-signing-key'
|
||||
histories = {}
|
||||
failures = []
|
||||
active = set()
|
||||
lock = threading.Lock()
|
||||
bad_reply = False
|
||||
|
||||
class Ingress(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.respond({'code': 0})
|
||||
|
||||
def respond(self, payload, status=200):
|
||||
body = json.dumps(payload).encode()
|
||||
self.send_response(status)
|
||||
self.send_header('Content-Length', str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def do_POST(self):
|
||||
raw = self.rfile.read(int(self.headers['Content-Length']))
|
||||
stamp = self.headers['X-LB-Timestamp']
|
||||
expected = (
|
||||
'sha256=' + hmac.new(secret.encode(), stamp.encode() + b'.' + raw, hashlib.sha256).hexdigest()
|
||||
)
|
||||
payload = json.loads(raw)
|
||||
group = payload['session_id']
|
||||
fields = dict(re.findall(r'LB_(GROUP|TURN|TOKEN|PREVIOUS)=(\S+)', payload['message'][0]['text']))
|
||||
with lock:
|
||||
prior = histories.get(group, [])
|
||||
if (
|
||||
self.headers['X-LB-Signature'] != expected
|
||||
or group in active
|
||||
or payload['session_type'] != 'group'
|
||||
or fields['GROUP'] != group
|
||||
or int(fields['TURN']) != len(prior) + 1
|
||||
or fields['PREVIOUS'] != (prior[-1]['token'] if prior else 'none')
|
||||
):
|
||||
failures.append(payload)
|
||||
active.add(group)
|
||||
time.sleep(0.01)
|
||||
with lock:
|
||||
histories.setdefault(group, []).append(
|
||||
{'token': fields['TOKEN'], 'member': payload['sender']['id']}
|
||||
)
|
||||
active.remove(group)
|
||||
ack = 'missing' if bad_reply and group.endswith('g0000') else fields['TOKEN']
|
||||
self.respond(
|
||||
{
|
||||
'code': 0,
|
||||
'data': {
|
||||
'session_id': group,
|
||||
'reply_to': 'incoming-' + fields['TOKEN'],
|
||||
'message': [
|
||||
{'type': 'Plain', 'text': 'TEST_OK GROUP_'},
|
||||
{'type': 'Plain', 'text': 'ACK=' + ack},
|
||||
],
|
||||
},
|
||||
},
|
||||
status=500 if bad_reply == 'unknown' and group.endswith('g0000') else 200,
|
||||
)
|
||||
|
||||
def log_message(self, *_args):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(('127.0.0.1', 0), Ingress)
|
||||
thread = threading.Thread(target=server.serve_forever)
|
||||
thread.start()
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
cases = root / 'cases.json'
|
||||
cases.write_text(
|
||||
json.dumps(
|
||||
[
|
||||
{
|
||||
'name': 'group',
|
||||
'expect': 'TEST_OK',
|
||||
'http_group': {
|
||||
'url': f'http://127.0.0.1:{server.server_port}/bots/test/sync',
|
||||
'secret_env': 'LANGBOT_QA_GROUP_SECRET',
|
||||
'message': 'ai Test group history',
|
||||
},
|
||||
}
|
||||
]
|
||||
)
|
||||
)
|
||||
state = root / 'group-state.json'
|
||||
command = [
|
||||
sys.executable,
|
||||
str(Path(__file__).with_name('regression_traffic.py')),
|
||||
'--lbctl',
|
||||
'unused',
|
||||
'--config',
|
||||
'unused',
|
||||
'--cases',
|
||||
str(cases),
|
||||
'--duration',
|
||||
'0.8',
|
||||
'--rps',
|
||||
'40',
|
||||
'--workers',
|
||||
'3',
|
||||
'--cooldown',
|
||||
'0',
|
||||
'--members-per-group',
|
||||
'2',
|
||||
'--health',
|
||||
f'core=http://127.0.0.1:{server.server_port}/healthz',
|
||||
]
|
||||
env = {**os.environ, 'LANGBOT_QA_GROUP_SECRET': secret}
|
||||
summaries = []
|
||||
for groups in (4, 8):
|
||||
output = root / f'phase-{groups}'
|
||||
result = subprocess.run(
|
||||
[*command, '--groups', str(groups), '--group-state', str(state), '--output', str(output)],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
|
||||
summaries.append(json.loads((output / 'summary.json').read_text()))
|
||||
first = summaries[0]['groups']['sessions']
|
||||
second = summaries[1]['groups']['sessions']
|
||||
self.assertEqual(len(first), 4)
|
||||
self.assertEqual(len(second), 8)
|
||||
for group in first:
|
||||
self.assertGreater(second[group]['successful_turns_total'], first[group]['successful_turns_total'])
|
||||
self.assertEqual(len(second[group]['members_this_phase']), 2)
|
||||
self.assertFalse(failures)
|
||||
self.assertNotIn(secret, state.read_text())
|
||||
bad_reply = True
|
||||
output = root / 'failed'
|
||||
result = subprocess.run(
|
||||
[*command, '--groups', '4', '--output', str(output)],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
|
||||
summary = json.loads((output / 'summary.json').read_text())
|
||||
failed = next(item for group, item in summary['groups']['sessions'].items() if group.endswith('g0000'))
|
||||
self.assertTrue(failed['failed'])
|
||||
self.assertEqual(failed['requests_this_phase'], 1)
|
||||
bad_reply = 'unknown'
|
||||
result = subprocess.run(
|
||||
[*command, '--groups', '4', '--output', str(root / 'unknown')],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
self.assertEqual(result.returncode, 1, result.stdout + result.stderr)
|
||||
unknown = json.loads((root / 'unknown/summary.json').read_text())
|
||||
self.assertEqual(unknown['status'].get('unknown'), 1)
|
||||
interrupted = json.loads(state.read_text())
|
||||
interrupted['in_progress'] = True
|
||||
state.write_text(json.dumps(interrupted))
|
||||
result = subprocess.run(
|
||||
[*command, '--groups', '4', '--group-state', str(state), '--output', str(root / 'interrupted')],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=15,
|
||||
)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn('interrupted', result.stderr)
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,195 @@
|
||||
"""Exercise Job ownership and lifecycle with controlled CLI/Docker responses."""
|
||||
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import yaml
|
||||
|
||||
from regression_job import Job, main
|
||||
from test_regression_suite import FAKE_CLI, profile
|
||||
|
||||
|
||||
class ControlledJob(Job):
|
||||
def __init__(self, manifest, suite, path, output, health, fail_install=False):
|
||||
super().__init__(manifest, suite, path, output)
|
||||
self.health = health
|
||||
self.fail_install = fail_install
|
||||
self.calls = []
|
||||
self.env['LBCTL_TEST_ROOT'] = str(output.parent)
|
||||
|
||||
def command(self, args, timeout=900):
|
||||
self.calls.append(args)
|
||||
if args[: len(self.cli)] == self.cli:
|
||||
operation = args[len(self.cli) :]
|
||||
if operation[0] == 'install':
|
||||
self.deploy.mkdir()
|
||||
(self.deploy / 'data').mkdir()
|
||||
(self.deploy / 'compose.yaml').write_text('services: {}')
|
||||
(self.deploy / '.env').write_text(
|
||||
'LANGBOT_PLUGIN_RUNTIME_CONTROL_TOKEN=' + 't' * 32 + '\nLANGBOT_BOX_ENABLED=true\n'
|
||||
)
|
||||
record = {
|
||||
'deployment_id': 'owned-deployment',
|
||||
'compose_project': 'test-project',
|
||||
'compose_file': str(self.deploy / 'compose.yaml'),
|
||||
'data_dir': str(self.deploy / 'data'),
|
||||
'env_file': str(self.deploy / '.env'),
|
||||
'endpoint': self.health,
|
||||
}
|
||||
(self.deploy / 'deployment.yaml').write_text(yaml.safe_dump(record))
|
||||
# Simulate a timeout/failed final probe with no returned record.
|
||||
return (
|
||||
(1, json.dumps({'data': {}}), 'install failed')
|
||||
if self.fail_install
|
||||
else (0, json.dumps({'data': {'record': record}}), '')
|
||||
)
|
||||
if operation[0] == '--endpoint':
|
||||
return 0, json.dumps({'data': {'workspace_uuid': 'test-workspace'}}), ''
|
||||
if operation[0] == 'uninstall':
|
||||
shutil.rmtree(self.deploy / 'data')
|
||||
return 0, json.dumps({'data': {}}), ''
|
||||
if args[:2] == ['docker', 'ps']:
|
||||
if 'label=langbot.box=true' in args:
|
||||
return 0, '', ''
|
||||
return (
|
||||
0,
|
||||
'\n'.join(
|
||||
f'id-{name} {name}'
|
||||
for name in ('langbot', 'langbot_plugin_runtime', 'langbot_box', 'regression_mock')
|
||||
),
|
||||
'',
|
||||
)
|
||||
if args[:2] == ['docker', 'inspect']:
|
||||
if '{{.State.Pid}}' in args:
|
||||
return 0, str(os.getpid()), ''
|
||||
return 0, json.dumps({'network': {'IPAddress': '127.0.0.1'}}), ''
|
||||
if args[:2] == ['docker', 'compose']:
|
||||
if 'logs' in args:
|
||||
return 0, self.redact(self.env.get('LANGBOT_QA_API_KEY', '') + '\n' + 't' * 32), ''
|
||||
return 0, '', ''
|
||||
raise AssertionError(args)
|
||||
|
||||
@staticmethod
|
||||
def ready(url):
|
||||
pass
|
||||
|
||||
def prepare(self):
|
||||
suite = super().prepare()
|
||||
# Real traffic driver and HTTP health probe; infrastructure calls are controlled above.
|
||||
suite.profile['observe'] = {'health': {'core': self.health}}
|
||||
return suite
|
||||
|
||||
|
||||
class JobTests(unittest.TestCase):
|
||||
def test_lifecycle_success_and_partial_install_recovery(self):
|
||||
class Health(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
self.send_response(200)
|
||||
self.end_headers()
|
||||
self.wfile.write(b'{"code":0}')
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(('127.0.0.1', 0), Health)
|
||||
thread = threading.Thread(target=server.serve_forever)
|
||||
thread.start()
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
cli = root / 'lbctl-test'
|
||||
cli.write_text(f'#!{sys.executable}\n' + FAKE_CLI)
|
||||
cli.chmod(0o700)
|
||||
manifest = {'lbctl': str(cli), 'deployment': {'version': 'v4.11.0', 'port': 15351}}
|
||||
suite = profile()
|
||||
suite['resources'] = [{'id': 'chat', 'kind': 'pipeline', 'body': {'name': 'QA', 'config': {'ai': {}}}}]
|
||||
suite['cases'][0].update(args=['pipeline', 'run', '${chat}'], expect='TEST_OK')
|
||||
suite['phases'][0].update(duration_seconds=0.5, rps=4, workers=2)
|
||||
health = f'http://127.0.0.1:{server.server_port}'
|
||||
job = ControlledJob(manifest, suite, root / 'profile.json', root / 'success', health)
|
||||
result = job.run()
|
||||
self.assertTrue(result['passed'], result)
|
||||
self.assertEqual(result['phase'], 'cleaned')
|
||||
self.assertFalse((job.deploy / 'data').exists())
|
||||
self.assertFalse((job.private / 'api-key').exists())
|
||||
self.assertIn('--purge-data', job.calls[-1])
|
||||
self.assertNotIn('t' * 32, (job.output / 'services.log').read_text())
|
||||
|
||||
job = ControlledJob(manifest, suite, root / 'profile.json', root / 'failed', health, fail_install=True)
|
||||
result = job.run()
|
||||
self.assertFalse(result['passed'])
|
||||
self.assertEqual(job.state['record']['deployment_id'], 'owned-deployment')
|
||||
self.assertTrue((job.deploy / 'data').exists())
|
||||
self.assertFalse(any('uninstall' in call for call in job.calls))
|
||||
with (
|
||||
patch.object(sys, 'argv', ['regression_job.py', 'cleanup', '--output', str(job.output)]),
|
||||
patch('regression_job.Job', return_value=job),
|
||||
):
|
||||
main()
|
||||
self.assertFalse((job.deploy / 'data').exists())
|
||||
self.assertEqual(job.state['phase'], 'cleaned')
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
def test_cleanup_refuses_changed_deployment(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
job = Job({}, profile(), root / 'profile.json', root)
|
||||
job.private.mkdir()
|
||||
job.deploy.mkdir()
|
||||
(job.private / 'job-id').write_text(job.state['job_id'])
|
||||
record = {
|
||||
'deployment_id': 'old',
|
||||
'compose_file': str(job.deploy / 'compose.yaml'),
|
||||
'data_dir': str(job.deploy / 'data'),
|
||||
'env_file': str(job.deploy / '.env'),
|
||||
}
|
||||
job.state['record'] = record
|
||||
(job.deploy / 'deployment.yaml').write_text(yaml.safe_dump({**record, 'deployment_id': 'new'}))
|
||||
with self.assertRaisesRegex(RuntimeError, 'identity changed'):
|
||||
job.cleanup()
|
||||
|
||||
def test_sandbox_cleanup_uses_owned_mount_roots(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
job = Job({}, profile(), root / 'profile.json', root)
|
||||
job.state['record'] = {'data_dir': str(job.deploy / 'data')}
|
||||
containers = ['ours', 'other']
|
||||
|
||||
def checked(args, timeout=900):
|
||||
if args[1] == 'ps':
|
||||
return '\n'.join(containers)
|
||||
return '\n'.join(
|
||||
json.dumps(
|
||||
{
|
||||
'Id': container,
|
||||
'Mounts': [
|
||||
{
|
||||
'Source': str(job.deploy / 'data/box/tenant')
|
||||
if container != 'other'
|
||||
else '/unrelated/box/tenant'
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
for container in args[4:]
|
||||
)
|
||||
|
||||
job.checked = checked
|
||||
self.assertEqual(job.owned_sandboxes(), ['ours'])
|
||||
containers.append('later')
|
||||
self.assertEqual(job.owned_sandboxes(), ['ours', 'later'])
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,205 @@
|
||||
"""Challenge the smoke verdict independently of a successful model reply."""
|
||||
|
||||
import base64
|
||||
import copy
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
from types import SimpleNamespace
|
||||
import unittest
|
||||
|
||||
from regression_smoke import (
|
||||
UnresolvedRun,
|
||||
check_attachment,
|
||||
check_fields,
|
||||
check_mock_requests,
|
||||
check_run,
|
||||
completed_run,
|
||||
verify_cleanup,
|
||||
)
|
||||
from regression_suite import validate
|
||||
|
||||
|
||||
class SmokeOracleTest(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.db = sqlite3.connect(':memory:')
|
||||
self.db.row_factory = sqlite3.Row
|
||||
self.addCleanup(self.db.close)
|
||||
self.db.executescript("""
|
||||
CREATE TABLE agent_run (id INTEGER, run_id TEXT, workspace_id TEXT,
|
||||
conversation_id TEXT, status TEXT, status_reason TEXT, finished_at TEXT);
|
||||
CREATE TABLE agent_run_event (run_id TEXT, sequence INTEGER, type TEXT, data_json TEXT);
|
||||
INSERT INTO agent_run VALUES (1, 'run', 'workspace', 'conversation', 'completed', '', 'now');
|
||||
INSERT INTO agent_run_event VALUES ('run', 1, 'run.completed', '{}');
|
||||
""")
|
||||
|
||||
def run_row(self, workspace='workspace', after=0):
|
||||
return completed_run(self.db, workspace, 'conversation', after, 0)
|
||||
|
||||
def test_terminal_status_and_event_are_both_required(self):
|
||||
self.assertTrue(check_run(self.db, self.run_row(), {})['terminal_event'])
|
||||
self.db.execute("UPDATE agent_run SET status='failed'")
|
||||
with self.assertRaises(ValueError):
|
||||
check_run(self.db, self.run_row(), {})
|
||||
self.db.execute("UPDATE agent_run SET status='completed'")
|
||||
self.db.execute('DELETE FROM agent_run_event')
|
||||
with self.assertRaises(ValueError):
|
||||
check_run(self.db, self.run_row(), {})
|
||||
|
||||
def test_missing_stale_wrong_workspace_and_ambiguous_runs_are_unknown(self):
|
||||
for workspace, after in [('other', 0), ('workspace', 1)]:
|
||||
with self.subTest(workspace=workspace, after=after), self.assertRaises(UnresolvedRun):
|
||||
self.run_row(workspace, after)
|
||||
self.db.execute("UPDATE agent_run SET status='running', finished_at=NULL")
|
||||
with self.assertRaises(UnresolvedRun):
|
||||
self.run_row()
|
||||
self.db.execute('INSERT INTO agent_run SELECT * FROM agent_run')
|
||||
with self.assertRaises(UnresolvedRun):
|
||||
self.run_row()
|
||||
|
||||
def test_box_requires_actual_backend_success_and_output(self):
|
||||
case = {'box_exec': {'count': 1, 'backend': 'docker', 'stdout': 'sentinel'}}
|
||||
good = {'backend': 'docker', 'exit_code': 0, 'stdout': 'sentinel'}
|
||||
for result, passes in [
|
||||
(good, True),
|
||||
({**good, 'backend': 'host'}, False),
|
||||
({**good, 'exit_code': 1}, False),
|
||||
({**good, 'stdout': ''}, False),
|
||||
(None, False),
|
||||
]:
|
||||
self.db.execute('DELETE FROM agent_run_event WHERE sequence=2')
|
||||
self.db.execute(
|
||||
'INSERT INTO agent_run_event VALUES (?, ?, ?, ?)',
|
||||
('run', 2, 'tool.call.completed', json.dumps({'tool_name': 'exec', 'result': result})),
|
||||
)
|
||||
with self.subTest(result=result):
|
||||
if passes:
|
||||
check_run(self.db, self.run_row(), case)
|
||||
else:
|
||||
with self.assertRaises(ValueError):
|
||||
check_run(self.db, self.run_row(), case)
|
||||
with self.assertRaises(ValueError):
|
||||
check_run(self.db, self.run_row(), {'box_exec': {**case['box_exec'], 'count': 2}})
|
||||
|
||||
def test_attachment_requires_exact_bytes_and_single_file(self):
|
||||
expected = {'name': 'out.txt', 'text': 'sentinel'}
|
||||
part = {'type': 'File', 'name': 'out.txt', 'base64': base64.b64encode(b'sentinel').decode()}
|
||||
data = {'replies': [{'message_chain': [part]}]}
|
||||
self.assertTrue(check_attachment(data, expected)['content_matched'])
|
||||
for parts in [
|
||||
[],
|
||||
[part, part],
|
||||
[{**part, 'base64': '!!!'}],
|
||||
[{**part, 'base64': 'd3Jvbmc='}],
|
||||
[{**part, 'name': 'other'}],
|
||||
[None],
|
||||
]:
|
||||
with self.subTest(parts=parts), self.assertRaises(ValueError):
|
||||
check_attachment({'replies': [{'message_chain': parts}]}, expected)
|
||||
|
||||
def test_fields_check_empty_results_and_missing_paths(self):
|
||||
check_fields({'results': []}, {'results': []})
|
||||
check_fields({'results': [{'metadata': {'kb': 'owned'}}]}, {'results.0.metadata.kb': 'owned'})
|
||||
for data in [{'results': [1]}, {}, {'results': None}]:
|
||||
with self.subTest(data=data), self.assertRaises(ValueError):
|
||||
check_fields(data, {'results': []})
|
||||
with self.assertRaises(ValueError):
|
||||
check_fields({'results': None}, {'results.0.metadata.kb': 'owned'})
|
||||
|
||||
def test_fallback_must_call_failed_primary_then_streaming_backup(self):
|
||||
expected = [
|
||||
{'model': 'primary', 'status': 429, 'stream': True},
|
||||
{'model': 'backup', 'status': 200, 'stream': True},
|
||||
]
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
path = Path(directory) / 'requests.jsonl'
|
||||
prefix = b'earlier request ignored\n'
|
||||
for rows in [
|
||||
expected,
|
||||
list(reversed(expected)),
|
||||
[expected[0], {**expected[1], 'stream': False}],
|
||||
[expected[0], {**expected[1], 'status': 503}],
|
||||
]:
|
||||
path.write_bytes(
|
||||
prefix
|
||||
+ b''.join(
|
||||
(
|
||||
json.dumps(
|
||||
{
|
||||
'path': '/v1/chat/completions',
|
||||
'status': row['status'],
|
||||
'request': {'model': row['model'], 'stream': row['stream']},
|
||||
}
|
||||
)
|
||||
+ '\n'
|
||||
).encode()
|
||||
for row in rows
|
||||
)
|
||||
)
|
||||
if rows == expected:
|
||||
self.assertEqual(check_mock_requests(path, len(prefix), expected), expected)
|
||||
else:
|
||||
with self.assertRaises(ValueError):
|
||||
check_mock_requests(path, len(prefix), expected)
|
||||
|
||||
def test_expected_tool_failure_requires_exact_exit_and_stderr(self):
|
||||
case = {'box_exec': {'count': 1, 'backend': 'docker', 'stdout': 'out', 'exit_code': 7, 'stderr': 'err'}}
|
||||
for exit_code, stderr in [(7, 'err'), (0, 'err'), (7, '')]:
|
||||
self.db.execute('DELETE FROM agent_run_event WHERE sequence=2')
|
||||
self.db.execute(
|
||||
'INSERT INTO agent_run_event VALUES (?, ?, ?, ?)',
|
||||
(
|
||||
'run',
|
||||
2,
|
||||
'tool.call.completed',
|
||||
json.dumps(
|
||||
{
|
||||
'tool_name': 'exec',
|
||||
'result': {'backend': 'docker', 'stdout': 'out', 'exit_code': exit_code, 'stderr': stderr},
|
||||
}
|
||||
),
|
||||
),
|
||||
)
|
||||
if exit_code == 7 and stderr == 'err':
|
||||
check_run(self.db, self.run_row(), case)
|
||||
else:
|
||||
with self.assertRaises(ValueError):
|
||||
check_run(self.db, self.run_row(), case)
|
||||
|
||||
def test_cleanup_readback_rejects_resources_still_visible(self):
|
||||
deleted = [{'kind': 'pipeline', 'uuid': 'owned'}]
|
||||
evidence = {}
|
||||
for visible in ['unrelated', 'owned']:
|
||||
suite = SimpleNamespace(
|
||||
call=lambda args: {'pipelines': [{'uuid': visible}]},
|
||||
write=lambda name, value: evidence.update({name: value}),
|
||||
)
|
||||
if visible == 'owned':
|
||||
with self.assertRaises(ValueError):
|
||||
verify_cleanup(suite, deleted)
|
||||
else:
|
||||
verify_cleanup(suite, deleted)
|
||||
self.assertEqual(evidence['cleanup-checks.json'][0]['remaining'], [])
|
||||
suite.call = lambda args: {'unexpected': []}
|
||||
with self.assertRaises(ValueError):
|
||||
verify_cleanup(suite, deleted)
|
||||
|
||||
def test_profile_supports_mixed_cases_and_rejects_ignored_oracles(self):
|
||||
profile = json.loads(Path(__file__).with_name('regression_profiles').joinpath('smoke.json').read_text())
|
||||
validate(profile)
|
||||
for changes in [{'thresholds': {'min_rps_ratio': 1}}, {'smoke': {'ledger_db': ''}}, {'phases': None}]:
|
||||
with self.subTest(changes=changes), self.assertRaises(ValueError):
|
||||
validate({**profile, **changes})
|
||||
wrong = copy.deepcopy(profile)
|
||||
next(case for case in wrong['cases'] if case['name'] == 'rag_retrieve')['box_exec'] = {
|
||||
'count': 1,
|
||||
'backend': 'docker',
|
||||
'stdout': 'x',
|
||||
}
|
||||
with self.assertRaises(ValueError):
|
||||
validate(wrong)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -0,0 +1,309 @@
|
||||
"""Focused checks for profile validation, ownership and the CLI preparation sequence."""
|
||||
|
||||
import copy
|
||||
from contextlib import redirect_stdout
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from regression_suite import Suite, main, merge, resolve, validate
|
||||
|
||||
|
||||
FAKE_CLI = r"""
|
||||
import json, os, sys
|
||||
from pathlib import Path
|
||||
args = sys.argv[5:]
|
||||
root = Path(os.environ['LBCTL_TEST_ROOT'])
|
||||
state_path = root / 'cli-state.json'
|
||||
state = json.loads(state_path.read_text()) if state_path.exists() else {}
|
||||
body = json.loads(Path(args[args.index('--file') + 1]).read_text()) if '--file' in args else {}
|
||||
with (root / 'cli-calls.jsonl').open('a') as log:
|
||||
log.write(json.dumps({'args': args, 'body': body}) + '\n')
|
||||
if args == ['whoami']:
|
||||
data = {'workspace_uuid': 'test-workspace'}
|
||||
elif args[:2] == ['pipeline', 'run']:
|
||||
assert args[2] in state
|
||||
data = {'status': 'completed', 'reply': 'TEST_OK'}
|
||||
elif args[1] == 'delete':
|
||||
del state[args[2]]
|
||||
state_path.write_text(json.dumps(state))
|
||||
data = {}
|
||||
elif args[:2] == ['pipeline', 'get']:
|
||||
data = state[args[2]]
|
||||
elif args[1] in ['create', 'apply']:
|
||||
uuid = body.get('uuid', f'created-{len(state)}')
|
||||
data = {'uuid': uuid, 'name': body.get('name', 'test'), 'description': '', 'config': body.get('config', {'ai': {'default': True}})}
|
||||
state[uuid] = data
|
||||
state_path.write_text(json.dumps(state))
|
||||
else:
|
||||
raise RuntimeError(args)
|
||||
print(json.dumps({'ok': True, 'data': data}))
|
||||
"""
|
||||
|
||||
|
||||
def profile():
|
||||
return {
|
||||
'version': 1,
|
||||
'cli': {'config': 'lbctl.yaml'},
|
||||
'cases': [{'name': 'chat', 'args': ['pipeline', 'run', 'existing'], 'expect': 'OK'}],
|
||||
'phases': [{'name': 'normal', 'duration_seconds': 1, 'rps': 2, 'workers': 4}],
|
||||
'observe': {'health': {'core': 'http://127.0.0.1:5300/healthz'}},
|
||||
}
|
||||
|
||||
|
||||
class ProfileTests(unittest.TestCase):
|
||||
def test_created_bot_verification_failure_can_only_defer_bot_readback(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
suite = Suite(profile(), root / 'profile.json', root)
|
||||
response = {
|
||||
'data': {'operation': 'bot.create', 'uuid': 'owned-bot', 'bot': {'uuid': 'owned-bot'}},
|
||||
'error': {'type': 'verification_failed'},
|
||||
}
|
||||
result = subprocess.CompletedProcess([], 1, json.dumps(response), '')
|
||||
with patch('regression_suite.subprocess.run', return_value=result):
|
||||
with self.assertRaises(RuntimeError):
|
||||
suite.call(['bot', 'create'], {'name': 'QA'})
|
||||
self.assertEqual(
|
||||
suite.call(['bot', 'create'], {'name': 'QA'}, defer_bot_readback=True)['uuid'], 'owned-bot'
|
||||
)
|
||||
with self.assertRaises(RuntimeError):
|
||||
suite.call(['pipeline', 'apply'], {}, defer_bot_readback=True)
|
||||
|
||||
def test_group_bot_is_owned_before_raw_readback_and_warmed_before_load(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
current = profile()
|
||||
current['resources'] = [
|
||||
{
|
||||
'id': 'chat_bot',
|
||||
'kind': 'bot',
|
||||
'body': {
|
||||
'name': 'QA',
|
||||
'adapter': 'http_bot',
|
||||
'enable': True,
|
||||
'adapter_config': {'inbound_secret': '${group_secret}'},
|
||||
'event_bindings': [{'target_type': 'pipeline', 'target_uuid': 'pipeline-a'}],
|
||||
},
|
||||
}
|
||||
]
|
||||
current['cases'] = [
|
||||
{
|
||||
'name': 'chat',
|
||||
'expect': 'OK',
|
||||
'http_group': {
|
||||
'url': 'http://localhost/bots/${chat_bot}/sync',
|
||||
'message': 'ai Test',
|
||||
'secret_env': 'LANGBOT_QA_GROUP_SECRET',
|
||||
},
|
||||
}
|
||||
]
|
||||
current['phases'][0]['groups'] = 1
|
||||
validate(current)
|
||||
for correct_route in (False, True):
|
||||
suite = Suite(current, root / 'profile.json', root)
|
||||
|
||||
def call(args, body=None, **kwargs):
|
||||
if args == ['whoami']:
|
||||
return {'workspace_uuid': 'workspace-a'}
|
||||
if args == ['bot', 'create']:
|
||||
self.assertTrue(kwargs['defer_bot_readback'])
|
||||
return {'uuid': 'owned-bot'}
|
||||
raise AssertionError(args)
|
||||
|
||||
suite.call = call
|
||||
actual = copy.deepcopy(current['resources'][0]['body'])
|
||||
actual['adapter_config']['inbound_secret'] = '***'
|
||||
if not correct_route:
|
||||
actual['event_bindings'][0]['target_uuid'] = 'wrong-pipeline'
|
||||
with (
|
||||
patch.object(suite, 'read_bot', return_value=actual),
|
||||
patch('regression_suite.group_request', return_value={'code': 0}) as warmup,
|
||||
):
|
||||
if correct_route:
|
||||
suite.prepare()
|
||||
warmup.assert_called_once()
|
||||
self.assertNotIn(suite.variables['group_secret'], (root / 'resources.json').read_text())
|
||||
else:
|
||||
with self.assertRaisesRegex(RuntimeError, 'Bot readback differs'):
|
||||
suite.prepare()
|
||||
warmup.assert_not_called()
|
||||
self.assertEqual(
|
||||
json.loads((root / 'owned.json').read_text())['resources'],
|
||||
[{'kind': 'bot', 'uuid': 'owned-bot'}],
|
||||
)
|
||||
|
||||
def test_normal_profile_and_overrides_contract(self):
|
||||
validate(json.loads(Path(__file__).with_name('regression_profiles').joinpath('normal.json').read_text()))
|
||||
bad = profile()
|
||||
for field, value in [('rps', float('nan')), ('workers', 33), ('duration_seconds', -1)]:
|
||||
with self.subTest(field=field):
|
||||
current = copy.deepcopy(bad)
|
||||
current['phases'][0][field] = value
|
||||
with self.assertRaises(ValueError):
|
||||
validate(current)
|
||||
|
||||
def test_missing_refs_and_existing_resource_overwrite_are_rejected(self):
|
||||
current = profile()
|
||||
current['cases'][0]['args'][-1] = '${missing}'
|
||||
with self.assertRaises(KeyError):
|
||||
validate(current)
|
||||
current = profile()
|
||||
current['resources'] = [{'id': 'pipeline', 'kind': 'pipeline', 'body': {'uuid': 'existing'}}]
|
||||
with self.assertRaisesRegex(ValueError, 'uuid is forbidden'):
|
||||
validate(current)
|
||||
|
||||
def test_merge_keeps_defaults_but_replaces_lists(self):
|
||||
base = {'ai': {'enabled': True, 'models': ['old']}, 'output': {'x': 1}}
|
||||
actual = merge(base, {'ai': {'models': ['new']}})
|
||||
self.assertEqual(actual, {'ai': {'enabled': True, 'models': ['new']}, 'output': {'x': 1}})
|
||||
self.assertEqual(base['ai']['models'], ['old'])
|
||||
self.assertEqual(resolve(['${id}', {'id': '${id}'}], {'id': 'created'}), ['created', {'id': 'created'}])
|
||||
|
||||
def test_ownership_recorded_before_configuration_failure(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
current = profile()
|
||||
current['resources'] = [{'id': 'chat', 'kind': 'pipeline', 'body': {'name': 'QA', 'config': {'ai': {}}}}]
|
||||
suite = Suite(current, root / 'profile.json', root)
|
||||
calls = []
|
||||
|
||||
def call(args, body=None):
|
||||
calls.append((args, body))
|
||||
if args == ['whoami']:
|
||||
return {'workspace_uuid': 'workspace-a'}
|
||||
if args == ['pipeline', 'apply'] and 'uuid' not in body:
|
||||
return {'uuid': 'owned-pipeline'}
|
||||
if args[:2] == ['pipeline', 'get']:
|
||||
return {'uuid': 'owned-pipeline', 'name': 'QA', 'description': '', 'config': {'ai': {'old': 1}}}
|
||||
if args == ['pipeline', 'apply']:
|
||||
raise RuntimeError('configuration failed')
|
||||
return {}
|
||||
|
||||
suite.call = call
|
||||
with self.assertRaisesRegex(RuntimeError, 'configuration failed'):
|
||||
suite.prepare()
|
||||
self.assertEqual(
|
||||
json.loads((root / 'owned.json').read_text())['resources'],
|
||||
[{'kind': 'pipeline', 'uuid': 'owned-pipeline'}],
|
||||
)
|
||||
self.assertEqual(suite.cleanup(), [])
|
||||
self.assertEqual(calls[-1][0], ['pipeline', 'delete', 'owned-pipeline', '--yes'])
|
||||
self.assertEqual(json.loads((root / 'owned.json').read_text())['resources'], [])
|
||||
|
||||
def test_cleanup_refuses_another_workspace(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
(root / 'owned.json').write_text(json.dumps({'workspace_uuid': 'other', 'resources': []}))
|
||||
suite = Suite(profile(), root / 'profile.json', root)
|
||||
suite.call = lambda args: {'workspace_uuid': 'current'}
|
||||
with self.assertRaisesRegex(RuntimeError, 'Workspace differs'):
|
||||
suite.cleanup()
|
||||
|
||||
def test_aborted_run_is_not_treated_as_successful_cleanup(self):
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
config = root / 'profile.json'
|
||||
config.write_text(json.dumps(profile()))
|
||||
for error_type in (KeyboardInterrupt, TypeError):
|
||||
output = root / error_type.__name__
|
||||
|
||||
def interrupt(suite):
|
||||
suite.variables['workspace_uuid'] = 'test-workspace'
|
||||
suite.journal()
|
||||
raise error_type
|
||||
|
||||
argv = ['regression_suite.py', 'run', '--profile', str(config), '--output', str(output)]
|
||||
with (
|
||||
patch.object(sys, 'argv', argv),
|
||||
patch.object(Suite, 'prepare', interrupt),
|
||||
patch.object(Suite, 'cleanup') as cleanup,
|
||||
redirect_stdout(io.StringIO()),
|
||||
):
|
||||
with self.assertRaises(SystemExit if error_type is KeyboardInterrupt else TypeError):
|
||||
main()
|
||||
cleanup.assert_not_called()
|
||||
self.assertFalse(json.loads((output / 'summary.json').read_text())['passed'])
|
||||
|
||||
def test_subprocess_phases_and_cleanup_with_controlled_cli(self):
|
||||
class Health(BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
payload = b'{"code":0}'
|
||||
self.send_response(200)
|
||||
self.send_header('Content-Length', str(len(payload)))
|
||||
self.end_headers()
|
||||
self.wfile.write(payload)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
server = ThreadingHTTPServer(('127.0.0.1', 0), Health)
|
||||
thread = threading.Thread(target=server.serve_forever)
|
||||
thread.start()
|
||||
try:
|
||||
with tempfile.TemporaryDirectory() as folder:
|
||||
root = Path(folder)
|
||||
cli = root / 'lbctl-test'
|
||||
cli.write_text(f'#!{sys.executable}\n' + FAKE_CLI)
|
||||
cli.chmod(0o700)
|
||||
current = profile()
|
||||
current['cli']['executable'] = str(cli)
|
||||
current['resources'] = [
|
||||
{'id': 'provider', 'kind': 'provider', 'body': {'name': 'test'}},
|
||||
{
|
||||
'id': 'pipeline',
|
||||
'kind': 'pipeline',
|
||||
'body': {'name': 'test', 'config': {'ai': {'model': '${provider}'}}},
|
||||
},
|
||||
]
|
||||
current['cases'] = [
|
||||
{'name': 'chat', 'args': ['pipeline', 'run', '${pipeline}'], 'expect': 'TEST_OK', 'weight': 2},
|
||||
{'name': 'box', 'args': ['pipeline', 'run', '${pipeline}'], 'expect': 'TEST_OK', 'weight': 1},
|
||||
]
|
||||
current['phases'] = [
|
||||
{'name': name, 'duration_seconds': 0.5, 'rps': 12, 'workers': 3} for name in ('low', 'high')
|
||||
]
|
||||
current['observe'] = {'health': {'core': f'http://127.0.0.1:{server.server_port}/healthz'}}
|
||||
current['cleanup'] = 'always'
|
||||
config = root / 'profile.json'
|
||||
command = [
|
||||
sys.executable,
|
||||
str(Path(__file__).with_name('regression_suite.py')),
|
||||
'run',
|
||||
'--profile',
|
||||
str(config),
|
||||
]
|
||||
environment = {**os.environ, 'LBCTL_TEST_ROOT': folder}
|
||||
for enforce_latency in (False, True):
|
||||
current['thresholds'] = {'max_p95_seconds': {'chat': 0.001}} if enforce_latency else {}
|
||||
config.write_text(json.dumps(current))
|
||||
output = root / ('failure' if enforce_latency else 'success')
|
||||
result = subprocess.run(
|
||||
[*command, '--output', str(output)], env=environment, capture_output=True, text=True, timeout=30
|
||||
)
|
||||
summary = json.loads((output / 'summary.json').read_text())
|
||||
self.assertEqual(result.returncode, int(enforce_latency), result.stdout + result.stderr)
|
||||
self.assertEqual(summary['passed'], not enforce_latency)
|
||||
self.assertEqual(len(summary['phases']), 2)
|
||||
for phase in summary['phases']:
|
||||
self.assertGreater(phase['achieved_rps'], 0)
|
||||
self.assertEqual(phase['health_failures'], 0)
|
||||
self.assertGreaterEqual(phase['cases']['chat']['requests'], phase['cases']['box']['requests'])
|
||||
self.assertEqual(json.loads((output / 'owned.json').read_text())['resources'], [])
|
||||
self.assertEqual(json.loads((root / 'cli-state.json').read_text()), {})
|
||||
finally:
|
||||
server.shutdown()
|
||||
server.server_close()
|
||||
thread.join()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user