From 7ff04002a2241abeb477fad63ba167c2a9278b70 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <1051233107@qq.com> Date: Sat, 31 Oct 2026 12:17:04 +0800 Subject: [PATCH] refactor(tools): organize regression harness packages (#2828) Reorganize regression tools into explicit packages and update all repository references. --- .github/workflows/core-smoke.yml | 4 +- .github/workflows/daily-endurance.yml | 4 +- tools/README.md | 103 +++++++++++------- tools/__init__.py | 0 .../{ => contracts}/regression_contracts.json | 0 tools/entrypoints/__init__.py | 0 tools/{ => entrypoints}/mock_llm.py | 2 +- .../{ => entrypoints}/regression_analysis.py | 12 +- tools/{ => entrypoints}/regression_ci.py | 42 ++++--- .../{ => entrypoints}/regression_coverage.py | 4 +- tools/{ => entrypoints}/regression_gate.py | 7 +- tools/{ => entrypoints}/regression_job.py | 12 +- tools/{ => entrypoints}/regression_suite.py | 23 ++-- tools/{ => entrypoints}/regression_traffic.py | 12 +- tools/{ => fixtures}/mock_llm_scenarios.json | 0 .../components/pages/probe/index.html | 0 .../components/pages/probe/probe.py | 0 .../components/pages/probe/probe.yaml | 0 .../plugin_lifecycle/main.py | 0 tools/mock-backends.md | 4 +- .../endurance.json | 0 .../groups.json | 0 .../job.json | 0 .../normal.json | 0 .../smoke.json | 0 tools/published-plugins.md | 10 +- tools/regression/__init__.py | 0 .../{ => regression}/regression_api_cases.py | 17 ++- .../{ => regression}/regression_api_checks.py | 6 +- .../regression_api_resources.py | 6 +- .../{ => regression}/regression_cli_cases.py | 8 +- .../regression_execution_cases.py | 4 +- .../regression_isolation_cases.py | 16 +-- .../regression_lifecycle_cases.py | 14 +-- .../regression_local_cases.py | 16 +-- .../regression_plugin_cases.py | 12 +- .../regression_plugin_lifecycle.py | 10 +- tools/{ => regression}/regression_plugins.py | 0 .../regression_resource_cases.py | 6 +- .../{ => regression}/regression_scenarios.py | 14 +-- tools/{ => regression}/regression_smoke.py | 10 +- tools/{ => regression}/regression_state.py | 14 +-- tools/tests/__init__.py | 0 tools/{ => tests}/test_mock_llm.py | 8 +- tools/{ => tests}/test_regression_analysis.py | 2 +- tools/{ => tests}/test_regression_api.py | 34 +++--- tools/{ => tests}/test_regression_ci.py | 13 ++- .../{ => tests}/test_regression_cli_cases.py | 23 ++-- .../{ => tests}/test_regression_containers.py | 2 +- tools/{ => tests}/test_regression_coverage.py | 22 ++-- .../test_regression_execution_cases.py | 12 +- tools/{ => tests}/test_regression_gate.py | 18 ++- tools/{ => tests}/test_regression_groups.py | 9 +- .../{ => tests}/test_regression_isolation.py | 10 +- tools/{ => tests}/test_regression_job.py | 21 ++-- .../test_regression_lifecycle_cases.py | 14 +-- .../test_regression_load_ledger.py | 6 +- .../test_regression_local_cases.py | 17 +-- .../test_regression_plugin_cases.py | 4 +- .../test_regression_plugin_lifecycle.py | 35 ++++-- tools/{ => tests}/test_regression_plugins.py | 2 +- .../test_regression_resource_cases.py | 17 +-- .../{ => tests}/test_regression_scenarios.py | 18 +-- tools/{ => tests}/test_regression_smoke.py | 8 +- tools/{ => tests}/test_regression_state.py | 6 +- tools/{ => tests}/test_regression_suite.py | 20 +++- 66 files changed, 413 insertions(+), 300 deletions(-) create mode 100644 tools/__init__.py rename tools/{ => contracts}/regression_contracts.json (100%) create mode 100644 tools/entrypoints/__init__.py rename tools/{ => entrypoints}/mock_llm.py (99%) rename tools/{ => entrypoints}/regression_analysis.py (96%) rename tools/{ => entrypoints}/regression_ci.py (92%) rename tools/{ => entrypoints}/regression_coverage.py (99%) rename tools/{ => entrypoints}/regression_gate.py (95%) rename tools/{ => entrypoints}/regression_job.py (97%) rename tools/{ => entrypoints}/regression_suite.py (98%) rename tools/{ => entrypoints}/regression_traffic.py (98%) rename tools/{ => fixtures}/mock_llm_scenarios.json (100%) rename tools/{regression_fixtures => fixtures}/plugin_lifecycle/components/pages/probe/index.html (100%) rename tools/{regression_fixtures => fixtures}/plugin_lifecycle/components/pages/probe/probe.py (100%) rename tools/{regression_fixtures => fixtures}/plugin_lifecycle/components/pages/probe/probe.yaml (100%) rename tools/{regression_fixtures => fixtures}/plugin_lifecycle/main.py (100%) rename tools/{regression_profiles => profiles}/endurance.json (100%) rename tools/{regression_profiles => profiles}/groups.json (100%) rename tools/{regression_profiles => profiles}/job.json (100%) rename tools/{regression_profiles => profiles}/normal.json (100%) rename tools/{regression_profiles => profiles}/smoke.json (100%) create mode 100644 tools/regression/__init__.py rename tools/{ => regression}/regression_api_cases.py (95%) rename tools/{ => regression}/regression_api_checks.py (94%) rename tools/{ => regression}/regression_api_resources.py (98%) rename tools/{ => regression}/regression_cli_cases.py (94%) rename tools/{ => regression}/regression_execution_cases.py (99%) rename tools/{ => regression}/regression_isolation_cases.py (96%) rename tools/{ => regression}/regression_lifecycle_cases.py (97%) rename tools/{ => regression}/regression_local_cases.py (89%) rename tools/{ => regression}/regression_plugin_cases.py (96%) rename tools/{ => regression}/regression_plugin_lifecycle.py (97%) rename tools/{ => regression}/regression_plugins.py (100%) rename tools/{ => regression}/regression_resource_cases.py (98%) rename tools/{ => regression}/regression_scenarios.py (95%) rename tools/{ => regression}/regression_smoke.py (98%) rename tools/{ => regression}/regression_state.py (95%) create mode 100644 tools/tests/__init__.py rename tools/{ => tests}/test_mock_llm.py (98%) rename tools/{ => tests}/test_regression_analysis.py (98%) rename tools/{ => tests}/test_regression_api.py (90%) rename tools/{ => tests}/test_regression_ci.py (96%) rename tools/{ => tests}/test_regression_cli_cases.py (78%) rename tools/{ => tests}/test_regression_containers.py (99%) rename tools/{ => tests}/test_regression_coverage.py (95%) rename tools/{ => tests}/test_regression_execution_cases.py (97%) rename tools/{ => tests}/test_regression_gate.py (81%) rename tools/{ => tests}/test_regression_groups.py (96%) rename tools/{ => tests}/test_regression_isolation.py (91%) rename tools/{ => tests}/test_regression_job.py (94%) rename tools/{ => tests}/test_regression_lifecycle_cases.py (96%) rename tools/{ => tests}/test_regression_load_ledger.py (94%) rename tools/{ => tests}/test_regression_local_cases.py (89%) rename tools/{ => tests}/test_regression_plugin_cases.py (99%) rename tools/{ => tests}/test_regression_plugin_lifecycle.py (76%) rename tools/{ => tests}/test_regression_plugins.py (99%) rename tools/{ => tests}/test_regression_resource_cases.py (93%) rename tools/{ => tests}/test_regression_scenarios.py (93%) rename tools/{ => tests}/test_regression_smoke.py (96%) rename tools/{ => tests}/test_regression_state.py (90%) rename tools/{ => tests}/test_regression_suite.py (95%) diff --git a/.github/workflows/core-smoke.yml b/.github/workflows/core-smoke.yml index f440c6b5e..08b12afce 100644 --- a/.github/workflows/core-smoke.yml +++ b/.github/workflows/core-smoke.yml @@ -28,7 +28,7 @@ jobs: python-version: '3.12' - run: python -m pip install PyYAML==6.0.3 - name: Check regression tools - run: timeout 60s python -m unittest discover -s tools -p 'test_*.py' -v + run: timeout 60s python -m unittest discover -s tools/tests -p 'test_*.py' -v smoke: name: Core smoke (60s) @@ -59,7 +59,7 @@ jobs: docker pull python:3.12-alpine - name: Prepare real services and run the 60-second smoke run: | - uv run --no-sync python tools/regression_ci.py \ + uv run --no-sync python -m tools.entrypoints.regression_ci \ --lbctl .smoke-bin/lbctl_linux_amd64 --plugins plugins --output .qa/core-smoke - name: Publish gate result if: always() diff --git a/.github/workflows/daily-endurance.yml b/.github/workflows/daily-endurance.yml index bef7c0674..81e8212bb 100644 --- a/.github/workflows/daily-endurance.yml +++ b/.github/workflows/daily-endurance.yml @@ -64,7 +64,7 @@ jobs: read -r -a extras <<< "$EXTRA_PLUGINS" args=() if [[ "$RESTART_SERVICES" == true ]]; then args+=(--restart-services); fi - uv run --no-sync python tools/regression_ci.py \ + uv run --no-sync python -m tools.entrypoints.regression_ci \ --suite daily --scenario-groups "${groups[@]}" "${args[@]}" \ --lbctl .endurance-bin/lbctl_linux_amd64 --plugins plugins --extra-plugins "${extras[@]}" --output .qa/daily - name: Publish observations @@ -188,7 +188,7 @@ jobs: docker pull python:3.12-alpine - name: Run API contracts run: | - uv run --no-sync python tools/regression_ci.py \ + uv run --no-sync python -m tools.entrypoints.regression_ci \ --suite api --api-seed "$GITHUB_RUN_ID" \ --lbctl .api-bin/lbctl_linux_amd64 --plugins plugins --output .qa/api - name: Publish API result diff --git a/tools/README.md b/tools/README.md index 2e3f42a94..9831ad24f 100644 --- a/tools/README.md +++ b/tools/README.md @@ -1,10 +1,29 @@ # LangBot regression tools +## Layout and entrypoints + +Run module entrypoints from the repository root: + +```sh +uv run --no-sync python -m tools.entrypoints.regression_ci --suite smoke --lbctl /path/to/lbctl --plugins plugins --output .qa/smoke +uv run --no-sync python -m tools.entrypoints.regression_job validate --profile tools/profiles/job.json +uv run --no-sync python -m tools.entrypoints.regression_suite validate --profile tools/profiles/groups.json +``` + +The directory boundaries are intentional: + +- `entrypoints/` contains executable orchestration and reporting commands. +- `regression/` contains shared harness code and scenario-domain modules. +- `profiles/` contains suite inputs; `fixtures/` contains mock and plugin data; `contracts/` contains the behavior catalog. +- `tests/` contains the tool and harness tests. Run them with `python -m unittest discover -s tools/tests -p 'test_*.py' -v`. + +Imports use the `tools.entrypoints` and `tools.regression` package paths. Keep commands rooted at the repository so profile and fixture paths resolve consistently. + Run the deterministic OpenAI Chat Completions endpoint with Python 3.11+, without API credentials or third-party dependencies: ```sh -python3 tools/mock_llm.py --port 18080 --record /tmp/langbot-mock-requests.jsonl +python3 -m tools.entrypoints.mock_llm --port 18080 --record /tmp/langbot-mock-requests.jsonl ``` In LangBot, create an `openai-chat-completions` Provider whose base URL is @@ -37,7 +56,7 @@ acknowledgement in the model input, and rejects another group's history. Final responses carry a per-turn acknowledgement checked by the traffic driver. Add fixed response/tool sequences with `--scenarios path/to/scenarios.json`, using -`mock_llm_scenarios.json` as the example. `expect_tool_output` checks a substring +`fixtures/mock_llm_scenarios.json` as the example. `expect_tool_output` checks a substring in the previous tool results; `expect_prompt` checks the submitted messages. The server returns tool calls; it never executes their commands. Request recordings contain prompts, tool results, responses, @@ -46,7 +65,7 @@ status, and disconnects, but no HTTP headers. Use test data only. Run the focused HTTP checks on the Linux test host: ```sh -python3 -m unittest discover -s tools -p test_mock_llm.py -v +python3 -m unittest discover -s tools/tests -p test_mock_llm.py -v ``` These checks validate the mock server. A LangBot regression must additionally @@ -59,7 +78,7 @@ token hashes do not establish semantic retrieval quality. ## Normal traffic and resource collection -`regression_traffic.py` executes a JSON list of normal `lbctl` cases at a selected +`entrypoints/regression_traffic.py` executes a JSON list of normal `lbctl` cases at a selected rate. Each case has a name, CLI arguments, and an expected response substring: ```json @@ -70,7 +89,7 @@ Prepare a Workspace-bound CLI context and its API key environment variable, then run on the Linux test host: ```sh -python3 tools/regression_traffic.py \ +python3 -m tools.entrypoints.regression_traffic \ --lbctl /path/to/lbctl --config /path/to/lbctl.yaml \ --cases /path/to/cases.json --output /path/to/new-results-directory \ --duration 600 --rps 2 --workers 4 --cooldown 30 \ @@ -98,11 +117,11 @@ any failed health probe or process/container resource read. ## Configured regression suites -[`regression_suite.py`](regression_suite.py) turns a JSON profile into a repeatable +[`entrypoints/regression_suite.py`](entrypoints/regression_suite.py) turns a JSON profile into a repeatable resource preparation, traffic, observation, verdict, and cleanup sequence. Start -with [`regression_profiles/groups.json`](regression_profiles/groups.json) for +with [`profiles/groups.json`](profiles/groups.json) for continuous group conversations. The earlier -[`normal.json`](regression_profiles/normal.json) profile retains independent CLI +[`normal.json`](profiles/normal.json) profile retains independent CLI requests; parallel CLI workers alone do not simulate persistent groups. Use a dedicated QA Workspace. First provision the instance with `lbctl install` @@ -115,9 +134,9 @@ paths resolve against the profile directory, so adjust them if you move it. Run on the Linux test host: ```sh -python3 tools/regression_suite.py validate --profile tools/regression_profiles/groups.json -python3 tools/regression_suite.py run \ - --profile tools/regression_profiles/groups.json --output .qa/regression/run-001 +python3 -m tools.entrypoints.regression_suite validate --profile tools/profiles/groups.json +python3 -m tools.entrypoints.regression_suite run \ + --profile tools/profiles/groups.json --output .qa/regression/run-001 ``` `validate` checks profile syntax and references without contacting the instance. @@ -159,8 +178,8 @@ Failed runs retain resources by default. After inspection, clean them with the same profile and credentials: ```sh -python3 tools/regression_suite.py cleanup \ - --profile tools/regression_profiles/groups.json --output .qa/regression/run-001 +python3 -m tools.entrypoints.regression_suite cleanup \ + --profile tools/profiles/groups.json --output .qa/regression/run-001 ``` Cleanup checks the Workspace against `owned.json` and records remaining items @@ -169,7 +188,7 @@ contains `profile.json`, resolved `resources.json`, and one directory per phase with the original request/resource records. The root `summary.json` includes target/achieved RPS, phase results, threshold failures, and the final verdict. -The `regression_suite.py` entry reuses an existing instance and backend processes. +The `entrypoints/regression_suite.py` entry reuses an existing instance and backend processes. Use the complete lifecycle Job below to own deployment, credentials and backends. The independent `normal.json` cases use a new-session CLI entry. Group traffic uses the signed HTTP Bot adapter instead, as described below. Real IM adapter @@ -177,7 +196,7 @@ acceptance and resource-leak verdicts need separate scenarios. Test the suite plumbing on Linux with: ```sh -python3 -m unittest discover -s tools -p test_regression_suite.py -v +python3 -m unittest discover -s tools/tests -p test_regression_suite.py -v ``` ### Persistent group traffic @@ -229,8 +248,8 @@ before load. The creation is not retried. ## Complete lifecycle Job -[`regression_job.py`](regression_job.py) owns the deployment as well as the suite. -Use [`regression_profiles/job.json`](regression_profiles/job.json) to pin the Core +[`entrypoints/regression_job.py`](entrypoints/regression_job.py) owns the deployment as well as the suite. +Use [`profiles/job.json`](profiles/job.json) to pin the Core Release and select the suite and plugin input directory. The job runs this chain: 1. `lbctl install --profile all` creates a fresh deployment inside the run directory. @@ -258,9 +277,9 @@ also reserves host port 5401 and 2280–2285, independently of its HTTP port. Set the plugin directory and the `lbctl` executable in `job.json`, then run: ```sh -uv run --no-sync python tools/regression_job.py validate --profile tools/regression_profiles/job.json -uv run --no-sync python tools/regression_job.py run \ - --profile tools/regression_profiles/job.json --output .qa/regression/job-001 +uv run --no-sync python -m tools.entrypoints.regression_job validate --profile tools/profiles/job.json +uv run --no-sync python -m tools.entrypoints.regression_job run \ + --profile tools/profiles/job.json --output .qa/regression/job-001 ``` `--duration`, `--rps` and `--workers` override every load phase. Each output @@ -271,7 +290,7 @@ deployment data after removing containers. Failed Jobs retain their environment by default; after inspection use: ```sh -uv run --no-sync python tools/regression_job.py cleanup \ +uv run --no-sync python -m tools.entrypoints.regression_job cleanup \ --output .qa/regression/job-001 ``` @@ -288,7 +307,7 @@ LangBot/plugin/Box/RAG overhead, not real-model inference cost or retrieval quality. Run the Job plumbing checks on Linux with: ```sh -uv run --no-sync python -m unittest discover -s tools -p 'test_*.py' -v +uv run --no-sync python -m unittest discover -s tools/tests -p 'test_*.py' -v ``` ## Local plugin lifecycle @@ -296,7 +315,7 @@ uv run --no-sync python -m unittest discover -s tools -p 'test_*.py' -v Run the separate plugin lifecycle Job on a disposable Linux stack: ```sh -uv run --no-sync python tools/regression_ci.py --suite plugin-lifecycle \ +uv run --no-sync python -m tools.entrypoints.regression_ci --suite plugin-lifecycle \ --lbctl /path/to/lbctl --output .qa/plugin-lifecycle-001 ``` @@ -322,7 +341,7 @@ under `gate/suite/`. ## Finite core smoke -Use [`regression_profiles/smoke.json`](regression_profiles/smoke.json) for a finite +Use [`profiles/smoke.json`](profiles/smoke.json) for a finite functional check on an existing Core. It prepares 35 resources through `lbctl`, ingests HTML and plain-text fixtures through GeneralParsers, then executes 28 cases / 43 requests. Reports group the verdicts by stage: @@ -341,7 +360,7 @@ Cleanup is checked separately: delete journaled API resources and confirm their UUIDs are absent from CLI list readback. Evidence goes to `cleanup-checks.json`; cleanup failures affect the overall verdict. -Start a dedicated `mock_llm.py` with the bundled scenarios and `--record` as +Start a dedicated `entrypoints/mock_llm.py` with the bundled scenarios and `--record` as shown above. Copy the smoke profile and configure the CLI executable/config, service URLs, plugin archives, `vars.mock_records` to that mock recording file, and `vars.core_db` to the target Core's existing SQLite `langbot.db`. The suite @@ -356,8 +375,8 @@ deterministic mock traffic; Core, plugins, Box, document parsing and retrieval execute normally. ```sh -uv run --no-sync python tools/regression_suite.py validate --profile /path/to/smoke.json -uv run --no-sync python tools/regression_suite.py run \ +uv run --no-sync python -m tools.entrypoints.regression_suite validate --profile /path/to/smoke.json +uv run --no-sync python -m tools.entrypoints.regression_suite run \ --profile /path/to/smoke.json --output .qa/smoke-001 ``` @@ -377,7 +396,7 @@ created resources. Successful runs delete those API resources. Failures retain them by default; after inspection use: ```sh -uv run --no-sync python tools/regression_suite.py cleanup \ +uv run --no-sync python -m tools.entrypoints.regression_suite cleanup \ --profile /path/to/smoke.json --output .qa/smoke-001 ``` @@ -415,7 +434,7 @@ workflow takes longer than a minute; it has a separate 15-minute timeout. On an existing prepared stack, use: ```sh -uv run --no-sync python tools/regression_gate.py \ +uv run --no-sync python -m tools.entrypoints.regression_gate \ --profile /path/to/smoke.json --output .qa/gate-001 ``` @@ -507,7 +526,7 @@ performance baseline. No automatic comparison between runs is enabled. To run the same entrypoint on a disposable Linux host with an idle Docker daemon: ```sh -uv run --no-sync python tools/regression_ci.py --suite daily --restart-services \ +uv run --no-sync python -m tools.entrypoints.regression_ci --suite daily --restart-services \ --lbctl /path/to/lbctl --plugins plugins \ --extra-plugins misc/EssentialCommands misc/FAQManager --output .qa/daily-001 ``` @@ -532,7 +551,7 @@ its Box containers; a process guard timeout can interrupt API cleanup. Run the finite state job on the same disposable Linux stack: ```sh -uv run --no-sync python tools/regression_ci.py --suite state \ +uv run --no-sync python -m tools.entrypoints.regression_ci --suite state \ --lbctl /path/to/lbctl --plugins plugins --output .qa/state-001 ``` @@ -563,7 +582,7 @@ evidence for 30 days. To replay locally on a disposable Linux/Docker host, pass the recorded seed: ```sh -uv run --no-sync python tools/regression_ci.py --suite api --api-seed 20261028 \ +uv run --no-sync python -m tools.entrypoints.regression_ci --suite api --api-seed 20261028 \ --lbctl /path/to/lbctl --plugins plugins --output .qa/api-001 ``` @@ -595,7 +614,7 @@ configuration unchanged. Any failed contract or cleanup makes the job fail. Run this finite job on a disposable Linux/Docker host: ```sh -uv run --no-sync python tools/regression_ci.py --suite isolation \ +uv run --no-sync python -m tools.entrypoints.regression_ci --suite isolation \ --lbctl /path/to/lbctl --plugins plugins --output .qa/isolation-001 ``` @@ -623,7 +642,7 @@ high concurrency. Run on the same disposable Linux host: ```sh -uv run --no-sync python tools/regression_ci.py --suite cli \ +uv run --no-sync python -m tools.entrypoints.regression_ci --suite cli \ --lbctl /path/to/lbctl --plugins plugins --output .qa/cli-001 ``` @@ -640,7 +659,7 @@ The suite is opt-in and adds no daily schedule or PR gate. ## Sustained regression and baseline comparison -[`regression_profiles/endurance.json`](regression_profiles/endurance.json) runs +[`profiles/endurance.json`](profiles/endurance.json) runs nine minutes of mixed chat/Box/RAG traffic at 1 request/second across four stable groups, followed by one minute without traffic to check recovery. Configure its CLI context, plugin inputs, model endpoints, Core SQLite path and Core/Runtime/Box @@ -649,9 +668,9 @@ PID files as for smoke. Set `plugins: []` to reuse initialized plugins. Configur have Docker inspection access. Do not start another Box runtime on a shared daemon. ```sh -uv run --no-sync python tools/regression_suite.py validate \ +uv run --no-sync python -m tools.entrypoints.regression_suite validate \ --profile /path/to/endurance.json -uv run --no-sync python tools/regression_suite.py run \ +uv run --no-sync python -m tools.entrypoints.regression_suite run \ --profile /path/to/endurance.json --output .qa/endurance-baseline # For an hour including the one-minute cooldown, use --duration 3540. ``` @@ -689,7 +708,7 @@ resources after inspection. To compare two successful runs from the same reserved test host: ```sh -uv run --no-sync python tools/regression_analysis.py \ +uv run --no-sync python -m tools.entrypoints.regression_analysis \ --baseline .qa/endurance-baseline --candidate .qa/endurance-candidate \ --output .qa/comparison.json ``` @@ -713,7 +732,7 @@ fresh-deployment path still needs separate end-to-end validation. ## Business behavior coverage -`regression_contracts.json` declares behavior obligations independently of how +`contracts/regression_contracts.json` declares behavior obligations independently of how many requests a workload repeats. It includes explicit unimplemented risks (cancellation, restart recovery, credential revocation, Workspace isolation, PostgreSQL parity, real adapters and embeddings). This is coverage of the @@ -741,7 +760,7 @@ inventory. Full source locations and any unresolved declarations are retained in `coverage.json`; new declarations appear without editing the case list. ```sh -python tools/regression_coverage.py --run .qa/smoke/suite \ +python -m tools.entrypoints.regression_coverage --run .qa/smoke/suite \ --run .qa/endurance/suite --output .qa/behavior-coverage ``` @@ -761,7 +780,7 @@ fixtures (86 requests with two groups and three turns). It adds signed ingress rejection, API authentication, group reset/history isolation, tool recovery and empty retrieval cases. Save its returned profile alongside the existing fixture paths, bind `cli.config` to the target Workspace, and run it through -`regression_suite.py`. Its three `mock-daily-*` models require the matching +`entrypoints/regression_suite.py`. Its three `mock-daily-*` models require the matching expanded mock scenario fixtures. The profile exercises one authenticated Workspace; OSS does not support creating a second Workspace. Multiple resources or group IDs do not establish cross-Workspace isolation. @@ -778,7 +797,7 @@ Add this to a finite smoke/scenario profile to run its ordered cases repeatedly: } ``` -Run it with the existing `regression_suite.py run --profile ... --output ...` +Run it with `python -m tools.entrypoints.regression_suite run --profile ... --output ...` entry point. Installation and plugin readiness precede the shared clock; the clock starts immediately before creating scenario resources. It covers preparation, case execution, owned-resource cleanup and final quiet sampling. The existing diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/regression_contracts.json b/tools/contracts/regression_contracts.json similarity index 100% rename from tools/regression_contracts.json rename to tools/contracts/regression_contracts.json diff --git a/tools/entrypoints/__init__.py b/tools/entrypoints/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/mock_llm.py b/tools/entrypoints/mock_llm.py similarity index 99% rename from tools/mock_llm.py rename to tools/entrypoints/mock_llm.py index 8bd037279..0d6947d1c 100644 --- a/tools/mock_llm.py +++ b/tools/entrypoints/mock_llm.py @@ -531,7 +531,7 @@ def main(): parser.add_argument( '--scenarios', type=Path, - default=Path(__file__).with_name('mock_llm_scenarios.json'), + default=Path(__file__).resolve().parents[1] / 'fixtures' / 'mock_llm_scenarios.json', ) parser.add_argument( '--record', diff --git a/tools/regression_analysis.py b/tools/entrypoints/regression_analysis.py similarity index 96% rename from tools/regression_analysis.py rename to tools/entrypoints/regression_analysis.py index 42222cdf1..c01240796 100644 --- a/tools/regression_analysis.py +++ b/tools/entrypoints/regression_analysis.py @@ -125,7 +125,7 @@ def resource_verdict(directory, limits): def workload_contract(profile, profile_path): """Fingerprint stimuli and load geometry, excluding target addresses and generated UUIDs.""" - from regression_suite import resolve + from tools.entrypoints.regression_suite import resolve profile_path = Path(profile_path).resolve() variables = { @@ -157,14 +157,18 @@ def workload_contract(profile, profile_path): contract = {key: profile.get(key) for key in ('phases', 'thresholds', 'resource_limits')} contract.update(resources=resources, cases=cases) contract['observer_host'] = {'name': platform.node(), 'machine': platform.machine(), 'cpus': os.cpu_count()} + oracle_paths = { + 'regression_traffic.py': Path(__file__).resolve().parent / 'regression_traffic.py', + 'regression_smoke.py': Path(__file__).resolve().parents[1] / 'regression' / 'regression_smoke.py', + 'regression_analysis.py': Path(__file__).resolve(), + } contract['oracle_sha256'] = { - name: hashlib.sha256(Path(__file__).with_name(name).read_bytes()).hexdigest() - for name in ('regression_traffic.py', 'regression_smoke.py', 'regression_analysis.py') + name: hashlib.sha256(path.read_bytes()).hexdigest() for name, path in oracle_paths.items() } contract['timing_contract'] = 'dispatch-after-ledger-snapshot-v1' contract['verified_runs'] = bool(profile.get('verification')) contract['mock_scenarios_sha256'] = hashlib.sha256( - Path(__file__).with_name('mock_llm_scenarios.json').read_bytes() + (Path(__file__).resolve().parents[1] / 'fixtures' / 'mock_llm_scenarios.json').read_bytes() ).hexdigest() return contract diff --git a/tools/regression_ci.py b/tools/entrypoints/regression_ci.py similarity index 92% rename from tools/regression_ci.py rename to tools/entrypoints/regression_ci.py index a9ad01412..3a7428787 100644 --- a/tools/regression_ci.py +++ b/tools/entrypoints/regression_ci.py @@ -13,14 +13,19 @@ import time import yaml -from regression_gate import supervise -from regression_coverage import write_report -from regression_plugins import PLUGIN_IDS, install_plugins, prepare_published_plugins, prepare_source_plugins -from regression_local_cases import DEFAULT_GROUPS, api, require -from regression_suite import Suite -from regression_traffic import owned_box_containers, read_endpoint_snapshot +from tools.entrypoints.regression_gate import supervise +from tools.entrypoints.regression_coverage import write_report +from tools.regression.regression_plugins import ( + PLUGIN_IDS, + install_plugins, + prepare_published_plugins, + prepare_source_plugins, +) +from tools.regression.regression_local_cases import DEFAULT_GROUPS, api, require +from tools.entrypoints.regression_suite import Suite +from tools.entrypoints.regression_traffic import owned_box_containers, read_endpoint_snapshot -ROOT = Path(__file__).resolve().parents[1] +ROOT = Path(__file__).resolve().parents[2] SUITE_BUDGETS = { 'smoke': 60, 'scenarios': 300, @@ -37,34 +42,34 @@ from tests.e2e.utils.config_factory import create_minimal_config, create_test_di def suite_profile(name, run_id, groups=(), api_seed=1): - profile_path = ROOT / 'tools/regression_profiles' / ('endurance.json' if name == 'endurance' else 'smoke.json') + profile_path = ROOT / 'tools/profiles' / ('endurance.json' if name == 'endurance' else 'smoke.json') if name in ('scenarios', 'daily'): - from regression_scenarios import expanded_profile + from tools.regression.regression_scenarios import expanded_profile profile = expanded_profile(groups) else: profile = json.loads(profile_path.read_text()) if name == 'state': - from regression_state import state_profile + from tools.regression.regression_state import state_profile profile = state_profile(profile) if name == 'api': - from regression_api_cases import add_cases + from tools.regression.regression_api_cases import add_cases profile.update(resources=[], cases=[], coverage_suite='api') profile['vars']['api_seed'] = api_seed add_cases(profile) if name == 'isolation': - from regression_isolation_cases import isolation_profile + from tools.regression.regression_isolation_cases import isolation_profile profile = isolation_profile(profile) if name == 'plugin-lifecycle': - from regression_plugin_lifecycle import add_cases + from tools.regression.regression_plugin_lifecycle import add_cases profile.update(resources=[], cases=[], coverage_suite='plugin-lifecycle') add_cases(profile) if name == 'cli': - from regression_cli_cases import configure + from tools.regression.regression_cli_cases import configure configure(profile) for resource in profile['resources']: @@ -252,7 +257,7 @@ def run(args): 'mock', [ sys.executable, - str(ROOT / 'tools/mock_llm.py'), + str(ROOT / 'tools/entrypoints/mock_llm.py'), '--port', '18080', '--record', @@ -324,7 +329,7 @@ def run(args): restart_result = None restart_elapsed = 0 if args.restart_services: - from regression_lifecycle_cases import run_restart_checks + from tools.regression.regression_lifecycle_cases import run_restart_checks started = time.monotonic() restart_result = run_restart_checks(profile, profile_path, root / 'restart-checks', env, restart_owned) @@ -341,14 +346,15 @@ def run(args): generated = private / f'{args.suite}.json' generated.write_text(json.dumps(profile, indent=2)) if args.suite == 'state': - from regression_state import run_state_checks + from tools.regression.regression_state import run_state_checks result = run_state_checks(profile, profile_path, root / 'gate/suite', env, restart_owned) else: result = supervise( [ sys.executable, - str(ROOT / 'tools/regression_suite.py'), + '-m', + 'tools.entrypoints.regression_suite', 'run', '--profile', str(generated), diff --git a/tools/regression_coverage.py b/tools/entrypoints/regression_coverage.py similarity index 99% rename from tools/regression_coverage.py rename to tools/entrypoints/regression_coverage.py index 4efc732cf..e359e9b3f 100644 --- a/tools/regression_coverage.py +++ b/tools/entrypoints/regression_coverage.py @@ -10,7 +10,7 @@ import re from urllib.parse import urlsplit from pathlib import Path -CATALOG = Path(__file__).with_name('regression_contracts.json') +CATALOG = Path(__file__).resolve().parents[1] / 'contracts' / 'regression_contracts.json' STATES = ('passed', 'failed', 'incomplete', 'skipped', 'not_run', 'missing_test') @@ -94,7 +94,7 @@ def collect(directory): def route_inventory(catalog, root=None): """Discover RouterGroup HTTP declarations; links express test intent, not endpoint hits.""" - root = Path(root) if root else Path(__file__).resolve().parents[1] + root = Path(root) if root else Path(__file__).resolve().parents[2] routes, unresolved = [], [] for path in sorted((root / 'src/langbot/pkg').rglob('*.py')): if 'group_class' not in path.read_text(): diff --git a/tools/regression_gate.py b/tools/entrypoints/regression_gate.py similarity index 95% rename from tools/regression_gate.py rename to tools/entrypoints/regression_gate.py index 2e3217b8b..531d2bd3e 100644 --- a/tools/regression_gate.py +++ b/tools/entrypoints/regression_gate.py @@ -10,8 +10,8 @@ import subprocess import sys import time -from regression_suite import Suite, validate -from regression_traffic import read_process_snapshot +from tools.entrypoints.regression_suite import Suite, validate +from tools.entrypoints.regression_traffic import read_process_snapshot def supervise(command, output, budget, pids, env=None): @@ -91,7 +91,8 @@ def main(): result = supervise( [ sys.executable, - str(Path(__file__).with_name('regression_suite.py')), + '-m', + 'tools.entrypoints.regression_suite', 'run', '--profile', str(args.profile.resolve()), diff --git a/tools/regression_job.py b/tools/entrypoints/regression_job.py similarity index 97% rename from tools/regression_job.py rename to tools/entrypoints/regression_job.py index df671539d..a1fa658a3 100644 --- a/tools/regression_job.py +++ b/tools/entrypoints/regression_job.py @@ -16,9 +16,9 @@ import urllib.request import yaml -from regression_suite import Suite, number, resolve, validate -from regression_plugins import install_plugins, prepare_published_plugins -from regression_traffic import owned_box_containers +from tools.entrypoints.regression_suite import Suite, number, resolve, validate +from tools.regression.regression_plugins import install_plugins, prepare_published_plugins +from tools.entrypoints.regression_traffic import owned_box_containers KEY_ENV = 'LANGBOT_QA_API_KEY' @@ -285,10 +285,10 @@ class Job: }, 'regression_mock': { 'image': self.profile.get('mock_image', 'python:3.12-alpine'), - 'volumes': [f'{Path(__file__).parent.resolve()}:/regression:ro', f'{mock_results}:/results'], + 'volumes': [f'{Path(__file__).resolve().parents[1]}:/regression:ro', f'{mock_results}:/results'], 'command': [ 'python', - '/regression/mock_llm.py', + '/regression/entrypoints/mock_llm.py', '--host', '0.0.0.0', '--port', @@ -365,7 +365,7 @@ class Job: self.save() suite.write('profile.json', profile) if 'smoke' not in profile: - from regression_analysis import workload_contract + from tools.entrypoints.regression_analysis import workload_contract suite.write('workload.json', workload_contract(profile, self.suite_path)) suite.prepare() diff --git a/tools/regression_suite.py b/tools/entrypoints/regression_suite.py similarity index 98% rename from tools/regression_suite.py rename to tools/entrypoints/regression_suite.py index 98e614631..05aa5e865 100644 --- a/tools/regression_suite.py +++ b/tools/entrypoints/regression_suite.py @@ -15,7 +15,7 @@ import urllib.error import urllib.request from uuid import uuid4 -from regression_traffic import group_request +from tools.entrypoints.regression_traffic import group_request KINDS = {'provider', 'model', 'pipeline', 'knowledge-base', 'bot'} @@ -100,7 +100,7 @@ def validate(profile): if profile.get('performance_mode', 'gate') not in {'gate', 'observe'}: raise ValueError('performance_mode must be gate or observe') if 'scenario_groups' in profile: - from regression_local_cases import GROUPS + from tools.regression.regression_local_cases import GROUPS groups = profile['scenario_groups'] if ( @@ -263,7 +263,7 @@ def validate(profile): ): raise ValueError('reset_group requires an accepted group case') if 'scenario' in case: - from regression_local_cases import modules + from tools.regression.regression_local_cases import modules handlers = modules() if ( @@ -422,7 +422,7 @@ def validate(profile): if 'resource_limits' in profile: if 'smoke' in profile and execution is None: raise ValueError('resource_limits requires measured load and cooldown phases') - from regression_analysis import validate_resource_limits + from tools.entrypoints.regression_analysis import validate_resource_limits validate_resource_limits(profile['resource_limits']) observed = resolve(profile.get('observe', {}), variables) @@ -532,7 +532,7 @@ class Suite: if self.execution: self.execution.finish(result) if self.profile.get('coverage_suite'): - from regression_coverage import write_report + from tools.entrypoints.regression_coverage import write_report coverage = write_report([self.output], self.output) result['errors'].extend(coverage['evidence_errors']) @@ -628,7 +628,7 @@ class Suite: if missing: raise RuntimeError(f'Required plugins are not initialized: {sorted(missing)}') if self.profile.get('execution'): - from regression_scenarios import ExecutionWindow + from tools.regression.regression_scenarios import ExecutionWindow self.execution = ExecutionWindow(self) self.execution.start() @@ -717,7 +717,7 @@ class Suite: saved['resources'].remove(resource) self.write('owned.json', saved) if 'smoke' in self.profile or 'verification' in self.profile: - from regression_smoke import verify_cleanup + from tools.regression.regression_smoke import verify_cleanup try: verify_cleanup(self, [item for item in original if item not in saved['resources']]) @@ -727,7 +727,7 @@ class Suite: def run(self): if 'smoke' in self.profile: - from regression_smoke import run_smoke + from tools.regression.regression_smoke import run_smoke return run_smoke(self) cases = [] @@ -782,7 +782,8 @@ class Suite: destination = self.output / phase['name'] command = [ sys.executable, - str(Path(__file__).with_name('regression_traffic.py')), + '-m', + 'tools.entrypoints.regression_traffic', '--lbctl', self.cli[0], '--config', @@ -820,7 +821,7 @@ class Suite: ) results.append(summary) if self.profile.get('resource_limits'): - from regression_analysis import resource_verdict + from tools.entrypoints.regression_analysis import resource_verdict summary['resources'] = resource_verdict(destination, self.profile['resource_limits']) errors.extend(f'{phase["name"]}: {error}' for error in summary['resources']['errors']) @@ -881,7 +882,7 @@ def main(): args.output.mkdir(parents=True, exist_ok=False, mode=0o700) suite.write('profile.json', profile) if 'smoke' not in profile: - from regression_analysis import workload_contract + from tools.entrypoints.regression_analysis import workload_contract suite.write('workload.json', workload_contract(profile, args.profile)) result = {'phases': [], 'errors': []} diff --git a/tools/regression_traffic.py b/tools/entrypoints/regression_traffic.py similarity index 98% rename from tools/regression_traffic.py rename to tools/entrypoints/regression_traffic.py index 8ac8e9520..e992e6e79 100644 --- a/tools/regression_traffic.py +++ b/tools/entrypoints/regression_traffic.py @@ -3,7 +3,7 @@ import argparse from collections import Counter -from regression_coverage import checked, check_response +from tools.entrypoints.regression_coverage import checked, check_response from concurrent.futures import ThreadPoolExecutor import json import hashlib @@ -20,7 +20,7 @@ import urllib.error import urllib.request import uuid -sys.path.insert(0, str(Path(__file__).resolve().parents[1] / 'scripts')) +sys.path.insert(0, str(Path(__file__).resolve().parents[2] / 'scripts')) from cloud_runtime_soak import read_endpoint_snapshot, read_process_snapshot @@ -276,7 +276,13 @@ def main(): parser.error('ledger verification requires an existing SQLite file and workspace') if not math.isfinite(args.verification_timeout) or args.verification_timeout <= 0: parser.error('verification timeout must be positive') - from regression_smoke import UnresolvedRun, check_run, check_attachment, check_fields, completed_run + from tools.regression.regression_smoke import ( + UnresolvedRun, + check_run, + check_attachment, + check_fields, + completed_run, + ) args.output.mkdir(parents=True, exist_ok=False) pids = {name: int(pid) for name, pid in (value.split('=', 1) for value in args.pid)} diff --git a/tools/mock_llm_scenarios.json b/tools/fixtures/mock_llm_scenarios.json similarity index 100% rename from tools/mock_llm_scenarios.json rename to tools/fixtures/mock_llm_scenarios.json diff --git a/tools/regression_fixtures/plugin_lifecycle/components/pages/probe/index.html b/tools/fixtures/plugin_lifecycle/components/pages/probe/index.html similarity index 100% rename from tools/regression_fixtures/plugin_lifecycle/components/pages/probe/index.html rename to tools/fixtures/plugin_lifecycle/components/pages/probe/index.html diff --git a/tools/regression_fixtures/plugin_lifecycle/components/pages/probe/probe.py b/tools/fixtures/plugin_lifecycle/components/pages/probe/probe.py similarity index 100% rename from tools/regression_fixtures/plugin_lifecycle/components/pages/probe/probe.py rename to tools/fixtures/plugin_lifecycle/components/pages/probe/probe.py diff --git a/tools/regression_fixtures/plugin_lifecycle/components/pages/probe/probe.yaml b/tools/fixtures/plugin_lifecycle/components/pages/probe/probe.yaml similarity index 100% rename from tools/regression_fixtures/plugin_lifecycle/components/pages/probe/probe.yaml rename to tools/fixtures/plugin_lifecycle/components/pages/probe/probe.yaml diff --git a/tools/regression_fixtures/plugin_lifecycle/main.py b/tools/fixtures/plugin_lifecycle/main.py similarity index 100% rename from tools/regression_fixtures/plugin_lifecycle/main.py rename to tools/fixtures/plugin_lifecycle/main.py diff --git a/tools/mock-backends.md b/tools/mock-backends.md index b62ee1f70..931f180b8 100644 --- a/tools/mock-backends.md +++ b/tools/mock-backends.md @@ -3,7 +3,7 @@ Run the test backend with Python 3.11 or newer: ```sh -python tools/mock_llm.py --port 18080 --record /tmp/mock-requests.jsonl +python -m tools.entrypoints.mock_llm --port 18080 --record /tmp/mock-requests.jsonl ``` Point the LangBot OpenAI-compatible provider at `http://127.0.0.1:18080/v1` @@ -98,5 +98,5 @@ count as a successful fault test. Records contain request bodies; use test data. Run the HTTP tests without installing LangBot dependencies: ```sh -python -m unittest discover -s tools -p test_mock_llm.py -v +python -m unittest discover -s tools/tests -p test_mock_llm.py -v ``` diff --git a/tools/regression_profiles/endurance.json b/tools/profiles/endurance.json similarity index 100% rename from tools/regression_profiles/endurance.json rename to tools/profiles/endurance.json diff --git a/tools/regression_profiles/groups.json b/tools/profiles/groups.json similarity index 100% rename from tools/regression_profiles/groups.json rename to tools/profiles/groups.json diff --git a/tools/regression_profiles/job.json b/tools/profiles/job.json similarity index 100% rename from tools/regression_profiles/job.json rename to tools/profiles/job.json diff --git a/tools/regression_profiles/normal.json b/tools/profiles/normal.json similarity index 100% rename from tools/regression_profiles/normal.json rename to tools/profiles/normal.json diff --git a/tools/regression_profiles/smoke.json b/tools/profiles/smoke.json similarity index 100% rename from tools/regression_profiles/smoke.json rename to tools/profiles/smoke.json diff --git a/tools/published-plugins.md b/tools/published-plugins.md index 27550cc55..74fa081a1 100644 --- a/tools/published-plugins.md +++ b/tools/published-plugins.md @@ -1,6 +1,6 @@ # Published plugins in regression runs -`regression_ci.py` and `regression_job.py` default to the latest published +`tools.entrypoints.regression_ci` and `tools.entrypoints.regression_job` default to the latest published LocalAgent, LangRAG and GeneralParsers packages from the official LangBot marketplace. Plugin preparation happens before business testing. Each run reads `latest_version` once for each plugin, downloads that exact version, and fails @@ -10,19 +10,19 @@ There is no fallback to older releases, cached packages or a source checkout. The source runner needs the normal Core/SDK dependencies, Docker and `lbctl`: ```sh -python tools/regression_ci.py --lbctl /path/to/lbctl --output /path/to/new-run +python -m tools.entrypoints.regression_ci --lbctl /path/to/lbctl --output /path/to/new-run ``` The released deployment runner uses its existing job profile: ```sh -python tools/regression_job.py run --profile tools/regression_profiles/job.json --output /path/to/new-run +python -m tools.entrypoints.regression_job run --profile tools/profiles/job.json --output /path/to/new-run ``` The job's default `plugin_mode` is `published`; its suite's `plugins` entries are replaced by the published preparation. Set `"plugin_mode": "profile"` in the job JSON to explicitly use the suite's existing local/GitHub/marketplace -inputs. For source development, `regression_ci.py --plugins /path/to/langbot-plugins` +inputs. For source development, `python -m tools.entrypoints.regression_ci --plugins /path/to/langbot-plugins` retains packaging from a source checkout. Each published run keeps `plugins/manifest.json` and the downloaded ZIPs under @@ -35,7 +35,7 @@ Dependency or SDK incompatibility is a preparation failure and is not downgraded To replay an earlier run, pass its manifest to the source runner: ```sh -python tools/regression_ci.py --lbctl /path/to/lbctl --output /path/to/replay --plugin-manifest /path/to/earlier/plugins/manifest.json +python -m tools.entrypoints.regression_ci --lbctl /path/to/lbctl --output /path/to/replay --plugin-manifest /path/to/earlier/plugins/manifest.json ``` For the deployment runner, set `"plugin_manifest": "/path/to/earlier/plugins/manifest.json"` diff --git a/tools/regression/__init__.py b/tools/regression/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/regression_api_cases.py b/tools/regression/regression_api_cases.py similarity index 95% rename from tools/regression_api_cases.py rename to tools/regression/regression_api_cases.py index cf00fd396..1aefbc58b 100644 --- a/tools/regression_api_cases.py +++ b/tools/regression/regression_api_cases.py @@ -3,10 +3,17 @@ import json import random -from regression_api_checks import check, owned_resource, request, tracked_resource, unique_name, user_headers -from regression_coverage import checked -from regression_local_cases import require -from regression_resource_cases import cleanup, journal +from tools.regression.regression_api_checks import ( + check, + owned_resource, + request, + tracked_resource, + unique_name, + user_headers, +) +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_local_cases import require +from tools.regression.regression_resource_cases import cleanup, journal CASE_NAMES = ('api_llm', 'api_embedding', 'api_rerank', 'api_pipeline', 'api_credentials', 'api_skills', 'api_mcp') @@ -193,7 +200,7 @@ def credentials_case(suite, row): def run_case(suite, case, row): - from regression_api_resources import mcp_case, skills_case + from tools.regression.regression_api_resources import mcp_case, skills_case name = case['scenario'] row['api_seed'] = suite.profile['vars']['api_seed'] diff --git a/tools/regression_api_checks.py b/tools/regression/regression_api_checks.py similarity index 94% rename from tools/regression_api_checks.py rename to tools/regression/regression_api_checks.py index e249b48c3..af404ec71 100644 --- a/tools/regression_api_checks.py +++ b/tools/regression/regression_api_checks.py @@ -3,9 +3,9 @@ from contextlib import contextmanager from uuid import uuid4 -from regression_coverage import checked -from regression_local_cases import api, require -from regression_resource_cases import cleanup, journal +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_local_cases import api, require +from tools.regression.regression_resource_cases import cleanup, journal def check(row, name, condition, message): diff --git a/tools/regression_api_resources.py b/tools/regression/regression_api_resources.py similarity index 98% rename from tools/regression_api_resources.py rename to tools/regression/regression_api_resources.py index 2bd520204..8dabc3a1e 100644 --- a/tools/regression_api_resources.py +++ b/tools/regression/regression_api_resources.py @@ -2,9 +2,9 @@ from contextlib import contextmanager -from regression_api_checks import check, request, unique_name, user_headers -from regression_local_cases import api, require -from regression_resource_cases import cleanup, journal +from tools.regression.regression_api_checks import check, request, unique_name, user_headers +from tools.regression.regression_local_cases import api, require +from tools.regression.regression_resource_cases import cleanup, journal @contextmanager diff --git a/tools/regression_cli_cases.py b/tools/regression/regression_cli_cases.py similarity index 94% rename from tools/regression_cli_cases.py rename to tools/regression/regression_cli_cases.py index 175109a15..5bf082110 100644 --- a/tools/regression_cli_cases.py +++ b/tools/regression/regression_cli_cases.py @@ -2,10 +2,10 @@ import subprocess -from regression_api_checks import check, request, tracked_resource, unique_name -from regression_coverage import checked -from regression_local_cases import require -from regression_resource_cases import journal +from tools.regression.regression_api_checks import check, request, tracked_resource, unique_name +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_local_cases import require +from tools.regression.regression_resource_cases import journal CASE_NAMES = ('cli.identity', 'cli.pipeline_roundtrip', 'cli.invalid_credential') diff --git a/tools/regression_execution_cases.py b/tools/regression/regression_execution_cases.py similarity index 99% rename from tools/regression_execution_cases.py rename to tools/regression/regression_execution_cases.py index d130a250e..486c4a42b 100644 --- a/tools/regression_execution_cases.py +++ b/tools/regression/regression_execution_cases.py @@ -9,8 +9,8 @@ import sys import urllib.request from uuid import uuid4 -from regression_coverage import checked -from regression_local_cases import api, endpoint, headers, record_run, require +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_local_cases import api, endpoint, headers, record_run, require GROUP = 'faults' RUNNER = 'plugin:langbot-team/LocalAgent/default' diff --git a/tools/regression_isolation_cases.py b/tools/regression/regression_isolation_cases.py similarity index 96% rename from tools/regression_isolation_cases.py rename to tools/regression/regression_isolation_cases.py index d1a62f9dd..8dfa61526 100644 --- a/tools/regression_isolation_cases.py +++ b/tools/regression/regression_isolation_cases.py @@ -9,13 +9,13 @@ import threading import secrets from uuid import uuid4 -import regression_execution_cases as execution -from regression_api_checks import owned_resource -from regression_coverage import checked -from regression_lifecycle_cases import after_id, ledger -from regression_local_cases import record_run, require -from regression_smoke import check_run, completed_run -from regression_traffic import group_request +import tools.regression.regression_execution_cases as execution +from tools.regression.regression_api_checks import owned_resource +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_lifecycle_cases import after_id, ledger +from tools.regression.regression_local_cases import record_run, require +from tools.regression.regression_smoke import check_run, completed_run +from tools.entrypoints.regression_traffic import group_request CASE_NAMES = ('isolation.same_pipeline', 'isolation.cross_pipeline', 'isolation.cancel_neighbor') @@ -168,7 +168,7 @@ def verify_wire(suite, row, receipts): path = Path(suite.path(suite.profile['smoke']['mock_records'])) records = [json.loads(line) for line in path.read_text().splitlines()] owned = {receipt['group'] for receipt in receipts} - from mock_llm import GROUP_TURN, group_ack + from tools.entrypoints.mock_llm import GROUP_TURN, group_ack matched = [] for record in records: diff --git a/tools/regression_lifecycle_cases.py b/tools/regression/regression_lifecycle_cases.py similarity index 97% rename from tools/regression_lifecycle_cases.py rename to tools/regression/regression_lifecycle_cases.py index e08c3e7fd..c97d8dce4 100644 --- a/tools/regression_lifecycle_cases.py +++ b/tools/regression/regression_lifecycle_cases.py @@ -12,8 +12,8 @@ import sqlite3 import threading import time -from regression_coverage import checked -from regression_local_cases import ScenarioUnavailable, api, record_run, require +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_local_cases import ScenarioUnavailable, api, record_run, require GROUP = 'lifecycle' CASE_NAMES = {'callback_retry_identity'} @@ -47,7 +47,7 @@ def after_id(suite): def completed(suite, row, session_id, after, expected='completed'): - from regression_smoke import completed_run + from tools.regression.regression_smoke import completed_run with closing(ledger(suite)) as connection: run = completed_run( @@ -295,9 +295,9 @@ def restart_profile(profile): def run_restart_checks(profile, profile_path, output, env, restart_owned): """Run owned-service restart checks once, before fixed-PID resource sampling starts.""" - from regression_smoke import UnresolvedRun, reconcile_scenario_runs - from regression_suite import Suite, resolve - from regression_traffic import group_request + from tools.regression.regression_smoke import UnresolvedRun, reconcile_scenario_runs + from tools.entrypoints.regression_suite import Suite, resolve + from tools.entrypoints.regression_traffic import group_request started = time.monotonic() deadline = started + 120 @@ -409,7 +409,7 @@ def run_restart_checks(profile, profile_path, output, env, restart_owned): 'response' not in outcome, 'Interrupted request returned a successful business response' ), ) - from regression_smoke import completed_run + from tools.regression.regression_smoke import completed_run with closing(ledger(suite)) as connection: terminal = completed_run( diff --git a/tools/regression_local_cases.py b/tools/regression/regression_local_cases.py similarity index 89% rename from tools/regression_local_cases.py rename to tools/regression/regression_local_cases.py index 0860bd9fc..66b227b8a 100644 --- a/tools/regression_local_cases.py +++ b/tools/regression/regression_local_cases.py @@ -10,14 +10,14 @@ import urllib.error import urllib.request GROUPS = { - 'faults': 'regression_execution_cases', - 'resources': 'regression_resource_cases', - 'lifecycle': 'regression_lifecycle_cases', - 'plugins': 'regression_plugin_cases', - 'api': 'regression_api_cases', - 'isolation': 'regression_isolation_cases', - 'plugin-lifecycle': 'regression_plugin_lifecycle', - 'cli': 'regression_cli_cases', + 'faults': 'tools.regression.regression_execution_cases', + 'resources': 'tools.regression.regression_resource_cases', + 'lifecycle': 'tools.regression.regression_lifecycle_cases', + 'plugins': 'tools.regression.regression_plugin_cases', + 'api': 'tools.regression.regression_api_cases', + 'isolation': 'tools.regression.regression_isolation_cases', + 'plugin-lifecycle': 'tools.regression.regression_plugin_lifecycle', + 'cli': 'tools.regression.regression_cli_cases', } diff --git a/tools/regression_plugin_cases.py b/tools/regression/regression_plugin_cases.py similarity index 96% rename from tools/regression_plugin_cases.py rename to tools/regression/regression_plugin_cases.py index 2c59fbb93..3c590d93a 100644 --- a/tools/regression_plugin_cases.py +++ b/tools/regression/regression_plugin_cases.py @@ -6,12 +6,12 @@ import json from pathlib import Path from uuid import uuid4 -from regression_coverage import checked -from regression_lifecycle_cases import ledger -from regression_local_cases import api, require -from regression_resource_cases import cleanup, journal -from regression_smoke import check_mock_requests, signed_headers -from regression_suite import resolve +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_lifecycle_cases import ledger +from tools.regression.regression_local_cases import api, require +from tools.regression.regression_resource_cases import cleanup, journal +from tools.regression.regression_smoke import check_mock_requests, signed_headers +from tools.entrypoints.regression_suite import resolve GROUP = 'plugins' CASE_NAMES = {'plugins.command_version', 'plugins.faq_crud'} diff --git a/tools/regression_plugin_lifecycle.py b/tools/regression/regression_plugin_lifecycle.py similarity index 97% rename from tools/regression_plugin_lifecycle.py rename to tools/regression/regression_plugin_lifecycle.py index 3ddaa1a4e..331a130f0 100644 --- a/tools/regression_plugin_lifecycle.py +++ b/tools/regression/regression_plugin_lifecycle.py @@ -10,13 +10,13 @@ import zipfile import yaml -from regression_api_checks import check, request -from regression_coverage import checked -from regression_local_cases import api, auth_headers, endpoint, require -from regression_resource_cases import cleanup, journal +from tools.regression.regression_api_checks import check, request +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_local_cases import api, auth_headers, endpoint, require +from tools.regression.regression_resource_cases import cleanup, journal CASE_NAMES = ('plugin_upgrade', 'plugin_rejected_package', 'plugin_failed_activation') -FIXTURE = Path(__file__).with_name('regression_fixtures') / 'plugin_lifecycle' +FIXTURE = Path(__file__).resolve().parents[1] / 'fixtures' / 'plugin_lifecycle' AUTHOR = 'regression-qa' diff --git a/tools/regression_plugins.py b/tools/regression/regression_plugins.py similarity index 100% rename from tools/regression_plugins.py rename to tools/regression/regression_plugins.py diff --git a/tools/regression_resource_cases.py b/tools/regression/regression_resource_cases.py similarity index 98% rename from tools/regression_resource_cases.py rename to tools/regression/regression_resource_cases.py index c1db30c48..d8283a942 100644 --- a/tools/regression_resource_cases.py +++ b/tools/regression/regression_resource_cases.py @@ -7,9 +7,9 @@ import sys import tempfile from uuid import uuid4 -from regression_coverage import checked -from regression_local_cases import ScenarioUnavailable, api, require -from regression_suite import resolve +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_local_cases import ScenarioUnavailable, api, require +from tools.entrypoints.regression_suite import resolve GROUP = 'resources' CASE_NAMES = {'auth_revocation', 'agent_direct_execution', 'retrieval_delete_visibility'} diff --git a/tools/regression_scenarios.py b/tools/regression/regression_scenarios.py similarity index 95% rename from tools/regression_scenarios.py rename to tools/regression/regression_scenarios.py index a7e37deef..dcb0f05a0 100644 --- a/tools/regression_scenarios.py +++ b/tools/regression/regression_scenarios.py @@ -7,12 +7,12 @@ import threading import time from pathlib import Path -ROOT = Path(__file__).parent +ROOT = Path(__file__).resolve().parents[1] RUNNER = 'plugin:langbot-team/LocalAgent/default' def expanded_profile(groups=()): - profile = json.loads((ROOT / 'regression_profiles/smoke.json').read_text()) + profile = json.loads((ROOT / 'profiles/smoke.json').read_text()) profile['coverage_suite'] = 'scenarios' profile['smoke']['group_turns'] = 3 resources = {r['id']: r for r in profile['resources']} @@ -103,7 +103,7 @@ def expanded_profile(groups=()): ) profile['cases'].append(case) if groups: - from regression_local_cases import add_cases + from tools.regression.regression_local_cases import add_cases add_cases(profile, groups) return profile @@ -144,8 +144,8 @@ class ExecutionWindow: return time.monotonic() < self.admission_deadline def sample(self): - from regression_suite import resolve - from regression_traffic import ( + from tools.entrypoints.regression_suite import resolve + from tools.entrypoints.regression_traffic import ( owned_box_containers, read_endpoint_snapshot, read_process_snapshot, @@ -222,7 +222,7 @@ class ExecutionWindow: result['errors'].append('Resource sampler did not stop before its drain deadline') result['errors'].extend(dict.fromkeys(self.errors)) if limits := self.suite.profile.get('resource_limits'): - from regression_analysis import resource_verdict + from tools.entrypoints.regression_analysis import resource_verdict try: verdict = resource_verdict(self.suite.output, limits) @@ -233,7 +233,7 @@ class ExecutionWindow: error = f'Resource recovery evidence unavailable: {exc}' result['errors'].append(error) result.setdefault('incomplete', []).append(error) - from regression_smoke import reconcile_scenario_runs + from tools.regression.regression_smoke import reconcile_scenario_runs reconcile_scenario_runs(self.suite, result, key='ledger_closeout') self.mark('finished') diff --git a/tools/regression_smoke.py b/tools/regression/regression_smoke.py similarity index 98% rename from tools/regression_smoke.py rename to tools/regression/regression_smoke.py index 2d7a32e19..f3b95d098 100644 --- a/tools/regression_smoke.py +++ b/tools/regression/regression_smoke.py @@ -14,9 +14,9 @@ import urllib.error import urllib.request from uuid import uuid4 -from regression_traffic import group_request, percentile, read_endpoint_snapshot, reconcile_runs -from regression_coverage import checked, check_response -from regression_local_cases import ScenarioUnavailable +from tools.entrypoints.regression_traffic import group_request, percentile, read_endpoint_snapshot, reconcile_runs +from tools.entrypoints.regression_coverage import checked, check_response +from tools.regression.regression_local_cases import ScenarioUnavailable class UnresolvedRun(RuntimeError): @@ -234,7 +234,7 @@ def verify_cleanup(suite, deleted): def run_cycle(suite, rows, health, cycle): # Imported here because Suite dispatches to this module after profile validation. - from regression_suite import resolve + from tools.entrypoints.regression_suite import resolve settings = suite.profile['smoke'] path = Path(suite.path(settings['ledger_db'])).resolve() @@ -318,7 +318,7 @@ def run_cycle(suite, rows, health, cycle): offset = mock_path.stat().st_size if case.get('mock_requests') and mock_path.exists() else 0 try: if 'scenario' in case: - from regression_local_cases import run_case + from tools.regression.regression_local_cases import run_case data = run_case(suite, case, row) elif rejected: diff --git a/tools/regression_state.py b/tools/regression/regression_state.py similarity index 95% rename from tools/regression_state.py rename to tools/regression/regression_state.py index 3b4deba1d..5e544c854 100644 --- a/tools/regression_state.py +++ b/tools/regression/regression_state.py @@ -6,13 +6,13 @@ from pathlib import Path import secrets import time -from regression_api_checks import check, request -from regression_coverage import checked -from regression_lifecycle_cases import after_id, completed -from regression_local_cases import require -from regression_smoke import reconcile_scenario_runs -from regression_suite import Suite, contains_fields, resolve, validate -from regression_traffic import group_request +from tools.regression.regression_api_checks import check, request +from tools.entrypoints.regression_coverage import checked +from tools.regression.regression_lifecycle_cases import after_id, completed +from tools.regression.regression_local_cases import require +from tools.regression.regression_smoke import reconcile_scenario_runs +from tools.entrypoints.regression_suite import Suite, contains_fields, resolve, validate +from tools.entrypoints.regression_traffic import group_request def state_profile(profile): diff --git a/tools/tests/__init__.py b/tools/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tools/test_mock_llm.py b/tools/tests/test_mock_llm.py similarity index 98% rename from tools/test_mock_llm.py rename to tools/tests/test_mock_llm.py index 2f31f329d..784c42ab4 100644 --- a/tools/test_mock_llm.py +++ b/tools/tests/test_mock_llm.py @@ -14,8 +14,8 @@ from unittest.mock import patch from urllib.error import HTTPError from urllib.request import Request, urlopen -import mock_llm -from mock_llm import MockServer, validate_scenarios +import tools.entrypoints.mock_llm as mock_llm +from tools.entrypoints.mock_llm import MockServer, validate_scenarios class MockLLMTest(unittest.TestCase): @@ -24,7 +24,9 @@ class MockLLMTest(unittest.TestCase): cls.tmp = tempfile.TemporaryDirectory() cls.record = Path(cls.tmp.name) / 'requests.jsonl' cls.record.touch() - scenarios = json.loads(Path(__file__).with_name('mock_llm_scenarios.json').read_text()) + scenarios = json.loads( + (Path(__file__).resolve().parents[1] / 'fixtures' / 'mock_llm_scenarios.json').read_text() + ) scenarios['test-stall-recovery'] = { **scenarios['mock-stream-stall'], 'stall_seconds': 0.2, diff --git a/tools/test_regression_analysis.py b/tools/tests/test_regression_analysis.py similarity index 98% rename from tools/test_regression_analysis.py rename to tools/tests/test_regression_analysis.py index f74708753..1892f2e14 100644 --- a/tools/test_regression_analysis.py +++ b/tools/tests/test_regression_analysis.py @@ -6,7 +6,7 @@ from pathlib import Path import tempfile import unittest -from regression_analysis import compare, container_metrics, resource_verdict, workload_contract +from tools.entrypoints.regression_analysis import compare, container_metrics, resource_verdict, workload_contract class AnalysisTests(unittest.TestCase): diff --git a/tools/test_regression_api.py b/tools/tests/test_regression_api.py similarity index 90% rename from tools/test_regression_api.py rename to tools/tests/test_regression_api.py index 8a61f2edb..7ad111be6 100644 --- a/tools/test_regression_api.py +++ b/tools/tests/test_regression_api.py @@ -6,13 +6,13 @@ import tempfile import unittest from unittest.mock import patch -from regression_api_cases import run_case -from regression_api_checks import owned_resource, request -from regression_api_resources import _expect_rejection -from regression_ci import suite_profile -from regression_local_cases import DEFAULT_GROUPS -from regression_scenarios import expanded_profile -from regression_suite import Suite, validate +from tools.regression.regression_api_cases import run_case +from tools.regression.regression_api_checks import owned_resource, request +from tools.regression.regression_api_resources import _expect_rejection +from tools.entrypoints.regression_ci import suite_profile +from tools.regression.regression_local_cases import DEFAULT_GROUPS +from tools.regression.regression_scenarios import expanded_profile +from tools.entrypoints.regression_suite import Suite, validate class ApiContractTests(unittest.TestCase): @@ -37,7 +37,7 @@ class ApiContractTests(unittest.TestCase): def test_http_success_cannot_hide_business_failure_or_wrong_status(self): for actual in ((200, {'code': -1}), (201, {'code': 0}), (400, {'code': 0}), (400, {})): row = {} - with self.subTest(actual=actual), patch('regression_api_checks.api', return_value=actual): + with self.subTest(actual=actual), patch('tools.regression.regression_api_checks.api', return_value=actual): with self.assertRaises(ValueError): request(None, row, 'GET', '/api/test', expected=400 if actual[0] == 400 else 200) self.assertEqual(row['assertions']['http_1'], 'failed') @@ -48,7 +48,9 @@ class ApiContractTests(unittest.TestCase): suite = SimpleNamespace(profile={'vars': {'api_seed': 1}}) row = {} - with patch('regression_api_cases.credentials_case', side_effect=ValueError('Revoked key still works')): + with patch( + 'tools.regression.regression_api_cases.credentials_case', side_effect=ValueError('Revoked key still works') + ): with self.assertRaisesRegex(ValueError, 'Revoked key'): run_case(suite, {'scenario': 'api_credentials', 'expect': 'OK'}, row) self.assertEqual(row['assertions']['api_chain'], 'failed') @@ -60,7 +62,9 @@ class ApiContractTests(unittest.TestCase): row['state_checked'] = True raise ValueError('Stored state changed') - with patch('regression_api_resources.request', side_effect=ValueError('Expected 400, got 200')): + with patch( + 'tools.regression.regression_api_resources.request', side_effect=ValueError('Expected 400, got 200') + ): with self.assertRaisesRegex(ValueError, 'Expected 400, got 200'): _expect_rejection(None, row, 'PUT', '/api/test', {}, expected=400, headers={}, verify=verify) self.assertTrue(row['state_checked']) @@ -95,7 +99,7 @@ class ApiContractTests(unittest.TestCase): } return 200, {'code': 0} - with patch('regression_api_checks.api', side_effect=api): + with patch('tools.regression.regression_api_checks.api', side_effect=api): with self.assertRaisesRegex(TimeoutError, 'Lost response'): with owned_resource( suite, row, '/collection', {'name': 'qa-owned'}, item_key='item', list_key='items' @@ -146,8 +150,8 @@ class ApiContractTests(unittest.TestCase): error = 'expected HTTP 400' if status == 200 else 'Invalid key request changed key list' with ( - patch('regression_api_checks.api', side_effect=api), - patch('regression_api_cases.unique_name', return_value='qa-api-owned-key'), + patch('tools.regression.regression_api_checks.api', side_effect=api), + patch('tools.regression.regression_api_cases.unique_name', return_value='qa-api-owned-key'), self.assertRaisesRegex(ValueError, error), ): run_case(suite, {'scenario': 'api_credentials', 'expect': 'OK'}, row) @@ -219,8 +223,8 @@ class ApiContractTests(unittest.TestCase): error = 'expected HTTP 403' if status == 200 else 'Read-only key changed resources' with ( - patch('regression_api_checks.api', side_effect=api), - patch('regression_api_cases.unique_name', return_value='qa-api-owned-key'), + patch('tools.regression.regression_api_checks.api', side_effect=api), + patch('tools.regression.regression_api_cases.unique_name', return_value='qa-api-owned-key'), self.assertRaisesRegex(ValueError, error), ): run_case(suite, {'scenario': 'api_credentials', 'expect': 'OK'}, row) diff --git a/tools/test_regression_ci.py b/tools/tests/test_regression_ci.py similarity index 96% rename from tools/test_regression_ci.py rename to tools/tests/test_regression_ci.py index 83b3ee06c..4bafaa082 100644 --- a/tools/test_regression_ci.py +++ b/tools/tests/test_regression_ci.py @@ -10,8 +10,8 @@ import unittest import yaml -from regression_ci import SUITE_BUDGETS, suite_profile -from regression_suite import validate +from tools.entrypoints.regression_ci import SUITE_BUDGETS, suite_profile +from tools.entrypoints.regression_suite import validate class CIProfileTest(unittest.TestCase): @@ -23,7 +23,8 @@ class CIProfileTest(unittest.TestCase): process = subprocess.run( [ sys.executable, - str(Path(__file__).with_name('regression_ci.py')), + '-m', + 'tools.entrypoints.regression_ci', '--suite', 'smoke', '--scenario-groups', @@ -91,7 +92,7 @@ class CIProfileTest(unittest.TestCase): class DailySummaryTest(unittest.TestCase): def test_summary_reports_observations_without_hiding_outer_failures(self): - workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml' + workflow = Path(__file__).resolve().parents[2] / '.github/workflows/daily-endurance.yml' steps = yaml.safe_load(workflow.read_text())['jobs']['endurance']['steps'] command = next(step['run'] for step in steps if step.get('name') == 'Publish observations') suite = { @@ -144,7 +145,7 @@ class DailySummaryTest(unittest.TestCase): self.assertNotIn('Daily regression: PASS', text) def test_api_summary_keeps_preparation_and_outer_failures_visible(self): - workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml' + workflow = Path(__file__).resolve().parents[2] / '.github/workflows/daily-endurance.yml' steps = yaml.safe_load(workflow.read_text())['jobs']['api-contracts']['steps'] command = next(step['run'] for step in steps if step.get('name') == 'Publish API result') suite = {'passed': True, 'requests': 14, 'status': {'passed': 14}, 'elapsed_seconds': 7.3} @@ -191,7 +192,7 @@ class DailySummaryTest(unittest.TestCase): self.assertNotIn('Daily API contracts: PASS', text) def test_core_summary_records_preparation_failure_and_selected_plugins(self): - workflow = Path(__file__).resolve().parents[1] / '.github/workflows/core-smoke.yml' + workflow = Path(__file__).resolve().parents[2] / '.github/workflows/core-smoke.yml' steps = yaml.safe_load(workflow.read_text())['jobs']['smoke']['steps'] command = next(step['run'] for step in steps if step.get('name') == 'Publish gate result') with tempfile.TemporaryDirectory() as directory: diff --git a/tools/test_regression_cli_cases.py b/tools/tests/test_regression_cli_cases.py similarity index 78% rename from tools/test_regression_cli_cases.py rename to tools/tests/test_regression_cli_cases.py index d22d0a226..dc4bb8939 100644 --- a/tools/test_regression_cli_cases.py +++ b/tools/tests/test_regression_cli_cases.py @@ -7,9 +7,9 @@ import json import unittest from unittest.mock import Mock, patch -from regression_cli_cases import identity, invalid_credential, pipeline_roundtrip -from regression_ci import suite_profile -from regression_suite import Suite, validate +from tools.regression.regression_cli_cases import identity, invalid_credential, pipeline_roundtrip +from tools.entrypoints.regression_ci import suite_profile +from tools.entrypoints.regression_suite import Suite, validate class CLICompatibilityTests(unittest.TestCase): @@ -31,7 +31,8 @@ class CLICompatibilityTests(unittest.TestCase): timeout=lambda value: value, ) with patch( - 'regression_cli_cases.subprocess.run', return_value=SimpleNamespace(returncode=0, stdout='{}', stderr='') + 'tools.regression.regression_cli_cases.subprocess.run', + return_value=SimpleNamespace(returncode=0, stdout='{}', stderr=''), ): with self.assertRaisesRegex(ValueError, 'success with an invalid credential'): invalid_credential(suite, {}) @@ -45,8 +46,11 @@ class CLICompatibilityTests(unittest.TestCase): suite.call = Mock(return_value={'uuid': 'reported-but-not-created'}) row = {} with ( - patch('regression_cli_cases.request', return_value={'pipelines': []}), - patch('regression_api_checks.api', return_value=(200, {'code': 0, 'data': {'pipelines': []}})), + patch('tools.regression.regression_cli_cases.request', return_value={'pipelines': []}), + patch( + 'tools.regression.regression_api_checks.api', + return_value=(200, {'code': 0, 'data': {'pipelines': []}}), + ), ): with self.assertRaisesRegex(ValueError, 'exactly one owned Pipeline'): pipeline_roundtrip(suite, row) @@ -61,7 +65,10 @@ class CLICompatibilityTests(unittest.TestCase): {'error': {'type': 'failed'}, 'data': {'uuid': 'owned'}}, ): process = SimpleNamespace(returncode=0, stdout=json.dumps(response), stderr='') - with self.subTest(response=response), patch('regression_suite.subprocess.run', return_value=process): + with ( + self.subTest(response=response), + patch('tools.entrypoints.regression_suite.subprocess.run', return_value=process), + ): with self.assertRaisesRegex(RuntimeError, 'failure response'): suite.call(['pipeline', 'get', 'owned']) process = SimpleNamespace( @@ -71,7 +78,7 @@ class CLICompatibilityTests(unittest.TestCase): ), stderr='', ) - with patch('regression_suite.subprocess.run', return_value=process): + with patch('tools.entrypoints.regression_suite.subprocess.run', return_value=process): self.assertEqual(suite.call(['pipeline', 'get', 'owned'])['pipeline']['uuid'], 'owned') def test_profile_is_finite_and_uses_no_shared_resources(self): diff --git a/tools/test_regression_containers.py b/tools/tests/test_regression_containers.py similarity index 99% rename from tools/test_regression_containers.py rename to tools/tests/test_regression_containers.py index 56a241227..9be8a214b 100644 --- a/tools/test_regression_containers.py +++ b/tools/tests/test_regression_containers.py @@ -6,7 +6,7 @@ from types import SimpleNamespace import unittest from unittest.mock import Mock -from regression_traffic import owned_box_containers, sample_containers +from tools.entrypoints.regression_traffic import owned_box_containers, sample_containers def description(container, mount): diff --git a/tools/test_regression_coverage.py b/tools/tests/test_regression_coverage.py similarity index 95% rename from tools/test_regression_coverage.py rename to tools/tests/test_regression_coverage.py index 86c7efa30..8ca3d47bf 100644 --- a/tools/test_regression_coverage.py +++ b/tools/tests/test_regression_coverage.py @@ -6,7 +6,15 @@ from pathlib import Path import tempfile import unittest -from regression_coverage import checked, collect, http_observations, load_catalog, report, route_inventory, write_report +from tools.entrypoints.regression_coverage import ( + checked, + collect, + http_observations, + load_catalog, + report, + route_inventory, + write_report, +) def catalog(): @@ -241,7 +249,7 @@ class CoverageTest(unittest.TestCase): self.assertNotIn('secret', json.dumps(result)) def test_suite_finalization_rejects_missing_evidence_for_direct_callers(self): - from regression_suite import Suite + from tools.entrypoints.regression_suite import Suite with tempfile.TemporaryDirectory() as directory: root = Path(directory) @@ -288,18 +296,18 @@ class Routes: for binding in contract['bindings']: name = binding['profile'] if name not in profiles: - path = Path(__file__).with_name('regression_profiles') / f'{name}.json' + path = Path(__file__).resolve().parents[1] / 'profiles' / f'{name}.json' if name in ('api', 'state', 'isolation', 'plugin-lifecycle', 'cli'): - from regression_ci import suite_profile + from tools.entrypoints.regression_ci import suite_profile profile, _ = suite_profile(name, 'catalog-test') elif name == 'scenarios': - from regression_scenarios import expanded_profile - from regression_local_cases import DEFAULT_GROUPS + from tools.regression.regression_scenarios import expanded_profile + from tools.regression.regression_local_cases import DEFAULT_GROUPS profile = expanded_profile(DEFAULT_GROUPS) if 'lifecycle' in DEFAULT_GROUPS: - from regression_lifecycle_cases import restart_profile + from tools.regression.regression_lifecycle_cases import restart_profile profile['cases'] += restart_profile(profile)['cases'] else: diff --git a/tools/test_regression_execution_cases.py b/tools/tests/test_regression_execution_cases.py similarity index 97% rename from tools/test_regression_execution_cases.py rename to tools/tests/test_regression_execution_cases.py index 8baf83cd5..01cd5701e 100644 --- a/tools/test_regression_execution_cases.py +++ b/tools/tests/test_regression_execution_cases.py @@ -14,8 +14,8 @@ from unittest.mock import patch from urllib.error import HTTPError from urllib.request import Request, urlopen -import regression_execution_cases as cases -from mock_llm import MockServer +import tools.regression.regression_execution_cases as cases +from tools.entrypoints.mock_llm import MockServer class ExecutionCasesTests(unittest.TestCase): @@ -221,7 +221,9 @@ class ExecutionCasesTests(unittest.TestCase): self.assertEqual(self.row['assertions']['no_late_side_effect'], 'failed') def test_recovery_command_distinguishes_missing_setup_and_late_side_effect(self): - scenarios = json.loads(Path(cases.__file__).with_name('mock_llm_scenarios.json').read_text()) + scenarios = json.loads( + (Path(cases.__file__).resolve().parents[1] / 'fixtures' / 'mock_llm_scenarios.json').read_text() + ) command = scenarios['mock-local-interrupt-check']['steps'][0]['tool_calls'][0]['arguments']['command'] command = command.removeprefix('sleep 9; ').replace( '/workspace/langbot-{mock_scope}', str(self.root / 'fixture') @@ -306,7 +308,9 @@ class ExecutionCasesTests(unittest.TestCase): thread.join() def test_real_mock_http_scopes_isolate_tool_paths_and_reject_unsafe_substitution(self): - scenarios = json.loads(Path(cases.__file__).with_name('mock_llm_scenarios.json').read_text()) + scenarios = json.loads( + (Path(cases.__file__).resolve().parents[1] / 'fixtures' / 'mock_llm_scenarios.json').read_text() + ) with MockServer(('127.0.0.1', 0), scenarios) as server: thread = threading.Thread(target=server.serve_forever) thread.start() diff --git a/tools/test_regression_gate.py b/tools/tests/test_regression_gate.py similarity index 81% rename from tools/test_regression_gate.py rename to tools/tests/test_regression_gate.py index 360796f8d..c1c1f3cf0 100644 --- a/tools/test_regression_gate.py +++ b/tools/tests/test_regression_gate.py @@ -8,7 +8,7 @@ import tempfile import unittest from unittest.mock import patch -from regression_gate import supervise +from tools.entrypoints.regression_gate import supervise class GateTest(unittest.TestCase): @@ -33,7 +33,7 @@ class GateTest(unittest.TestCase): self.assertLess(result['elapsed_seconds'], 2) def test_teardown_failure_overrides_successful_smoke_verdict(self): - from regression_ci import main + from tools.entrypoints.regression_ci import main with tempfile.TemporaryDirectory() as directory: output = Path(directory) / 'ci' @@ -45,9 +45,19 @@ class GateTest(unittest.TestCase): raise RuntimeError('sandbox teardown failed') with ( - patch('regression_ci.run', side_effect=fail_teardown), + patch('tools.entrypoints.regression_ci.run', side_effect=fail_teardown), patch.object( - sys, 'argv', ['regression_ci', '--output', str(output), '--lbctl', 'unused', '--plugins', 'unused'] + sys, + 'argv', + [ + 'tools.entrypoints.regression_ci', + '--output', + str(output), + '--lbctl', + 'unused', + '--plugins', + 'unused', + ], ), self.assertRaises(RuntimeError), ): diff --git a/tools/test_regression_groups.py b/tools/tests/test_regression_groups.py similarity index 96% rename from tools/test_regression_groups.py rename to tools/tests/test_regression_groups.py index 3d7bd5d0c..bad6ec579 100644 --- a/tools/test_regression_groups.py +++ b/tools/tests/test_regression_groups.py @@ -14,7 +14,7 @@ import threading import time import unittest -from regression_suite import Suite, validate +from tools.entrypoints.regression_suite import Suite, validate class GroupTests(unittest.TestCase): @@ -37,7 +37,7 @@ class GroupTests(unittest.TestCase): thread.start() try: with tempfile.TemporaryDirectory() as folder: - current = json.loads((Path(__file__).with_name('regression_profiles') / 'groups.json').read_text()) + current = json.loads((Path(__file__).resolve().parents[1] / 'profiles' / 'groups.json').read_text()) suite = Suite(current, Path(folder) / 'profile.json', Path(folder), env={'TEST_API_KEY': 'test-key'}) suite.variables['workspace_uuid'] = 'bound-workspace' suite.connection = { @@ -52,7 +52,7 @@ class GroupTests(unittest.TestCase): thread.join() def test_groups_profile_and_invalid_modes(self): - path = Path(__file__).with_name('regression_profiles') / 'groups.json' + path = Path(__file__).resolve().parents[1] / 'profiles' / 'groups.json' profile = json.loads(path.read_text()) validate(profile) for field, value in [('groups', 3), ('members_per_group', 0), ('groups', 4097)]: @@ -151,7 +151,8 @@ class GroupTests(unittest.TestCase): state = root / 'group-state.json' command = [ sys.executable, - str(Path(__file__).with_name('regression_traffic.py')), + '-m', + 'tools.entrypoints.regression_traffic', '--lbctl', 'unused', '--config', diff --git a/tools/test_regression_isolation.py b/tools/tests/test_regression_isolation.py similarity index 91% rename from tools/test_regression_isolation.py rename to tools/tests/test_regression_isolation.py index c39449423..be345deeb 100644 --- a/tools/test_regression_isolation.py +++ b/tools/tests/test_regression_isolation.py @@ -7,11 +7,11 @@ import tempfile from types import SimpleNamespace import unittest -from regression_ci import suite_profile -from regression_isolation_cases import CASE_NAMES, check_reply, overlapping, verify_wire -from regression_local_cases import DEFAULT_GROUPS -from regression_scenarios import expanded_profile -from regression_suite import validate +from tools.entrypoints.regression_ci import suite_profile +from tools.regression.regression_isolation_cases import CASE_NAMES, check_reply, overlapping, verify_wire +from tools.regression.regression_local_cases import DEFAULT_GROUPS +from tools.regression.regression_scenarios import expanded_profile +from tools.entrypoints.regression_suite import validate class IsolationTests(unittest.TestCase): diff --git a/tools/test_regression_job.py b/tools/tests/test_regression_job.py similarity index 94% rename from tools/test_regression_job.py rename to tools/tests/test_regression_job.py index a20caf0bf..3997104fe 100644 --- a/tools/test_regression_job.py +++ b/tools/tests/test_regression_job.py @@ -14,10 +14,10 @@ from unittest.mock import patch import yaml -from regression_job import Job, load_profile, main -from regression_scenarios import ExecutionWindow -from regression_suite import Suite -from test_regression_suite import FAKE_CLI, profile +from tools.entrypoints.regression_job import Job, load_profile, main +from tools.regression.regression_scenarios import ExecutionWindow +from tools.entrypoints.regression_suite import Suite +from tools.tests.test_regression_suite import FAKE_CLI, profile class ControlledJob(Job): @@ -96,7 +96,10 @@ class JobTests(unittest.TestCase): with tempfile.TemporaryDirectory() as folder: root = Path(folder) job = ControlledJob({}, profile(), root / 'profile.json', root / 'run', 'unused') - with patch('regression_job.prepare_published_plugins', side_effect=RuntimeError('release unavailable')): + with patch( + 'tools.entrypoints.regression_job.prepare_published_plugins', + side_effect=RuntimeError('release unavailable'), + ): result = job.run() self.assertFalse(result['passed']) self.assertFalse(result['completed']) @@ -169,8 +172,10 @@ class JobTests(unittest.TestCase): self.assertTrue((job.deploy / 'data').exists()) self.assertFalse(any('uninstall' in call for call in job.calls)) with ( - patch.object(sys, 'argv', ['regression_job.py', 'cleanup', '--output', str(job.output)]), - patch('regression_job.Job', return_value=job), + patch.object( + sys, 'argv', ['tools.entrypoints.regression_job', 'cleanup', '--output', str(job.output)] + ), + patch('tools.entrypoints.regression_job.Job', return_value=job), ): main() self.assertFalse((job.deploy / 'data').exists()) @@ -235,7 +240,7 @@ class JobTests(unittest.TestCase): patch.object(job, 'prepare', side_effect=prepare), patch.object(job, 'collect', side_effect=lambda: events.append('collect')), patch.object(job, 'cleanup', side_effect=lambda: events.append('teardown')), - patch('regression_traffic.read_endpoint_snapshot', return_value={'http.ok': 1}), + patch('tools.entrypoints.regression_traffic.read_endpoint_snapshot', return_value={'http.ok': 1}), ): result = job.run() self.assertFalse(result['passed']) diff --git a/tools/test_regression_lifecycle_cases.py b/tools/tests/test_regression_lifecycle_cases.py similarity index 96% rename from tools/test_regression_lifecycle_cases.py rename to tools/tests/test_regression_lifecycle_cases.py index ff88d8fe0..f133c9324 100644 --- a/tools/test_regression_lifecycle_cases.py +++ b/tools/tests/test_regression_lifecycle_cases.py @@ -13,7 +13,7 @@ from unittest.mock import patch import urllib.error import urllib.request -import regression_lifecycle_cases as lifecycle +import tools.regression.regression_lifecycle_cases as lifecycle class LifecycleTest(unittest.TestCase): @@ -166,7 +166,7 @@ class LifecycleTest(unittest.TestCase): self.assertNotIn('callback_retry', row['assertions']) def test_restart_profile_reuses_fixture_resources_without_mutating_input(self): - path = Path(__file__).with_name('regression_profiles') / 'smoke.json' + path = Path(__file__).resolve().parents[1] / 'profiles' / 'smoke.json' original = json.loads(path.read_text()) original['execution'] = {'total_seconds': 1800, 'drain_seconds': 120, 'cooldown_seconds': 60} snapshot = json.dumps(original, sort_keys=True) @@ -180,7 +180,7 @@ class LifecycleTest(unittest.TestCase): self.assertEqual(len(profile['resources']), 7) def test_restart_preflight_records_real_ledger_rows_and_preserves_failed_history(self): - from regression_suite import Suite + from tools.entrypoints.regression_suite import Suite for fail_history in (False, True): with self.subTest(fail_history=fail_history), tempfile.TemporaryDirectory() as directory: @@ -191,7 +191,7 @@ class LifecycleTest(unittest.TestCase): 'CREATE TABLE agent_run (id INTEGER PRIMARY KEY, run_id TEXT, workspace_id TEXT, ' 'conversation_id TEXT, status TEXT, finished_at TEXT, status_reason TEXT)' ) - profile_path = Path(__file__).with_name('regression_profiles') / 'smoke.json' + profile_path = Path(__file__).resolve().parents[1] / 'profiles' / 'smoke.json' profile = json.loads(profile_path.read_text()) profile['smoke']['ledger_db'] = str(path) release = threading.Event() @@ -251,7 +251,7 @@ class LifecycleTest(unittest.TestCase): with ( patch.object(Suite, 'prepare', prepare), patch.object(Suite, 'call', return_value={'workspace_uuid': 'workspace'}), - patch('regression_traffic.group_request', side_effect=group_request), + patch('tools.entrypoints.regression_traffic.group_request', side_effect=group_request), patch.object(lifecycle, 'api', side_effect=public_api), ): result = lifecycle.run_restart_checks( @@ -276,9 +276,9 @@ class LifecycleTest(unittest.TestCase): self.assertEqual(sum(len(row['scenario_runs']) for row in result['cases']), 4) def test_restart_missing_user_token_is_incomplete_before_preparation(self): - from regression_suite import Suite + from tools.entrypoints.regression_suite import Suite - path = Path(__file__).with_name('regression_profiles') / 'smoke.json' + path = Path(__file__).resolve().parents[1] / 'profiles' / 'smoke.json' profile = json.loads(path.read_text()) with tempfile.TemporaryDirectory() as directory, patch.object(Suite, 'prepare') as prepare: result = lifecycle.run_restart_checks(profile, path, Path(directory) / 'preflight', {}, None) diff --git a/tools/test_regression_load_ledger.py b/tools/tests/test_regression_load_ledger.py similarity index 94% rename from tools/test_regression_load_ledger.py rename to tools/tests/test_regression_load_ledger.py index 9fb021984..77ae33584 100644 --- a/tools/test_regression_load_ledger.py +++ b/tools/tests/test_regression_load_ledger.py @@ -13,7 +13,7 @@ import unittest from unittest.mock import patch, Mock from uuid import uuid4 -from regression_traffic import main, reconcile_runs, sample_containers +from tools.entrypoints.regression_traffic import main, reconcile_runs, sample_containers class LoadLedgerTests(unittest.TestCase): @@ -121,8 +121,8 @@ class LoadLedgerTests(unittest.TestCase): with ( patch.object(sys, 'argv', argv), patch.dict('os.environ', {'TEST_GROUP_SECRET': 'test'}), - patch('regression_traffic.sqlite3.connect', side_effect=slow_connect), - patch('regression_traffic.group_request', side_effect=response), + patch('tools.entrypoints.regression_traffic.sqlite3.connect', side_effect=slow_connect), + patch('tools.entrypoints.regression_traffic.group_request', side_effect=response), redirect_stdout(io.StringIO()), ): main() diff --git a/tools/test_regression_local_cases.py b/tools/tests/test_regression_local_cases.py similarity index 89% rename from tools/test_regression_local_cases.py rename to tools/tests/test_regression_local_cases.py index 15c7baecd..ff9dc141d 100644 --- a/tools/test_regression_local_cases.py +++ b/tools/tests/test_regression_local_cases.py @@ -7,11 +7,11 @@ import tempfile import unittest from unittest.mock import patch -from regression_local_cases import DEFAULT_GROUPS, ScenarioUnavailable -from regression_scenarios import expanded_profile -from regression_smoke import run_smoke -from regression_suite import Suite, validate -from regression_traffic import reconcile_runs +from tools.regression.regression_local_cases import DEFAULT_GROUPS, ScenarioUnavailable +from tools.regression.regression_scenarios import expanded_profile +from tools.regression.regression_smoke import run_smoke +from tools.entrypoints.regression_suite import Suite, validate +from tools.entrypoints.regression_traffic import reconcile_runs class LocalScenarioTests(unittest.TestCase): @@ -48,7 +48,10 @@ class LocalScenarioTests(unittest.TestCase): 'observe': {'health': {}}, } suite = Suite(profile, root / 'profile.json', root) - with patch('regression_local_cases.run_case', side_effect=ScenarioUnavailable('Missing user credential')): + with patch( + 'tools.regression.regression_local_cases.run_case', + side_effect=ScenarioUnavailable('Missing user credential'), + ): result = run_smoke(suite) self.assertEqual(result['status'], {'skipped': 1}) self.assertEqual(result['errors'], result['incomplete']) @@ -74,7 +77,7 @@ class LocalScenarioTests(unittest.TestCase): row['cleanup_errors'] = ['Agent deletion returned HTTP 500'] raise RuntimeError('Request outcome unknown') - with patch('regression_local_cases.run_case', side_effect=fail): + with patch('tools.regression.regression_local_cases.run_case', side_effect=fail): result = run_smoke(suite) suite.finish(result) self.assertEqual(result['outcome'], 'FAIL') diff --git a/tools/test_regression_plugin_cases.py b/tools/tests/test_regression_plugin_cases.py similarity index 99% rename from tools/test_regression_plugin_cases.py rename to tools/tests/test_regression_plugin_cases.py index 52258ce02..6d4f97ca3 100644 --- a/tools/test_regression_plugin_cases.py +++ b/tools/tests/test_regression_plugin_cases.py @@ -11,8 +11,8 @@ from types import SimpleNamespace import unittest from unittest.mock import patch -import regression_plugin_cases as plugins -from regression_scenarios import expanded_profile +import tools.regression.regression_plugin_cases as plugins +from tools.regression.regression_scenarios import expanded_profile def success(value=None): diff --git a/tools/test_regression_plugin_lifecycle.py b/tools/tests/test_regression_plugin_lifecycle.py similarity index 76% rename from tools/test_regression_plugin_lifecycle.py rename to tools/tests/test_regression_plugin_lifecycle.py index 467ae1a20..1e4b0a4ae 100644 --- a/tools/test_regression_plugin_lifecycle.py +++ b/tools/tests/test_regression_plugin_lifecycle.py @@ -4,10 +4,10 @@ from types import SimpleNamespace import unittest from unittest.mock import Mock, patch -from regression_ci import suite_profile -from regression_local_cases import DEFAULT_GROUPS -from regression_plugin_lifecycle import failed_generation_safe, wait_task, workers_stopped -from regression_suite import validate +from tools.entrypoints.regression_ci import suite_profile +from tools.regression.regression_local_cases import DEFAULT_GROUPS +from tools.regression.regression_plugin_lifecycle import failed_generation_safe, wait_task, workers_stopped +from tools.entrypoints.regression_suite import validate class PluginLifecycleTests(unittest.TestCase): @@ -40,7 +40,10 @@ class PluginLifecycleTests(unittest.TestCase): [plugin('0.0.2', 'mounted'), (500, {'code': 0})], ): row = {} - with self.subTest(responses=responses), patch('regression_plugin_lifecycle.api', side_effect=responses): + with ( + self.subTest(responses=responses), + patch('tools.regression.regression_plugin_lifecycle.api', side_effect=responses), + ): with self.assertRaises(ValueError): failed_generation_safe(None, row, path, Mock()) self.assertEqual(row['assertions']['failed_generation_safe'], 'failed') @@ -50,7 +53,10 @@ class PluginLifecycleTests(unittest.TestCase): [plugin('0.0.2', 'mounted'), (400, {'code': -1})], ): row, verify_good = {}, Mock() - with self.subTest(responses=responses), patch('regression_plugin_lifecycle.api', side_effect=responses): + with ( + self.subTest(responses=responses), + patch('tools.regression.regression_plugin_lifecycle.api', side_effect=responses), + ): failed_generation_safe(None, row, path, verify_good) self.assertEqual(row['assertions']['failed_generation_safe'], 'passed') verify_good.assert_not_called() @@ -59,7 +65,7 @@ class PluginLifecycleTests(unittest.TestCase): row, verify_good = {}, Mock(side_effect=state_error) with ( self.subTest(state_error=state_error), - patch('regression_plugin_lifecycle.api', return_value=plugin('0.0.1', 'initialized')), + patch('tools.regression.regression_plugin_lifecycle.api', return_value=plugin('0.0.1', 'initialized')), ): if state_error: with self.assertRaisesRegex(ValueError, 'lost configuration'): @@ -88,28 +94,33 @@ class PluginLifecycleTests(unittest.TestCase): {'id': 7, 'status': 'failed', 'error': None}, {'id': 8, 'status': 'failed', 'error': {'type': 'task_failed'}}, ): - with self.subTest(task=task), patch('regression_plugin_lifecycle.request', return_value=task): + with ( + self.subTest(task=task), + patch('tools.regression.regression_plugin_lifecycle.request', return_value=task), + ): with self.assertRaises(ValueError): wait_task(suite, {}, 7, 'failed', 'broken_upgrade') row = {} task = {'id': 7, 'status': 'failed', 'error': {'type': 'task_failed'}} - with patch('regression_plugin_lifecycle.request', return_value=task): + with patch('tools.regression.regression_plugin_lifecycle.request', return_value=task): wait_task(suite, row, 7, 'failed', 'broken_upgrade') self.assertEqual(row['assertions']['broken_upgrade_task'], 'passed') self.assertEqual(row['plugin_tasks'][0]['status'], 'failed') def test_live_worker_is_a_cleanup_failure_but_reused_pid_is_not(self): suite = SimpleNamespace(timeout=lambda seconds: 0) - with patch('regression_plugin_lifecycle.process_identity', return_value='same-start'): + with patch('tools.regression.regression_plugin_lifecycle.process_identity', return_value='same-start'): with self.assertRaisesRegex(ValueError, 'worker remains'): workers_stopped(suite, {'100': 'same-start'}) for identity in (None, 'new-start'): - with patch('regression_plugin_lifecycle.process_identity', return_value=identity): + with patch('tools.regression.regression_plugin_lifecycle.process_identity', return_value=identity): workers_stopped(suite, {'100': 'same-start'}) def test_unfinished_task_cannot_be_accepted(self): suite = SimpleNamespace(timeout=lambda seconds: 0) - with patch('regression_plugin_lifecycle.request', Mock(return_value={'id': 7, 'status': 'running'})): + with patch( + 'tools.regression.regression_plugin_lifecycle.request', Mock(return_value={'id': 7, 'status': 'running'}) + ): with self.assertRaisesRegex(TimeoutError, 'did not finish'): wait_task(suite, {}, 7, 'succeeded', 'install') diff --git a/tools/test_regression_plugins.py b/tools/tests/test_regression_plugins.py similarity index 99% rename from tools/test_regression_plugins.py rename to tools/tests/test_regression_plugins.py index 6ae525ebe..5be9ef1b3 100644 --- a/tools/test_regression_plugins.py +++ b/tools/tests/test_regression_plugins.py @@ -14,7 +14,7 @@ import zipfile import yaml -import regression_plugins as plugins +import tools.regression.regression_plugins as plugins def archive(plugin_id, version='1.2.3'): diff --git a/tools/test_regression_resource_cases.py b/tools/tests/test_regression_resource_cases.py similarity index 93% rename from tools/test_regression_resource_cases.py rename to tools/tests/test_regression_resource_cases.py index 3e039ee6b..34aaaaf91 100644 --- a/tools/test_regression_resource_cases.py +++ b/tools/tests/test_regression_resource_cases.py @@ -7,8 +7,8 @@ from types import SimpleNamespace import unittest from unittest.mock import patch -from regression_local_cases import ScenarioUnavailable -from regression_resource_cases import run_case +from tools.regression.regression_local_cases import ScenarioUnavailable +from tools.regression.regression_resource_cases import run_case def success(data=None): @@ -38,7 +38,7 @@ class ResourceCasesTests(unittest.TestCase): def test_missing_management_session_skips_without_creating_credentials(self): self.suite.env.clear() - with patch('regression_resource_cases.api') as request, self.assertRaises(ScenarioUnavailable): + with patch('tools.regression.regression_resource_cases.api') as request, self.assertRaises(ScenarioUnavailable): run_case(self.suite, {'scenario': 'auth_revocation'}, {}) request.assert_not_called() @@ -65,7 +65,7 @@ class ResourceCasesTests(unittest.TestCase): ) return success({'key': key}) - with patch('regression_resource_cases.api', side_effect=request): + with patch('tools.regression.regression_resource_cases.api', side_effect=request): case = {'scenario': 'auth_revocation', 'expect': 'REVOKED'} if stale_authorization: with self.assertRaisesRegex(ValueError, 'still authorizes'): @@ -112,7 +112,7 @@ class ResourceCasesTests(unittest.TestCase): return success() return (404, {'code': -1}) if state.get('deleted') else success({'agent': state}) - with patch('regression_resource_cases.api', side_effect=request): + with patch('tools.regression.regression_resource_cases.api', side_effect=request): case = {'scenario': 'agent_direct_execution', 'expect': 'DIRECT_OK'} if missing_event: with self.assertRaisesRegex(ValueError, 'completion event'): @@ -149,7 +149,10 @@ class ResourceCasesTests(unittest.TestCase): return success() return (404, {'code': -1}) if state.get('deleted') else success({'agent': state}) - with patch('regression_resource_cases.api', side_effect=request), self.assertRaises(ConnectionResetError): + with ( + patch('tools.regression.regression_resource_cases.api', side_effect=request), + self.assertRaises(ConnectionResetError), + ): run_case(self.suite, {'scenario': 'agent_direct_execution', 'expect': 'OK'}, row) self.assertEqual(sum(method == 'POST' for method, _ in calls), 1) self.assertEqual(row['owned_resources'][0]['state'], 'deleted') @@ -194,7 +197,7 @@ class ResourceCasesTests(unittest.TestCase): ) self.suite.call = ingest - with patch('regression_resource_cases.api', side_effect=request): + with patch('tools.regression.regression_resource_cases.api', side_effect=request): case = {'scenario': 'retrieval_delete_visibility', 'expect': 'DELETED'} if stale_vectors: with self.assertRaisesRegex(ValueError, 'remains visible'): diff --git a/tools/test_regression_scenarios.py b/tools/tests/test_regression_scenarios.py similarity index 93% rename from tools/test_regression_scenarios.py rename to tools/tests/test_regression_scenarios.py index 9d74d4d45..5321e6940 100644 --- a/tools/test_regression_scenarios.py +++ b/tools/tests/test_regression_scenarios.py @@ -8,9 +8,9 @@ import tempfile import unittest from unittest.mock import patch -from regression_scenarios import ExecutionWindow, expanded_profile -from regression_smoke import run_smoke -from regression_suite import Suite, validate +from tools.regression.regression_scenarios import ExecutionWindow, expanded_profile +from tools.regression.regression_smoke import run_smoke +from tools.entrypoints.regression_suite import Suite, validate class ScenarioExecutionTests(unittest.TestCase): @@ -52,7 +52,7 @@ class ScenarioExecutionTests(unittest.TestCase): return value self.suite.call = call - with patch('regression_smoke.read_endpoint_snapshot', return_value={'http.ok': 1}): + with patch('tools.regression.regression_smoke.read_endpoint_snapshot', return_value={'http.ok': 1}): result = run_smoke(self.suite) return result, calls @@ -94,7 +94,9 @@ class ScenarioExecutionTests(unittest.TestCase): 'http_probe': {'url': 'http://unused.invalid', 'auth': 'missing'}, 'expect': '401', } - with patch('regression_smoke.api_probe', side_effect=ConnectionResetError('peer reset')) as probe: + with patch( + 'tools.regression.regression_smoke.api_probe', side_effect=ConnectionResetError('peer reset') + ) as probe: result, calls = self.execute(iter(['OK'] * 4)) self.assertEqual(probe.call_count, 1) self.assertEqual(len(calls), 2) @@ -123,7 +125,7 @@ class ScenarioExecutionTests(unittest.TestCase): connection.execute('DELETE FROM agent_run') connection.execute('DELETE FROM agent_run_event') window = self.suite.execution = ExecutionWindow(self.suite) - with patch('regression_smoke.read_endpoint_snapshot', return_value={'http.ok': 1}): + with patch('tools.regression.regression_smoke.read_endpoint_snapshot', return_value={'http.ok': 1}): result = run_smoke(self.suite) self.assertTrue(result['ledger']['matched']) self.assertEqual(result['errors'], []) @@ -186,7 +188,7 @@ class ScenarioExecutionTests(unittest.TestCase): def test_cli_timeout_shrinks_to_shared_deadline_and_cleanup_reuses_it(self): self.now = 87 value = subprocess.CompletedProcess([], 0, '{"data": {}}', '') - with patch('regression_suite.subprocess.run', return_value=value) as run: + with patch('tools.entrypoints.regression_suite.subprocess.run', return_value=value) as run: self.suite.call(['provider', 'list']) self.assertEqual(run.call_args.kwargs['timeout'], 3) self.now = 90 @@ -205,7 +207,7 @@ class ScenarioExecutionTests(unittest.TestCase): self.profile['observe']['pids'] = {'missing': 999999999} window = ExecutionWindow(self.suite) result = {'errors': []} - with patch('regression_traffic.read_endpoint_snapshot', return_value={'http.ok': 1}): + with patch('tools.entrypoints.regression_traffic.read_endpoint_snapshot', return_value={'http.ok': 1}): window.start() window.finish(result) self.assertTrue(any('process sample failed' in error for error in result['errors'])) diff --git a/tools/test_regression_smoke.py b/tools/tests/test_regression_smoke.py similarity index 96% rename from tools/test_regression_smoke.py rename to tools/tests/test_regression_smoke.py index faf7ed79a..2c5942aed 100644 --- a/tools/test_regression_smoke.py +++ b/tools/tests/test_regression_smoke.py @@ -11,7 +11,7 @@ import unittest from unittest.mock import patch from urllib.error import HTTPError -from regression_smoke import ( +from tools.regression.regression_smoke import ( UnresolvedRun, check_attachment, check_fields, @@ -21,7 +21,7 @@ from regression_smoke import ( verify_cleanup, reset_group, ) -from regression_suite import validate +from tools.entrypoints.regression_suite import validate class SmokeOracleTest(unittest.TestCase): @@ -42,7 +42,7 @@ class SmokeOracleTest(unittest.TestCase): def test_received_reset_error_is_a_failed_contract_not_unknown_transport(self): case = {'http_group': {'url': 'http://localhost/bots/test/sync', 'secret_env': 'SECRET'}} - with patch('regression_smoke.urllib.request.build_opener') as build: + with patch('tools.regression.regression_smoke.urllib.request.build_opener') as build: build.return_value.open.side_effect = HTTPError('reset', 500, 'error', {}, None) with self.assertRaisesRegex(ValueError, 'Reset returned HTTP 500'): reset_group(case, 'group', {'SECRET': 'test'}) @@ -199,7 +199,7 @@ class SmokeOracleTest(unittest.TestCase): verify_cleanup(suite, deleted) def test_profile_supports_mixed_cases_and_rejects_ignored_oracles(self): - profile = json.loads(Path(__file__).with_name('regression_profiles').joinpath('smoke.json').read_text()) + profile = json.loads((Path(__file__).resolve().parents[1] / 'profiles' / 'smoke.json').read_text()) validate(profile) for changes in [{'thresholds': {'min_rps_ratio': 1}}, {'smoke': {'ledger_db': ''}}, {'phases': None}]: with self.subTest(changes=changes), self.assertRaises(ValueError): diff --git a/tools/test_regression_state.py b/tools/tests/test_regression_state.py similarity index 90% rename from tools/test_regression_state.py rename to tools/tests/test_regression_state.py index 49d523e0a..3f67c807b 100644 --- a/tools/test_regression_state.py +++ b/tools/tests/test_regression_state.py @@ -3,9 +3,9 @@ import copy import unittest -from regression_ci import SUITE_BUDGETS, suite_profile -from regression_state import document_absent, document_visible -from regression_suite import validate +from tools.entrypoints.regression_ci import SUITE_BUDGETS, suite_profile +from tools.regression.regression_state import document_absent, document_visible +from tools.entrypoints.regression_suite import validate class StateLifecycleTest(unittest.TestCase): diff --git a/tools/test_regression_suite.py b/tools/tests/test_regression_suite.py similarity index 95% rename from tools/test_regression_suite.py rename to tools/tests/test_regression_suite.py index 6d8b59a26..b9d1321de 100644 --- a/tools/test_regression_suite.py +++ b/tools/tests/test_regression_suite.py @@ -14,7 +14,7 @@ import threading import unittest from unittest.mock import patch -from regression_suite import Suite, main, merge, resolve, validate +from tools.entrypoints.regression_suite import Suite, main, merge, resolve, validate FAKE_CLI = r""" @@ -69,7 +69,7 @@ class ProfileTests(unittest.TestCase): 'error': {'type': 'verification_failed'}, } result = subprocess.CompletedProcess([], 1, json.dumps(response), '') - with patch('regression_suite.subprocess.run', return_value=result): + with patch('tools.entrypoints.regression_suite.subprocess.run', return_value=result): with self.assertRaises(RuntimeError): suite.call(['bot', 'create'], {'name': 'QA'}) self.assertEqual( @@ -126,7 +126,7 @@ class ProfileTests(unittest.TestCase): actual['event_bindings'][0]['target_uuid'] = 'wrong-pipeline' with ( patch.object(suite, 'read_bot', return_value=actual), - patch('regression_suite.group_request', return_value={'code': 0}) as warmup, + patch('tools.entrypoints.regression_suite.group_request', return_value={'code': 0}) as warmup, ): if correct_route: suite.prepare() @@ -142,7 +142,7 @@ class ProfileTests(unittest.TestCase): ) def test_normal_profile_and_overrides_contract(self): - validate(json.loads(Path(__file__).with_name('regression_profiles').joinpath('normal.json').read_text())) + validate(json.loads((Path(__file__).resolve().parents[1] / 'profiles' / 'normal.json').read_text())) bad = profile() for field, value in [('rps', float('nan')), ('workers', 33), ('duration_seconds', -1)]: with self.subTest(field=field): @@ -221,7 +221,14 @@ class ProfileTests(unittest.TestCase): suite.journal() raise error_type - argv = ['regression_suite.py', 'run', '--profile', str(config), '--output', str(output)] + argv = [ + 'tools.entrypoints.regression_suite', + 'run', + '--profile', + str(config), + '--output', + str(output), + ] with ( patch.object(sys, 'argv', argv), patch.object(Suite, 'prepare', interrupt), @@ -276,7 +283,8 @@ class ProfileTests(unittest.TestCase): config = root / 'profile.json' command = [ sys.executable, - str(Path(__file__).with_name('regression_suite.py')), + '-m', + 'tools.entrypoints.regression_suite', 'run', '--profile', str(config),