mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-29 20:05:25 +08:00
Merge pull request #2806 from langbot-app/ci/daily-api-contract
ci(daily): add API contract regression job
This commit is contained in:
huanghuoguoguo
@@ -158,3 +158,94 @@ jobs:
|
||||
.qa/daily/*.log
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
api-contracts:
|
||||
name: Daily API contracts
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 15
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.12'
|
||||
- uses: astral-sh/setup-uv@v4
|
||||
with:
|
||||
enable-cache: true
|
||||
- name: Install locked dependencies
|
||||
run: uv sync --dev --frozen
|
||||
- name: Prepare CLI and sandbox image
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
run: |
|
||||
mkdir -p .api-bin
|
||||
gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .api-bin \
|
||||
--pattern lbctl_linux_amd64 --pattern checksums.txt
|
||||
cd .api-bin
|
||||
sha256sum --check --ignore-missing checksums.txt
|
||||
chmod +x lbctl_linux_amd64
|
||||
docker pull python:3.12-alpine
|
||||
- name: Run API contracts
|
||||
run: |
|
||||
uv run --no-sync python tools/regression_ci.py \
|
||||
--suite api --api-seed "$GITHUB_RUN_ID" \
|
||||
--lbctl .api-bin/lbctl_linux_amd64 --plugins plugins --output .qa/api
|
||||
- name: Publish API result
|
||||
if: always()
|
||||
run: |
|
||||
python - <<'PY_SUMMARY'
|
||||
import json, os
|
||||
from pathlib import Path
|
||||
|
||||
root = Path('.qa/api')
|
||||
def read(path):
|
||||
source = root / path
|
||||
return json.loads(source.read_text()) if source.exists() else {}
|
||||
|
||||
gate = read('gate/gate.json')
|
||||
suite = read('gate/suite/summary.json')
|
||||
coverage = read('gate/suite/coverage.json')
|
||||
manifest = read('plugins/manifest.json')
|
||||
passed = gate.get('passed') is True and suite.get('passed') is True
|
||||
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out:
|
||||
out.write(f"Daily API contracts: {'PASS' if passed else 'FAIL / incomplete'}\n\n")
|
||||
out.write(f"Core: `{os.environ['GITHUB_SHA']}`; shuffle seed: `{gate.get('api_seed', os.environ['GITHUB_RUN_ID'])}`.\n\n")
|
||||
out.write('Fixed order followed by a seeded chain shuffle, with sequential operations inside each chain. The execution guard is 600 seconds; dependency and service preparation is separate. This job is not a required PR check.\n\n')
|
||||
out.write(f"Cases: {suite.get('requests', 'not reported')}; statuses: {suite.get('status', 'not reported')}; suite execution: {suite.get('elapsed_seconds', 'not reported')}s.\n\n")
|
||||
out.write(f"API behavior contracts: {coverage.get('areas', {}).get('api', 'not reported')}.\n\n")
|
||||
out.write(f"Observed HTTP route templates (including preparation): {len(coverage['http_observations']['routes']) if 'http_observations' in coverage else 'not reported'}. A route hit does not establish all behavior behind it.\n\n")
|
||||
for plugin in manifest.get('plugins', []):
|
||||
out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n")
|
||||
errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', []))
|
||||
if gate.get('timed_out'):
|
||||
errors.append('API suite exceeded its 600-second outer process guard.')
|
||||
if gate.get('infrastructure_error'):
|
||||
errors.append(gate['infrastructure_error'])
|
||||
if not gate or not suite:
|
||||
errors.append('Missing verdict; inspect preparation and service logs.')
|
||||
for error in errors:
|
||||
out.write(f"- {error}\n")
|
||||
PY_SUMMARY
|
||||
- name: Preserve API verdicts and evidence
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: daily-api-${{ github.run_id }}-${{ github.run_attempt }}
|
||||
path: |
|
||||
.qa/api/gate/gate.json
|
||||
.qa/api/gate/resources.jsonl
|
||||
.qa/api/gate/suite.log
|
||||
.qa/api/gate/suite/summary.json
|
||||
.qa/api/gate/suite/profile.json
|
||||
.qa/api/gate/suite/cases.json
|
||||
.qa/api/gate/suite/health.json
|
||||
.qa/api/gate/suite/cleanup-checks.json
|
||||
.qa/api/gate/suite/resource-case-*.json
|
||||
.qa/api/gate/suite/coverage.json
|
||||
.qa/api/gate/suite/coverage.md
|
||||
.qa/api/plugins/manifest.json
|
||||
.qa/api/plugins/*-build.log
|
||||
.qa/api/*.log
|
||||
if-no-files-found: warn
|
||||
retention-days: 30
|
||||
|
||||
+41
-1
@@ -486,7 +486,7 @@ uv run --no-sync python tools/regression_ci.py --suite daily --restart-services
|
||||
`--suite smoke` retains the 60-second gate. `--suite scenarios` runs the shared
|
||||
expanded profile once under a 300-second guard. `--suite endurance` retains the
|
||||
separate ten-minute load profile under a 900-second guard; its CI entry point also
|
||||
reports throughput/P95 without applying their thresholds. All four modes use
|
||||
reports throughput/P95 without applying their thresholds. All suites use
|
||||
published plugins when `--plugins` is omitted. Both workflows explicitly select
|
||||
`--plugins plugins` so source changes are tested in the same PR. Daily additionally
|
||||
selects `misc/EssentialCommands misc/FAQManager`; manual dispatch can change the
|
||||
@@ -498,6 +498,46 @@ bind their own extensions to avoid changing unrelated prompts/tools. The suite a
|
||||
even after failures. The CI owner then stops its service processes and removes
|
||||
its Box containers; a process guard timeout can interrupt API cleanup.
|
||||
|
||||
## Daily API contract job
|
||||
|
||||
`daily-endurance.yml` runs API contracts alongside the 30-minute endurance job
|
||||
on separate runners, at the existing daily schedule and on manual dispatch.
|
||||
The API job uses the GitHub run ID as its shuffle seed, so each daily run explores
|
||||
a different order and a rerun keeps the same seed. It has a 15-minute job timeout,
|
||||
including preparation, and is not a required PR check. Its summary and separate
|
||||
`daily-api-<run_id>-<attempt>` artifact retain failures, order, coverage and cleanup
|
||||
evidence for 30 days.
|
||||
|
||||
To replay locally on a disposable Linux/Docker host, pass the recorded seed:
|
||||
|
||||
```sh
|
||||
uv run --no-sync python tools/regression_ci.py --suite api --api-seed 20261028 \
|
||||
--lbctl /path/to/lbctl --plugins plugins --output .qa/api-001
|
||||
```
|
||||
|
||||
This separate suite executes seven independent operation chains once in fixed
|
||||
order, then once in seed-shuffled order (14 case rows). Operations within each
|
||||
chain retain their create/read/update/delete dependencies. Reuse the seed to
|
||||
replay chain ordering; resource names and credentials remain unique. Preparation
|
||||
is outside the 600-second execution guard; a completed run ends immediately.
|
||||
The default PR smoke and daily scenario groups do not include these cases.
|
||||
|
||||
The chains cover Provider and LLM/embedding/rerank model CRUD, Pipeline extension
|
||||
validation, API key permissions/revocation, Skill files and revision conflicts,
|
||||
and disabled MCP configuration. Checks require matching HTTP/business status,
|
||||
read back stored state, and reject unexpected mutation after invalid requests.
|
||||
Resources carry unique ownership markers and are deleted in `finally`; API keys
|
||||
are revoked. Cleanup errors remain failures. MCP fixtures stay disabled and use
|
||||
loopback addresses. This job does not exercise external MCP servers, real model
|
||||
inference, concurrent API mutations, or cross-Workspace isolation.
|
||||
|
||||
Inspect `gate/gate.json` for the verdict, seed and exact case order;
|
||||
`gate/suite/cases.json` for HTTP steps, assertions and cleanup results; and
|
||||
`gate/suite/coverage.json` for observed route templates and behavior contracts.
|
||||
An HTTP route hit includes rejected requests and is not proof of all behavior
|
||||
behind that route. Invalid MCP modes must return HTTP 400 and leave the original
|
||||
configuration unchanged. Any failed contract or cleanup makes the job fail.
|
||||
|
||||
## Sustained regression and baseline comparison
|
||||
|
||||
[`regression_profiles/endurance.json`](regression_profiles/endurance.json) runs
|
||||
|
||||
@@ -0,0 +1,213 @@
|
||||
"""Finite API contracts: fixed first pass, then reproducibly shuffled independent chains."""
|
||||
|
||||
import json
|
||||
import random
|
||||
|
||||
from regression_api_checks import check, owned_resource, request, tracked_resource, unique_name, user_headers
|
||||
from regression_coverage import checked
|
||||
from regression_local_cases import require
|
||||
from regression_resource_cases import cleanup, journal
|
||||
|
||||
CASE_NAMES = ('api_llm', 'api_embedding', 'api_rerank', 'api_pipeline', 'api_credentials', 'api_skills', 'api_mcp')
|
||||
|
||||
|
||||
def add_cases(profile):
|
||||
require(profile.get('coverage_suite') == 'api', 'API contracts require their dedicated suite')
|
||||
shuffled = list(CASE_NAMES)
|
||||
seed = profile['vars']['api_seed']
|
||||
random.Random(seed).shuffle(shuffled)
|
||||
for phase, names in (('fixed', CASE_NAMES), ('shuffled', shuffled)):
|
||||
for name in names:
|
||||
profile['cases'].append(
|
||||
{'name': f'{phase}_{name}', 'scenario': name, 'stage': 'api', 'expect': 'API_CONTRACT_OK'}
|
||||
)
|
||||
profile['scenario_groups'] = ['api']
|
||||
|
||||
|
||||
def model_case(suite, row, model_type):
|
||||
providers = '/api/v1/provider/providers'
|
||||
name, secret = unique_name(), unique_name()
|
||||
suite.secrets.append(secret)
|
||||
body = {
|
||||
'name': name,
|
||||
'requester': 'jina-rerank' if model_type == 'rerank' else 'openai-chat-completions',
|
||||
'base_url': suite.variables['mock_base_url'],
|
||||
'api_keys': [secret],
|
||||
}
|
||||
with owned_resource(suite, row, providers, body, item_key='provider', list_key='providers') as (path, uid):
|
||||
visible = request(suite, row, 'GET', path + '?include_secret=true')['provider']
|
||||
check(
|
||||
row,
|
||||
'provider_identity',
|
||||
visible['name'] == name and visible['api_keys'] == [secret],
|
||||
'Provider identity or secret changed',
|
||||
)
|
||||
redacted = request(suite, row, 'GET', path + '?include_secret=false')['provider']
|
||||
check(row, 'provider_redacted', secret not in json.dumps(redacted), 'Provider secret was disclosed')
|
||||
request(suite, row, 'GET', path + '?include_secret=invalid', expected=400)
|
||||
request(suite, row, 'PUT', path, {'name': name + '-updated'})
|
||||
updated = request(suite, row, 'GET', path + '?include_secret=true')['provider']
|
||||
check(
|
||||
row,
|
||||
'provider_update',
|
||||
updated['name'] == name + '-updated' and updated['api_keys'] == [secret],
|
||||
'Provider metadata update lost its secret',
|
||||
)
|
||||
collection = '/api/v1/provider/models/' + model_type
|
||||
model_name = unique_name()
|
||||
with owned_resource(
|
||||
suite, row, collection, {'name': model_name, 'provider_uuid': uid}, item_key='model', list_key='models'
|
||||
) as (model_path, model_id):
|
||||
model = request(suite, row, 'GET', model_path)['model']
|
||||
check(
|
||||
row,
|
||||
'model_identity',
|
||||
model['name'] == model_name and model['provider_uuid'] == uid,
|
||||
'Model did not retain its provider binding',
|
||||
)
|
||||
items = request(suite, row, 'GET', collection + '?provider_uuid=' + uid)['models']
|
||||
check(row, 'model_list', [m['uuid'] for m in items] == [model_id], 'Provider model filter is incorrect')
|
||||
request(suite, row, 'PUT', model_path, {'name': model_name + '-updated'})
|
||||
model = request(suite, row, 'GET', model_path)['model']
|
||||
check(
|
||||
row,
|
||||
'model_update',
|
||||
model['name'] == model_name + '-updated' and model['provider_uuid'] == uid,
|
||||
'Model metadata update lost its binding',
|
||||
)
|
||||
request(suite, row, 'DELETE', path, expected=400)
|
||||
still = request(suite, row, 'GET', model_path)['model']
|
||||
check(row, 'rejected_delete_preserves_model', still == model, 'Rejected provider deletion mutated child')
|
||||
request(suite, row, 'GET', path)
|
||||
items = request(suite, row, 'GET', collection + '?provider_uuid=' + uid)['models']
|
||||
check(row, 'model_deleted_from_list', not items, 'Deleted model remains listed')
|
||||
|
||||
|
||||
def pipeline_case(suite, row):
|
||||
collection, name = '/api/v1/pipelines', unique_name()
|
||||
with owned_resource(suite, row, collection, {'name': name}, item_key='pipeline', list_key='pipelines') as (
|
||||
path,
|
||||
uid,
|
||||
):
|
||||
pipeline = request(suite, row, 'GET', path)['pipeline']
|
||||
check(
|
||||
row,
|
||||
'pipeline_identity',
|
||||
pipeline['name'] == name and pipeline['uuid'] == uid,
|
||||
'Pipeline creation readback differs',
|
||||
)
|
||||
items = request(suite, row, 'GET', collection)['pipelines']
|
||||
check(row, 'pipeline_list', sum(p['uuid'] == uid for p in items) == 1, 'Pipeline missing or duplicated')
|
||||
request(suite, row, 'PUT', path, {'name': name + '-updated', 'description': 'API contract 中文'})
|
||||
updated = request(suite, row, 'GET', path)['pipeline']
|
||||
check(
|
||||
row,
|
||||
'pipeline_update',
|
||||
updated['name'] == name + '-updated' and updated['description'] == 'API contract 中文',
|
||||
'Pipeline update readback differs',
|
||||
)
|
||||
bindings = dict(
|
||||
bound_plugins=[],
|
||||
bound_mcp_servers=[],
|
||||
bound_skills=[],
|
||||
enable_all_plugins=False,
|
||||
enable_all_mcp_servers=False,
|
||||
enable_all_skills=False,
|
||||
)
|
||||
request(suite, row, 'PUT', path + '/extensions', bindings)
|
||||
before = request(suite, row, 'GET', path + '/extensions')
|
||||
check(row, 'extension_readback', all(before[k] == v for k, v in bindings.items()), 'Extension bindings differ')
|
||||
for invalid in ({'enable_all_plugins': 'false'}, {'bound_skills': [42]}, {'bound_plugins': [{}]}):
|
||||
request(suite, row, 'PUT', path + '/extensions', invalid, expected=400)
|
||||
after = request(suite, row, 'GET', path + '/extensions')
|
||||
check(row, 'rejected_extension_' + next(iter(invalid)), after == before, 'Rejected binding changed state')
|
||||
for invalid in ({}, {'message': ''}, {'message': 42}):
|
||||
request(suite, row, 'POST', path + '/run', invalid, expected=400)
|
||||
after = request(suite, row, 'GET', path)['pipeline']
|
||||
check(row, 'invalid_run_preserves_config', after['config'] == updated['config'], 'Invalid run changed config')
|
||||
|
||||
|
||||
def credentials_case(suite, row):
|
||||
manager, collection, name = user_headers(suite), '/api/v1/apikeys', unique_name()
|
||||
for headers in ({}, {'Authorization': 'Bearer qa-invalid'}):
|
||||
request(suite, row, 'GET', '/api/v1/pipelines', headers=headers, expected=401)
|
||||
owned = {'kind': 'api-key', 'name': name, 'state': 'creating'}
|
||||
row.setdefault('owned_resources', []).append(owned)
|
||||
journal(suite, row)
|
||||
|
||||
def revoke():
|
||||
if 'id' not in owned:
|
||||
keys = request(suite, row, 'GET', collection, headers=manager)['keys']
|
||||
matches = [k for k in keys if k['name'] in (name, name + '-updated')]
|
||||
require(len(matches) <= 1, 'Ambiguous owned key')
|
||||
if not matches:
|
||||
owned['state'] = 'absent'
|
||||
return
|
||||
owned['id'] = matches[0]['id']
|
||||
path = collection + '/' + str(owned['id'])
|
||||
current = request(suite, row, 'GET', path, headers=manager)['key']
|
||||
require(current['name'] in (name, name + '-updated'), 'Key ownership changed')
|
||||
if current['status'] != 'revoked':
|
||||
request(suite, row, 'DELETE', path, headers=manager)
|
||||
current = request(suite, row, 'GET', path, headers=manager)['key']
|
||||
check(
|
||||
row,
|
||||
'revoked_readback',
|
||||
current['status'] == 'revoked' and current['secret_available'] is False and 'key' not in current,
|
||||
'Revoked key remains active or exposes its secret',
|
||||
)
|
||||
owned['state'] = 'revoked'
|
||||
|
||||
try:
|
||||
before = request(suite, row, 'GET', collection, headers=manager)['keys']
|
||||
for invalid in ({'scopes': ['qa.invalid']}, {'expires_at': 'not-a-date'}):
|
||||
request(suite, row, 'POST', collection, {'name': name, **invalid}, headers=manager, expected=400)
|
||||
after = request(suite, row, 'GET', collection, headers=manager)['keys']
|
||||
check(row, 'invalid_key_not_created', before == after, 'Invalid key request changed key list')
|
||||
key = request(suite, row, 'POST', collection, {'name': name, 'scopes': ['resource.view']}, headers=manager)[
|
||||
'key'
|
||||
]
|
||||
owned.update(id=key['id'], state='active')
|
||||
journal(suite, row)
|
||||
secret = key['key']
|
||||
suite.secrets.append(secret)
|
||||
child = {'X-API-Key': secret}
|
||||
path = collection + '/' + str(key['id'])
|
||||
request(suite, row, 'PUT', path, {'name': name + '-updated'}, headers=manager)
|
||||
current = request(suite, row, 'GET', path, headers=manager)['key']
|
||||
check(
|
||||
row,
|
||||
'key_metadata_redacted',
|
||||
current['name'] == name + '-updated' and 'key' not in current,
|
||||
'Key metadata update or one-time-secret contract failed',
|
||||
)
|
||||
with tracked_resource(suite, row, '/api/v1/pipelines', name, item_key='pipeline', list_key='pipelines'):
|
||||
before = request(suite, row, 'GET', '/api/v1/pipelines', headers=child)['pipelines']
|
||||
request(suite, row, 'POST', '/api/v1/pipelines', {'name': name}, headers=child, expected=403)
|
||||
after = request(suite, row, 'GET', '/api/v1/pipelines', headers=child)['pipelines']
|
||||
check(row, 'readonly_denial_preserves_state', before == after, 'Read-only key changed resources')
|
||||
revoke()
|
||||
request(suite, row, 'GET', '/api/v1/pipelines', headers=child, expected=401)
|
||||
finally:
|
||||
cleanup(suite, row, 'key_cleanup', revoke)
|
||||
|
||||
|
||||
def run_case(suite, case, row):
|
||||
from regression_api_resources import mcp_case, skills_case
|
||||
|
||||
name = case['scenario']
|
||||
row['api_seed'] = suite.profile['vars']['api_seed']
|
||||
|
||||
def execute():
|
||||
if name in ('api_llm', 'api_embedding', 'api_rerank'):
|
||||
model_case(suite, row, name.removeprefix('api_'))
|
||||
else:
|
||||
{
|
||||
'api_pipeline': pipeline_case,
|
||||
'api_credentials': credentials_case,
|
||||
'api_skills': skills_case,
|
||||
'api_mcp': mcp_case,
|
||||
}[name](suite, row)
|
||||
|
||||
checked(row, 'api_chain', execute)
|
||||
return {'result': case['expect']}
|
||||
@@ -0,0 +1,84 @@
|
||||
"""HTTP and ownership checks shared by the local API contract cases."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
from uuid import uuid4
|
||||
|
||||
from regression_coverage import checked
|
||||
from regression_local_cases import api, require
|
||||
from regression_resource_cases import cleanup, journal
|
||||
|
||||
|
||||
def check(row, name, condition, message):
|
||||
return checked(row, name, lambda: require(condition, message))
|
||||
|
||||
|
||||
def request(suite, row, method, path, body=None, *, expected=200, headers=None):
|
||||
step = {'method': method, 'path': path, 'expected_status': expected}
|
||||
row.setdefault('http_steps', []).append(step)
|
||||
status, response = api(suite, method, path, body, headers=headers)
|
||||
step['status'] = status
|
||||
check(
|
||||
row,
|
||||
f'http_{len(row["http_steps"])}',
|
||||
status == expected
|
||||
and isinstance(response, dict)
|
||||
and ((response.get('code') == 0) if expected < 400 else (response.get('code') not in (None, 0))),
|
||||
f'{method} {path}: expected HTTP {expected} with matching business status, got {status}',
|
||||
)
|
||||
return response.get('data', {}) if expected < 400 else response
|
||||
|
||||
|
||||
def unique_name():
|
||||
return 'qa-api-' + uuid4().hex
|
||||
|
||||
|
||||
def user_headers(suite):
|
||||
token = suite.env.get('LANGBOT_QA_USER_TOKEN')
|
||||
require(bool(token), 'API contracts require the disposable test account token')
|
||||
if token not in suite.secrets:
|
||||
suite.secrets.append(token)
|
||||
return {'Authorization': 'Bearer ' + token}
|
||||
|
||||
|
||||
@contextmanager
|
||||
def tracked_resource(suite, row, collection, name, *, item_key, list_key):
|
||||
"""Clean an owned UUID resource even if its creation unexpectedly succeeds or times out."""
|
||||
owned = {'kind': item_key, 'name': name, 'state': 'creating'}
|
||||
row.setdefault('owned_resources', []).append(owned)
|
||||
journal(suite, row)
|
||||
|
||||
def remove():
|
||||
if 'uuid' not in owned:
|
||||
items = request(suite, row, 'GET', collection)[list_key]
|
||||
matches = [item for item in items if item.get('name') in (name, name + '-updated')]
|
||||
require(len(matches) <= 1, 'Ambiguous owned resource identity')
|
||||
if not matches:
|
||||
owned['state'] = 'absent'
|
||||
return
|
||||
owned['uuid'] = matches[0]['uuid']
|
||||
path = collection + '/' + owned['uuid']
|
||||
status, response = api(suite, 'GET', path)
|
||||
if status != 404:
|
||||
require(status == 200 and response.get('code') == 0, 'Cannot read resource before cleanup')
|
||||
item = response['data'][item_key]
|
||||
require(
|
||||
item.get('uuid') == owned['uuid'] and item.get('name') in (name, name + '-updated'),
|
||||
'Resource ownership changed before cleanup',
|
||||
)
|
||||
request(suite, row, 'DELETE', path)
|
||||
request(suite, row, 'GET', path, expected=404)
|
||||
owned['state'] = 'deleted'
|
||||
|
||||
try:
|
||||
yield owned
|
||||
finally:
|
||||
cleanup(suite, row, 'cleanup_' + item_key + '_' + name, remove)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def owned_resource(suite, row, collection, body, *, item_key, list_key):
|
||||
with tracked_resource(suite, row, collection, body['name'], item_key=item_key, list_key=list_key) as owned:
|
||||
result = request(suite, row, 'POST', collection, body)
|
||||
owned.update(uuid=result['uuid'], state='created')
|
||||
journal(suite, row)
|
||||
yield collection + '/' + owned['uuid'], owned['uuid']
|
||||
@@ -0,0 +1,310 @@
|
||||
"""Local Skill publications and disabled MCP configuration HTTP contracts."""
|
||||
|
||||
from contextlib import contextmanager
|
||||
|
||||
from regression_api_checks import check, request, unique_name, user_headers
|
||||
from regression_local_cases import api, require
|
||||
from regression_resource_cases import cleanup, journal
|
||||
|
||||
|
||||
@contextmanager
|
||||
def _named_resource(suite, row, collection, body, item_key, ownership_field, headers):
|
||||
name = body['name']
|
||||
path = collection + '/' + name
|
||||
request(suite, row, 'GET', path, expected=404, headers=headers)
|
||||
owned = {'kind': item_key, 'name': name, 'state': 'creating'}
|
||||
row.setdefault('owned_resources', []).append(owned)
|
||||
journal(suite, row)
|
||||
|
||||
def remove():
|
||||
status, response = api(suite, 'GET', path, headers=headers)
|
||||
if status != 404:
|
||||
require(status == 200 and response.get('code') == 0, 'Cannot read owned resource before cleanup')
|
||||
item = response['data'][item_key]
|
||||
require(
|
||||
item.get('name') == name and item.get(ownership_field) == body[ownership_field],
|
||||
'Named resource ownership changed before cleanup',
|
||||
)
|
||||
if 'uuid' in owned:
|
||||
require(item.get('uuid') == owned['uuid'], 'Named resource identity changed before cleanup')
|
||||
request(suite, row, 'DELETE', path, headers=headers)
|
||||
request(suite, row, 'GET', path, expected=404, headers=headers)
|
||||
owned['state'] = 'deleted'
|
||||
|
||||
try:
|
||||
created = request(suite, row, 'POST', collection, body, headers=headers)
|
||||
owned['state'] = 'created'
|
||||
if created.get('uuid'):
|
||||
owned['uuid'] = created['uuid']
|
||||
journal(suite, row)
|
||||
yield path, created
|
||||
finally:
|
||||
cleanup(suite, row, 'cleanup_' + item_key, remove)
|
||||
|
||||
|
||||
def _expect_rejection(suite, row, method, path, body, *, expected, headers, verify):
|
||||
try:
|
||||
request(suite, row, method, path, body, expected=expected, headers=headers)
|
||||
except Exception:
|
||||
try:
|
||||
verify()
|
||||
except Exception:
|
||||
# Both assertions are recorded; retain the original HTTP failure.
|
||||
pass
|
||||
raise
|
||||
verify()
|
||||
|
||||
|
||||
def skills_case(suite, row):
|
||||
headers = user_headers(suite)
|
||||
name = unique_name()
|
||||
instructions = '# API fixture\n\nRead the local reference file.'
|
||||
body = {
|
||||
'name': name,
|
||||
'display_name': 'API contract fixture',
|
||||
'description': 'Owned Skill fixture ' + name,
|
||||
'instructions': instructions,
|
||||
}
|
||||
with _named_resource(suite, row, '/api/v1/skills', body, 'skill', 'description', headers) as (path, created):
|
||||
skill = created['skill']
|
||||
initial_revision = skill.get('revision')
|
||||
check(
|
||||
row,
|
||||
'skill_created',
|
||||
skill.get('name') == name and bool(initial_revision),
|
||||
'Skill identity/revision missing',
|
||||
)
|
||||
readback = request(suite, row, 'GET', path, headers=headers)['skill']
|
||||
check(row, 'skill_readback', readback == skill, 'Created Skill differs from its readback')
|
||||
listed = request(suite, row, 'GET', '/api/v1/skills', headers=headers)['skills']
|
||||
check(row, 'skill_listed', sum(item.get('name') == name for item in listed) == 1, 'Skill listing is not unique')
|
||||
preview = request(suite, row, 'GET', path + '/preview', headers=headers)
|
||||
check(row, 'skill_preview', preview.get('instructions') == instructions, 'Skill preview lost instructions')
|
||||
|
||||
file_path = path + '/files/references/fixture.txt'
|
||||
content = 'Skill API fixture: 中文\n' + name + '\n'
|
||||
written = request(
|
||||
suite, row, 'PUT', file_path, {'content': content, 'base_revision': initial_revision}, headers=headers
|
||||
)
|
||||
file_revision = written.get('revision')
|
||||
check(
|
||||
row,
|
||||
'skill_file_published',
|
||||
bool(file_revision)
|
||||
and file_revision != initial_revision
|
||||
and written.get('bytes_written') == len(content.encode()),
|
||||
'Skill file write did not publish the expected UTF-8 bytes as a new revision',
|
||||
)
|
||||
file_readback = request(suite, row, 'GET', file_path, headers=headers)
|
||||
check(
|
||||
row,
|
||||
'skill_file_readback',
|
||||
file_readback.get('content') == content
|
||||
and file_readback.get('path') == 'references/fixture.txt'
|
||||
and file_readback.get('skill', {}).get('name') == name,
|
||||
'Skill file readback differs from the written content or identity',
|
||||
)
|
||||
files = request(suite, row, 'GET', path + '/files?path=references', headers=headers)
|
||||
check(
|
||||
row,
|
||||
'skill_files_listed',
|
||||
any(
|
||||
item.get('path') == 'references/fixture.txt'
|
||||
and item.get('size') == len(content.encode())
|
||||
and item.get('is_dir') is False
|
||||
for item in files.get('entries', [])
|
||||
),
|
||||
'Written Skill file is missing from the directory listing',
|
||||
)
|
||||
|
||||
updated_instructions = '# Updated API fixture\n\nKeep the local reference file.'
|
||||
updated = request(
|
||||
suite,
|
||||
row,
|
||||
'PUT',
|
||||
path,
|
||||
{'instructions': updated_instructions, 'base_revision': file_revision},
|
||||
headers=headers,
|
||||
)['skill']
|
||||
check(
|
||||
row,
|
||||
'skill_updated',
|
||||
updated.get('instructions') == updated_instructions and updated.get('revision') != file_revision,
|
||||
'Skill update did not publish changed instructions',
|
||||
)
|
||||
preview = request(suite, row, 'GET', path + '/preview', headers=headers)
|
||||
check(
|
||||
row, 'skill_updated_preview', preview.get('instructions') == updated_instructions, 'Skill preview is stale'
|
||||
)
|
||||
unchanged = request(
|
||||
suite,
|
||||
row,
|
||||
'PUT',
|
||||
path,
|
||||
{'instructions': updated_instructions, 'base_revision': updated['revision']},
|
||||
headers=headers,
|
||||
)['skill']
|
||||
check(
|
||||
row,
|
||||
'skill_update_idempotent',
|
||||
unchanged.get('revision') == updated['revision'],
|
||||
'No-op Skill update changed content revision',
|
||||
)
|
||||
before_conflict = request(suite, row, 'GET', path, headers=headers)['skill']
|
||||
|
||||
def verify_stale_update():
|
||||
after_conflict = request(suite, row, 'GET', path, headers=headers)['skill']
|
||||
check(
|
||||
row,
|
||||
'skill_stale_update_preserved',
|
||||
after_conflict == before_conflict,
|
||||
'Rejected Skill update changed the published state',
|
||||
)
|
||||
|
||||
_expect_rejection(
|
||||
suite,
|
||||
row,
|
||||
'PUT',
|
||||
path,
|
||||
{'instructions': 'stale metadata must not win', 'base_revision': initial_revision},
|
||||
expected=409,
|
||||
headers=headers,
|
||||
verify=verify_stale_update,
|
||||
)
|
||||
before_file_conflict = request(suite, row, 'GET', file_path, headers=headers)
|
||||
before_file_skill = request(suite, row, 'GET', path, headers=headers)['skill']
|
||||
|
||||
def verify_stale_file():
|
||||
after_file_conflict = request(suite, row, 'GET', file_path, headers=headers)
|
||||
after_conflict = request(suite, row, 'GET', path, headers=headers)['skill']
|
||||
check(
|
||||
row,
|
||||
'skill_stale_file_preserved',
|
||||
after_file_conflict == before_file_conflict and after_conflict == before_file_skill,
|
||||
'Rejected Skill file write changed the file or published revision',
|
||||
)
|
||||
|
||||
# The file route currently maps revision conflicts to HTTP 400.
|
||||
_expect_rejection(
|
||||
suite,
|
||||
row,
|
||||
'PUT',
|
||||
file_path,
|
||||
{'content': 'stale file must not win', 'base_revision': file_revision},
|
||||
expected=400,
|
||||
headers=headers,
|
||||
verify=verify_stale_file,
|
||||
)
|
||||
request(suite, row, 'DELETE', path, headers=headers)
|
||||
request(suite, row, 'GET', path, expected=404, headers=headers)
|
||||
request(suite, row, 'DELETE', path, expected=400, headers=headers)
|
||||
request(suite, row, 'GET', path, expected=404, headers=headers)
|
||||
check(
|
||||
row,
|
||||
'skill_deleted',
|
||||
not any(
|
||||
item.get('name') == name
|
||||
for item in request(suite, row, 'GET', '/api/v1/skills', headers=headers)['skills']
|
||||
),
|
||||
'Deleted Skill remains listed',
|
||||
)
|
||||
return {'skill_name': name, 'revision': before_conflict['revision']}
|
||||
|
||||
|
||||
def _mcp_config(server):
|
||||
return {
|
||||
key: server.get(key) for key in ('uuid', 'workspace_uuid', 'name', 'enable', 'mode', 'extra_args', 'readme')
|
||||
}
|
||||
|
||||
|
||||
def mcp_case(suite, row):
|
||||
headers = user_headers(suite)
|
||||
name = unique_name()
|
||||
collection = '/api/v1/mcp/servers'
|
||||
body = {
|
||||
'name': name,
|
||||
'enable': False,
|
||||
'mode': 'remote',
|
||||
'extra_args': {'url': 'http://127.0.0.1:9/mcp'},
|
||||
'readme': 'Owned disabled MCP fixture ' + name,
|
||||
}
|
||||
with _named_resource(suite, row, collection, body, 'server', 'readme', headers) as (path, created):
|
||||
server = request(suite, row, 'GET', path, headers=headers)['server']
|
||||
check(
|
||||
row,
|
||||
'mcp_created_disabled',
|
||||
bool(created.get('uuid'))
|
||||
and server.get('uuid') == created['uuid']
|
||||
and server.get('workspace_uuid') == suite.variables['workspace_uuid']
|
||||
and all(server.get(key) == value for key, value in body.items()),
|
||||
'MCP create/readback changed identity, disabled state or configuration',
|
||||
)
|
||||
servers = request(suite, row, 'GET', collection, headers=headers)['servers']
|
||||
check(
|
||||
row,
|
||||
'mcp_listed',
|
||||
sum(item.get('name') == name and item.get('uuid') == created['uuid'] for item in servers) == 1,
|
||||
'MCP listing did not return the created server exactly once',
|
||||
)
|
||||
update = {'enable': False, 'extra_args': {'url': 'http://127.0.0.1:9/updated-mcp'}}
|
||||
request(suite, row, 'PUT', path, update, headers=headers)
|
||||
updated = request(suite, row, 'GET', path, headers=headers)['server']
|
||||
check(
|
||||
row,
|
||||
'mcp_updated_disabled',
|
||||
_mcp_config(updated) == {**_mcp_config(server), **update},
|
||||
'MCP update changed unrelated configuration or enabled the server',
|
||||
)
|
||||
request(suite, row, 'PUT', path, update, headers=headers)
|
||||
repeated = request(suite, row, 'GET', path, headers=headers)['server']
|
||||
check(
|
||||
row,
|
||||
'mcp_update_idempotent',
|
||||
_mcp_config(repeated) == _mcp_config(updated),
|
||||
'Repeated MCP update changed configuration',
|
||||
)
|
||||
|
||||
def verify_duplicate():
|
||||
after_duplicate = request(suite, row, 'GET', path, headers=headers)['server']
|
||||
servers = request(suite, row, 'GET', collection, headers=headers)['servers']
|
||||
check(
|
||||
row,
|
||||
'mcp_duplicate_preserved',
|
||||
_mcp_config(after_duplicate) == _mcp_config(updated)
|
||||
and sum(item.get('name') == name for item in servers) == 1,
|
||||
'Duplicate MCP create changed the original or added another server',
|
||||
)
|
||||
|
||||
_expect_rejection(suite, row, 'POST', collection, body, expected=400, headers=headers, verify=verify_duplicate)
|
||||
|
||||
def verify_invalid_transport():
|
||||
after_invalid = request(suite, row, 'GET', path, headers=headers)['server']
|
||||
row['mcp_transport'] = {'before': updated['mode'], 'after': after_invalid['mode']}
|
||||
check(
|
||||
row,
|
||||
'mcp_invalid_transport_preserved',
|
||||
_mcp_config(after_invalid) == _mcp_config(updated),
|
||||
'Invalid MCP transport changed stored configuration',
|
||||
)
|
||||
|
||||
_expect_rejection(
|
||||
suite,
|
||||
row,
|
||||
'PUT',
|
||||
path,
|
||||
{'mode': 'qa-invalid-transport', 'enable': False},
|
||||
expected=400,
|
||||
headers=headers,
|
||||
verify=verify_invalid_transport,
|
||||
)
|
||||
request(suite, row, 'DELETE', path, headers=headers)
|
||||
request(suite, row, 'GET', path, expected=404, headers=headers)
|
||||
request(suite, row, 'DELETE', path, expected=404, headers=headers)
|
||||
servers = request(suite, row, 'GET', collection, headers=headers)['servers']
|
||||
check(
|
||||
row,
|
||||
'mcp_deleted',
|
||||
not any(item.get('name') == name for item in servers),
|
||||
'Deleted MCP server remains listed',
|
||||
)
|
||||
return {'mcp_name': name, 'uuid': created['uuid']}
|
||||
+19
-7
@@ -16,17 +16,17 @@ import yaml
|
||||
from regression_gate import supervise
|
||||
from regression_coverage import write_report
|
||||
from regression_plugins import PLUGIN_IDS, install_plugins, prepare_published_plugins, prepare_source_plugins
|
||||
from regression_local_cases import GROUPS, api, require
|
||||
from regression_local_cases import DEFAULT_GROUPS, api, require
|
||||
from regression_suite import Suite
|
||||
from regression_traffic import owned_box_containers, read_endpoint_snapshot
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
SUITE_BUDGETS = {'smoke': 60, 'scenarios': 300, 'daily': 1860, 'endurance': 900}
|
||||
SUITE_BUDGETS = {'smoke': 60, 'scenarios': 300, 'daily': 1860, 'endurance': 900, 'api': 600}
|
||||
sys.path.insert(0, str(ROOT))
|
||||
from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402
|
||||
|
||||
|
||||
def suite_profile(name, run_id, groups=()):
|
||||
def suite_profile(name, run_id, groups=(), api_seed=1):
|
||||
profile_path = ROOT / 'tools/regression_profiles' / ('endurance.json' if name == 'endurance' else 'smoke.json')
|
||||
if name in ('scenarios', 'daily'):
|
||||
from regression_scenarios import expanded_profile
|
||||
@@ -34,6 +34,12 @@ def suite_profile(name, run_id, groups=()):
|
||||
profile = expanded_profile(groups)
|
||||
else:
|
||||
profile = json.loads(profile_path.read_text())
|
||||
if name == 'api':
|
||||
from regression_api_cases import add_cases
|
||||
|
||||
profile.update(resources=[], cases=[], coverage_suite='api')
|
||||
profile['vars']['api_seed'] = api_seed
|
||||
add_cases(profile)
|
||||
for resource in profile['resources']:
|
||||
if resource['kind'] == 'pipeline':
|
||||
resource['body'].setdefault(
|
||||
@@ -172,10 +178,12 @@ def run(args):
|
||||
config_path.write_text(yaml.safe_dump(config))
|
||||
groups = args.scenario_groups
|
||||
if groups is None:
|
||||
groups = list(GROUPS) if args.suite in ('scenarios', 'daily') else []
|
||||
groups = list(DEFAULT_GROUPS) if args.suite in ('scenarios', 'daily') else []
|
||||
if args.suite == 'api':
|
||||
groups = ['api']
|
||||
if 'plugins' in groups and not {'misc/EssentialCommands', 'misc/FAQManager'} <= set(args.extra_plugins):
|
||||
groups.remove('plugins')
|
||||
profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups)
|
||||
profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups, args.api_seed)
|
||||
cli_config = private / 'lbctl.yaml'
|
||||
cli_config.write_text(
|
||||
yaml.safe_dump(
|
||||
@@ -270,7 +278,7 @@ def run(args):
|
||||
cli_data = yaml.safe_load(cli_config.read_text())
|
||||
cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid']
|
||||
cli_config.write_text(yaml.safe_dump(cli_data))
|
||||
if 'resources' in groups or args.restart_services:
|
||||
if {'resources', 'api'} & set(groups) or args.restart_services:
|
||||
# Account creation is restricted to this runner's newly created private database.
|
||||
credentials = {'user': '[email protected]', 'password': secrets.token_urlsafe(32)}
|
||||
suite.secrets.append(credentials['password'])
|
||||
@@ -324,6 +332,9 @@ def run(args):
|
||||
coverage = write_report(evidence, root / 'gate/suite', http_logs=[root / 'core.log'])
|
||||
result['coverage_errors'] = coverage['evidence_errors']
|
||||
result['scenario_groups'] = groups
|
||||
if args.suite == 'api':
|
||||
result['api_seed'] = args.api_seed
|
||||
result['case_order'] = [case['name'] for case in profile['cases']]
|
||||
result['passed'] = result['passed'] and not result['coverage_errors']
|
||||
(root / 'gate/gate.json').write_text(json.dumps(result, indent=2) + '\n')
|
||||
return result
|
||||
@@ -357,9 +368,10 @@ def main():
|
||||
parser.add_argument(
|
||||
'--scenario-groups',
|
||||
nargs='*',
|
||||
choices=tuple(GROUPS),
|
||||
choices=DEFAULT_GROUPS,
|
||||
help='Local groups for scenarios/daily (default: all); an empty list selects only the base cases',
|
||||
)
|
||||
parser.add_argument('--api-seed', type=int, default=1, help='Replay the API suite chain shuffle (default: 1)')
|
||||
parser.add_argument('--restart-services', action='store_true', help='Check owned Core/Runtime restarts once')
|
||||
args = parser.parse_args()
|
||||
if args.extra_plugins and args.plugins is None:
|
||||
|
||||
@@ -378,6 +378,62 @@
|
||||
"area": "retrieval", "kind": "blackbox", "priority": "P1",
|
||||
"description": "An empty knowledge base allows chat without injecting other knowledge fixtures",
|
||||
"bindings": [{"profile": "scenarios", "case": "empty_rag_chat", "checks": ["response"]}]
|
||||
},
|
||||
{"id": "api.model_llm",
|
||||
"area": "api",
|
||||
"kind": "blackbox",
|
||||
"priority": "P1",
|
||||
"description": "llm model and provider CRUD preserve bindings, secrets and children on rejected deletion",
|
||||
"bindings": [{"profile": "api", "case": "fixed_api_llm", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}, {"profile": "api", "case": "shuffled_api_llm", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}]
|
||||
},
|
||||
{"id": "api.model_embedding",
|
||||
"area": "api",
|
||||
"kind": "blackbox",
|
||||
"priority": "P1",
|
||||
"description": "embedding model and provider CRUD preserve bindings, secrets and children on rejected deletion",
|
||||
"bindings": [{"profile": "api", "case": "fixed_api_embedding", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}, {"profile": "api", "case": "shuffled_api_embedding", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}]
|
||||
},
|
||||
{"id": "api.model_rerank",
|
||||
"area": "api",
|
||||
"kind": "blackbox",
|
||||
"priority": "P1",
|
||||
"description": "rerank model and provider CRUD preserve bindings, secrets and children on rejected deletion",
|
||||
"bindings": [{"profile": "api", "case": "fixed_api_rerank", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}, {"profile": "api", "case": "shuffled_api_rerank", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}]
|
||||
},
|
||||
{"id": "api.pipeline",
|
||||
"area": "api",
|
||||
"kind": "blackbox",
|
||||
"priority": "P1",
|
||||
"description": "Pipeline CRUD and extension updates preserve state after invalid bindings and run inputs",
|
||||
"bindings": [{"profile": "api", "case": "fixed_api_pipeline", "checks": ["api_chain", "pipeline_identity", "pipeline_list", "pipeline_update", "extension_readback", "rejected_extension_enable_all_plugins", "rejected_extension_bound_skills", "rejected_extension_bound_plugins", "invalid_run_preserves_config"]}, {"profile": "api", "case": "shuffled_api_pipeline", "checks": ["api_chain", "pipeline_identity", "pipeline_list", "pipeline_update", "extension_readback", "rejected_extension_enable_all_plugins", "rejected_extension_bound_skills", "rejected_extension_bound_plugins", "invalid_run_preserves_config"]}]
|
||||
},
|
||||
{"id": "api.credentials",
|
||||
"area": "api",
|
||||
"kind": "blackbox",
|
||||
"priority": "P1",
|
||||
"description": "Read-only and revoked keys cannot write or read protected resources; secrets are returned only once",
|
||||
"bindings": [{"profile": "api", "case": "fixed_api_credentials", "checks": ["api_chain", "invalid_key_not_created", "key_metadata_redacted", "readonly_denial_preserves_state", "revoked_readback"]}, {"profile": "api", "case": "shuffled_api_credentials", "checks": ["api_chain", "invalid_key_not_created", "key_metadata_redacted", "readonly_denial_preserves_state", "revoked_readback"]}]
|
||||
},
|
||||
{"id": "api.skills",
|
||||
"area": "api",
|
||||
"kind": "blackbox",
|
||||
"priority": "P1",
|
||||
"description": "Skill publication and UTF-8 files retain revision and content after stale writes",
|
||||
"bindings": [{"profile": "api", "case": "fixed_api_skills", "checks": ["api_chain", "skill_created", "skill_file_published", "skill_updated", "skill_update_idempotent", "skill_stale_update_preserved", "skill_stale_file_preserved", "skill_deleted"]}, {"profile": "api", "case": "shuffled_api_skills", "checks": ["api_chain", "skill_created", "skill_file_published", "skill_updated", "skill_update_idempotent", "skill_stale_update_preserved", "skill_stale_file_preserved", "skill_deleted"]}]
|
||||
},
|
||||
{"id": "api.mcp_crud",
|
||||
"area": "api",
|
||||
"kind": "blackbox",
|
||||
"priority": "P1",
|
||||
"description": "Disabled MCP configurations retain identity and state across CRUD and duplicate requests",
|
||||
"bindings": [{"profile": "api", "case": "fixed_api_mcp", "checks": ["mcp_created_disabled", "mcp_listed", "mcp_updated_disabled", "mcp_update_idempotent", "mcp_duplicate_preserved", "cleanup_server"]}, {"profile": "api", "case": "shuffled_api_mcp", "checks": ["mcp_created_disabled", "mcp_listed", "mcp_updated_disabled", "mcp_update_idempotent", "mcp_duplicate_preserved", "cleanup_server"]}]
|
||||
},
|
||||
{"id": "api.mcp_validation",
|
||||
"area": "api",
|
||||
"kind": "blackbox",
|
||||
"priority": "P1",
|
||||
"description": "Invalid MCP transport is rejected without changing stored configuration",
|
||||
"bindings": [{"profile": "api", "case": "fixed_api_mcp", "checks": ["api_chain", "mcp_invalid_transport_preserved"]}, {"profile": "api", "case": "shuffled_api_mcp", "checks": ["api_chain", "mcp_invalid_transport_preserved"]}]
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -14,9 +14,13 @@ GROUPS = {
|
||||
'resources': 'regression_resource_cases',
|
||||
'lifecycle': 'regression_lifecycle_cases',
|
||||
'plugins': 'regression_plugin_cases',
|
||||
'api': 'regression_api_cases',
|
||||
}
|
||||
|
||||
|
||||
DEFAULT_GROUPS = ('faults', 'resources', 'lifecycle', 'plugins')
|
||||
|
||||
|
||||
class ScenarioUnavailable(RuntimeError):
|
||||
"""An explicit prerequisite is missing; this is not a passed assertion."""
|
||||
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Challenge API job oracles and replay/cleanup boundaries without a running Core."""
|
||||
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from regression_api_cases import run_case
|
||||
from regression_api_checks import owned_resource, request
|
||||
from regression_api_resources import _expect_rejection
|
||||
from regression_ci import suite_profile
|
||||
from regression_local_cases import DEFAULT_GROUPS
|
||||
from regression_scenarios import expanded_profile
|
||||
from regression_suite import Suite, validate
|
||||
|
||||
|
||||
class ApiContractTests(unittest.TestCase):
|
||||
def test_replay_and_job_isolation(self):
|
||||
first, _ = suite_profile('api', 'run-a', api_seed=20261028)
|
||||
replay, _ = suite_profile('api', 'run-b', api_seed=20261028)
|
||||
other, _ = suite_profile('api', 'run-c', api_seed=2)
|
||||
validate(first)
|
||||
self.assertEqual(first['cases'], replay['cases'])
|
||||
self.assertNotEqual(first['cases'], other['cases'])
|
||||
half = len(first['cases']) // 2
|
||||
fixed, shuffled = first['cases'][:half], first['cases'][half:]
|
||||
self.assertEqual(Counter(c['scenario'] for c in fixed), Counter(c['scenario'] for c in shuffled))
|
||||
self.assertEqual(set(Counter(c['scenario'] for c in fixed).values()), {1})
|
||||
self.assertEqual(first['resources'], [])
|
||||
self.assertNotIn('api', DEFAULT_GROUPS)
|
||||
daily = expanded_profile(DEFAULT_GROUPS)
|
||||
self.assertFalse(any(c.get('scenario', '').startswith('api_') for c in daily['cases']))
|
||||
with self.assertRaisesRegex(ValueError, 'dedicated suite'):
|
||||
expanded_profile(['api'])
|
||||
|
||||
def test_http_success_cannot_hide_business_failure_or_wrong_status(self):
|
||||
for actual in ((200, {'code': -1}), (201, {'code': 0}), (400, {'code': 0}), (400, {})):
|
||||
row = {}
|
||||
with self.subTest(actual=actual), patch('regression_api_checks.api', return_value=actual):
|
||||
with self.assertRaises(ValueError):
|
||||
request(None, row, 'GET', '/api/test', expected=400 if actual[0] == 400 else 200)
|
||||
self.assertEqual(row['assertions']['http_1'], 'failed')
|
||||
self.assertEqual(row['http_steps'][0]['status'], actual[0])
|
||||
|
||||
def test_late_failure_cannot_mark_the_api_chain_passed(self):
|
||||
from types import SimpleNamespace
|
||||
|
||||
suite = SimpleNamespace(profile={'vars': {'api_seed': 1}})
|
||||
row = {}
|
||||
with patch('regression_api_cases.credentials_case', side_effect=ValueError('Revoked key still works')):
|
||||
with self.assertRaisesRegex(ValueError, 'Revoked key'):
|
||||
run_case(suite, {'scenario': 'api_credentials', 'expect': 'OK'}, row)
|
||||
self.assertEqual(row['assertions']['api_chain'], 'failed')
|
||||
|
||||
def test_rejection_keeps_http_failure_when_state_check_also_fails(self):
|
||||
row = {}
|
||||
|
||||
def verify():
|
||||
row['state_checked'] = True
|
||||
raise ValueError('Stored state changed')
|
||||
|
||||
with patch('regression_api_resources.request', side_effect=ValueError('Expected 400, got 200')):
|
||||
with self.assertRaisesRegex(ValueError, 'Expected 400, got 200'):
|
||||
_expect_rejection(None, row, 'PUT', '/api/test', {}, expected=400, headers={}, verify=verify)
|
||||
self.assertTrue(row['state_checked'])
|
||||
|
||||
def test_lost_create_response_cleans_only_exact_owned_resource(self):
|
||||
for changed in (False, True):
|
||||
with self.subTest(changed=changed), tempfile.TemporaryDirectory() as directory:
|
||||
suite = Suite(
|
||||
{'version': 1, 'cli': {'config': 'unused'}, 'cases': [], 'vars': {'workspace_uuid': 'test'}},
|
||||
Path(directory) / 'profile.json',
|
||||
Path(directory),
|
||||
)
|
||||
row, calls = {}, []
|
||||
|
||||
def api(suite, method, path, body=None, headers=None):
|
||||
calls.append((method, path))
|
||||
if method == 'POST':
|
||||
raise TimeoutError('Lost response after create')
|
||||
if path == '/collection':
|
||||
return 200, {
|
||||
'code': 0,
|
||||
'data': {
|
||||
'items': [{'name': 'qa-owned', 'uuid': 'owned'}, {'name': 'unrelated', 'uuid': 'other'}]
|
||||
},
|
||||
}
|
||||
if calls[-2][0] == 'DELETE':
|
||||
return 404, {'code': -1}
|
||||
if method == 'GET':
|
||||
return 200, {
|
||||
'code': 0,
|
||||
'data': {'item': {'name': 'unrelated' if changed else 'qa-owned', 'uuid': 'owned'}},
|
||||
}
|
||||
return 200, {'code': 0}
|
||||
|
||||
with patch('regression_api_checks.api', side_effect=api):
|
||||
with self.assertRaisesRegex(TimeoutError, 'Lost response'):
|
||||
with owned_resource(
|
||||
suite, row, '/collection', {'name': 'qa-owned'}, item_key='item', list_key='items'
|
||||
):
|
||||
self.fail('Create timeout must not enter the contract body')
|
||||
deleted = [path for method, path in calls if method == 'DELETE']
|
||||
self.assertEqual(deleted, [] if changed else ['/collection/owned'])
|
||||
if changed:
|
||||
self.assertTrue(row['cleanup_errors'])
|
||||
else:
|
||||
self.assertEqual(row['owned_resources'][0]['state'], 'deleted')
|
||||
|
||||
def test_invalid_key_side_effects_fail_and_revoke_only_the_owned_key(self):
|
||||
for status, invalid_field in ((200, 'scopes'), (400, 'scopes'), (200, 'expires_at'), (400, 'expires_at')):
|
||||
with self.subTest(status=status, field=invalid_field), tempfile.TemporaryDirectory() as directory:
|
||||
suite = Suite(
|
||||
{
|
||||
'version': 1,
|
||||
'cli': {'config': 'unused'},
|
||||
'cases': [],
|
||||
'vars': {'workspace_uuid': 'test', 'api_seed': 1},
|
||||
},
|
||||
Path(directory) / 'profile.json',
|
||||
Path(directory),
|
||||
env={'LANGBOT_QA_USER_TOKEN': 'manager-secret'},
|
||||
)
|
||||
other = {'id': 1, 'name': 'qa-api-other-key', 'status': 'active', 'secret_available': False}
|
||||
keys, row, deleted = {1: dict(other)}, {}, []
|
||||
|
||||
def api(suite, method, path, body=None, headers=None):
|
||||
if path == '/api/v1/pipelines':
|
||||
return 401, {'code': -1}
|
||||
if path == '/api/v1/apikeys':
|
||||
if method == 'GET':
|
||||
return 200, {'code': 0, 'data': {'keys': [dict(key) for key in keys.values()]}}
|
||||
self.assertEqual(method, 'POST')
|
||||
if invalid_field not in body:
|
||||
return 400, {'code': -1}
|
||||
keys[7] = {'id': 7, 'name': body['name'], 'status': 'active', 'secret_available': False}
|
||||
return status, {'code': 0 if status == 200 else -1, 'data': {'key': dict(keys[7])}}
|
||||
self.assertEqual(path, '/api/v1/apikeys/7')
|
||||
if method == 'DELETE':
|
||||
deleted.append(path)
|
||||
keys[7]['status'] = 'revoked'
|
||||
return 200, {'code': 0}
|
||||
self.assertEqual(method, 'GET')
|
||||
return 200, {'code': 0, 'data': {'key': dict(keys[7])}}
|
||||
|
||||
error = 'expected HTTP 400' if status == 200 else 'Invalid key request changed key list'
|
||||
with (
|
||||
patch('regression_api_checks.api', side_effect=api),
|
||||
patch('regression_api_cases.unique_name', return_value='qa-api-owned-key'),
|
||||
self.assertRaisesRegex(ValueError, error),
|
||||
):
|
||||
run_case(suite, {'scenario': 'api_credentials', 'expect': 'OK'}, row)
|
||||
self.assertEqual(row['assertions']['api_chain'], 'failed')
|
||||
self.assertEqual(row['assertions']['key_cleanup'], 'passed')
|
||||
self.assertFalse(row.get('cleanup_errors'))
|
||||
self.assertEqual(deleted, ['/api/v1/apikeys/7'])
|
||||
self.assertEqual(keys[7]['status'], 'revoked')
|
||||
self.assertEqual(keys[1], other)
|
||||
self.assertEqual(set(keys), {1, 7})
|
||||
self.assertEqual(row['owned_resources'][0]['state'], 'revoked')
|
||||
|
||||
def test_readonly_pipeline_side_effects_fail_and_cleanup_preserves_other_resources(self):
|
||||
for status in (200, 403):
|
||||
with self.subTest(status=status), tempfile.TemporaryDirectory() as directory:
|
||||
suite = Suite(
|
||||
{
|
||||
'version': 1,
|
||||
'cli': {'config': 'unused'},
|
||||
'cases': [],
|
||||
'vars': {'workspace_uuid': 'test', 'api_seed': 1},
|
||||
},
|
||||
Path(directory) / 'profile.json',
|
||||
Path(directory),
|
||||
env={'LANGBOT_QA_USER_TOKEN': 'manager-secret'},
|
||||
)
|
||||
other_key = {'id': 1, 'name': 'qa-api-other-key', 'status': 'active', 'secret_available': False}
|
||||
other_pipeline = {'uuid': 'other-pipeline', 'name': 'qa-api-other-pipeline'}
|
||||
keys, pipelines = {1: dict(other_key)}, {'other-pipeline': dict(other_pipeline)}
|
||||
row, deleted = {}, []
|
||||
|
||||
def api(suite, method, path, body=None, headers=None):
|
||||
if path == '/api/v1/apikeys':
|
||||
if method == 'GET':
|
||||
return 200, {'code': 0, 'data': {'keys': [dict(key) for key in keys.values()]}}
|
||||
self.assertEqual(method, 'POST')
|
||||
if body.get('scopes') == ['qa.invalid'] or 'expires_at' in body:
|
||||
return 400, {'code': -1}
|
||||
keys[7] = {'id': 7, 'name': body['name'], 'status': 'active', 'secret_available': False}
|
||||
return 200, {'code': 0, 'data': {'key': {**keys[7], 'key': 'child-secret'}}}
|
||||
if path == '/api/v1/apikeys/7':
|
||||
if method == 'PUT':
|
||||
keys[7]['name'] = body['name']
|
||||
return 200, {'code': 0}
|
||||
if method == 'DELETE':
|
||||
deleted.append(path)
|
||||
keys[7]['status'] = 'revoked'
|
||||
return 200, {'code': 0}
|
||||
self.assertEqual(method, 'GET')
|
||||
return 200, {'code': 0, 'data': {'key': dict(keys[7])}}
|
||||
if path == '/api/v1/pipelines':
|
||||
if headers in ({}, {'Authorization': 'Bearer qa-invalid'}):
|
||||
return 401, {'code': -1}
|
||||
if method == 'GET':
|
||||
return 200, {'code': 0, 'data': {'pipelines': [dict(item) for item in pipelines.values()]}}
|
||||
self.assertEqual(method, 'POST')
|
||||
self.assertEqual(headers, {'X-API-Key': 'child-secret'})
|
||||
pipelines['owned-pipeline'] = {'uuid': 'owned-pipeline', 'name': body['name']}
|
||||
return status, {'code': 0 if status == 200 else -1, 'data': {'uuid': 'owned-pipeline'}}
|
||||
self.assertEqual(path, '/api/v1/pipelines/owned-pipeline')
|
||||
if method == 'DELETE':
|
||||
deleted.append(path)
|
||||
del pipelines['owned-pipeline']
|
||||
return 200, {'code': 0}
|
||||
self.assertEqual(method, 'GET')
|
||||
if 'owned-pipeline' not in pipelines:
|
||||
return 404, {'code': -1}
|
||||
return 200, {'code': 0, 'data': {'pipeline': dict(pipelines['owned-pipeline'])}}
|
||||
|
||||
error = 'expected HTTP 403' if status == 200 else 'Read-only key changed resources'
|
||||
with (
|
||||
patch('regression_api_checks.api', side_effect=api),
|
||||
patch('regression_api_cases.unique_name', return_value='qa-api-owned-key'),
|
||||
self.assertRaisesRegex(ValueError, error),
|
||||
):
|
||||
run_case(suite, {'scenario': 'api_credentials', 'expect': 'OK'}, row)
|
||||
self.assertEqual(row['assertions']['api_chain'], 'failed')
|
||||
self.assertEqual(row['assertions']['key_cleanup'], 'passed')
|
||||
self.assertEqual(row['assertions']['cleanup_pipeline_qa-api-owned-key'], 'passed')
|
||||
self.assertFalse(row.get('cleanup_errors'))
|
||||
self.assertEqual(deleted, ['/api/v1/pipelines/owned-pipeline', '/api/v1/apikeys/7'])
|
||||
self.assertEqual(keys[7]['status'], 'revoked')
|
||||
self.assertEqual(keys[1], other_key)
|
||||
self.assertEqual(set(keys), {1, 7})
|
||||
self.assertEqual(pipelines, {'other-pipeline': other_pipeline})
|
||||
self.assertEqual(
|
||||
{item['kind']: item['state'] for item in row['owned_resources']},
|
||||
{'api-key': 'revoked', 'pipeline': 'deleted'},
|
||||
)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -143,6 +143,53 @@ class DailySummaryTest(unittest.TestCase):
|
||||
else:
|
||||
self.assertNotIn('Daily regression: PASS', text)
|
||||
|
||||
def test_api_summary_keeps_preparation_and_outer_failures_visible(self):
|
||||
workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml'
|
||||
steps = yaml.safe_load(workflow.read_text())['jobs']['api-contracts']['steps']
|
||||
command = next(step['run'] for step in steps if step.get('name') == 'Publish API result')
|
||||
suite = {'passed': True, 'requests': 14, 'status': {'passed': 14}, 'elapsed_seconds': 7.3}
|
||||
for gate, actual_suite, expected in [
|
||||
({'passed': True, 'api_seed': 123}, suite, 'Daily API contracts: PASS'),
|
||||
(
|
||||
{'passed': False, 'infrastructure_error': 'plugin installation failed'},
|
||||
None,
|
||||
'plugin installation failed',
|
||||
),
|
||||
({'passed': False, 'timed_out': True}, suite, '600-second outer process guard'),
|
||||
({'passed': False, 'resource_errors': ['sampling failed']}, suite, 'sampling failed'),
|
||||
({'passed': False, 'coverage_errors': ['missing evidence']}, suite, 'missing evidence'),
|
||||
({'passed': False}, {'passed': False, 'errors': ['MCP mode persisted']}, 'MCP mode persisted'),
|
||||
(None, None, 'Missing verdict'),
|
||||
]:
|
||||
with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory:
|
||||
root = Path(directory)
|
||||
target = root / '.qa/api/gate/suite'
|
||||
target.mkdir(parents=True)
|
||||
if gate is not None:
|
||||
(target.parent / 'gate.json').write_text(json.dumps(gate))
|
||||
if actual_suite is not None:
|
||||
(target / 'summary.json').write_text(json.dumps(actual_suite))
|
||||
summary = root / 'summary.md'
|
||||
subprocess.run(
|
||||
['bash', '-e', '-c', command],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
env={
|
||||
**os.environ,
|
||||
'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'],
|
||||
'GITHUB_STEP_SUMMARY': str(summary),
|
||||
'GITHUB_SHA': 'tested-core',
|
||||
'GITHUB_RUN_ID': '456',
|
||||
},
|
||||
)
|
||||
text = summary.read_text()
|
||||
self.assertIn(expected, text)
|
||||
self.assertIn('not reported', text)
|
||||
self.assertIn('shuffle seed: `123`' if gate and gate.get('passed') else 'shuffle seed: `456`', text)
|
||||
if not gate or not gate.get('passed'):
|
||||
self.assertNotIn('Daily API contracts: PASS', text)
|
||||
|
||||
def test_core_summary_records_preparation_failure_and_selected_plugins(self):
|
||||
workflow = Path(__file__).resolve().parents[1] / '.github/workflows/core-smoke.yml'
|
||||
steps = yaml.safe_load(workflow.read_text())['jobs']['smoke']['steps']
|
||||
|
||||
@@ -289,12 +289,16 @@ class Routes:
|
||||
name = binding['profile']
|
||||
if name not in profiles:
|
||||
path = Path(__file__).with_name('regression_profiles') / f'{name}.json'
|
||||
if name == 'scenarios':
|
||||
from regression_scenarios import expanded_profile
|
||||
from regression_local_cases import GROUPS
|
||||
if name == 'api':
|
||||
from regression_ci import suite_profile
|
||||
|
||||
profile = expanded_profile(GROUPS)
|
||||
if 'lifecycle' in GROUPS:
|
||||
profile, _ = suite_profile('api', 'catalog-test')
|
||||
elif name == 'scenarios':
|
||||
from regression_scenarios import expanded_profile
|
||||
from regression_local_cases import DEFAULT_GROUPS
|
||||
|
||||
profile = expanded_profile(DEFAULT_GROUPS)
|
||||
if 'lifecycle' in DEFAULT_GROUPS:
|
||||
from regression_lifecycle_cases import restart_profile
|
||||
|
||||
profile['cases'] += restart_profile(profile)['cases']
|
||||
|
||||
@@ -7,7 +7,7 @@ import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from regression_local_cases import GROUPS, ScenarioUnavailable
|
||||
from regression_local_cases import DEFAULT_GROUPS, ScenarioUnavailable
|
||||
from regression_scenarios import expanded_profile
|
||||
from regression_smoke import run_smoke
|
||||
from regression_suite import Suite, validate
|
||||
@@ -17,7 +17,7 @@ from regression_traffic import reconcile_runs
|
||||
class LocalScenarioTests(unittest.TestCase):
|
||||
def test_group_switches_keep_base_cases_and_reject_unknown_handlers(self):
|
||||
base = expanded_profile()
|
||||
for group in GROUPS:
|
||||
for group in DEFAULT_GROUPS:
|
||||
profile = expanded_profile([group])
|
||||
validate(profile)
|
||||
self.assertEqual(profile['scenario_groups'], [group])
|
||||
|
||||
Reference in New Issue
Block a user