From b19315b078e89aca8b4785e7dd5c570c41307d22 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Wed, 28 Oct 2026 23:04:28 +0800 Subject: [PATCH 1/2] test: add isolated API contract regression job --- tools/README.md | 33 +++ tools/regression_api_cases.py | 213 ++++++++++++++++++ tools/regression_api_checks.py | 84 ++++++++ tools/regression_api_resources.py | 310 +++++++++++++++++++++++++++ tools/regression_ci.py | 26 ++- tools/regression_contracts.json | 56 +++++ tools/regression_local_cases.py | 4 + tools/test_regression_api.py | 243 +++++++++++++++++++++ tools/test_regression_coverage.py | 14 +- tools/test_regression_local_cases.py | 4 +- 10 files changed, 973 insertions(+), 14 deletions(-) create mode 100644 tools/regression_api_cases.py create mode 100644 tools/regression_api_checks.py create mode 100644 tools/regression_api_resources.py create mode 100644 tools/test_regression_api.py diff --git a/tools/README.md b/tools/README.md index cdb11e2f4..b5037aa1e 100644 --- a/tools/README.md +++ b/tools/README.md @@ -498,6 +498,39 @@ bind their own extensions to avoid changing unrelated prompts/tools. The suite a even after failures. The CI owner then stops its service processes and removes its Box containers; a process guard timeout can interrupt API cleanup. +## Local API contract job + +Run on the same disposable Linux/Docker host used by `regression_ci.py`: + +```sh +uv run --no-sync python tools/regression_ci.py --suite api --api-seed 20261028 \ + --lbctl /path/to/lbctl --plugins plugins --output .qa/api-001 +``` + +This separate suite executes seven independent operation chains once in fixed +order, then once in seed-shuffled order (14 case rows). Operations within each +chain retain their create/read/update/delete dependencies. Reuse the seed to +replay chain ordering; resource names and credentials remain unique. Preparation +is outside the 600-second execution guard; a completed run ends immediately. +The default PR smoke and daily scenario groups do not include these cases. + +The chains cover Provider and LLM/embedding/rerank model CRUD, Pipeline extension +validation, API key permissions/revocation, Skill files and revision conflicts, +and disabled MCP configuration. Checks require matching HTTP/business status, +read back stored state, and reject unexpected mutation after invalid requests. +Resources carry unique ownership markers and are deleted in `finally`; API keys +are revoked. Cleanup errors remain failures. MCP fixtures stay disabled and use +loopback addresses. This job does not exercise external MCP servers, real model +inference, concurrent API mutations, or cross-Workspace isolation. + +Inspect `gate/gate.json` for the verdict, seed and exact case order; +`gate/suite/cases.json` for HTTP steps, assertions and cleanup results; and +`gate/suite/coverage.json` for observed route templates and behavior contracts. +An HTTP route hit includes rejected requests and is not proof of all behavior +behind that route. The job intentionally fails on invalid MCP modes accepted +and persisted by Core; fixing that product validation should make the existing +negative case pass without changing its expected result. + ## Sustained regression and baseline comparison [`regression_profiles/endurance.json`](regression_profiles/endurance.json) runs diff --git a/tools/regression_api_cases.py b/tools/regression_api_cases.py new file mode 100644 index 000000000..cf00fd396 --- /dev/null +++ b/tools/regression_api_cases.py @@ -0,0 +1,213 @@ +"""Finite API contracts: fixed first pass, then reproducibly shuffled independent chains.""" + +import json +import random + +from regression_api_checks import check, owned_resource, request, tracked_resource, unique_name, user_headers +from regression_coverage import checked +from regression_local_cases import require +from regression_resource_cases import cleanup, journal + +CASE_NAMES = ('api_llm', 'api_embedding', 'api_rerank', 'api_pipeline', 'api_credentials', 'api_skills', 'api_mcp') + + +def add_cases(profile): + require(profile.get('coverage_suite') == 'api', 'API contracts require their dedicated suite') + shuffled = list(CASE_NAMES) + seed = profile['vars']['api_seed'] + random.Random(seed).shuffle(shuffled) + for phase, names in (('fixed', CASE_NAMES), ('shuffled', shuffled)): + for name in names: + profile['cases'].append( + {'name': f'{phase}_{name}', 'scenario': name, 'stage': 'api', 'expect': 'API_CONTRACT_OK'} + ) + profile['scenario_groups'] = ['api'] + + +def model_case(suite, row, model_type): + providers = '/api/v1/provider/providers' + name, secret = unique_name(), unique_name() + suite.secrets.append(secret) + body = { + 'name': name, + 'requester': 'jina-rerank' if model_type == 'rerank' else 'openai-chat-completions', + 'base_url': suite.variables['mock_base_url'], + 'api_keys': [secret], + } + with owned_resource(suite, row, providers, body, item_key='provider', list_key='providers') as (path, uid): + visible = request(suite, row, 'GET', path + '?include_secret=true')['provider'] + check( + row, + 'provider_identity', + visible['name'] == name and visible['api_keys'] == [secret], + 'Provider identity or secret changed', + ) + redacted = request(suite, row, 'GET', path + '?include_secret=false')['provider'] + check(row, 'provider_redacted', secret not in json.dumps(redacted), 'Provider secret was disclosed') + request(suite, row, 'GET', path + '?include_secret=invalid', expected=400) + request(suite, row, 'PUT', path, {'name': name + '-updated'}) + updated = request(suite, row, 'GET', path + '?include_secret=true')['provider'] + check( + row, + 'provider_update', + updated['name'] == name + '-updated' and updated['api_keys'] == [secret], + 'Provider metadata update lost its secret', + ) + collection = '/api/v1/provider/models/' + model_type + model_name = unique_name() + with owned_resource( + suite, row, collection, {'name': model_name, 'provider_uuid': uid}, item_key='model', list_key='models' + ) as (model_path, model_id): + model = request(suite, row, 'GET', model_path)['model'] + check( + row, + 'model_identity', + model['name'] == model_name and model['provider_uuid'] == uid, + 'Model did not retain its provider binding', + ) + items = request(suite, row, 'GET', collection + '?provider_uuid=' + uid)['models'] + check(row, 'model_list', [m['uuid'] for m in items] == [model_id], 'Provider model filter is incorrect') + request(suite, row, 'PUT', model_path, {'name': model_name + '-updated'}) + model = request(suite, row, 'GET', model_path)['model'] + check( + row, + 'model_update', + model['name'] == model_name + '-updated' and model['provider_uuid'] == uid, + 'Model metadata update lost its binding', + ) + request(suite, row, 'DELETE', path, expected=400) + still = request(suite, row, 'GET', model_path)['model'] + check(row, 'rejected_delete_preserves_model', still == model, 'Rejected provider deletion mutated child') + request(suite, row, 'GET', path) + items = request(suite, row, 'GET', collection + '?provider_uuid=' + uid)['models'] + check(row, 'model_deleted_from_list', not items, 'Deleted model remains listed') + + +def pipeline_case(suite, row): + collection, name = '/api/v1/pipelines', unique_name() + with owned_resource(suite, row, collection, {'name': name}, item_key='pipeline', list_key='pipelines') as ( + path, + uid, + ): + pipeline = request(suite, row, 'GET', path)['pipeline'] + check( + row, + 'pipeline_identity', + pipeline['name'] == name and pipeline['uuid'] == uid, + 'Pipeline creation readback differs', + ) + items = request(suite, row, 'GET', collection)['pipelines'] + check(row, 'pipeline_list', sum(p['uuid'] == uid for p in items) == 1, 'Pipeline missing or duplicated') + request(suite, row, 'PUT', path, {'name': name + '-updated', 'description': 'API contract 中文'}) + updated = request(suite, row, 'GET', path)['pipeline'] + check( + row, + 'pipeline_update', + updated['name'] == name + '-updated' and updated['description'] == 'API contract 中文', + 'Pipeline update readback differs', + ) + bindings = dict( + bound_plugins=[], + bound_mcp_servers=[], + bound_skills=[], + enable_all_plugins=False, + enable_all_mcp_servers=False, + enable_all_skills=False, + ) + request(suite, row, 'PUT', path + '/extensions', bindings) + before = request(suite, row, 'GET', path + '/extensions') + check(row, 'extension_readback', all(before[k] == v for k, v in bindings.items()), 'Extension bindings differ') + for invalid in ({'enable_all_plugins': 'false'}, {'bound_skills': [42]}, {'bound_plugins': [{}]}): + request(suite, row, 'PUT', path + '/extensions', invalid, expected=400) + after = request(suite, row, 'GET', path + '/extensions') + check(row, 'rejected_extension_' + next(iter(invalid)), after == before, 'Rejected binding changed state') + for invalid in ({}, {'message': ''}, {'message': 42}): + request(suite, row, 'POST', path + '/run', invalid, expected=400) + after = request(suite, row, 'GET', path)['pipeline'] + check(row, 'invalid_run_preserves_config', after['config'] == updated['config'], 'Invalid run changed config') + + +def credentials_case(suite, row): + manager, collection, name = user_headers(suite), '/api/v1/apikeys', unique_name() + for headers in ({}, {'Authorization': 'Bearer qa-invalid'}): + request(suite, row, 'GET', '/api/v1/pipelines', headers=headers, expected=401) + owned = {'kind': 'api-key', 'name': name, 'state': 'creating'} + row.setdefault('owned_resources', []).append(owned) + journal(suite, row) + + def revoke(): + if 'id' not in owned: + keys = request(suite, row, 'GET', collection, headers=manager)['keys'] + matches = [k for k in keys if k['name'] in (name, name + '-updated')] + require(len(matches) <= 1, 'Ambiguous owned key') + if not matches: + owned['state'] = 'absent' + return + owned['id'] = matches[0]['id'] + path = collection + '/' + str(owned['id']) + current = request(suite, row, 'GET', path, headers=manager)['key'] + require(current['name'] in (name, name + '-updated'), 'Key ownership changed') + if current['status'] != 'revoked': + request(suite, row, 'DELETE', path, headers=manager) + current = request(suite, row, 'GET', path, headers=manager)['key'] + check( + row, + 'revoked_readback', + current['status'] == 'revoked' and current['secret_available'] is False and 'key' not in current, + 'Revoked key remains active or exposes its secret', + ) + owned['state'] = 'revoked' + + try: + before = request(suite, row, 'GET', collection, headers=manager)['keys'] + for invalid in ({'scopes': ['qa.invalid']}, {'expires_at': 'not-a-date'}): + request(suite, row, 'POST', collection, {'name': name, **invalid}, headers=manager, expected=400) + after = request(suite, row, 'GET', collection, headers=manager)['keys'] + check(row, 'invalid_key_not_created', before == after, 'Invalid key request changed key list') + key = request(suite, row, 'POST', collection, {'name': name, 'scopes': ['resource.view']}, headers=manager)[ + 'key' + ] + owned.update(id=key['id'], state='active') + journal(suite, row) + secret = key['key'] + suite.secrets.append(secret) + child = {'X-API-Key': secret} + path = collection + '/' + str(key['id']) + request(suite, row, 'PUT', path, {'name': name + '-updated'}, headers=manager) + current = request(suite, row, 'GET', path, headers=manager)['key'] + check( + row, + 'key_metadata_redacted', + current['name'] == name + '-updated' and 'key' not in current, + 'Key metadata update or one-time-secret contract failed', + ) + with tracked_resource(suite, row, '/api/v1/pipelines', name, item_key='pipeline', list_key='pipelines'): + before = request(suite, row, 'GET', '/api/v1/pipelines', headers=child)['pipelines'] + request(suite, row, 'POST', '/api/v1/pipelines', {'name': name}, headers=child, expected=403) + after = request(suite, row, 'GET', '/api/v1/pipelines', headers=child)['pipelines'] + check(row, 'readonly_denial_preserves_state', before == after, 'Read-only key changed resources') + revoke() + request(suite, row, 'GET', '/api/v1/pipelines', headers=child, expected=401) + finally: + cleanup(suite, row, 'key_cleanup', revoke) + + +def run_case(suite, case, row): + from regression_api_resources import mcp_case, skills_case + + name = case['scenario'] + row['api_seed'] = suite.profile['vars']['api_seed'] + + def execute(): + if name in ('api_llm', 'api_embedding', 'api_rerank'): + model_case(suite, row, name.removeprefix('api_')) + else: + { + 'api_pipeline': pipeline_case, + 'api_credentials': credentials_case, + 'api_skills': skills_case, + 'api_mcp': mcp_case, + }[name](suite, row) + + checked(row, 'api_chain', execute) + return {'result': case['expect']} diff --git a/tools/regression_api_checks.py b/tools/regression_api_checks.py new file mode 100644 index 000000000..e249b48c3 --- /dev/null +++ b/tools/regression_api_checks.py @@ -0,0 +1,84 @@ +"""HTTP and ownership checks shared by the local API contract cases.""" + +from contextlib import contextmanager +from uuid import uuid4 + +from regression_coverage import checked +from regression_local_cases import api, require +from regression_resource_cases import cleanup, journal + + +def check(row, name, condition, message): + return checked(row, name, lambda: require(condition, message)) + + +def request(suite, row, method, path, body=None, *, expected=200, headers=None): + step = {'method': method, 'path': path, 'expected_status': expected} + row.setdefault('http_steps', []).append(step) + status, response = api(suite, method, path, body, headers=headers) + step['status'] = status + check( + row, + f'http_{len(row["http_steps"])}', + status == expected + and isinstance(response, dict) + and ((response.get('code') == 0) if expected < 400 else (response.get('code') not in (None, 0))), + f'{method} {path}: expected HTTP {expected} with matching business status, got {status}', + ) + return response.get('data', {}) if expected < 400 else response + + +def unique_name(): + return 'qa-api-' + uuid4().hex + + +def user_headers(suite): + token = suite.env.get('LANGBOT_QA_USER_TOKEN') + require(bool(token), 'API contracts require the disposable test account token') + if token not in suite.secrets: + suite.secrets.append(token) + return {'Authorization': 'Bearer ' + token} + + +@contextmanager +def tracked_resource(suite, row, collection, name, *, item_key, list_key): + """Clean an owned UUID resource even if its creation unexpectedly succeeds or times out.""" + owned = {'kind': item_key, 'name': name, 'state': 'creating'} + row.setdefault('owned_resources', []).append(owned) + journal(suite, row) + + def remove(): + if 'uuid' not in owned: + items = request(suite, row, 'GET', collection)[list_key] + matches = [item for item in items if item.get('name') in (name, name + '-updated')] + require(len(matches) <= 1, 'Ambiguous owned resource identity') + if not matches: + owned['state'] = 'absent' + return + owned['uuid'] = matches[0]['uuid'] + path = collection + '/' + owned['uuid'] + status, response = api(suite, 'GET', path) + if status != 404: + require(status == 200 and response.get('code') == 0, 'Cannot read resource before cleanup') + item = response['data'][item_key] + require( + item.get('uuid') == owned['uuid'] and item.get('name') in (name, name + '-updated'), + 'Resource ownership changed before cleanup', + ) + request(suite, row, 'DELETE', path) + request(suite, row, 'GET', path, expected=404) + owned['state'] = 'deleted' + + try: + yield owned + finally: + cleanup(suite, row, 'cleanup_' + item_key + '_' + name, remove) + + +@contextmanager +def owned_resource(suite, row, collection, body, *, item_key, list_key): + with tracked_resource(suite, row, collection, body['name'], item_key=item_key, list_key=list_key) as owned: + result = request(suite, row, 'POST', collection, body) + owned.update(uuid=result['uuid'], state='created') + journal(suite, row) + yield collection + '/' + owned['uuid'], owned['uuid'] diff --git a/tools/regression_api_resources.py b/tools/regression_api_resources.py new file mode 100644 index 000000000..2bd520204 --- /dev/null +++ b/tools/regression_api_resources.py @@ -0,0 +1,310 @@ +"""Local Skill publications and disabled MCP configuration HTTP contracts.""" + +from contextlib import contextmanager + +from regression_api_checks import check, request, unique_name, user_headers +from regression_local_cases import api, require +from regression_resource_cases import cleanup, journal + + +@contextmanager +def _named_resource(suite, row, collection, body, item_key, ownership_field, headers): + name = body['name'] + path = collection + '/' + name + request(suite, row, 'GET', path, expected=404, headers=headers) + owned = {'kind': item_key, 'name': name, 'state': 'creating'} + row.setdefault('owned_resources', []).append(owned) + journal(suite, row) + + def remove(): + status, response = api(suite, 'GET', path, headers=headers) + if status != 404: + require(status == 200 and response.get('code') == 0, 'Cannot read owned resource before cleanup') + item = response['data'][item_key] + require( + item.get('name') == name and item.get(ownership_field) == body[ownership_field], + 'Named resource ownership changed before cleanup', + ) + if 'uuid' in owned: + require(item.get('uuid') == owned['uuid'], 'Named resource identity changed before cleanup') + request(suite, row, 'DELETE', path, headers=headers) + request(suite, row, 'GET', path, expected=404, headers=headers) + owned['state'] = 'deleted' + + try: + created = request(suite, row, 'POST', collection, body, headers=headers) + owned['state'] = 'created' + if created.get('uuid'): + owned['uuid'] = created['uuid'] + journal(suite, row) + yield path, created + finally: + cleanup(suite, row, 'cleanup_' + item_key, remove) + + +def _expect_rejection(suite, row, method, path, body, *, expected, headers, verify): + try: + request(suite, row, method, path, body, expected=expected, headers=headers) + except Exception: + try: + verify() + except Exception: + # Both assertions are recorded; retain the original HTTP failure. + pass + raise + verify() + + +def skills_case(suite, row): + headers = user_headers(suite) + name = unique_name() + instructions = '# API fixture\n\nRead the local reference file.' + body = { + 'name': name, + 'display_name': 'API contract fixture', + 'description': 'Owned Skill fixture ' + name, + 'instructions': instructions, + } + with _named_resource(suite, row, '/api/v1/skills', body, 'skill', 'description', headers) as (path, created): + skill = created['skill'] + initial_revision = skill.get('revision') + check( + row, + 'skill_created', + skill.get('name') == name and bool(initial_revision), + 'Skill identity/revision missing', + ) + readback = request(suite, row, 'GET', path, headers=headers)['skill'] + check(row, 'skill_readback', readback == skill, 'Created Skill differs from its readback') + listed = request(suite, row, 'GET', '/api/v1/skills', headers=headers)['skills'] + check(row, 'skill_listed', sum(item.get('name') == name for item in listed) == 1, 'Skill listing is not unique') + preview = request(suite, row, 'GET', path + '/preview', headers=headers) + check(row, 'skill_preview', preview.get('instructions') == instructions, 'Skill preview lost instructions') + + file_path = path + '/files/references/fixture.txt' + content = 'Skill API fixture: 中文\n' + name + '\n' + written = request( + suite, row, 'PUT', file_path, {'content': content, 'base_revision': initial_revision}, headers=headers + ) + file_revision = written.get('revision') + check( + row, + 'skill_file_published', + bool(file_revision) + and file_revision != initial_revision + and written.get('bytes_written') == len(content.encode()), + 'Skill file write did not publish the expected UTF-8 bytes as a new revision', + ) + file_readback = request(suite, row, 'GET', file_path, headers=headers) + check( + row, + 'skill_file_readback', + file_readback.get('content') == content + and file_readback.get('path') == 'references/fixture.txt' + and file_readback.get('skill', {}).get('name') == name, + 'Skill file readback differs from the written content or identity', + ) + files = request(suite, row, 'GET', path + '/files?path=references', headers=headers) + check( + row, + 'skill_files_listed', + any( + item.get('path') == 'references/fixture.txt' + and item.get('size') == len(content.encode()) + and item.get('is_dir') is False + for item in files.get('entries', []) + ), + 'Written Skill file is missing from the directory listing', + ) + + updated_instructions = '# Updated API fixture\n\nKeep the local reference file.' + updated = request( + suite, + row, + 'PUT', + path, + {'instructions': updated_instructions, 'base_revision': file_revision}, + headers=headers, + )['skill'] + check( + row, + 'skill_updated', + updated.get('instructions') == updated_instructions and updated.get('revision') != file_revision, + 'Skill update did not publish changed instructions', + ) + preview = request(suite, row, 'GET', path + '/preview', headers=headers) + check( + row, 'skill_updated_preview', preview.get('instructions') == updated_instructions, 'Skill preview is stale' + ) + unchanged = request( + suite, + row, + 'PUT', + path, + {'instructions': updated_instructions, 'base_revision': updated['revision']}, + headers=headers, + )['skill'] + check( + row, + 'skill_update_idempotent', + unchanged.get('revision') == updated['revision'], + 'No-op Skill update changed content revision', + ) + before_conflict = request(suite, row, 'GET', path, headers=headers)['skill'] + + def verify_stale_update(): + after_conflict = request(suite, row, 'GET', path, headers=headers)['skill'] + check( + row, + 'skill_stale_update_preserved', + after_conflict == before_conflict, + 'Rejected Skill update changed the published state', + ) + + _expect_rejection( + suite, + row, + 'PUT', + path, + {'instructions': 'stale metadata must not win', 'base_revision': initial_revision}, + expected=409, + headers=headers, + verify=verify_stale_update, + ) + before_file_conflict = request(suite, row, 'GET', file_path, headers=headers) + before_file_skill = request(suite, row, 'GET', path, headers=headers)['skill'] + + def verify_stale_file(): + after_file_conflict = request(suite, row, 'GET', file_path, headers=headers) + after_conflict = request(suite, row, 'GET', path, headers=headers)['skill'] + check( + row, + 'skill_stale_file_preserved', + after_file_conflict == before_file_conflict and after_conflict == before_file_skill, + 'Rejected Skill file write changed the file or published revision', + ) + + # The file route currently maps revision conflicts to HTTP 400. + _expect_rejection( + suite, + row, + 'PUT', + file_path, + {'content': 'stale file must not win', 'base_revision': file_revision}, + expected=400, + headers=headers, + verify=verify_stale_file, + ) + request(suite, row, 'DELETE', path, headers=headers) + request(suite, row, 'GET', path, expected=404, headers=headers) + request(suite, row, 'DELETE', path, expected=400, headers=headers) + request(suite, row, 'GET', path, expected=404, headers=headers) + check( + row, + 'skill_deleted', + not any( + item.get('name') == name + for item in request(suite, row, 'GET', '/api/v1/skills', headers=headers)['skills'] + ), + 'Deleted Skill remains listed', + ) + return {'skill_name': name, 'revision': before_conflict['revision']} + + +def _mcp_config(server): + return { + key: server.get(key) for key in ('uuid', 'workspace_uuid', 'name', 'enable', 'mode', 'extra_args', 'readme') + } + + +def mcp_case(suite, row): + headers = user_headers(suite) + name = unique_name() + collection = '/api/v1/mcp/servers' + body = { + 'name': name, + 'enable': False, + 'mode': 'remote', + 'extra_args': {'url': 'http://127.0.0.1:9/mcp'}, + 'readme': 'Owned disabled MCP fixture ' + name, + } + with _named_resource(suite, row, collection, body, 'server', 'readme', headers) as (path, created): + server = request(suite, row, 'GET', path, headers=headers)['server'] + check( + row, + 'mcp_created_disabled', + bool(created.get('uuid')) + and server.get('uuid') == created['uuid'] + and server.get('workspace_uuid') == suite.variables['workspace_uuid'] + and all(server.get(key) == value for key, value in body.items()), + 'MCP create/readback changed identity, disabled state or configuration', + ) + servers = request(suite, row, 'GET', collection, headers=headers)['servers'] + check( + row, + 'mcp_listed', + sum(item.get('name') == name and item.get('uuid') == created['uuid'] for item in servers) == 1, + 'MCP listing did not return the created server exactly once', + ) + update = {'enable': False, 'extra_args': {'url': 'http://127.0.0.1:9/updated-mcp'}} + request(suite, row, 'PUT', path, update, headers=headers) + updated = request(suite, row, 'GET', path, headers=headers)['server'] + check( + row, + 'mcp_updated_disabled', + _mcp_config(updated) == {**_mcp_config(server), **update}, + 'MCP update changed unrelated configuration or enabled the server', + ) + request(suite, row, 'PUT', path, update, headers=headers) + repeated = request(suite, row, 'GET', path, headers=headers)['server'] + check( + row, + 'mcp_update_idempotent', + _mcp_config(repeated) == _mcp_config(updated), + 'Repeated MCP update changed configuration', + ) + + def verify_duplicate(): + after_duplicate = request(suite, row, 'GET', path, headers=headers)['server'] + servers = request(suite, row, 'GET', collection, headers=headers)['servers'] + check( + row, + 'mcp_duplicate_preserved', + _mcp_config(after_duplicate) == _mcp_config(updated) + and sum(item.get('name') == name for item in servers) == 1, + 'Duplicate MCP create changed the original or added another server', + ) + + _expect_rejection(suite, row, 'POST', collection, body, expected=400, headers=headers, verify=verify_duplicate) + + def verify_invalid_transport(): + after_invalid = request(suite, row, 'GET', path, headers=headers)['server'] + row['mcp_transport'] = {'before': updated['mode'], 'after': after_invalid['mode']} + check( + row, + 'mcp_invalid_transport_preserved', + _mcp_config(after_invalid) == _mcp_config(updated), + 'Invalid MCP transport changed stored configuration', + ) + + _expect_rejection( + suite, + row, + 'PUT', + path, + {'mode': 'qa-invalid-transport', 'enable': False}, + expected=400, + headers=headers, + verify=verify_invalid_transport, + ) + request(suite, row, 'DELETE', path, headers=headers) + request(suite, row, 'GET', path, expected=404, headers=headers) + request(suite, row, 'DELETE', path, expected=404, headers=headers) + servers = request(suite, row, 'GET', collection, headers=headers)['servers'] + check( + row, + 'mcp_deleted', + not any(item.get('name') == name for item in servers), + 'Deleted MCP server remains listed', + ) + return {'mcp_name': name, 'uuid': created['uuid']} diff --git a/tools/regression_ci.py b/tools/regression_ci.py index f83c9c885..95d4de210 100644 --- a/tools/regression_ci.py +++ b/tools/regression_ci.py @@ -16,17 +16,17 @@ import yaml from regression_gate import supervise from regression_coverage import write_report from regression_plugins import PLUGIN_IDS, install_plugins, prepare_published_plugins, prepare_source_plugins -from regression_local_cases import GROUPS, api, require +from regression_local_cases import DEFAULT_GROUPS, api, require from regression_suite import Suite from regression_traffic import owned_box_containers, read_endpoint_snapshot ROOT = Path(__file__).resolve().parents[1] -SUITE_BUDGETS = {'smoke': 60, 'scenarios': 300, 'daily': 1860, 'endurance': 900} +SUITE_BUDGETS = {'smoke': 60, 'scenarios': 300, 'daily': 1860, 'endurance': 900, 'api': 600} sys.path.insert(0, str(ROOT)) from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402 -def suite_profile(name, run_id, groups=()): +def suite_profile(name, run_id, groups=(), api_seed=1): profile_path = ROOT / 'tools/regression_profiles' / ('endurance.json' if name == 'endurance' else 'smoke.json') if name in ('scenarios', 'daily'): from regression_scenarios import expanded_profile @@ -34,6 +34,12 @@ def suite_profile(name, run_id, groups=()): profile = expanded_profile(groups) else: profile = json.loads(profile_path.read_text()) + if name == 'api': + from regression_api_cases import add_cases + + profile.update(resources=[], cases=[], coverage_suite='api') + profile['vars']['api_seed'] = api_seed + add_cases(profile) for resource in profile['resources']: if resource['kind'] == 'pipeline': resource['body'].setdefault( @@ -172,10 +178,12 @@ def run(args): config_path.write_text(yaml.safe_dump(config)) groups = args.scenario_groups if groups is None: - groups = list(GROUPS) if args.suite in ('scenarios', 'daily') else [] + groups = list(DEFAULT_GROUPS) if args.suite in ('scenarios', 'daily') else [] + if args.suite == 'api': + groups = ['api'] if 'plugins' in groups and not {'misc/EssentialCommands', 'misc/FAQManager'} <= set(args.extra_plugins): groups.remove('plugins') - profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups) + profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups, args.api_seed) cli_config = private / 'lbctl.yaml' cli_config.write_text( yaml.safe_dump( @@ -270,7 +278,7 @@ def run(args): cli_data = yaml.safe_load(cli_config.read_text()) cli_data['contexts']['smoke']['expected_workspace_uuid'] = identity['workspace_uuid'] cli_config.write_text(yaml.safe_dump(cli_data)) - if 'resources' in groups or args.restart_services: + if {'resources', 'api'} & set(groups) or args.restart_services: # Account creation is restricted to this runner's newly created private database. credentials = {'user': 'regression@langbot.test', 'password': secrets.token_urlsafe(32)} suite.secrets.append(credentials['password']) @@ -324,6 +332,9 @@ def run(args): coverage = write_report(evidence, root / 'gate/suite', http_logs=[root / 'core.log']) result['coverage_errors'] = coverage['evidence_errors'] result['scenario_groups'] = groups + if args.suite == 'api': + result['api_seed'] = args.api_seed + result['case_order'] = [case['name'] for case in profile['cases']] result['passed'] = result['passed'] and not result['coverage_errors'] (root / 'gate/gate.json').write_text(json.dumps(result, indent=2) + '\n') return result @@ -357,9 +368,10 @@ def main(): parser.add_argument( '--scenario-groups', nargs='*', - choices=tuple(GROUPS), + choices=DEFAULT_GROUPS, help='Local groups for scenarios/daily (default: all); an empty list selects only the base cases', ) + parser.add_argument('--api-seed', type=int, default=1, help='Replay the API suite chain shuffle (default: 1)') parser.add_argument('--restart-services', action='store_true', help='Check owned Core/Runtime restarts once') args = parser.parse_args() if args.extra_plugins and args.plugins is None: diff --git a/tools/regression_contracts.json b/tools/regression_contracts.json index da894dcbc..2597653b9 100644 --- a/tools/regression_contracts.json +++ b/tools/regression_contracts.json @@ -378,6 +378,62 @@ "area": "retrieval", "kind": "blackbox", "priority": "P1", "description": "An empty knowledge base allows chat without injecting other knowledge fixtures", "bindings": [{"profile": "scenarios", "case": "empty_rag_chat", "checks": ["response"]}] + }, + {"id": "api.model_llm", + "area": "api", + "kind": "blackbox", + "priority": "P1", + "description": "llm model and provider CRUD preserve bindings, secrets and children on rejected deletion", + "bindings": [{"profile": "api", "case": "fixed_api_llm", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}, {"profile": "api", "case": "shuffled_api_llm", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}] + }, + {"id": "api.model_embedding", + "area": "api", + "kind": "blackbox", + "priority": "P1", + "description": "embedding model and provider CRUD preserve bindings, secrets and children on rejected deletion", + "bindings": [{"profile": "api", "case": "fixed_api_embedding", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}, {"profile": "api", "case": "shuffled_api_embedding", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}] + }, + {"id": "api.model_rerank", + "area": "api", + "kind": "blackbox", + "priority": "P1", + "description": "rerank model and provider CRUD preserve bindings, secrets and children on rejected deletion", + "bindings": [{"profile": "api", "case": "fixed_api_rerank", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}, {"profile": "api", "case": "shuffled_api_rerank", "checks": ["api_chain", "provider_identity", "provider_redacted", "provider_update", "model_identity", "model_list", "model_update", "rejected_delete_preserves_model", "model_deleted_from_list"]}] + }, + {"id": "api.pipeline", + "area": "api", + "kind": "blackbox", + "priority": "P1", + "description": "Pipeline CRUD and extension updates preserve state after invalid bindings and run inputs", + "bindings": [{"profile": "api", "case": "fixed_api_pipeline", "checks": ["api_chain", "pipeline_identity", "pipeline_list", "pipeline_update", "extension_readback", "rejected_extension_enable_all_plugins", "rejected_extension_bound_skills", "rejected_extension_bound_plugins", "invalid_run_preserves_config"]}, {"profile": "api", "case": "shuffled_api_pipeline", "checks": ["api_chain", "pipeline_identity", "pipeline_list", "pipeline_update", "extension_readback", "rejected_extension_enable_all_plugins", "rejected_extension_bound_skills", "rejected_extension_bound_plugins", "invalid_run_preserves_config"]}] + }, + {"id": "api.credentials", + "area": "api", + "kind": "blackbox", + "priority": "P1", + "description": "Read-only and revoked keys cannot write or read protected resources; secrets are returned only once", + "bindings": [{"profile": "api", "case": "fixed_api_credentials", "checks": ["api_chain", "invalid_key_not_created", "key_metadata_redacted", "readonly_denial_preserves_state", "revoked_readback"]}, {"profile": "api", "case": "shuffled_api_credentials", "checks": ["api_chain", "invalid_key_not_created", "key_metadata_redacted", "readonly_denial_preserves_state", "revoked_readback"]}] + }, + {"id": "api.skills", + "area": "api", + "kind": "blackbox", + "priority": "P1", + "description": "Skill publication and UTF-8 files retain revision and content after stale writes", + "bindings": [{"profile": "api", "case": "fixed_api_skills", "checks": ["api_chain", "skill_created", "skill_file_published", "skill_updated", "skill_update_idempotent", "skill_stale_update_preserved", "skill_stale_file_preserved", "skill_deleted"]}, {"profile": "api", "case": "shuffled_api_skills", "checks": ["api_chain", "skill_created", "skill_file_published", "skill_updated", "skill_update_idempotent", "skill_stale_update_preserved", "skill_stale_file_preserved", "skill_deleted"]}] + }, + {"id": "api.mcp_crud", + "area": "api", + "kind": "blackbox", + "priority": "P1", + "description": "Disabled MCP configurations retain identity and state across CRUD and duplicate requests", + "bindings": [{"profile": "api", "case": "fixed_api_mcp", "checks": ["mcp_created_disabled", "mcp_listed", "mcp_updated_disabled", "mcp_update_idempotent", "mcp_duplicate_preserved", "cleanup_server"]}, {"profile": "api", "case": "shuffled_api_mcp", "checks": ["mcp_created_disabled", "mcp_listed", "mcp_updated_disabled", "mcp_update_idempotent", "mcp_duplicate_preserved", "cleanup_server"]}] + }, + {"id": "api.mcp_validation", + "area": "api", + "kind": "blackbox", + "priority": "P1", + "description": "Invalid MCP transport is rejected without changing stored configuration", + "bindings": [{"profile": "api", "case": "fixed_api_mcp", "checks": ["api_chain", "mcp_invalid_transport_preserved"]}, {"profile": "api", "case": "shuffled_api_mcp", "checks": ["api_chain", "mcp_invalid_transport_preserved"]}] } ] } diff --git a/tools/regression_local_cases.py b/tools/regression_local_cases.py index dc25dcff2..b935f4527 100644 --- a/tools/regression_local_cases.py +++ b/tools/regression_local_cases.py @@ -14,9 +14,13 @@ GROUPS = { 'resources': 'regression_resource_cases', 'lifecycle': 'regression_lifecycle_cases', 'plugins': 'regression_plugin_cases', + 'api': 'regression_api_cases', } +DEFAULT_GROUPS = ('faults', 'resources', 'lifecycle', 'plugins') + + class ScenarioUnavailable(RuntimeError): """An explicit prerequisite is missing; this is not a passed assertion.""" diff --git a/tools/test_regression_api.py b/tools/test_regression_api.py new file mode 100644 index 000000000..8a61f2edb --- /dev/null +++ b/tools/test_regression_api.py @@ -0,0 +1,243 @@ +"""Challenge API job oracles and replay/cleanup boundaries without a running Core.""" + +from collections import Counter +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from regression_api_cases import run_case +from regression_api_checks import owned_resource, request +from regression_api_resources import _expect_rejection +from regression_ci import suite_profile +from regression_local_cases import DEFAULT_GROUPS +from regression_scenarios import expanded_profile +from regression_suite import Suite, validate + + +class ApiContractTests(unittest.TestCase): + def test_replay_and_job_isolation(self): + first, _ = suite_profile('api', 'run-a', api_seed=20261028) + replay, _ = suite_profile('api', 'run-b', api_seed=20261028) + other, _ = suite_profile('api', 'run-c', api_seed=2) + validate(first) + self.assertEqual(first['cases'], replay['cases']) + self.assertNotEqual(first['cases'], other['cases']) + half = len(first['cases']) // 2 + fixed, shuffled = first['cases'][:half], first['cases'][half:] + self.assertEqual(Counter(c['scenario'] for c in fixed), Counter(c['scenario'] for c in shuffled)) + self.assertEqual(set(Counter(c['scenario'] for c in fixed).values()), {1}) + self.assertEqual(first['resources'], []) + self.assertNotIn('api', DEFAULT_GROUPS) + daily = expanded_profile(DEFAULT_GROUPS) + self.assertFalse(any(c.get('scenario', '').startswith('api_') for c in daily['cases'])) + with self.assertRaisesRegex(ValueError, 'dedicated suite'): + expanded_profile(['api']) + + def test_http_success_cannot_hide_business_failure_or_wrong_status(self): + for actual in ((200, {'code': -1}), (201, {'code': 0}), (400, {'code': 0}), (400, {})): + row = {} + with self.subTest(actual=actual), patch('regression_api_checks.api', return_value=actual): + with self.assertRaises(ValueError): + request(None, row, 'GET', '/api/test', expected=400 if actual[0] == 400 else 200) + self.assertEqual(row['assertions']['http_1'], 'failed') + self.assertEqual(row['http_steps'][0]['status'], actual[0]) + + def test_late_failure_cannot_mark_the_api_chain_passed(self): + from types import SimpleNamespace + + suite = SimpleNamespace(profile={'vars': {'api_seed': 1}}) + row = {} + with patch('regression_api_cases.credentials_case', side_effect=ValueError('Revoked key still works')): + with self.assertRaisesRegex(ValueError, 'Revoked key'): + run_case(suite, {'scenario': 'api_credentials', 'expect': 'OK'}, row) + self.assertEqual(row['assertions']['api_chain'], 'failed') + + def test_rejection_keeps_http_failure_when_state_check_also_fails(self): + row = {} + + def verify(): + row['state_checked'] = True + raise ValueError('Stored state changed') + + with patch('regression_api_resources.request', side_effect=ValueError('Expected 400, got 200')): + with self.assertRaisesRegex(ValueError, 'Expected 400, got 200'): + _expect_rejection(None, row, 'PUT', '/api/test', {}, expected=400, headers={}, verify=verify) + self.assertTrue(row['state_checked']) + + def test_lost_create_response_cleans_only_exact_owned_resource(self): + for changed in (False, True): + with self.subTest(changed=changed), tempfile.TemporaryDirectory() as directory: + suite = Suite( + {'version': 1, 'cli': {'config': 'unused'}, 'cases': [], 'vars': {'workspace_uuid': 'test'}}, + Path(directory) / 'profile.json', + Path(directory), + ) + row, calls = {}, [] + + def api(suite, method, path, body=None, headers=None): + calls.append((method, path)) + if method == 'POST': + raise TimeoutError('Lost response after create') + if path == '/collection': + return 200, { + 'code': 0, + 'data': { + 'items': [{'name': 'qa-owned', 'uuid': 'owned'}, {'name': 'unrelated', 'uuid': 'other'}] + }, + } + if calls[-2][0] == 'DELETE': + return 404, {'code': -1} + if method == 'GET': + return 200, { + 'code': 0, + 'data': {'item': {'name': 'unrelated' if changed else 'qa-owned', 'uuid': 'owned'}}, + } + return 200, {'code': 0} + + with patch('regression_api_checks.api', side_effect=api): + with self.assertRaisesRegex(TimeoutError, 'Lost response'): + with owned_resource( + suite, row, '/collection', {'name': 'qa-owned'}, item_key='item', list_key='items' + ): + self.fail('Create timeout must not enter the contract body') + deleted = [path for method, path in calls if method == 'DELETE'] + self.assertEqual(deleted, [] if changed else ['/collection/owned']) + if changed: + self.assertTrue(row['cleanup_errors']) + else: + self.assertEqual(row['owned_resources'][0]['state'], 'deleted') + + def test_invalid_key_side_effects_fail_and_revoke_only_the_owned_key(self): + for status, invalid_field in ((200, 'scopes'), (400, 'scopes'), (200, 'expires_at'), (400, 'expires_at')): + with self.subTest(status=status, field=invalid_field), tempfile.TemporaryDirectory() as directory: + suite = Suite( + { + 'version': 1, + 'cli': {'config': 'unused'}, + 'cases': [], + 'vars': {'workspace_uuid': 'test', 'api_seed': 1}, + }, + Path(directory) / 'profile.json', + Path(directory), + env={'LANGBOT_QA_USER_TOKEN': 'manager-secret'}, + ) + other = {'id': 1, 'name': 'qa-api-other-key', 'status': 'active', 'secret_available': False} + keys, row, deleted = {1: dict(other)}, {}, [] + + def api(suite, method, path, body=None, headers=None): + if path == '/api/v1/pipelines': + return 401, {'code': -1} + if path == '/api/v1/apikeys': + if method == 'GET': + return 200, {'code': 0, 'data': {'keys': [dict(key) for key in keys.values()]}} + self.assertEqual(method, 'POST') + if invalid_field not in body: + return 400, {'code': -1} + keys[7] = {'id': 7, 'name': body['name'], 'status': 'active', 'secret_available': False} + return status, {'code': 0 if status == 200 else -1, 'data': {'key': dict(keys[7])}} + self.assertEqual(path, '/api/v1/apikeys/7') + if method == 'DELETE': + deleted.append(path) + keys[7]['status'] = 'revoked' + return 200, {'code': 0} + self.assertEqual(method, 'GET') + return 200, {'code': 0, 'data': {'key': dict(keys[7])}} + + error = 'expected HTTP 400' if status == 200 else 'Invalid key request changed key list' + with ( + patch('regression_api_checks.api', side_effect=api), + patch('regression_api_cases.unique_name', return_value='qa-api-owned-key'), + self.assertRaisesRegex(ValueError, error), + ): + run_case(suite, {'scenario': 'api_credentials', 'expect': 'OK'}, row) + self.assertEqual(row['assertions']['api_chain'], 'failed') + self.assertEqual(row['assertions']['key_cleanup'], 'passed') + self.assertFalse(row.get('cleanup_errors')) + self.assertEqual(deleted, ['/api/v1/apikeys/7']) + self.assertEqual(keys[7]['status'], 'revoked') + self.assertEqual(keys[1], other) + self.assertEqual(set(keys), {1, 7}) + self.assertEqual(row['owned_resources'][0]['state'], 'revoked') + + def test_readonly_pipeline_side_effects_fail_and_cleanup_preserves_other_resources(self): + for status in (200, 403): + with self.subTest(status=status), tempfile.TemporaryDirectory() as directory: + suite = Suite( + { + 'version': 1, + 'cli': {'config': 'unused'}, + 'cases': [], + 'vars': {'workspace_uuid': 'test', 'api_seed': 1}, + }, + Path(directory) / 'profile.json', + Path(directory), + env={'LANGBOT_QA_USER_TOKEN': 'manager-secret'}, + ) + other_key = {'id': 1, 'name': 'qa-api-other-key', 'status': 'active', 'secret_available': False} + other_pipeline = {'uuid': 'other-pipeline', 'name': 'qa-api-other-pipeline'} + keys, pipelines = {1: dict(other_key)}, {'other-pipeline': dict(other_pipeline)} + row, deleted = {}, [] + + def api(suite, method, path, body=None, headers=None): + if path == '/api/v1/apikeys': + if method == 'GET': + return 200, {'code': 0, 'data': {'keys': [dict(key) for key in keys.values()]}} + self.assertEqual(method, 'POST') + if body.get('scopes') == ['qa.invalid'] or 'expires_at' in body: + return 400, {'code': -1} + keys[7] = {'id': 7, 'name': body['name'], 'status': 'active', 'secret_available': False} + return 200, {'code': 0, 'data': {'key': {**keys[7], 'key': 'child-secret'}}} + if path == '/api/v1/apikeys/7': + if method == 'PUT': + keys[7]['name'] = body['name'] + return 200, {'code': 0} + if method == 'DELETE': + deleted.append(path) + keys[7]['status'] = 'revoked' + return 200, {'code': 0} + self.assertEqual(method, 'GET') + return 200, {'code': 0, 'data': {'key': dict(keys[7])}} + if path == '/api/v1/pipelines': + if headers in ({}, {'Authorization': 'Bearer qa-invalid'}): + return 401, {'code': -1} + if method == 'GET': + return 200, {'code': 0, 'data': {'pipelines': [dict(item) for item in pipelines.values()]}} + self.assertEqual(method, 'POST') + self.assertEqual(headers, {'X-API-Key': 'child-secret'}) + pipelines['owned-pipeline'] = {'uuid': 'owned-pipeline', 'name': body['name']} + return status, {'code': 0 if status == 200 else -1, 'data': {'uuid': 'owned-pipeline'}} + self.assertEqual(path, '/api/v1/pipelines/owned-pipeline') + if method == 'DELETE': + deleted.append(path) + del pipelines['owned-pipeline'] + return 200, {'code': 0} + self.assertEqual(method, 'GET') + if 'owned-pipeline' not in pipelines: + return 404, {'code': -1} + return 200, {'code': 0, 'data': {'pipeline': dict(pipelines['owned-pipeline'])}} + + error = 'expected HTTP 403' if status == 200 else 'Read-only key changed resources' + with ( + patch('regression_api_checks.api', side_effect=api), + patch('regression_api_cases.unique_name', return_value='qa-api-owned-key'), + self.assertRaisesRegex(ValueError, error), + ): + run_case(suite, {'scenario': 'api_credentials', 'expect': 'OK'}, row) + self.assertEqual(row['assertions']['api_chain'], 'failed') + self.assertEqual(row['assertions']['key_cleanup'], 'passed') + self.assertEqual(row['assertions']['cleanup_pipeline_qa-api-owned-key'], 'passed') + self.assertFalse(row.get('cleanup_errors')) + self.assertEqual(deleted, ['/api/v1/pipelines/owned-pipeline', '/api/v1/apikeys/7']) + self.assertEqual(keys[7]['status'], 'revoked') + self.assertEqual(keys[1], other_key) + self.assertEqual(set(keys), {1, 7}) + self.assertEqual(pipelines, {'other-pipeline': other_pipeline}) + self.assertEqual( + {item['kind']: item['state'] for item in row['owned_resources']}, + {'api-key': 'revoked', 'pipeline': 'deleted'}, + ) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_regression_coverage.py b/tools/test_regression_coverage.py index f0d7f5870..a0d87e941 100644 --- a/tools/test_regression_coverage.py +++ b/tools/test_regression_coverage.py @@ -289,12 +289,16 @@ class Routes: name = binding['profile'] if name not in profiles: path = Path(__file__).with_name('regression_profiles') / f'{name}.json' - if name == 'scenarios': - from regression_scenarios import expanded_profile - from regression_local_cases import GROUPS + if name == 'api': + from regression_ci import suite_profile - profile = expanded_profile(GROUPS) - if 'lifecycle' in GROUPS: + profile, _ = suite_profile('api', 'catalog-test') + elif name == 'scenarios': + from regression_scenarios import expanded_profile + from regression_local_cases import DEFAULT_GROUPS + + profile = expanded_profile(DEFAULT_GROUPS) + if 'lifecycle' in DEFAULT_GROUPS: from regression_lifecycle_cases import restart_profile profile['cases'] += restart_profile(profile)['cases'] diff --git a/tools/test_regression_local_cases.py b/tools/test_regression_local_cases.py index e1a42e040..15c7baecd 100644 --- a/tools/test_regression_local_cases.py +++ b/tools/test_regression_local_cases.py @@ -7,7 +7,7 @@ import tempfile import unittest from unittest.mock import patch -from regression_local_cases import GROUPS, ScenarioUnavailable +from regression_local_cases import DEFAULT_GROUPS, ScenarioUnavailable from regression_scenarios import expanded_profile from regression_smoke import run_smoke from regression_suite import Suite, validate @@ -17,7 +17,7 @@ from regression_traffic import reconcile_runs class LocalScenarioTests(unittest.TestCase): def test_group_switches_keep_base_cases_and_reject_unknown_handlers(self): base = expanded_profile() - for group in GROUPS: + for group in DEFAULT_GROUPS: profile = expanded_profile([group]) validate(profile) self.assertEqual(profile['scenario_groups'], [group]) From f5924164d20a6a22c93607a2eee8d669f6c8b832 Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Thu, 29 Oct 2026 11:00:11 +0800 Subject: [PATCH 2/2] ci(daily): run API contracts alongside endurance --- .github/workflows/daily-endurance.yml | 91 +++++++++++++++++++++++++++ tools/README.md | 19 ++++-- tools/test_regression_ci.py | 47 ++++++++++++++ 3 files changed, 151 insertions(+), 6 deletions(-) diff --git a/.github/workflows/daily-endurance.yml b/.github/workflows/daily-endurance.yml index f11f86482..bef7c0674 100644 --- a/.github/workflows/daily-endurance.yml +++ b/.github/workflows/daily-endurance.yml @@ -158,3 +158,94 @@ jobs: .qa/daily/*.log if-no-files-found: warn retention-days: 30 + + api-contracts: + name: Daily API contracts + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + with: + persist-credentials: false + - uses: actions/setup-python@v5 + with: + python-version: '3.12' + - uses: astral-sh/setup-uv@v4 + with: + enable-cache: true + - name: Install locked dependencies + run: uv sync --dev --frozen + - name: Prepare CLI and sandbox image + env: + GH_TOKEN: ${{ github.token }} + run: | + mkdir -p .api-bin + gh release download v0.3.0 --repo langbot-app/langbot-cli --dir .api-bin \ + --pattern lbctl_linux_amd64 --pattern checksums.txt + cd .api-bin + sha256sum --check --ignore-missing checksums.txt + chmod +x lbctl_linux_amd64 + docker pull python:3.12-alpine + - name: Run API contracts + run: | + uv run --no-sync python tools/regression_ci.py \ + --suite api --api-seed "$GITHUB_RUN_ID" \ + --lbctl .api-bin/lbctl_linux_amd64 --plugins plugins --output .qa/api + - name: Publish API result + if: always() + run: | + python - <<'PY_SUMMARY' + import json, os + from pathlib import Path + + root = Path('.qa/api') + def read(path): + source = root / path + return json.loads(source.read_text()) if source.exists() else {} + + gate = read('gate/gate.json') + suite = read('gate/suite/summary.json') + coverage = read('gate/suite/coverage.json') + manifest = read('plugins/manifest.json') + passed = gate.get('passed') is True and suite.get('passed') is True + with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as out: + out.write(f"Daily API contracts: {'PASS' if passed else 'FAIL / incomplete'}\n\n") + out.write(f"Core: `{os.environ['GITHUB_SHA']}`; shuffle seed: `{gate.get('api_seed', os.environ['GITHUB_RUN_ID'])}`.\n\n") + out.write('Fixed order followed by a seeded chain shuffle, with sequential operations inside each chain. The execution guard is 600 seconds; dependency and service preparation is separate. This job is not a required PR check.\n\n') + out.write(f"Cases: {suite.get('requests', 'not reported')}; statuses: {suite.get('status', 'not reported')}; suite execution: {suite.get('elapsed_seconds', 'not reported')}s.\n\n") + out.write(f"API behavior contracts: {coverage.get('areas', {}).get('api', 'not reported')}.\n\n") + out.write(f"Observed HTTP route templates (including preparation): {len(coverage['http_observations']['routes']) if 'http_observations' in coverage else 'not reported'}. A route hit does not establish all behavior behind it.\n\n") + for plugin in manifest.get('plugins', []): + out.write(f"Plugin: `{plugin['id']}@{plugin['version']}`; installation readback: `{plugin.get('installed', 'not confirmed')}`.\n\n") + errors = list(suite.get('errors', [])) + list(gate.get('resource_errors', [])) + list(gate.get('coverage_errors', [])) + if gate.get('timed_out'): + errors.append('API suite exceeded its 600-second outer process guard.') + if gate.get('infrastructure_error'): + errors.append(gate['infrastructure_error']) + if not gate or not suite: + errors.append('Missing verdict; inspect preparation and service logs.') + for error in errors: + out.write(f"- {error}\n") + PY_SUMMARY + - name: Preserve API verdicts and evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: daily-api-${{ github.run_id }}-${{ github.run_attempt }} + path: | + .qa/api/gate/gate.json + .qa/api/gate/resources.jsonl + .qa/api/gate/suite.log + .qa/api/gate/suite/summary.json + .qa/api/gate/suite/profile.json + .qa/api/gate/suite/cases.json + .qa/api/gate/suite/health.json + .qa/api/gate/suite/cleanup-checks.json + .qa/api/gate/suite/resource-case-*.json + .qa/api/gate/suite/coverage.json + .qa/api/gate/suite/coverage.md + .qa/api/plugins/manifest.json + .qa/api/plugins/*-build.log + .qa/api/*.log + if-no-files-found: warn + retention-days: 30 diff --git a/tools/README.md b/tools/README.md index b5037aa1e..0c3f8d5cb 100644 --- a/tools/README.md +++ b/tools/README.md @@ -486,7 +486,7 @@ uv run --no-sync python tools/regression_ci.py --suite daily --restart-services `--suite smoke` retains the 60-second gate. `--suite scenarios` runs the shared expanded profile once under a 300-second guard. `--suite endurance` retains the separate ten-minute load profile under a 900-second guard; its CI entry point also -reports throughput/P95 without applying their thresholds. All four modes use +reports throughput/P95 without applying their thresholds. All suites use published plugins when `--plugins` is omitted. Both workflows explicitly select `--plugins plugins` so source changes are tested in the same PR. Daily additionally selects `misc/EssentialCommands misc/FAQManager`; manual dispatch can change the @@ -498,9 +498,17 @@ bind their own extensions to avoid changing unrelated prompts/tools. The suite a even after failures. The CI owner then stops its service processes and removes its Box containers; a process guard timeout can interrupt API cleanup. -## Local API contract job +## Daily API contract job -Run on the same disposable Linux/Docker host used by `regression_ci.py`: +`daily-endurance.yml` runs API contracts alongside the 30-minute endurance job +on separate runners, at the existing daily schedule and on manual dispatch. +The API job uses the GitHub run ID as its shuffle seed, so each daily run explores +a different order and a rerun keeps the same seed. It has a 15-minute job timeout, +including preparation, and is not a required PR check. Its summary and separate +`daily-api--` artifact retain failures, order, coverage and cleanup +evidence for 30 days. + +To replay locally on a disposable Linux/Docker host, pass the recorded seed: ```sh uv run --no-sync python tools/regression_ci.py --suite api --api-seed 20261028 \ @@ -527,9 +535,8 @@ Inspect `gate/gate.json` for the verdict, seed and exact case order; `gate/suite/cases.json` for HTTP steps, assertions and cleanup results; and `gate/suite/coverage.json` for observed route templates and behavior contracts. An HTTP route hit includes rejected requests and is not proof of all behavior -behind that route. The job intentionally fails on invalid MCP modes accepted -and persisted by Core; fixing that product validation should make the existing -negative case pass without changing its expected result. +behind that route. Invalid MCP modes must return HTTP 400 and leave the original +configuration unchanged. Any failed contract or cleanup makes the job fail. ## Sustained regression and baseline comparison diff --git a/tools/test_regression_ci.py b/tools/test_regression_ci.py index 354835fcb..83b3ee06c 100644 --- a/tools/test_regression_ci.py +++ b/tools/test_regression_ci.py @@ -143,6 +143,53 @@ class DailySummaryTest(unittest.TestCase): else: self.assertNotIn('Daily regression: PASS', text) + def test_api_summary_keeps_preparation_and_outer_failures_visible(self): + workflow = Path(__file__).resolve().parents[1] / '.github/workflows/daily-endurance.yml' + steps = yaml.safe_load(workflow.read_text())['jobs']['api-contracts']['steps'] + command = next(step['run'] for step in steps if step.get('name') == 'Publish API result') + suite = {'passed': True, 'requests': 14, 'status': {'passed': 14}, 'elapsed_seconds': 7.3} + for gate, actual_suite, expected in [ + ({'passed': True, 'api_seed': 123}, suite, 'Daily API contracts: PASS'), + ( + {'passed': False, 'infrastructure_error': 'plugin installation failed'}, + None, + 'plugin installation failed', + ), + ({'passed': False, 'timed_out': True}, suite, '600-second outer process guard'), + ({'passed': False, 'resource_errors': ['sampling failed']}, suite, 'sampling failed'), + ({'passed': False, 'coverage_errors': ['missing evidence']}, suite, 'missing evidence'), + ({'passed': False}, {'passed': False, 'errors': ['MCP mode persisted']}, 'MCP mode persisted'), + (None, None, 'Missing verdict'), + ]: + with self.subTest(gate=gate), tempfile.TemporaryDirectory() as directory: + root = Path(directory) + target = root / '.qa/api/gate/suite' + target.mkdir(parents=True) + if gate is not None: + (target.parent / 'gate.json').write_text(json.dumps(gate)) + if actual_suite is not None: + (target / 'summary.json').write_text(json.dumps(actual_suite)) + summary = root / 'summary.md' + subprocess.run( + ['bash', '-e', '-c', command], + cwd=root, + check=True, + capture_output=True, + env={ + **os.environ, + 'PATH': str(Path(sys.executable).parent) + os.pathsep + os.environ['PATH'], + 'GITHUB_STEP_SUMMARY': str(summary), + 'GITHUB_SHA': 'tested-core', + 'GITHUB_RUN_ID': '456', + }, + ) + text = summary.read_text() + self.assertIn(expected, text) + self.assertIn('not reported', text) + self.assertIn('shuffle seed: `123`' if gate and gate.get('passed') else 'shuffle seed: `456`', text) + if not gate or not gate.get('passed'): + self.assertNotIn('Daily API contracts: PASS', text) + def test_core_summary_records_preparation_failure_and_selected_plugins(self): workflow = Path(__file__).resolve().parents[1] / '.github/workflows/core-smoke.yml' steps = yaml.safe_load(workflow.read_text())['jobs']['smoke']['steps']