diff --git a/tools/README.md b/tools/README.md index 0c3f8d5cb..db9c6c3b5 100644 --- a/tools/README.md +++ b/tools/README.md @@ -538,6 +538,26 @@ An HTTP route hit includes rejected requests and is not proof of all behavior behind that route. Invalid MCP modes must return HTTP 400 and leave the original configuration unchanged. Any failed contract or cleanup makes the job fail. +## CLI compatibility check + +Run on the same disposable Linux host: + +```sh +uv run --no-sync python tools/regression_ci.py --suite cli \ + --lbctl /path/to/lbctl --plugins plugins --output .qa/cli-001 +``` + +The finite job checks +Workspace identity, capability discovery, Pipeline create/read/update/delete, +and invalid credentials. It uses the supplied CLI binary and records its version; +select the latest stable release when validating the evolving CLI. + +Assertions allow additional output fields and changes to human-readable wording. +Workspace identity, created resource identity, actual backend state, cleanup and +nonzero exit on authentication failure remain required. HTTP readback provides +an independent oracle, so an exit code of zero alone cannot pass a mutation. +The suite is opt-in and adds no daily schedule or PR gate. + ## Sustained regression and baseline comparison [`regression_profiles/endurance.json`](regression_profiles/endurance.json) runs diff --git a/tools/regression_ci.py b/tools/regression_ci.py index 95d4de210..0b9bef10f 100644 --- a/tools/regression_ci.py +++ b/tools/regression_ci.py @@ -21,7 +21,7 @@ from regression_suite import Suite from regression_traffic import owned_box_containers, read_endpoint_snapshot ROOT = Path(__file__).resolve().parents[1] -SUITE_BUDGETS = {'smoke': 60, 'scenarios': 300, 'daily': 1860, 'endurance': 900, 'api': 600} +SUITE_BUDGETS = {'smoke': 60, 'scenarios': 300, 'daily': 1860, 'endurance': 900, 'api': 600, 'cli': 180} sys.path.insert(0, str(ROOT)) from tests.e2e.utils.config_factory import create_minimal_config, create_test_directories # noqa: E402 @@ -34,6 +34,10 @@ def suite_profile(name, run_id, groups=(), api_seed=1): profile = expanded_profile(groups) else: profile = json.loads(profile_path.read_text()) + if name == 'cli': + from regression_cli_cases import configure + + configure(profile) if name == 'api': from regression_api_cases import add_cases @@ -179,8 +183,8 @@ def run(args): groups = args.scenario_groups if groups is None: groups = list(DEFAULT_GROUPS) if args.suite in ('scenarios', 'daily') else [] - if args.suite == 'api': - groups = ['api'] + if args.suite in ('api', 'cli'): + groups = [args.suite] if 'plugins' in groups and not {'misc/EssentialCommands', 'misc/FAQManager'} <= set(args.extra_plugins): groups.remove('plugins') profile, profile_path = suite_profile(args.suite, secrets.token_hex(8), groups, args.api_seed) diff --git a/tools/regression_cli_cases.py b/tools/regression_cli_cases.py new file mode 100644 index 000000000..175109a15 --- /dev/null +++ b/tools/regression_cli_cases.py @@ -0,0 +1,115 @@ +"""Published CLI compatibility checks with strict backend state and flexible presentation.""" + +import subprocess + +from regression_api_checks import check, request, tracked_resource, unique_name +from regression_coverage import checked +from regression_local_cases import require +from regression_resource_cases import journal + +CASE_NAMES = ('cli.identity', 'cli.pipeline_roundtrip', 'cli.invalid_credential') + + +def configure(profile): + profile.update(resources=[], cases=[], coverage_suite='cli', scenario_groups=['cli']) + profile['cases'] = [ + {'name': name, 'scenario': name, 'stage': 'cli', 'expect': 'CLI_COMPAT_OK'} for name in CASE_NAMES + ] + + +def identity(suite, row): + row['cli_version'] = suite.call(['version']) + result = suite.call(['whoami']) + check( + row, + 'workspace_identity', + result.get('workspace_uuid') == suite.variables['workspace_uuid'], + 'CLI resolved a different Workspace', + ) + capabilities = suite.call(['capabilities']) + check(row, 'capability_discovery', bool(capabilities), 'CLI capability discovery returned no data') + + +def pipeline_roundtrip(suite, row): + name, collection = unique_name(), '/api/v1/pipelines' + with tracked_resource(suite, row, collection, name, item_key='pipeline', list_key='pipelines') as owned: + created = suite.call(['pipeline', 'apply'], {'name': name, 'description': 'CLI fixture δΈ­ζ–‡'}) + matches = [p for p in request(suite, row, 'GET', collection)['pipelines'] if p['name'] == name] + require(len(matches) == 1, 'CLI create did not persist exactly one owned Pipeline') + uid = matches[0]['uuid'] + owned.update(uuid=uid, state='created') + journal(suite, row) + path = collection + '/' + uid + check( + row, + 'create_identity', + created.get('pipeline', created).get('uuid') == uid, + 'CLI create omitted the created resource identity', + ) + current = suite.call(['pipeline', 'get', uid]) + check( + row, + 'cli_get', + current.get('pipeline', current).get('uuid') == uid, + 'CLI get omitted the owned entity identity', + ) + listed = suite.call(['pipeline', 'list']) + items = listed.get('pipelines', []) if isinstance(listed, dict) else listed + check( + row, + 'cli_list', + isinstance(items, list) and any(item.get('uuid') == uid for item in items), + 'CLI list omitted the owned entity identity', + ) + before = request(suite, row, 'GET', path)['pipeline'] + check(row, 'create_state', before['description'] == 'CLI fixture δΈ­ζ–‡', 'CLI create changed submitted content') + suite.call( + ['pipeline', 'apply'], + { + 'uuid': uid, + 'name': name + '-updated', + 'description': 'CLI updated 🌍', + 'config': before['config'], + }, + ) + after = request(suite, row, 'GET', path)['pipeline'] + check( + row, + 'update_state', + after['name'] == name + '-updated' + and after['description'] == 'CLI updated 🌍' + and after['config'] == before['config'], + 'CLI update lost submitted or existing configuration', + ) + suite.call(['pipeline', 'delete', uid, '--yes']) + request(suite, row, 'GET', path, expected=404) + + +def invalid_credential(suite, row): + source = suite.connection.get('credential_source', '') + require(source.startswith('env:'), 'CLI credential probe requires the disposable environment credential') + result = subprocess.run( + [*suite.cli, 'pipeline', 'list'], + capture_output=True, + text=True, + timeout=suite.timeout(30), + env={**suite.env, source.removeprefix('env:'): 'qa-invalid-credential'}, + ) + row['rejected_exit_code'] = result.returncode + # Exit code is stable; error wording and JSON error details may evolve. + check(row, 'credential_rejected', result.returncode != 0, 'CLI reported success with an invalid credential') + check(row, 'error_signal', bool(result.stdout.strip() or result.stderr.strip()), 'CLI failure had no diagnostic') + suite.call(['pipeline', 'list']) + + +def run_case(suite, case, row): + checked( + row, + 'cli_chain', + lambda: { + 'cli.identity': identity, + 'cli.pipeline_roundtrip': pipeline_roundtrip, + 'cli.invalid_credential': invalid_credential, + }[case['scenario']](suite, row), + ) + return {'result': case['expect']} diff --git a/tools/regression_contracts.json b/tools/regression_contracts.json index 2597653b9..a3d48169c 100644 --- a/tools/regression_contracts.json +++ b/tools/regression_contracts.json @@ -434,6 +434,9 @@ "priority": "P1", "description": "Invalid MCP transport is rejected without changing stored configuration", "bindings": [{"profile": "api", "case": "fixed_api_mcp", "checks": ["api_chain", "mcp_invalid_transport_preserved"]}, {"profile": "api", "case": "shuffled_api_mcp", "checks": ["api_chain", "mcp_invalid_transport_preserved"]}] - } + }, + {"id": "cli.identity", "area": "cli", "kind": "blackbox", "priority": "P2", "description": "CLI resolves the intended Workspace and discovers capabilities without freezing presentation", "bindings": [{"profile": "cli", "case": "cli.identity", "checks": ["cli_chain", "workspace_identity", "capability_discovery"]}]}, + {"id": "cli.pipeline_roundtrip", "area": "cli", "kind": "blackbox", "priority": "P2", "description": "CLI mutations preserve submitted and existing Pipeline state, with independent HTTP readback", "bindings": [{"profile": "cli", "case": "cli.pipeline_roundtrip", "checks": ["cli_chain", "create_identity", "cli_get", "cli_list", "create_state", "update_state"]}]}, + {"id": "cli.invalid_credential", "area": "cli", "kind": "blackbox", "priority": "P2", "description": "CLI rejects invalid credentials with a nonzero exit and a diagnostic signal", "bindings": [{"profile": "cli", "case": "cli.invalid_credential", "checks": ["cli_chain", "credential_rejected", "error_signal"]}]} ] } diff --git a/tools/regression_local_cases.py b/tools/regression_local_cases.py index b935f4527..547344366 100644 --- a/tools/regression_local_cases.py +++ b/tools/regression_local_cases.py @@ -15,6 +15,7 @@ GROUPS = { 'lifecycle': 'regression_lifecycle_cases', 'plugins': 'regression_plugin_cases', 'api': 'regression_api_cases', + 'cli': 'regression_cli_cases', } diff --git a/tools/regression_suite.py b/tools/regression_suite.py index ba5bd0073..98e614631 100644 --- a/tools/regression_suite.py +++ b/tools/regression_suite.py @@ -568,6 +568,11 @@ class Suite: error = error.replace(secret, '[REDACTED]') raise RuntimeError(error[:3000]) response = json.loads(result.stdout) + if not isinstance(response, dict) or response.get('ok') is False or response.get('error'): + error = result.stdout + for secret in self.secrets: + error = error.replace(secret, '[REDACTED]') + raise RuntimeError('CLI returned a failure response: ' + error[:3000]) if args == ['whoami']: self.connection = response.get('meta') or {} return response['data'] diff --git a/tools/test_regression_cli_cases.py b/tools/test_regression_cli_cases.py new file mode 100644 index 000000000..d22d0a226 --- /dev/null +++ b/tools/test_regression_cli_cases.py @@ -0,0 +1,88 @@ +"""Keep CLI presentation flexible while rejecting wrong identity and false success.""" + +from pathlib import Path +from types import SimpleNamespace +import tempfile +import json +import unittest +from unittest.mock import Mock, patch + +from regression_cli_cases import identity, invalid_credential, pipeline_roundtrip +from regression_ci import suite_profile +from regression_suite import Suite, validate + + +class CLICompatibilityTests(unittest.TestCase): + def test_wrong_workspace_fails_even_when_cli_exits_successfully(self): + suite = SimpleNamespace( + call=Mock(side_effect=[{'version': 'future', 'new_field': True}, {'workspace_uuid': 'wrong'}]), + variables={'workspace_uuid': 'expected'}, + ) + row = {} + with self.assertRaisesRegex(ValueError, 'different Workspace'): + identity(suite, row) + self.assertEqual(row['assertions']['workspace_identity'], 'failed') + + def test_invalid_credential_must_have_nonzero_exit(self): + suite = SimpleNamespace( + connection={'credential_source': 'env:QA_KEY'}, + cli=['lbctl', '-o', 'json'], + env={'QA_KEY': 'secret'}, + timeout=lambda value: value, + ) + with patch( + 'regression_cli_cases.subprocess.run', return_value=SimpleNamespace(returncode=0, stdout='{}', stderr='') + ): + with self.assertRaisesRegex(ValueError, 'success with an invalid credential'): + invalid_credential(suite, {}) + + def test_successful_cli_create_without_backend_state_fails(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + suite = Suite( + {'cli': {'config': 'unused'}, 'cases': [], 'vars': {'workspace_uuid': 'test'}}, path / 'p.json', path + ) + suite.call = Mock(return_value={'uuid': 'reported-but-not-created'}) + row = {} + with ( + patch('regression_cli_cases.request', return_value={'pipelines': []}), + patch('regression_api_checks.api', return_value=(200, {'code': 0, 'data': {'pipelines': []}})), + ): + with self.assertRaisesRegex(ValueError, 'exactly one owned Pipeline'): + pipeline_roundtrip(suite, row) + self.assertEqual(row['owned_resources'][0]['state'], 'absent') + + def test_zero_exit_does_not_hide_an_explicit_failure_envelope(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) + suite = Suite({'cli': {'config': 'unused'}, 'cases': []}, path / 'p.json', path, env={}) + for response in ( + {'ok': False, 'data': {'requested_uuid': 'owned'}}, + {'error': {'type': 'failed'}, 'data': {'uuid': 'owned'}}, + ): + process = SimpleNamespace(returncode=0, stdout=json.dumps(response), stderr='') + with self.subTest(response=response), patch('regression_suite.subprocess.run', return_value=process): + with self.assertRaisesRegex(RuntimeError, 'failure response'): + suite.call(['pipeline', 'get', 'owned']) + process = SimpleNamespace( + returncode=0, + stdout=json.dumps( + {'ok': True, 'new_field': 'allowed', 'data': {'pipeline': {'uuid': 'owned', 'extra': 1}}} + ), + stderr='', + ) + with patch('regression_suite.subprocess.run', return_value=process): + self.assertEqual(suite.call(['pipeline', 'get', 'owned'])['pipeline']['uuid'], 'owned') + + def test_profile_is_finite_and_uses_no_shared_resources(self): + profile, _ = suite_profile('cli', 'unit-test') + validate(profile) + self.assertEqual(profile['resources'], []) + self.assertEqual(len(profile['cases']), 3) + self.assertNotIn('execution', profile) + smoke, _ = suite_profile('smoke', 'unit-test') + self.assertFalse(any(case.get('scenario', '').startswith('cli.') for case in smoke['cases'])) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/test_regression_coverage.py b/tools/test_regression_coverage.py index a0d87e941..3f54f9a94 100644 --- a/tools/test_regression_coverage.py +++ b/tools/test_regression_coverage.py @@ -289,10 +289,10 @@ class Routes: name = binding['profile'] if name not in profiles: path = Path(__file__).with_name('regression_profiles') / f'{name}.json' - if name == 'api': + if name in ('api', 'cli'): from regression_ci import suite_profile - profile, _ = suite_profile('api', 'catalog-test') + profile, _ = suite_profile(name, 'catalog-test') elif name == 'scenarios': from regression_scenarios import expanded_profile from regression_local_cases import DEFAULT_GROUPS