diff --git a/tools/README.md b/tools/README.md index 0d2a1838a..8aa270953 100644 --- a/tools/README.md +++ b/tools/README.md @@ -311,7 +311,10 @@ the Job stops when its finite cases finish. The current HTTP installation path does not promise automatic rollback after initialization failure. This Job verifies explicit recovery; the separate -`plugins.upgrade_recovery` coverage obligation remains unclaimed. It uses the +`plugins.upgrade_recovery` coverage obligation remains unclaimed. A failed +generation must be absent or reject Page calls. If Core instead restores the old +version automatically, that version must pass its full configuration, storage and +Page oracle. It uses the existing runner's stack preparation and local HTTP uploads, installs only the bundled fixtures without Marketplace access, and is not included in smoke or daily by default. Cases, task outcomes, worker identities and cleanup journals are saved diff --git a/tools/regression_contracts.json b/tools/regression_contracts.json index 50c6694e8..d4b560b31 100644 --- a/tools/regression_contracts.json +++ b/tools/regression_contracts.json @@ -270,7 +270,7 @@ {"id": "plugins.explicit_recovery", "area": "plugins", "kind": "blackbox", "priority": "P1", "description": "A failed plugin initialization reports failure and reinstalling a known good package restores persisted state and later upgrades", - "bindings": [{"profile": "plugin-lifecycle", "case": "plugin_failed_activation", "checks": ["broken_v2_task", "broken_v2_error", "restore_v1_task", "restored_v1_state", "recovered_v3_task", "v3_state", "plugin_cleanup"]}] + "bindings": [{"profile": "plugin-lifecycle", "case": "plugin_failed_activation", "checks": ["broken_v2_task", "broken_v2_error", "failed_generation_safe", "restore_v1_task", "restored_v1_state", "recovered_v3_task", "v3_state", "plugin_cleanup"]}] }, {"id": "plugins.observed_worker_cleanup", "area": "plugins", "kind": "graybox", "priority": "P1", diff --git a/tools/regression_plugin_lifecycle.py b/tools/regression_plugin_lifecycle.py index 5378e58a5..3ddaa1a4e 100644 --- a/tools/regression_plugin_lifecycle.py +++ b/tools/regression_plugin_lifecycle.py @@ -119,6 +119,48 @@ def metadata(plugin): return plugin['manifest']['manifest']['metadata'] +def failed_generation_safe(suite, row, path, verify_good): + def verify(): + status, response = api(suite, 'GET', path) + row['failed_generation'] = {'http_status': status, 'plugin': response.get('data', {}).get('plugin')} + if status == 404: + require(response.get('code') not in (None, 0), 'Missing plugin returned a successful business status') + return + require(status == 200 and response.get('code') == 0, 'Failed generation cannot be observed') + info = response.get('data', {}).get('plugin') + require(isinstance(info, dict), 'Failed generation metadata is missing') + identity = metadata(info) + require( + identity.get('author') == AUTHOR and identity.get('name') == path.rsplit('/', 1)[1], + 'Failed generation identity changed', + ) + if identity.get('version') == '0.0.1': + verify_good() # A future automatic restore must pass the complete old-version state oracle. + return + require( + identity.get('version') == '0.0.2' and info.get('status') in {'unmounted', 'mounted'}, + 'Failed plugin generation is initialized or has an unexpected version/state', + ) + page_status, page = api( + suite, + 'POST', + path + '/page-api', + { + 'page_id': 'probe', + 'endpoint': '/state', + 'method': 'GET', + 'body': None, + }, + ) + row['failed_generation']['page_status'] = page_status + require( + page_status in {400, 404, 409, 500, 503} and page.get('code') not in (None, 0), + 'Failed plugin generation remains callable', + ) + + checked(row, 'failed_generation_safe', verify) + + def workers_stopped(suite, workers): deadline = time.monotonic() + suite.timeout(10) while any(process_identity(int(pid)) == started for pid, started in workers.items()): @@ -231,9 +273,7 @@ def run_case(suite, case, row): installed('0.0.1', 'after_invalid_archive') elif case['scenario'] == 'plugin_failed_activation': upload(suite, row, package(name, '0.0.2', broken=True), 'broken_v2', expected='failed') - status, response = api(suite, 'GET', path) - row['failed_generation'] = {'http_status': status, 'plugin': response.get('data', {}).get('plugin')} - require(status in {200, 404}, 'Failed generation cannot be observed') + failed_generation_safe(suite, row, path, lambda: installed('0.0.1', 'automatic_restore_v1')) upload(suite, row, first, 'restore_v1') installed('0.0.1', 'restored_v1') upload(suite, row, package(name, '0.0.3'), 'recovered_v3') diff --git a/tools/test_regression_plugin_lifecycle.py b/tools/test_regression_plugin_lifecycle.py index ba5100e48..467ae1a20 100644 --- a/tools/test_regression_plugin_lifecycle.py +++ b/tools/test_regression_plugin_lifecycle.py @@ -6,11 +6,70 @@ from unittest.mock import Mock, patch from regression_ci import suite_profile from regression_local_cases import DEFAULT_GROUPS -from regression_plugin_lifecycle import wait_task, workers_stopped +from regression_plugin_lifecycle import failed_generation_safe, wait_task, workers_stopped from regression_suite import validate class PluginLifecycleTests(unittest.TestCase): + def test_failed_generation_cannot_pass_by_http_status_alone(self): + path = '/api/v1/plugins/regression-qa/probe' + + def plugin(version, status): + return ( + 200, + { + 'code': 0, + 'data': { + 'plugin': { + 'status': status, + 'manifest': { + 'manifest': { + 'metadata': {'author': 'regression-qa', 'name': 'probe', 'version': version}, + } + }, + } + }, + }, + ) + + for responses in ( + [(404, {'code': 0})], + [plugin('0.0.2', 'initialized')], + [plugin('0.0.9', 'mounted')], + [plugin('0.0.2', 'mounted'), (200, {'code': 0, 'data': {}})], + [plugin('0.0.2', 'mounted'), (500, {'code': 0})], + ): + row = {} + with self.subTest(responses=responses), patch('regression_plugin_lifecycle.api', side_effect=responses): + with self.assertRaises(ValueError): + failed_generation_safe(None, row, path, Mock()) + self.assertEqual(row['assertions']['failed_generation_safe'], 'failed') + + for responses in ( + [(404, {'code': -1})], + [plugin('0.0.2', 'mounted'), (400, {'code': -1})], + ): + row, verify_good = {}, Mock() + with self.subTest(responses=responses), patch('regression_plugin_lifecycle.api', side_effect=responses): + failed_generation_safe(None, row, path, verify_good) + self.assertEqual(row['assertions']['failed_generation_safe'], 'passed') + verify_good.assert_not_called() + + for state_error in (None, ValueError('Restored plugin lost configuration or stored data')): + row, verify_good = {}, Mock(side_effect=state_error) + with ( + self.subTest(state_error=state_error), + patch('regression_plugin_lifecycle.api', return_value=plugin('0.0.1', 'initialized')), + ): + if state_error: + with self.assertRaisesRegex(ValueError, 'lost configuration'): + failed_generation_safe(None, row, path, verify_good) + self.assertEqual(row['assertions']['failed_generation_safe'], 'failed') + else: + failed_generation_safe(None, row, path, verify_good) + self.assertEqual(row['assertions']['failed_generation_safe'], 'passed') + verify_good.assert_called_once_with() + def test_job_is_opt_in_and_does_not_expand_daily(self): profile, _ = suite_profile('plugin-lifecycle', 'run') validate(profile)