From 67419699b4c3b9893f260541cb7134a85687cf3b Mon Sep 17 00:00:00 2001 From: huanghuoguoguo <60681390+huanghuoguoguo@users.noreply.github.com> Date: Wed, 28 Oct 2026 17:11:57 +0800 Subject: [PATCH] test(regression): preserve cancellation evidence across Box retirement --- tools/mock_llm_scenarios.json | 6 ++-- tools/regression_execution_cases.py | 5 +++- tools/test_regression_execution_cases.py | 38 +++++++++++++++++++++--- 3 files changed, 41 insertions(+), 8 deletions(-) diff --git a/tools/mock_llm_scenarios.json b/tools/mock_llm_scenarios.json index 0f9aaf468..e6a6d091b 100644 --- a/tools/mock_llm_scenarios.json +++ b/tools/mock_llm_scenarios.json @@ -303,7 +303,7 @@ { "name": "exec", "arguments": { - "command": "test ! -e /tmp/langbot-{mock_scope}-ready && test ! -e /tmp/langbot-{mock_scope}-leak && printf SETUP_READY > /tmp/langbot-{mock_scope}-ready && cat /tmp/langbot-{mock_scope}-ready" + "command": "test ! -e /workspace/langbot-{mock_scope}-ready && test ! -e /workspace/langbot-{mock_scope}-leak && printf SETUP_READY > /workspace/langbot-{mock_scope}-ready && cat /workspace/langbot-{mock_scope}-ready" } } ] @@ -314,7 +314,7 @@ { "name": "exec", "arguments": { - "command": "sleep 8; printf LATE_SIDE_EFFECT > /tmp/langbot-{mock_scope}-leak", + "command": "sleep 8; printf LATE_SIDE_EFFECT > /workspace/langbot-{mock_scope}-leak", "timeout_sec": 20 } } @@ -333,7 +333,7 @@ { "name": "exec", "arguments": { - "command": "sleep 9; if test ! -f /tmp/langbot-{mock_scope}-ready; then printf INTERRUPT_SETUP_MISSING; exit 41; fi; if test -e /tmp/langbot-{mock_scope}-leak; then printf INTERRUPT_LATE_SIDE_EFFECT; exit 42; fi; rm /tmp/langbot-{mock_scope}-ready && printf INTERRUPT_CLEANUP_OK", + "command": "sleep 9; if test ! -f /workspace/langbot-{mock_scope}-ready; then printf INTERRUPT_SETUP_MISSING; exit 41; fi; if test -e /workspace/langbot-{mock_scope}-leak; then printf INTERRUPT_LATE_SIDE_EFFECT; exit 42; fi; rm /workspace/langbot-{mock_scope}-ready && printf INTERRUPT_CLEANUP_OK", "timeout_sec": 15 } } diff --git a/tools/regression_execution_cases.py b/tools/regression_execution_cases.py index afe666d0b..cfdbb372d 100644 --- a/tools/regression_execution_cases.py +++ b/tools/regression_execution_cases.py @@ -128,6 +128,7 @@ def _stream(suite, agent, scope, row, *, interrupt=None, box_session_id=None, fr 'command', '' ).startswith('sleep 8;'): row['box_session_id'] = checked(row, 'exec_active', lambda: _running_box(suite, box_session_id)) + row['box_before_interrupt'] = _sessions(suite).get(row['box_session_id']) if interrupt == 'cancel': break # Closing this public debug stream cancels its run. if frame.get('kind') in {'completed', 'error'}: @@ -299,13 +300,15 @@ def run_case(suite, case, row): 'exec_stopped', lambda: _poll( suite, - lambda: _sessions(suite).get(row['box_session_id'], {}).get('status') == 'idle', + lambda: _sessions(suite).get(row['box_session_id'], {}).get('status') in (None, 'idle'), 'Interrupted Box exec remains active', ), ) + row['box_after_interrupt'] = _sessions(suite).get(row['box_session_id']) config = _agent_config(suite, box_key, 'fault_interrupt_check_model', box=True) _data(suite, 'PUT', f'/api/v1/agents/{agent}', {'config': config}) recovery = _stream(suite, agent, scope, row, frame_key='recovery_frames') + row['box_after_recovery'] = _sessions(suite).get(row['box_session_id']) calls = _tool_events([frame['data'] for frame in recovery if frame.get('kind') == 'result'], 'exec') row['recovery_exec_results'] = calls try: diff --git a/tools/test_regression_execution_cases.py b/tools/test_regression_execution_cases.py index c42dda48c..c6d28284b 100644 --- a/tools/test_regression_execution_cases.py +++ b/tools/test_regression_execution_cases.py @@ -47,7 +47,15 @@ class ExecutionCasesTests(unittest.TestCase): ) def run_fixture( - self, scenario, *, tool_ok=True, fault=True, primary_status=None, recovery_status='completed', delete_ok=True + self, + scenario, + *, + tool_ok=True, + fault=True, + primary_status=None, + recovery_status='completed', + delete_ok=True, + recycled=False, ): expected = { 'execution.cancel_cleanup': 'cancelled', @@ -92,6 +100,8 @@ class ExecutionCasesTests(unittest.TestCase): if method == 'GET' and path == '/api/v1/agents/agent': return 404, {'code': -1} if path == '/api/v1/box/sessions': + if recycled and len(runs) == 1: + return 200, {'code': 0, 'data': []} return 200, {'code': 0, 'data': [{'session_id': box_session_id, 'status': 'idle'}]} if path.endswith('/events'): run_id = path.split('/')[-2] @@ -195,10 +205,27 @@ class ExecutionCasesTests(unittest.TestCase): self.assertEqual(self.row['recovery_exec_results'][0]['result']['exit_code'], 42) self.assertTrue(self.deleted) + def test_recycled_box_requires_the_same_persistent_file_check(self): + for scenario in ('execution.cancel_cleanup', 'execution.timeout_cleanup'): + with self.subTest(scenario=scenario): + with sqlite3.connect(self.ledger) as db: + db.execute('DELETE FROM agent_run') + result = self.run_fixture(scenario, recycled=True) + self.assertEqual(result['result'], 'FAULT_CASE_OK') + self.assertIsNone(self.row['box_after_interrupt']) + self.assertEqual(self.row['assertions']['no_late_side_effect'], 'passed') + with sqlite3.connect(self.ledger) as db: + db.execute('DELETE FROM agent_run') + with self.assertRaisesRegex(ValueError, 'INTERRUPT_LATE_SIDE_EFFECT'): + self.run_fixture(scenario, recycled=True, tool_ok=False, recovery_status='failed') + self.assertEqual(self.row['assertions']['no_late_side_effect'], 'failed') + def test_recovery_command_distinguishes_missing_setup_and_late_side_effect(self): scenarios = json.loads(Path(cases.__file__).with_name('mock_llm_scenarios.json').read_text()) command = scenarios['mock-local-interrupt-check']['steps'][0]['tool_calls'][0]['arguments']['command'] - command = command.removeprefix('sleep 9; ').replace('/tmp/langbot-{mock_scope}', str(self.root / 'fixture')) + command = command.removeprefix('sleep 9; ').replace( + '/workspace/langbot-{mock_scope}', str(self.root / 'fixture') + ) ready, leaked = self.root / 'fixture-ready', self.root / 'fixture-leak' for setup, leak, status, output in ( (False, False, 41, 'INTERRUPT_SETUP_MISSING'), @@ -253,7 +280,10 @@ class ExecutionCasesTests(unittest.TestCase): try: self.assertEqual(cases._stream(self.suite, 'agent', 'scope', {})[-1]['kind'], 'completed') row = {} - with patch.object(cases, '_running_box', return_value='box'): + with ( + patch.object(cases, '_running_box', return_value='box'), + patch.object(cases, '_sessions', return_value={'box': {'status': 'running'}}), + ): cancelled = cases._stream(self.suite, 'agent', 'scope', row, interrupt='cancel') self.assertEqual(len(cancelled), 1) self.assertEqual(row['box_session_id'], 'box') @@ -292,7 +322,7 @@ class ExecutionCasesTests(unittest.TestCase): for scope in ('run-a', 'run-b'): call = request(scope)['choices'][0]['message']['tool_calls'][0] command = json.loads(call['function']['arguments'])['command'] - self.assertIn(f'/tmp/langbot-{scope}-ready', command) + self.assertIn(f'/workspace/langbot-{scope}-ready', command) self.assertNotIn('{mock_scope}', command) commands.append(command) self.assertNotEqual(commands[0], commands[1])