mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-25 01:45:23 +08:00
huanghuoguoguo
5ec41f15a1
* test(tools): add deterministic lbctl regression job * test(tools): simulate persistent multi-group conversations * test(tools): add finite core smoke with terminal run and file oracles * test(tools): expand smoke coverage across execution stages * ci: add a timed real core smoke and lightweight tool checks * fix(ci): keep infrastructure failures visible in smoke verdicts
206 lines
8.9 KiB
Python
206 lines
8.9 KiB
Python
"""Challenge the smoke verdict independently of a successful model reply."""
|
|
|
|
import base64
|
|
import copy
|
|
import json
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
|
|
from regression_smoke import (
|
|
UnresolvedRun,
|
|
check_attachment,
|
|
check_fields,
|
|
check_mock_requests,
|
|
check_run,
|
|
completed_run,
|
|
verify_cleanup,
|
|
)
|
|
from regression_suite import validate
|
|
|
|
|
|
class SmokeOracleTest(unittest.TestCase):
|
|
def setUp(self):
|
|
self.db = sqlite3.connect(':memory:')
|
|
self.db.row_factory = sqlite3.Row
|
|
self.addCleanup(self.db.close)
|
|
self.db.executescript("""
|
|
CREATE TABLE agent_run (id INTEGER, run_id TEXT, workspace_id TEXT,
|
|
conversation_id TEXT, status TEXT, status_reason TEXT, finished_at TEXT);
|
|
CREATE TABLE agent_run_event (run_id TEXT, sequence INTEGER, type TEXT, data_json TEXT);
|
|
INSERT INTO agent_run VALUES (1, 'run', 'workspace', 'conversation', 'completed', '', 'now');
|
|
INSERT INTO agent_run_event VALUES ('run', 1, 'run.completed', '{}');
|
|
""")
|
|
|
|
def run_row(self, workspace='workspace', after=0):
|
|
return completed_run(self.db, workspace, 'conversation', after, 0)
|
|
|
|
def test_terminal_status_and_event_are_both_required(self):
|
|
self.assertTrue(check_run(self.db, self.run_row(), {})['terminal_event'])
|
|
self.db.execute("UPDATE agent_run SET status='failed'")
|
|
with self.assertRaises(ValueError):
|
|
check_run(self.db, self.run_row(), {})
|
|
self.db.execute("UPDATE agent_run SET status='completed'")
|
|
self.db.execute('DELETE FROM agent_run_event')
|
|
with self.assertRaises(ValueError):
|
|
check_run(self.db, self.run_row(), {})
|
|
|
|
def test_missing_stale_wrong_workspace_and_ambiguous_runs_are_unknown(self):
|
|
for workspace, after in [('other', 0), ('workspace', 1)]:
|
|
with self.subTest(workspace=workspace, after=after), self.assertRaises(UnresolvedRun):
|
|
self.run_row(workspace, after)
|
|
self.db.execute("UPDATE agent_run SET status='running', finished_at=NULL")
|
|
with self.assertRaises(UnresolvedRun):
|
|
self.run_row()
|
|
self.db.execute('INSERT INTO agent_run SELECT * FROM agent_run')
|
|
with self.assertRaises(UnresolvedRun):
|
|
self.run_row()
|
|
|
|
def test_box_requires_actual_backend_success_and_output(self):
|
|
case = {'box_exec': {'count': 1, 'backend': 'docker', 'stdout': 'sentinel'}}
|
|
good = {'backend': 'docker', 'exit_code': 0, 'stdout': 'sentinel'}
|
|
for result, passes in [
|
|
(good, True),
|
|
({**good, 'backend': 'host'}, False),
|
|
({**good, 'exit_code': 1}, False),
|
|
({**good, 'stdout': ''}, False),
|
|
(None, False),
|
|
]:
|
|
self.db.execute('DELETE FROM agent_run_event WHERE sequence=2')
|
|
self.db.execute(
|
|
'INSERT INTO agent_run_event VALUES (?, ?, ?, ?)',
|
|
('run', 2, 'tool.call.completed', json.dumps({'tool_name': 'exec', 'result': result})),
|
|
)
|
|
with self.subTest(result=result):
|
|
if passes:
|
|
check_run(self.db, self.run_row(), case)
|
|
else:
|
|
with self.assertRaises(ValueError):
|
|
check_run(self.db, self.run_row(), case)
|
|
with self.assertRaises(ValueError):
|
|
check_run(self.db, self.run_row(), {'box_exec': {**case['box_exec'], 'count': 2}})
|
|
|
|
def test_attachment_requires_exact_bytes_and_single_file(self):
|
|
expected = {'name': 'out.txt', 'text': 'sentinel'}
|
|
part = {'type': 'File', 'name': 'out.txt', 'base64': base64.b64encode(b'sentinel').decode()}
|
|
data = {'replies': [{'message_chain': [part]}]}
|
|
self.assertTrue(check_attachment(data, expected)['content_matched'])
|
|
for parts in [
|
|
[],
|
|
[part, part],
|
|
[{**part, 'base64': '!!!'}],
|
|
[{**part, 'base64': 'd3Jvbmc='}],
|
|
[{**part, 'name': 'other'}],
|
|
[None],
|
|
]:
|
|
with self.subTest(parts=parts), self.assertRaises(ValueError):
|
|
check_attachment({'replies': [{'message_chain': parts}]}, expected)
|
|
|
|
def test_fields_check_empty_results_and_missing_paths(self):
|
|
check_fields({'results': []}, {'results': []})
|
|
check_fields({'results': [{'metadata': {'kb': 'owned'}}]}, {'results.0.metadata.kb': 'owned'})
|
|
for data in [{'results': [1]}, {}, {'results': None}]:
|
|
with self.subTest(data=data), self.assertRaises(ValueError):
|
|
check_fields(data, {'results': []})
|
|
with self.assertRaises(ValueError):
|
|
check_fields({'results': None}, {'results.0.metadata.kb': 'owned'})
|
|
|
|
def test_fallback_must_call_failed_primary_then_streaming_backup(self):
|
|
expected = [
|
|
{'model': 'primary', 'status': 429, 'stream': True},
|
|
{'model': 'backup', 'status': 200, 'stream': True},
|
|
]
|
|
with tempfile.TemporaryDirectory() as directory:
|
|
path = Path(directory) / 'requests.jsonl'
|
|
prefix = b'earlier request ignored\n'
|
|
for rows in [
|
|
expected,
|
|
list(reversed(expected)),
|
|
[expected[0], {**expected[1], 'stream': False}],
|
|
[expected[0], {**expected[1], 'status': 503}],
|
|
]:
|
|
path.write_bytes(
|
|
prefix
|
|
+ b''.join(
|
|
(
|
|
json.dumps(
|
|
{
|
|
'path': '/v1/chat/completions',
|
|
'status': row['status'],
|
|
'request': {'model': row['model'], 'stream': row['stream']},
|
|
}
|
|
)
|
|
+ '\n'
|
|
).encode()
|
|
for row in rows
|
|
)
|
|
)
|
|
if rows == expected:
|
|
self.assertEqual(check_mock_requests(path, len(prefix), expected), expected)
|
|
else:
|
|
with self.assertRaises(ValueError):
|
|
check_mock_requests(path, len(prefix), expected)
|
|
|
|
def test_expected_tool_failure_requires_exact_exit_and_stderr(self):
|
|
case = {'box_exec': {'count': 1, 'backend': 'docker', 'stdout': 'out', 'exit_code': 7, 'stderr': 'err'}}
|
|
for exit_code, stderr in [(7, 'err'), (0, 'err'), (7, '')]:
|
|
self.db.execute('DELETE FROM agent_run_event WHERE sequence=2')
|
|
self.db.execute(
|
|
'INSERT INTO agent_run_event VALUES (?, ?, ?, ?)',
|
|
(
|
|
'run',
|
|
2,
|
|
'tool.call.completed',
|
|
json.dumps(
|
|
{
|
|
'tool_name': 'exec',
|
|
'result': {'backend': 'docker', 'stdout': 'out', 'exit_code': exit_code, 'stderr': stderr},
|
|
}
|
|
),
|
|
),
|
|
)
|
|
if exit_code == 7 and stderr == 'err':
|
|
check_run(self.db, self.run_row(), case)
|
|
else:
|
|
with self.assertRaises(ValueError):
|
|
check_run(self.db, self.run_row(), case)
|
|
|
|
def test_cleanup_readback_rejects_resources_still_visible(self):
|
|
deleted = [{'kind': 'pipeline', 'uuid': 'owned'}]
|
|
evidence = {}
|
|
for visible in ['unrelated', 'owned']:
|
|
suite = SimpleNamespace(
|
|
call=lambda args: {'pipelines': [{'uuid': visible}]},
|
|
write=lambda name, value: evidence.update({name: value}),
|
|
)
|
|
if visible == 'owned':
|
|
with self.assertRaises(ValueError):
|
|
verify_cleanup(suite, deleted)
|
|
else:
|
|
verify_cleanup(suite, deleted)
|
|
self.assertEqual(evidence['cleanup-checks.json'][0]['remaining'], [])
|
|
suite.call = lambda args: {'unexpected': []}
|
|
with self.assertRaises(ValueError):
|
|
verify_cleanup(suite, deleted)
|
|
|
|
def test_profile_supports_mixed_cases_and_rejects_ignored_oracles(self):
|
|
profile = json.loads(Path(__file__).with_name('regression_profiles').joinpath('smoke.json').read_text())
|
|
validate(profile)
|
|
for changes in [{'thresholds': {'min_rps_ratio': 1}}, {'smoke': {'ledger_db': ''}}, {'phases': None}]:
|
|
with self.subTest(changes=changes), self.assertRaises(ValueError):
|
|
validate({**profile, **changes})
|
|
wrong = copy.deepcopy(profile)
|
|
next(case for case in wrong['cases'] if case['name'] == 'rag_retrieve')['box_exec'] = {
|
|
'count': 1,
|
|
'backend': 'docker',
|
|
'stdout': 'x',
|
|
}
|
|
with self.assertRaises(ValueError):
|
|
validate(wrong)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|