mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-30 04:15:45 +08:00
256 lines
12 KiB
Python
256 lines
12 KiB
Python
"""Challenge plugin business oracles and cleanup without starting target services."""
|
|
|
|
import copy
|
|
import hashlib
|
|
import hmac
|
|
import json
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import tempfile
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
import regression_plugin_cases as plugins
|
|
from regression_scenarios import expanded_profile
|
|
|
|
|
|
def success(value=None):
|
|
return 200, {'code': 0, 'data': value}
|
|
|
|
|
|
class PluginCasesTests(unittest.TestCase):
|
|
def setUp(self):
|
|
temporary = tempfile.TemporaryDirectory()
|
|
self.addCleanup(temporary.cleanup)
|
|
self.root = Path(temporary.name)
|
|
self.profile = expanded_profile()
|
|
plugins.add_cases(self.profile)
|
|
self.records = self.root / 'mock.jsonl'
|
|
self.records.write_text(
|
|
json.dumps({'path': '/v1/chat/completions', 'request': {'model': 'earlier'}, 'status': 200}) + '\n'
|
|
)
|
|
self.database = self.root / 'ledger.db'
|
|
with sqlite3.connect(self.database) as connection:
|
|
connection.execute('CREATE TABLE agent_run (id INTEGER, workspace_id TEXT, bot_id TEXT)')
|
|
self.profile['smoke'].update(ledger_db=str(self.database), mock_records=str(self.records))
|
|
self.suite = SimpleNamespace(
|
|
profile=self.profile,
|
|
variables={
|
|
'workspace_uuid': 'workspace',
|
|
'chat_pipeline': 'original-pipeline',
|
|
'plugin_command_pipeline': 'isolated-pipeline',
|
|
'group_secret': 'private-signing-secret',
|
|
},
|
|
secrets=['private-signing-secret'],
|
|
env={},
|
|
path=lambda value: value,
|
|
timeout=lambda value: value,
|
|
write=lambda name, value: (self.root / name).write_text(json.dumps(value)),
|
|
)
|
|
|
|
def case(self, scenario):
|
|
return next(case for case in self.profile['cases'] if case.get('scenario') == scenario)
|
|
|
|
def test_profile_adds_only_one_isolated_pipeline_and_two_scenarios(self):
|
|
original = expanded_profile()
|
|
profile = copy.deepcopy(original)
|
|
plugins.add_cases(profile)
|
|
self.assertEqual(profile['resources'][:-1], original['resources'])
|
|
self.assertEqual(profile['cases'][:-2], original['cases'])
|
|
pipeline = profile['resources'][-1]
|
|
self.assertEqual(pipeline['id'], 'plugin_command_pipeline')
|
|
self.assertEqual(
|
|
pipeline['body']['extensions_preferences'],
|
|
{
|
|
'enable_all_plugins': False,
|
|
'plugins': [
|
|
{'author': 'langbot-team', 'name': name}
|
|
for name in ('LocalAgent', 'LangRAG', 'GeneralParsers', 'EssentialCommands')
|
|
],
|
|
},
|
|
)
|
|
self.assertEqual({case['scenario'] for case in profile['cases'][-2:]}, plugins.CASE_NAMES)
|
|
self.assertTrue(all('http_group' not in case for case in profile['cases'][-2:]))
|
|
|
|
def command_api(self, failure=None):
|
|
state = {}
|
|
|
|
def request(suite, method, path, body=None, headers=None):
|
|
if path == '/api/v1/pipelines/isolated-pipeline':
|
|
return success(
|
|
{
|
|
'pipeline': {
|
|
'extensions_preferences': {
|
|
'enable_all_plugins': failure == 'bindings',
|
|
'plugins': copy.deepcopy(plugins.COMMAND_PLUGINS),
|
|
}
|
|
}
|
|
}
|
|
)
|
|
if path == '/api/v1/system/info':
|
|
return success({'version': '9.5.1'})
|
|
if path == '/api/v1/platform/bots' and method == 'POST':
|
|
self.assertEqual(body['event_bindings'][0]['target_uuid'], 'isolated-pipeline')
|
|
self.assertEqual(body['plugin_processors'], [])
|
|
self.assertEqual(body['adapter_config']['inbound_secret'], 'private-signing-secret')
|
|
state.update(uuid='owned-bot', name=body['name'])
|
|
return success({'uuid': 'owned-bot'})
|
|
if path == '/bots/owned-bot/sync':
|
|
self.assertEqual(body['session_type'], 'person')
|
|
self.assertEqual(body['message'], [{'type': 'Plain', 'text': '!version'}])
|
|
signed = headers['X-LB-Timestamp'].encode() + b'.' + json.dumps(body).encode()
|
|
expected = 'sha256=' + hmac.new(b'private-signing-secret', signed, hashlib.sha256).hexdigest()
|
|
self.assertEqual(headers['X-LB-Signature'], expected)
|
|
if failure == 'run':
|
|
with sqlite3.connect(self.database) as connection:
|
|
connection.execute("INSERT INTO agent_run VALUES (1, 'workspace', 'owned-bot')")
|
|
if failure == 'model':
|
|
with self.records.open('a') as stream:
|
|
stream.write(
|
|
json.dumps({'path': '/v1/chat/completions', 'request': {'model': 'wrong'}, 'status': 200})
|
|
+ '\n'
|
|
)
|
|
return success(
|
|
{
|
|
'session_id': 'wrong-session' if failure == 'session' else body['session_id'],
|
|
'reply_to': 'in_message',
|
|
'message': [
|
|
{
|
|
'type': 'Plain',
|
|
'text': '[bot] LangBot Version: ' + ('9.4.0' if failure == 'version' else '9.5.1'),
|
|
}
|
|
],
|
|
}
|
|
)
|
|
if path == '/api/v1/platform/bots/owned-bot':
|
|
if method == 'DELETE':
|
|
state['deleted'] = True
|
|
return success() # The real Bot DELETE returns data=null.
|
|
return (404, {'code': -1}) if state.get('deleted') else success({'bot': dict(state)})
|
|
self.fail(f'Unexpected command API call: {method} {path}')
|
|
|
|
return request, state
|
|
|
|
def test_command_checks_signed_reply_without_requiring_a_run(self):
|
|
request, state = self.command_api()
|
|
row = {'after_id': 0}
|
|
with patch.object(plugins, 'api', side_effect=request):
|
|
result = plugins.run_case(self.suite, self.case('plugins.command_version'), row)
|
|
self.assertEqual(
|
|
result, {'result': 'LANGBOT_PLUGIN_COMMAND_OK', 'version': '9.5.1', 'agent_runs': 0, 'model_requests': 0}
|
|
)
|
|
self.assertTrue(state['deleted'])
|
|
self.assertNotIn('scenario_runs', row)
|
|
self.assertTrue(all(value == 'passed' for value in row['assertions'].values()))
|
|
|
|
def test_command_rejects_bad_reply_model_calls_runs_and_plugin_leakage(self):
|
|
for failure in ('version', 'session', 'model', 'run', 'bindings'):
|
|
with self.subTest(failure=failure):
|
|
with sqlite3.connect(self.database) as connection:
|
|
connection.execute('DELETE FROM agent_run')
|
|
request, state = self.command_api(failure)
|
|
row = {'after_id': 0}
|
|
with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ValueError):
|
|
plugins.run_case(self.suite, self.case('plugins.command_version'), row)
|
|
if failure != 'bindings':
|
|
self.assertTrue(state['deleted'])
|
|
self.assertIn('failed', row['assertions'].values())
|
|
|
|
def faq_api(self, failure=None, cleanup_failure=False):
|
|
unrelated = {'id': 'unrelated', 'question': 'Existing question', 'answer': 'Existing answer'}
|
|
state = {'entries': [unrelated], 'created': 0, 'deletes': []}
|
|
|
|
def request(suite, method, path, body=None, headers=None):
|
|
self.assertEqual((method, path, body['page_id']), ('POST', plugins.FAQ_API, 'manager'))
|
|
verb, endpoint, payload = body['method'], body['endpoint'], body['body']
|
|
if (verb, endpoint) == ('GET', '/entries'):
|
|
return success({'entries': copy.deepcopy(state['entries'])})
|
|
if (verb, endpoint) == ('POST', '/entries'):
|
|
state['created'] += 1
|
|
entry = {'id': 'owned-faq', **payload}
|
|
state['entries'].append(entry)
|
|
state['original'] = copy.deepcopy(entry)
|
|
if failure == 'lost_create':
|
|
raise ConnectionResetError('creation response lost')
|
|
return success({'entry': copy.deepcopy(entry)})
|
|
if (verb, endpoint) == ('PUT', '/entries'):
|
|
self.assertEqual(payload['id'], 'owned-faq')
|
|
if failure == 'update_error':
|
|
raise ValueError('primary update failure')
|
|
if failure != 'stale_update':
|
|
state['entries'][1].update(payload)
|
|
return success({'entry': copy.deepcopy(payload)})
|
|
if (verb, endpoint) == ('POST', '/search'):
|
|
rows = list(state['entries'])
|
|
if failure == 'stale_search' and state['deletes']:
|
|
rows.append(state['original'])
|
|
return success(
|
|
{
|
|
'results': copy.deepcopy(
|
|
[
|
|
item
|
|
for item in rows
|
|
if payload['query'] in item['question'] or payload['query'] in item['answer']
|
|
]
|
|
)
|
|
}
|
|
)
|
|
if (verb, endpoint) == ('DELETE', '/entries'):
|
|
self.assertEqual(payload, {'id': 'owned-faq'})
|
|
if cleanup_failure:
|
|
raise RuntimeError('cleanup private-signing-secret failed')
|
|
state['deletes'].append(payload['id'])
|
|
state['entries'] = [item for item in state['entries'] if item['id'] != payload['id']]
|
|
return success({'deleted': payload['id']})
|
|
self.fail(f'Unexpected Page API call: {body}')
|
|
|
|
return request, state
|
|
|
|
def test_faq_crud_checks_content_and_removes_only_its_entry(self):
|
|
request, state = self.faq_api()
|
|
row = {}
|
|
with patch.object(plugins, 'api', side_effect=request):
|
|
result = plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
|
|
self.assertEqual(result, {'result': 'LANGBOT_PLUGIN_FAQ_OK', 'entry_id': 'owned-faq', 'deleted': True})
|
|
self.assertEqual(state['deletes'], ['owned-faq'])
|
|
self.assertEqual([entry['id'] for entry in state['entries']], ['unrelated'])
|
|
self.assertTrue(all(value == 'passed' for value in row['assertions'].values()))
|
|
|
|
def test_faq_rejects_stale_readback_and_search(self):
|
|
for failure, assertion in (('stale_update', 'faq_update_read'), ('stale_search', 'faq_delete')):
|
|
with self.subTest(failure=failure):
|
|
request, state = self.faq_api(failure)
|
|
row = {}
|
|
with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ValueError):
|
|
plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
|
|
self.assertEqual(row['assertions'][assertion], 'failed')
|
|
self.assertEqual(state['deletes'], ['owned-faq'])
|
|
self.assertEqual([entry['id'] for entry in state['entries']], ['unrelated'])
|
|
|
|
def test_faq_lost_create_response_recovers_owned_id_without_retrying_creation(self):
|
|
request, state = self.faq_api('lost_create')
|
|
row = {}
|
|
with patch.object(plugins, 'api', side_effect=request), self.assertRaises(ConnectionResetError):
|
|
plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
|
|
self.assertEqual(state['created'], 1)
|
|
self.assertEqual(state['deletes'], ['owned-faq'])
|
|
self.assertEqual(row['owned_resources'][0]['state'], 'deleted')
|
|
|
|
def test_faq_preserves_original_error_and_redacts_cleanup_error(self):
|
|
request, _ = self.faq_api('update_error', cleanup_failure=True)
|
|
row = {}
|
|
with (
|
|
patch.object(plugins, 'api', side_effect=request),
|
|
self.assertRaisesRegex(ValueError, 'primary update failure'),
|
|
):
|
|
plugins.run_case(self.suite, self.case('plugins.faq_crud'), row)
|
|
self.assertEqual(row['cleanup_errors'], ['RuntimeError: cleanup [REDACTED] failed'])
|
|
evidence = (self.root / row['resource_journal']).read_text()
|
|
self.assertNotIn('private-signing-secret', evidence)
|
|
self.assertIn('owned-faq', evidence)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
unittest.main()
|