test: repair stale skill/sandbox tests for feat/sandbox

The skill subsystem moved to Tool-Call activation and a Box-managed
skill store; several tests still asserted removed APIs and a sys.modules
stub leaked across the suite. Full unit suite now green (was 23 failing).

- test_skill_tools: drop TestSkillManagerActivation (text-marker API
  removed); rewrite TestSkillActivationHelper around the current
  skill.activation.register_activated_skill; replace the CRUD
  TestSkillAuthoringToolLoader with TestSkillToolLoader covering the
  current activate/register_skill tools and sandbox-availability gating
- test_tool_manager_native: ToolManager attr is skill_tool_loader (not
  skill_authoring_tool_loader); native loader now exposes 6 tools
  (exec/read/write/edit/glob/grep) and requires initialize() with a
  backend-available get_status()
- test_localagent_sandbox_exec: remove obsolete activation-marker
  leakage tests and their helper providers
- test_model_service / pipeline conftest: give the mocks skill_mgr=None
  so PreProcessor's local-agent skill-binding guard short-circuits
- test_n8nsvapi: stop permanently overwriting sys.modules
  ('langbot.pkg.provider.runner' etc.); save and restore around the
  import so other modules get the real LocalAgentRunner base class

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Junyan Qin
2026-05-19 22:56:25 +08:00
parent 5773e8aa27
commit 18ad51e21e
6 changed files with 214 additions and 507 deletions
+3
View File
@@ -34,6 +34,9 @@ class MockApplication:
self.query_pool = self._create_mock_query_pool() self.query_pool = self._create_mock_query_pool()
self.instance_config = self._create_mock_instance_config() self.instance_config = self._create_mock_instance_config()
self.task_mgr = self._create_mock_task_manager() self.task_mgr = self._create_mock_task_manager()
# Skill manager is optional; PreProcessor only touches it for the
# local-agent runner. None keeps the skill-binding branch inert.
self.skill_mgr = None
def _create_mock_logger(self): def _create_mock_logger(self):
logger = Mock() logger = Mock()
+29 -10
View File
@@ -14,24 +14,40 @@ import json
import sys import sys
from unittest.mock import AsyncMock, MagicMock, Mock, patch from unittest.mock import AsyncMock, MagicMock, Mock, patch
# Break the circular import chain before importing n8nsvapi:
# n8nsvapi → runner → app → pipelinemgr → all runners → runner (partially init)
_mock_runner = MagicMock()
_mock_runner.runner_class = lambda name: (lambda cls: cls) # no-op decorator
_mock_runner.RequestRunner = object
sys.modules.setdefault('langbot.pkg.provider.runner', _mock_runner)
sys.modules.setdefault('langbot.pkg.core.app', MagicMock())
sys.modules.setdefault('langbot.pkg.utils.httpclient', MagicMock())
import pytest import pytest
import langbot_plugin.api.entities.builtin.provider.message as provider_message import langbot_plugin.api.entities.builtin.provider.message as provider_message
from langbot.pkg.provider.runners.n8nsvapi import N8nServiceAPIRunner
# Break the circular import chain while importing n8nsvapi:
# n8nsvapi → runner → app → pipelinemgr → all runners → runner (partially init)
# The stubs are restored immediately afterwards so this module does NOT pollute
# sys.modules for other test modules (e.g. ones importing the real
# LocalAgentRunner, which would otherwise inherit ``object`` and break).
_runner_stub = MagicMock()
_runner_stub.runner_class = lambda name: (lambda cls: cls) # no-op decorator
_runner_stub.RequestRunner = object
_import_stubs = {
'langbot.pkg.provider.runner': _runner_stub,
'langbot.pkg.core.app': MagicMock(),
'langbot.pkg.utils.httpclient': MagicMock(),
}
_saved_modules = {name: sys.modules.get(name) for name in _import_stubs}
for _name, _stub in _import_stubs.items():
sys.modules[_name] = _stub
try:
from langbot.pkg.provider.runners.n8nsvapi import N8nServiceAPIRunner
finally:
for _name, _original in _saved_modules.items():
if _original is None:
sys.modules.pop(_name, None)
else:
sys.modules[_name] = _original
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Helpers # Helpers
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def make_runner(output_key: str = 'response') -> N8nServiceAPIRunner: def make_runner(output_key: str = 'response') -> N8nServiceAPIRunner:
ap = Mock() ap = Mock()
ap.logger = Mock() ap.logger = Mock()
@@ -74,6 +90,7 @@ async def collect_chunks(runner: N8nServiceAPIRunner, chunks: list[bytes | str])
# _process_response: stream format (type:item/end) # _process_response: stream format (type:item/end)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_stream_format_single_item(): async def test_stream_format_single_item():
"""Single item + end in one chunk yields final chunk with full content.""" """Single item + end in one chunk yields final chunk with full content."""
@@ -153,6 +170,7 @@ async def test_stream_format_no_spurious_empty_yield():
# _process_response: plain JSON fallback # _process_response: plain JSON fallback
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_plain_json_with_output_key(): async def test_plain_json_with_output_key():
"""Plain JSON with matching output_key extracts value via output_key.""" """Plain JSON with matching output_key extracts value via output_key."""
@@ -223,6 +241,7 @@ async def test_invalid_json_returns_raw_text():
# _call_webhook: output type depends on is_stream # _call_webhook: output type depends on is_stream
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
def make_query(is_stream: bool): def make_query(is_stream: bool):
"""Build a minimal Query mock.""" """Build a minimal Query mock."""
query = Mock() query = Mock()
@@ -91,101 +91,6 @@ class RecordingStreamProvider:
return _stream() return _stream()
class ActivationProvider:
def __init__(self):
self.requests: list[dict] = []
async def invoke_llm(self, query, model, messages, funcs, extra_args=None, remove_think=None):
self.requests.append(
{
'messages': list(messages),
'funcs': list(funcs),
'remove_think': remove_think,
}
)
if len(self.requests) == 1:
return provider_message.Message(
role='assistant',
content='[ACTIVATE_SKILL: demo]\nI will use the skill.',
)
return provider_message.Message(
role='assistant',
content='final answer after activation',
)
class FailingActivationProvider:
def __init__(self):
self.requests: list[dict] = []
async def invoke_llm(self, query, model, messages, funcs, extra_args=None, remove_think=None):
self.requests.append(
{
'messages': list(messages),
'funcs': list(funcs),
'remove_think': remove_think,
}
)
if len(self.requests) == 1:
return provider_message.Message(
role='assistant',
content='[ACTIVATE_SKILL: demo]\nI will use the skill.',
)
raise RuntimeError('activation failed')
class ActivationStreamProvider:
def __init__(self):
self.stream_requests: list[dict] = []
def invoke_llm_stream(self, query, model, messages, funcs, extra_args=None, remove_think=None):
self.stream_requests.append(
{
'messages': list(messages),
'funcs': list(funcs),
'remove_think': remove_think,
}
)
async def _stream():
if len(self.stream_requests) == 1:
yield provider_message.MessageChunk(
role='assistant',
content='[ACTIVATE_SKILL: demo]\nI will use the skill.',
is_final=True,
)
return
yield provider_message.MessageChunk(
role='assistant',
content='final answer after activation',
is_final=True,
)
return _stream()
def make_skill_manager():
skill_data = {
'uuid': 'skill-demo',
'name': 'demo',
'instructions': 'Do the demo task.',
'type': 'skill',
'package_root': '/tmp/demo-skill',
'sandbox_timeout_sec': 120,
'sandbox_network': False,
}
return SimpleNamespace(
SKILL_ACTIVATION_MARKER='[ACTIVATE_SKILL:',
detect_skill_activations=Mock(
side_effect=lambda content: ['demo'] if '[ACTIVATE_SKILL: demo]' in (content or '') else []
),
build_activation_prompt_for_skills=Mock(return_value='skill prompt'),
get_skill_by_name=Mock(side_effect=lambda name: skill_data if name == 'demo' else None),
remove_activation_marker=Mock(side_effect=lambda content: (content or '').replace('[ACTIVATE_SKILL: demo]\n', '')),
)
def make_query() -> pipeline_query.Query: def make_query() -> pipeline_query.Query:
adapter = AsyncMock() adapter = AsyncMock()
adapter.is_stream_output_supported = AsyncMock(return_value=False) adapter.is_stream_output_supported = AsyncMock(return_value=False)
@@ -335,110 +240,3 @@ async def test_localagent_streaming_tool_error_yields_message_chunks():
assert all(isinstance(message, provider_message.MessageChunk) for message in results) assert all(isinstance(message, provider_message.MessageChunk) for message in results)
assert any(message.role == 'tool' and message.content == 'err: boom' for message in results) assert any(message.role == 'tool' and message.content == 'err: boom' for message in results)
@pytest.mark.asyncio
async def test_localagent_hides_activation_marker_before_follow_up_request():
provider = ActivationProvider()
model = SimpleNamespace(
provider=provider,
model_entity=SimpleNamespace(
uuid='test-model-uuid',
name='test-model',
abilities=['func_call'],
extra_args={},
),
)
app = SimpleNamespace(
logger=Mock(),
model_mgr=SimpleNamespace(get_model_by_uuid=AsyncMock(return_value=model)),
tool_mgr=SimpleNamespace(execute_func_call=AsyncMock()),
rag_mgr=SimpleNamespace(),
box_service=SimpleNamespace(get_system_guidance=Mock(return_value='sandbox guidance')),
skill_mgr=make_skill_manager(),
)
runner = LocalAgentRunner(app, pipeline_config={})
query = make_query()
query.use_funcs = []
results = [message async for message in runner.run(query)]
assert [(message.role, message.content) for message in results] == [
('assistant', 'final answer after activation')
]
assert len(provider.requests) == 2
assert provider.requests[1]['messages'][-2].content == 'I will use the skill.'
assert '[ACTIVATE_SKILL:' not in provider.requests[1]['messages'][-2].content
@pytest.mark.asyncio
async def test_localagent_activation_failure_rolls_back_query_state_and_sanitizes_response():
provider = FailingActivationProvider()
model = SimpleNamespace(
provider=provider,
model_entity=SimpleNamespace(
uuid='test-model-uuid',
name='test-model',
abilities=['func_call'],
extra_args={},
),
)
app = SimpleNamespace(
logger=Mock(),
model_mgr=SimpleNamespace(get_model_by_uuid=AsyncMock(return_value=model)),
tool_mgr=SimpleNamespace(execute_func_call=AsyncMock()),
rag_mgr=SimpleNamespace(),
box_service=SimpleNamespace(get_system_guidance=Mock(return_value='sandbox guidance')),
skill_mgr=make_skill_manager(),
)
runner = LocalAgentRunner(app, pipeline_config={})
query = make_query()
query.use_funcs = []
results = [message async for message in runner.run(query)]
assert [(message.role, message.content) for message in results] == [
('assistant', 'I will use the skill.')
]
assert query.use_funcs == []
assert query.variables == {}
@pytest.mark.asyncio
async def test_localagent_streaming_activation_does_not_leak_marker():
provider = ActivationStreamProvider()
model = SimpleNamespace(
provider=provider,
model_entity=SimpleNamespace(
uuid='test-model-uuid',
name='test-model',
abilities=['func_call'],
extra_args={},
),
)
adapter = AsyncMock()
adapter.is_stream_output_supported = AsyncMock(return_value=True)
app = SimpleNamespace(
logger=Mock(),
model_mgr=SimpleNamespace(get_model_by_uuid=AsyncMock(return_value=model)),
tool_mgr=SimpleNamespace(execute_func_call=AsyncMock()),
rag_mgr=SimpleNamespace(),
box_service=SimpleNamespace(get_system_guidance=Mock(return_value='sandbox guidance')),
skill_mgr=make_skill_manager(),
)
runner = LocalAgentRunner(app, pipeline_config={})
query = make_query()
query.adapter = adapter
query.use_funcs = []
results = [message async for message in runner.run(query)]
assert all(isinstance(message, provider_message.MessageChunk) for message in results)
assert [message.content for message in results] == ['final answer after activation']
@@ -69,6 +69,7 @@ async def test_updated_llm_model_is_immediately_usable_by_local_agent_pipeline()
ap.logger = Mock() ap.logger = Mock()
ap.persistence_mgr = SimpleNamespace(execute_async=AsyncMock()) ap.persistence_mgr = SimpleNamespace(execute_async=AsyncMock())
ap.tool_mgr = SimpleNamespace(get_all_tools=AsyncMock(return_value=[])) ap.tool_mgr = SimpleNamespace(get_all_tools=AsyncMock(return_value=[]))
ap.skill_mgr = None # PreProcessor only uses skill_mgr for the local-agent skill-binding branch
ap.plugin_connector = SimpleNamespace( ap.plugin_connector = SimpleNamespace(
emit_event=AsyncMock(return_value=SimpleNamespace(event=SimpleNamespace(default_prompt=[], prompt=[]))) emit_event=AsyncMock(return_value=SimpleNamespace(event=SimpleNamespace(default_prompt=[], prompt=[])))
) )
+165 -280
View File
@@ -79,52 +79,18 @@ class TestSkillManagerPackageLoading:
assert skill_data['description'] == 'Second' assert skill_data['description'] == 'Second'
class TestSkillManagerActivation:
def test_detect_skill_activations_returns_unique_ordered_skills(self):
from langbot.pkg.skill.manager import SkillManager
ap = _make_ap()
mgr = SkillManager(ap)
mgr.skills = {
'alpha': _make_skill_data(name='alpha'),
'beta': _make_skill_data(name='beta'),
}
response = '[ACTIVATE_SKILL: alpha]\n[ACTIVATE_SKILL: beta]\n[ACTIVATE_SKILL: alpha]\nLet me handle this.'
assert mgr.detect_skill_activations(response) == ['alpha', 'beta']
assert mgr.detect_skill_activation(response) == 'alpha'
def test_build_activation_prompt_for_skills_includes_runtime_guidance(self):
from langbot.pkg.skill.manager import SkillManager
ap = _make_ap()
mgr = SkillManager(ap)
mgr.skills = {
'primary': _make_skill_data(name='primary', instructions='Primary instructions'),
'aux': _make_skill_data(name='aux', instructions='Aux instructions'),
}
prompt = mgr.build_activation_prompt_for_skills(['primary', 'aux'])
assert 'Activated skills: primary, aux' in prompt
assert 'role="primary"' in prompt
assert 'role="auxiliary"' in prompt
assert '/workspace/.skills/<skill-name>' in prompt
def test_remove_activation_marker_removes_multiple_markers(self):
from langbot.pkg.skill.manager import SkillManager
ap = _make_ap()
mgr = SkillManager(ap)
response = '[ACTIVATE_SKILL: alpha]\n[ACTIVATE_SKILL: beta]\nFinal answer'
assert mgr.remove_activation_marker(response) == 'Final answer'
class TestSkillActivationHelper: class TestSkillActivationHelper:
def test_prepare_skill_activation_registers_only_explicit_activated_skills(self): """Skill activation is now Tool-Call based.
from langbot.pkg.skill.activation import prepare_skill_activation
The legacy text-marker mechanism (``[ACTIVATE_SKILL: x]`` detection,
``build_activation_prompt_for_skills``, ``remove_activation_marker``,
``prepare_skill_activation``) has been removed. Activation now goes
through ``skill.activation.register_activated_skill``, invoked by the
``activate`` Tool Call.
"""
def test_register_activated_skill_records_known_skill(self):
from langbot.pkg.skill.activation import register_activated_skill
from langbot.pkg.provider.tools.loaders.skill import ACTIVATED_SKILLS_KEY from langbot.pkg.provider.tools.loaders.skill import ACTIVATED_SKILLS_KEY
from langbot.pkg.skill.manager import SkillManager from langbot.pkg.skill.manager import SkillManager
@@ -132,45 +98,37 @@ class TestSkillActivationHelper:
mgr = SkillManager(ap) mgr = SkillManager(ap)
mgr.skills = { mgr.skills = {
'primary': _make_skill_data(name='primary', instructions='Primary instructions'), 'primary': _make_skill_data(name='primary', instructions='Primary instructions'),
'aux': _make_skill_data(name='aux', instructions='Aux instructions'),
} }
ap.skill_mgr = mgr ap.skill_mgr = mgr
query = SimpleNamespace(variables={}, use_funcs=[]) query = SimpleNamespace(variables={})
activation = prepare_skill_activation(
ap,
query,
'[ACTIVATE_SKILL: primary]\n[ACTIVATE_SKILL: aux]\nWorking on it.',
)
assert activation is not None assert register_activated_skill(ap, query, 'primary') is True
assert activation.activated_skill_names == ['primary', 'aux'] assert set(query.variables[ACTIVATED_SKILLS_KEY].keys()) == {'primary'}
assert activation.cleaned_content == 'Working on it.' assert query.variables[ACTIVATED_SKILLS_KEY]['primary']['name'] == 'primary'
assert set(query.variables[ACTIVATED_SKILLS_KEY].keys()) == {'primary', 'aux'}
def test_prepare_skill_activation_ignores_skills_not_bound_to_pipeline(self): def test_register_activated_skill_rejects_unknown_skill(self):
from langbot.pkg.skill.activation import prepare_skill_activation from langbot.pkg.skill.activation import register_activated_skill
from langbot.pkg.provider.tools.loaders.skill import ACTIVATED_SKILLS_KEY, PIPELINE_BOUND_SKILLS_KEY from langbot.pkg.provider.tools.loaders.skill import ACTIVATED_SKILLS_KEY
from langbot.pkg.skill.manager import SkillManager from langbot.pkg.skill.manager import SkillManager
ap = _make_ap() ap = _make_ap()
mgr = SkillManager(ap) mgr = SkillManager(ap)
mgr.skills = { mgr.skills = {'primary': _make_skill_data(name='primary')}
'primary': _make_skill_data(name='primary', instructions='Primary instructions'),
'hidden': _make_skill_data(name='hidden', instructions='Hidden instructions'),
}
ap.skill_mgr = mgr ap.skill_mgr = mgr
query = SimpleNamespace(variables={PIPELINE_BOUND_SKILLS_KEY: ['primary']}, use_funcs=[]) query = SimpleNamespace(variables={})
activation = prepare_skill_activation(
ap,
query,
'[ACTIVATE_SKILL: hidden]\n[ACTIVATE_SKILL: primary]\nWorking on it.',
)
assert activation is not None assert register_activated_skill(ap, query, 'missing') is False
assert activation.activated_skill_names == ['primary'] assert ACTIVATED_SKILLS_KEY not in query.variables
assert set(query.variables[ACTIVATED_SKILLS_KEY].keys()) == {'primary'}
def test_register_activated_skill_without_skill_manager_returns_false(self):
from langbot.pkg.skill.activation import register_activated_skill
ap = _make_ap() # no skill_mgr attribute
query = SimpleNamespace(variables={})
assert register_activated_skill(ap, query, 'primary') is False
class TestSkillPathHelpers: class TestSkillPathHelpers:
@@ -247,199 +205,97 @@ class TestSkillPathHelpers:
assert command.rstrip().endswith('python scripts/run.py') assert command.rstrip().endswith('python scripts/run.py')
class TestSkillAuthoringToolLoader: class TestSkillToolLoader:
"""The skill tool surface is now just ``activate`` + ``register_skill``.
The legacy CRUD authoring tools (create/list/get/update/delete/
import_skill_from_directory/reload_skills) were removed; skill CRUD is
handled by SkillService via the HTTP API / web UI instead.
"""
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_create_skill_creates_managed_prompt_only_skill(self): async def test_activate_returns_instructions_and_registers_skill(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import ( from langbot.pkg.provider.tools.loaders.skill_authoring import (
CREATE_SKILL_TOOL_NAME, ACTIVATE_SKILL_TOOL_NAME,
SkillAuthoringToolLoader, SkillToolLoader,
)
from langbot.pkg.provider.tools.loaders.skill import ACTIVATED_SKILLS_KEY
skill = _make_skill_data(name='demo', package_root='/data/skills/demo', instructions='Step 1')
ap = _make_ap()
ap.skill_mgr = SimpleNamespace(
skills={'demo': skill},
get_skill_by_name=lambda name: skill if name == 'demo' else None,
)
loader = SkillToolLoader(ap)
query = SimpleNamespace(variables={})
result = await loader.invoke_tool(ACTIVATE_SKILL_TOOL_NAME, {'skill_name': 'demo'}, query)
assert result['activated'] is True
assert result['skill_name'] == 'demo'
assert result['mount_path'] == '/workspace/.skills/demo'
assert 'Step 1' in result['content']
assert set(query.variables[ACTIVATED_SKILLS_KEY].keys()) == {'demo'}
@pytest.mark.asyncio
async def test_activate_unknown_skill_raises(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import (
ACTIVATE_SKILL_TOOL_NAME,
SkillToolLoader,
) )
ap = _make_ap() ap = _make_ap()
ap.skill_service = SimpleNamespace( ap.skill_mgr = SimpleNamespace(
create_skill=AsyncMock( skills={'demo': _make_skill_data(name='demo')},
return_value=_make_skill_data(name='prompt-skill', package_root='/data/skills/prompt-skill') get_skill_by_name=lambda name: None,
),
reload_skills=AsyncMock(),
list_skills=AsyncMock(return_value=[]),
) )
loader = SkillAuthoringToolLoader(ap) loader = SkillToolLoader(ap)
await loader.initialize()
result = await loader.invoke_tool( with pytest.raises(ValueError, match='not found'):
CREATE_SKILL_TOOL_NAME, await loader.invoke_tool(
{ ACTIVATE_SKILL_TOOL_NAME,
'name': 'prompt-skill', {'skill_name': 'ghost'},
'display_name': 'Prompt Skill', SimpleNamespace(variables={}),
'description': 'Prompt only skill', )
'instructions': 'Follow these steps carefully.',
},
SimpleNamespace(),
)
ap.skill_service.create_skill.assert_awaited_once_with(
{
'name': 'prompt-skill',
'display_name': 'Prompt Skill',
'description': 'Prompt only skill',
'instructions': 'Follow these steps carefully.',
}
)
assert result == {
'created': True,
'skill': _make_skill_data(name='prompt-skill', package_root='/data/skills/prompt-skill'),
}
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_list_skills_returns_managed_skills(self): async def test_register_skill_scans_directory_and_creates_skill(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import ( from langbot.pkg.provider.tools.loaders.skill_authoring import (
LIST_SKILLS_TOOL_NAME, REGISTER_SKILL_TOOL_NAME,
SkillAuthoringToolLoader, SkillToolLoader,
) )
ap = _make_ap()
ap.skill_service = SimpleNamespace(
list_skills=AsyncMock(return_value=[_make_skill_data(name='alpha'), _make_skill_data(name='beta')]),
)
loader = SkillAuthoringToolLoader(ap)
await loader.initialize()
result = await loader.invoke_tool(LIST_SKILLS_TOOL_NAME, {}, SimpleNamespace())
assert result == {
'skills': [_make_skill_data(name='alpha'), _make_skill_data(name='beta')],
'skill_names': ['alpha', 'beta'],
'count': 2,
}
@pytest.mark.asyncio
async def test_get_skill_returns_one_managed_skill(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import (
GET_SKILL_TOOL_NAME,
SkillAuthoringToolLoader,
)
ap = _make_ap()
ap.skill_service = SimpleNamespace(
get_skill=AsyncMock(return_value=_make_skill_data(name='time-now', package_root='/data/skills/time-now')),
)
loader = SkillAuthoringToolLoader(ap)
await loader.initialize()
result = await loader.invoke_tool(GET_SKILL_TOOL_NAME, {'name': 'time-now'}, SimpleNamespace())
ap.skill_service.get_skill.assert_awaited_once_with('time-now')
assert result == {
'skill': _make_skill_data(name='time-now', package_root='/data/skills/time-now'),
}
@pytest.mark.asyncio
async def test_update_skill_updates_managed_prompt_only_skill(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import (
UPDATE_SKILL_TOOL_NAME,
SkillAuthoringToolLoader,
)
ap = _make_ap()
ap.skill_service = SimpleNamespace(
create_skill=AsyncMock(),
update_skill=AsyncMock(
return_value=_make_skill_data(name='time-now', package_root='/data/skills/time-now')
),
reload_skills=AsyncMock(),
list_skills=AsyncMock(return_value=[]),
)
loader = SkillAuthoringToolLoader(ap)
await loader.initialize()
result = await loader.invoke_tool(
UPDATE_SKILL_TOOL_NAME,
{
'name': 'time-now',
'description': 'Fixed to Beijing time',
'instructions': 'Always use Asia/Shanghai and never offer other timezones.',
},
SimpleNamespace(),
)
ap.skill_service.update_skill.assert_awaited_once_with(
'time-now',
{
'name': 'time-now',
'description': 'Fixed to Beijing time',
'instructions': 'Always use Asia/Shanghai and never offer other timezones.',
},
)
assert result == {
'updated': True,
'skill': _make_skill_data(name='time-now', package_root='/data/skills/time-now'),
}
@pytest.mark.asyncio
async def test_delete_skill_deletes_managed_skill(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import (
DELETE_SKILL_TOOL_NAME,
SkillAuthoringToolLoader,
)
ap = _make_ap()
ap.skill_service = SimpleNamespace(
delete_skill=AsyncMock(return_value=True),
)
loader = SkillAuthoringToolLoader(ap)
await loader.initialize()
result = await loader.invoke_tool(DELETE_SKILL_TOOL_NAME, {'name': 'time-now'}, SimpleNamespace())
ap.skill_service.delete_skill.assert_awaited_once_with('time-now')
assert result == {
'deleted': True,
'skill_name': 'time-now',
}
@pytest.mark.asyncio
async def test_import_skill_from_directory_uses_workspace_path_and_service_import(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import (
IMPORT_SKILL_FROM_DIRECTORY_TOOL_NAME,
SkillAuthoringToolLoader,
)
ap = _make_ap()
ap.box_service = SimpleNamespace(default_workspace='/tmp/langbot-workspace')
ap.skill_service = SimpleNamespace(
scan_directory=Mock(
return_value={
'name': 'cloned-skill',
'display_name': 'Cloned Skill',
'description': 'Imported from clone',
'instructions': 'Do work',
}
),
create_skill=AsyncMock(return_value=_make_skill_data(name='cloned-skill', package_root='/repo/root')),
reload_skills=AsyncMock(),
list_skills=AsyncMock(return_value=[]),
)
loader = SkillAuthoringToolLoader(ap)
await loader.initialize()
with tempfile.TemporaryDirectory() as tmpdir: with tempfile.TemporaryDirectory() as tmpdir:
ap.box_service.default_workspace = tmpdir repo_dir = os.path.join(tmpdir, 'repo')
repo_dir = os.path.join(tmpdir, 'repos', 'cloned-skill')
os.makedirs(repo_dir) os.makedirs(repo_dir)
ap = _make_ap()
ap.box_service = SimpleNamespace(default_workspace=tmpdir, available=True)
ap.skill_service = SimpleNamespace(
scan_directory_async=AsyncMock(
return_value={
'name': 'cloned-skill',
'display_name': 'Cloned Skill',
'description': 'Imported from clone',
'instructions': 'Do work',
}
),
create_skill=AsyncMock(
return_value=_make_skill_data(name='cloned-skill', package_root=os.path.realpath(repo_dir))
),
)
loader = SkillToolLoader(ap)
result = await loader.invoke_tool( result = await loader.invoke_tool(
IMPORT_SKILL_FROM_DIRECTORY_TOOL_NAME, REGISTER_SKILL_TOOL_NAME,
{'path': '/workspace/repos/cloned-skill'}, {'path': '/workspace/repo'},
SimpleNamespace(), SimpleNamespace(),
) )
ap.skill_service.scan_directory.assert_called_once_with(os.path.realpath(repo_dir)) ap.skill_service.scan_directory_async.assert_awaited_once_with(os.path.realpath(repo_dir))
ap.skill_service.create_skill.assert_awaited_once_with( ap.skill_service.create_skill.assert_awaited_once_with(
{ {
'name': 'cloned-skill', 'name': 'cloned-skill',
@@ -449,59 +305,88 @@ class TestSkillAuthoringToolLoader:
'package_root': os.path.realpath(repo_dir), 'package_root': os.path.realpath(repo_dir),
} }
) )
assert result['imported'] is True assert result['registered'] is True
assert result['source_path'] == '/workspace/repos/cloned-skill' assert result['skill_name'] == 'cloned-skill'
assert result['source_path'] == '/workspace/repo'
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_import_skill_from_directory_rejects_workspace_escape(self): async def test_register_skill_rejects_workspace_escape(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import ( from langbot.pkg.provider.tools.loaders.skill_authoring import (
IMPORT_SKILL_FROM_DIRECTORY_TOOL_NAME, REGISTER_SKILL_TOOL_NAME,
SkillAuthoringToolLoader, SkillToolLoader,
) )
ap = _make_ap() with tempfile.TemporaryDirectory() as tmpdir:
ap.box_service = SimpleNamespace(default_workspace='/tmp/langbot-workspace') ap = _make_ap()
ap.skill_service = SimpleNamespace( ap.box_service = SimpleNamespace(default_workspace=tmpdir, available=True)
scan_directory=Mock(), ap.skill_service = SimpleNamespace(scan_directory_async=AsyncMock(), create_skill=AsyncMock())
create_skill=AsyncMock(),
reload_skills=AsyncMock(),
list_skills=AsyncMock(return_value=[]),
)
loader = SkillAuthoringToolLoader(ap) loader = SkillToolLoader(ap)
await loader.initialize()
with pytest.raises(ValueError, match='escapes the workspace boundary'): with pytest.raises(ValueError, match='escapes the workspace boundary'):
await loader.invoke_tool( await loader.invoke_tool(
IMPORT_SKILL_FROM_DIRECTORY_TOOL_NAME, REGISTER_SKILL_TOOL_NAME,
{'path': '/workspace/../../etc'}, {'path': '/workspace/../../etc'},
SimpleNamespace(), SimpleNamespace(),
) )
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_reload_skills_rescans_filesystem_and_returns_current_names(self): async def test_register_skill_requires_skill_service(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import ( from langbot.pkg.provider.tools.loaders.skill_authoring import (
RELOAD_SKILLS_TOOL_NAME, REGISTER_SKILL_TOOL_NAME,
SkillAuthoringToolLoader, SkillToolLoader,
) )
with tempfile.TemporaryDirectory() as tmpdir:
ap = _make_ap() # no skill_service attribute
ap.box_service = SimpleNamespace(default_workspace=tmpdir, available=True)
loader = SkillToolLoader(ap)
with pytest.raises(ValueError, match='Skill service not available'):
await loader.invoke_tool(
REGISTER_SKILL_TOOL_NAME,
{'path': '/workspace/foo'},
SimpleNamespace(),
)
@pytest.mark.asyncio
async def test_tools_hidden_when_sandbox_backend_unavailable(self):
from langbot.pkg.provider.tools.loaders.skill_authoring import SkillToolLoader
ap = _make_ap() ap = _make_ap()
ap.skill_service = SimpleNamespace( ap.skill_mgr = SimpleNamespace(skills={})
reload_skills=AsyncMock(), ap.box_service = SimpleNamespace(
list_skills=AsyncMock(return_value=[_make_skill_data(name='alpha'), _make_skill_data(name='beta')]), available=True,
get_status=AsyncMock(return_value={'backend': {'available': False}}),
) )
loader = SkillAuthoringToolLoader(ap) loader = SkillToolLoader(ap)
await loader.initialize() await loader.initialize()
result = await loader.invoke_tool(RELOAD_SKILLS_TOOL_NAME, {}, SimpleNamespace()) assert await loader.get_tools() == []
assert await loader.has_tool('activate') is False
assert await loader.has_tool('register_skill') is False
assert result == { @pytest.mark.asyncio
'reloaded': True, async def test_tools_exposed_when_sandbox_backend_available(self):
'skill_names': ['alpha', 'beta'], from langbot.pkg.provider.tools.loaders.skill_authoring import SkillToolLoader
'count': 2,
} ap = _make_ap()
ap.skill_service.reload_skills.assert_awaited_once_with() ap.skill_mgr = SimpleNamespace(skills={'demo': _make_skill_data(name='demo')})
ap.box_service = SimpleNamespace(
available=True,
get_status=AsyncMock(return_value={'backend': {'available': True}}),
)
loader = SkillToolLoader(ap)
await loader.initialize()
tools = await loader.get_tools()
assert sorted(tool.name for tool in tools) == ['activate', 'register_skill']
assert await loader.has_tool('activate') is True
assert await loader.has_tool('register_skill') is True
class TestNativeToolLoaderSkillPaths: class TestNativeToolLoaderSkillPaths:
@@ -3,7 +3,7 @@ from __future__ import annotations
import os import os
import tempfile import tempfile
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import Mock from unittest.mock import AsyncMock, Mock
import pytest import pytest
@@ -45,7 +45,7 @@ def make_tool(name: str) -> resource_tool.LLMTool:
async def test_tool_manager_omits_skill_authoring_tools_by_default(): async def test_tool_manager_omits_skill_authoring_tools_by_default():
manager = ToolManager(SimpleNamespace()) manager = ToolManager(SimpleNamespace())
manager.native_tool_loader = StubLoader([make_tool('exec')]) manager.native_tool_loader = StubLoader([make_tool('exec')])
manager.skill_authoring_tool_loader = StubLoader([make_tool('reload_skills')]) manager.skill_tool_loader = StubLoader([make_tool('activate')])
manager.plugin_tool_loader = StubLoader([make_tool('plugin_tool')]) manager.plugin_tool_loader = StubLoader([make_tool('plugin_tool')])
manager.mcp_tool_loader = StubLoader([make_tool('mcp_tool')]) manager.mcp_tool_loader = StubLoader([make_tool('mcp_tool')])
@@ -58,13 +58,13 @@ async def test_tool_manager_omits_skill_authoring_tools_by_default():
async def test_tool_manager_includes_skill_authoring_tools_when_requested(): async def test_tool_manager_includes_skill_authoring_tools_when_requested():
manager = ToolManager(SimpleNamespace()) manager = ToolManager(SimpleNamespace())
manager.native_tool_loader = StubLoader([make_tool('exec')]) manager.native_tool_loader = StubLoader([make_tool('exec')])
manager.skill_authoring_tool_loader = StubLoader([make_tool('reload_skills')]) manager.skill_tool_loader = StubLoader([make_tool('activate')])
manager.plugin_tool_loader = StubLoader([make_tool('plugin_tool')]) manager.plugin_tool_loader = StubLoader([make_tool('plugin_tool')])
manager.mcp_tool_loader = StubLoader([make_tool('mcp_tool')]) manager.mcp_tool_loader = StubLoader([make_tool('mcp_tool')])
tools = await manager.get_all_tools(include_skill_authoring=True) tools = await manager.get_all_tools(include_skill_authoring=True)
assert [tool.name for tool in tools] == ['exec', 'reload_skills', 'plugin_tool', 'mcp_tool'] assert [tool.name for tool in tools] == ['exec', 'activate', 'plugin_tool', 'mcp_tool']
@pytest.mark.asyncio @pytest.mark.asyncio
@@ -72,7 +72,7 @@ async def test_tool_manager_routes_native_tool_calls():
app = SimpleNamespace() app = SimpleNamespace()
manager = ToolManager(app) manager = ToolManager(app)
manager.native_tool_loader = StubLoader([make_tool('exec')], invoke_result={'backend': 'fake'}) manager.native_tool_loader = StubLoader([make_tool('exec')], invoke_result={'backend': 'fake'})
manager.skill_authoring_tool_loader = StubLoader([make_tool('reload_skills')]) manager.skill_tool_loader = StubLoader([make_tool('activate')])
manager.plugin_tool_loader = StubLoader([make_tool('plugin_tool')]) manager.plugin_tool_loader = StubLoader([make_tool('plugin_tool')])
manager.mcp_tool_loader = StubLoader([make_tool('mcp_tool')]) manager.mcp_tool_loader = StubLoader([make_tool('mcp_tool')])
@@ -86,23 +86,24 @@ async def test_native_tool_loader_hides_tools_when_box_unavailable():
loader = NativeToolLoader(SimpleNamespace(box_service=SimpleNamespace(available=False))) loader = NativeToolLoader(SimpleNamespace(box_service=SimpleNamespace(available=False)))
assert await loader.get_tools() == [] assert await loader.get_tools() == []
assert await loader.has_tool('exec') is False for tool_name in ('exec', 'read', 'write', 'edit', 'glob', 'grep'):
assert await loader.has_tool('read') is False assert await loader.has_tool(tool_name) is False
assert await loader.has_tool('write') is False
assert await loader.has_tool('edit') is False
@pytest.mark.asyncio @pytest.mark.asyncio
async def test_native_tool_loader_exposes_all_tools_when_box_available(): async def test_native_tool_loader_exposes_all_tools_when_box_available():
loader = NativeToolLoader(SimpleNamespace(box_service=SimpleNamespace(available=True))) box_service = SimpleNamespace(
available=True,
get_status=AsyncMock(return_value={'backend': {'available': True}}),
)
loader = NativeToolLoader(SimpleNamespace(box_service=box_service, logger=Mock()))
await loader.initialize()
tools = await loader.get_tools() tools = await loader.get_tools()
assert [tool.name for tool in tools] == ['exec', 'read', 'write', 'edit'] assert [tool.name for tool in tools] == ['exec', 'read', 'write', 'edit', 'glob', 'grep']
assert await loader.has_tool('exec') is True for tool_name in ('exec', 'read', 'write', 'edit', 'glob', 'grep'):
assert await loader.has_tool('read') is True assert await loader.has_tool(tool_name) is True
assert await loader.has_tool('write') is True
assert await loader.has_tool('edit') is True
# ── read/write/edit file tool tests ───────────────────────────── # ── read/write/edit file tool tests ─────────────────────────────