mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-22 08:25:21 +08:00
refactor(provider): decouple Groq message sanitization from reasoning family
Use the chat provider to apply transport-specific metadata cleanup. Keep finish_reason removal in the common message conversion path. Verify streaming and non-streaming HTTP payloads through LiteLLM, metadata preservation for other providers, and source message immutability.
This commit is contained in:
@@ -878,14 +878,6 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
|
||||
else:
|
||||
msg_dict.pop('provider_specific_fields', None)
|
||||
|
||||
if reasoning_family == 'groq':
|
||||
# Groq validates message and tool-call fields strictly and does
|
||||
# not accept LiteLLM's provider metadata envelope.
|
||||
msg_dict.pop('provider_specific_fields', None)
|
||||
for tool_call in msg_dict.get('tool_calls') or []:
|
||||
tool_call.pop('provider_specific_fields', None)
|
||||
tool_call.get('function', {}).pop('provider_specific_fields', None)
|
||||
|
||||
if isinstance(content, list):
|
||||
converted_parts = []
|
||||
for part in content:
|
||||
@@ -923,6 +915,17 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
|
||||
|
||||
return req_messages
|
||||
|
||||
@staticmethod
|
||||
def _adapt_messages_for_provider(messages: list[dict], chat_provider: str | None) -> None:
|
||||
"""Adapt serialized request messages in place for the chat transport."""
|
||||
if chat_provider == 'groq':
|
||||
# Groq rejects LiteLLM's metadata envelope on messages and tool calls.
|
||||
for message in messages:
|
||||
message.pop('provider_specific_fields', None)
|
||||
for tool_call in message.get('tool_calls') or []:
|
||||
tool_call.pop('provider_specific_fields', None)
|
||||
tool_call.get('function', {}).pop('provider_specific_fields', None)
|
||||
|
||||
_THINK_PATTERNS: tuple[str, ...] = (
|
||||
r'^\s*(?:(?!<think>).)*?</think>\s*',
|
||||
r'^\s*(?:(?!CRETIRE_REASONING_BEGINk).)*?CRETIRE_REASONING_ENDk\s*',
|
||||
@@ -1339,6 +1342,7 @@ class LiteLLMRequester(requester.ProviderAPIRequester):
|
||||
# OpenRouter shares requester metadata with OpenAI-compatible non-chat
|
||||
# operations, but chat needs its own adapter to preserve cache markers.
|
||||
chat_provider = 'openrouter' if self._uses_openrouter_protocol() else self._get_custom_llm_provider()
|
||||
self._adapt_messages_for_provider(req_messages, chat_provider)
|
||||
model_name = self._build_litellm_model_name(model.model_entity.name, chat_provider)
|
||||
api_key = model.provider.token_mgr.get_token()
|
||||
|
||||
|
||||
@@ -4,15 +4,17 @@ This includes provider_specific_fields round-trip coverage for GitHub issue
|
||||
#1899 and token counting preflight behavior for Runner context budgeting.
|
||||
"""
|
||||
|
||||
import json
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock, patch
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import langbot_plugin.api.entities.builtin.provider.message as provider_message
|
||||
import langbot_plugin.api.entities.builtin.resource.tool as resource_tool
|
||||
|
||||
from langbot.pkg.provider.modelmgr import requester as model_requester
|
||||
from langbot.pkg.provider.modelmgr.requesters.litellmchat import LiteLLMRequester
|
||||
from langbot.pkg.provider.modelmgr.requesters.litellmchat import LiteLLMRequester, _LeasedHTTPHandler
|
||||
|
||||
|
||||
def _make_requester() -> LiteLLMRequester:
|
||||
@@ -172,14 +174,9 @@ def test_response_metadata_removal_preserves_gemini_signatures():
|
||||
assert out[0]['provider_specific_fields'] == {'thought_signatures': ['signature']}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('stream', [False, True])
|
||||
async def test_groq_completion_drops_provider_metadata_from_replayed_messages(stream):
|
||||
req = _make_configured_requester()
|
||||
req.requester_cfg['custom_llm_provider'] = 'groq'
|
||||
model = _make_runtime_model()
|
||||
model.model_entity.name = 'openai/gpt-oss-120b'
|
||||
msg = provider_message.Message(
|
||||
@pytest.fixture
|
||||
def replayed_tool_message():
|
||||
return provider_message.Message(
|
||||
role='assistant',
|
||||
content='Hello',
|
||||
provider_specific_fields={'finish_reason': 'tool_calls', 'other_response_metadata': True},
|
||||
@@ -193,17 +190,90 @@ async def test_groq_completion_drops_provider_metadata_from_replayed_messages(st
|
||||
],
|
||||
)
|
||||
|
||||
args = await req._build_completion_args(model, [msg], None, {}, stream=stream)
|
||||
|
||||
assert args['model'] == 'groq/openai/gpt-oss-120b'
|
||||
assert args['messages'] == [
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('stream', [False, True])
|
||||
@pytest.mark.parametrize('chat_provider,family', [('groq', 'openai'), ('gemini', 'groq'), ('openrouter', 'groq')])
|
||||
async def test_completion_metadata_adaptation_uses_chat_provider(stream, chat_provider, family, replayed_tool_message):
|
||||
req = _make_configured_requester()
|
||||
req.requester_cfg['custom_llm_provider'] = chat_provider
|
||||
model = _make_runtime_model()
|
||||
model.model_entity.name = 'openai/gpt-oss-120b'
|
||||
msg = replayed_tool_message
|
||||
original = msg.model_dump()
|
||||
|
||||
with patch.object(req, '_reasoning_family', return_value=family):
|
||||
args = await req._build_completion_args(model, [msg], None, {}, stream=stream)
|
||||
|
||||
assert args['model'] == f'{chat_provider}/openai/gpt-oss-120b'
|
||||
expected = msg.model_dump(exclude_none=True)
|
||||
expected['provider_specific_fields'].pop('finish_reason')
|
||||
if chat_provider == 'groq':
|
||||
expected.pop('provider_specific_fields')
|
||||
expected['tool_calls'][0].pop('provider_specific_fields')
|
||||
assert args['messages'] == [expected]
|
||||
assert msg.model_dump() == original
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('stream', [False, True])
|
||||
async def test_groq_http_request_excludes_replayed_metadata(stream, replayed_tool_message):
|
||||
req = _make_configured_requester()
|
||||
req.requester_cfg.update(custom_llm_provider='groq', base_url='https://groq.test/openai/v1')
|
||||
model = _make_runtime_model()
|
||||
model.model_entity.name = 'openai/gpt-oss-120b'
|
||||
msg = replayed_tool_message
|
||||
original = msg.model_dump()
|
||||
bodies = []
|
||||
|
||||
def handle(request):
|
||||
assert request.url == 'https://groq.test/openai/v1/chat/completions'
|
||||
bodies.append(json.loads(request.content))
|
||||
response = {
|
||||
'id': 'completion-1',
|
||||
'object': 'chat.completion',
|
||||
'created': 1,
|
||||
'model': 'openai/gpt-oss-120b',
|
||||
'service_tier': 'on_demand',
|
||||
'choices': [{'index': 0, 'message': {'role': 'assistant', 'content': 'Done'}, 'finish_reason': 'stop'}],
|
||||
'usage': {'prompt_tokens': 1, 'completion_tokens': 1, 'total_tokens': 2},
|
||||
}
|
||||
if stream:
|
||||
response['object'] = 'chat.completion.chunk'
|
||||
response['choices'][0]['delta'] = response['choices'][0].pop('message')
|
||||
return httpx.Response(
|
||||
200,
|
||||
headers={'content-type': 'text/event-stream'},
|
||||
content=f'data: {json.dumps(response)}\n\ndata: [DONE]\n\n',
|
||||
)
|
||||
return httpx.Response(200, json=response)
|
||||
|
||||
messages = [
|
||||
provider_message.Message(role='user', content='Search'),
|
||||
msg,
|
||||
provider_message.Message(role='tool', tool_call_id='call-1', content='Found'),
|
||||
]
|
||||
async with httpx.AsyncClient(transport=httpx.MockTransport(handle)) as client:
|
||||
extra_args = {'client': _LeasedHTTPHandler(client, 10)}
|
||||
if stream:
|
||||
chunks = [chunk async for chunk in req.invoke_llm_stream(None, model, messages, extra_args=extra_args)]
|
||||
assert ''.join(chunk.content or '' for chunk in chunks) == 'Done'
|
||||
else:
|
||||
result, _ = await req.invoke_llm(None, model, messages, extra_args=extra_args)
|
||||
assert result.content == 'Done'
|
||||
|
||||
assert len(bodies) == 1
|
||||
assert bodies[0]['model'] == 'openai/gpt-oss-120b'
|
||||
assert bodies[0]['messages'] == [
|
||||
{'role': 'user', 'content': 'Search'},
|
||||
{
|
||||
'role': 'assistant',
|
||||
'content': 'Hello',
|
||||
'tool_calls': [{'id': 'call-1', 'type': 'function', 'function': {'name': 'search', 'arguments': '{}'}}],
|
||||
}
|
||||
},
|
||||
{'role': 'tool', 'tool_call_id': 'call-1', 'content': 'Found'},
|
||||
]
|
||||
assert msg.provider_specific_fields['finish_reason'] == 'tool_calls'
|
||||
assert msg.model_dump() == original
|
||||
|
||||
|
||||
def test_normalize_stream_tool_calls_preserves_provider_specific_fields():
|
||||
|
||||
Reference in New Issue
Block a user