mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-16 14:57:15 +00:00
feat(provider): support Codex subscriptions with ChatGPT sign-in (#2513)
* feat(provider): support Codex subscriptions with ChatGPT sign-in * style: format Codex live integration test * fix(provider): preserve Codex identity in temporary model tests * fix(web): portal provider selector without dialog overflow * fix(web): allow native scrolling in provider dropdown * fix(provider): surface safe Codex quota and upstream errors * fix(web): provide reliable Codex copy feedback in dialogs * feat(provider): confirm cascade deletion from edit dialog * fix(persistence): discard connections after failed commit * fix(web): polish provider loading and confirmation motion --------- Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com>
This commit is contained in:
Hyu
@@ -0,0 +1,200 @@
|
||||
"""Replay synthetic HTTP/SSE traffic through the real Codex requester."""
|
||||
|
||||
import json
|
||||
from collections import OrderedDict
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import langbot_plugin.api.entities.builtin.provider.message as pm
|
||||
|
||||
from langbot.pkg.provider.modelmgr.requesters.codex import CodexRequester, sse_events
|
||||
|
||||
|
||||
TOKENS = {'access_token': 'access-secret', 'account_id': 'account', 'connection_id': 'connection'}
|
||||
MODEL = SimpleNamespace(model_entity=SimpleNamespace(name='codex-test', extra_args={}, reasoning_config=None))
|
||||
|
||||
|
||||
def requester(monkeypatch, handler):
|
||||
real_client = httpx.AsyncClient
|
||||
monkeypatch.setattr(
|
||||
httpx, 'AsyncClient', lambda **kwargs: real_client(transport=httpx.MockTransport(handler), **kwargs)
|
||||
)
|
||||
obj = object.__new__(CodexRequester)
|
||||
obj.workspace, obj.provider = 'w', 'p'
|
||||
obj._replay = OrderedDict()
|
||||
obj.auth = SimpleNamespace(access=AsyncMock(return_value=TOKENS))
|
||||
return obj
|
||||
|
||||
|
||||
def stream(events):
|
||||
return httpx.Response(200, content=''.join('data: ' + json.dumps(event) + '\r\n\r\n' for event in events))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_text_tools_usage_and_scoped_opaque_replay(monkeypatch):
|
||||
call = {'type': 'function_call', 'call_id': 'call_1', 'name': 'lookup', 'arguments': '{"q":"test"}'}
|
||||
output = [
|
||||
{'type': 'reasoning', 'encrypted_content': 'opaque-secret'},
|
||||
call,
|
||||
{'type': 'message', 'role': 'assistant', 'content': [{'type': 'output_text', 'text': 'Hello'}]},
|
||||
]
|
||||
requests = []
|
||||
|
||||
def handler(request):
|
||||
requests.append(request)
|
||||
return stream(
|
||||
[
|
||||
{'type': 'response.created', 'response': {'id': 'resp_1'}},
|
||||
{'type': 'response.output_text.delta', 'delta': 'Hel'},
|
||||
{'type': 'response.output_text.delta', 'delta': 'lo'},
|
||||
{'type': 'response.function_call_arguments.delta', 'delta': '{broken'},
|
||||
{'type': 'response.output_item.done', 'item': call, 'output_index': 1},
|
||||
{
|
||||
'type': 'response.completed',
|
||||
'response': {
|
||||
'id': 'resp_1',
|
||||
'status': 'completed',
|
||||
'output': output,
|
||||
'usage': {'input_tokens': 4, 'output_tokens': 3, 'input_tokens_details': {'cached_tokens': 2}},
|
||||
},
|
||||
},
|
||||
]
|
||||
)
|
||||
|
||||
obj = requester(monkeypatch, handler)
|
||||
query = SimpleNamespace(query_id='q', variables=None)
|
||||
messages = [pm.Message(role='system', content='Be brief'), pm.Message(role='user', content='Hi')]
|
||||
message, usage = await obj.invoke_llm(query, MODEL, messages)
|
||||
assert message.content == 'Hello'
|
||||
assert len(message.tool_calls) == 1
|
||||
assert message.tool_calls[0].function.arguments == '{"q":"test"}'
|
||||
assert usage['total_tokens'] == 7
|
||||
assert query.variables['_stream_usage'] == usage
|
||||
assert 'opaque-secret' not in message.model_dump_json()
|
||||
body = json.loads(requests[0].content)
|
||||
assert body['store'] is False and body['stream'] is True
|
||||
assert body['instructions'] == 'Be brief'
|
||||
assert requests[0].url.path.endswith('/codex/responses')
|
||||
assert requests[0].headers['authorization'] == 'Bearer access-secret'
|
||||
assert requests[0].headers['originator'] == 'langbot'
|
||||
same = obj._body(query, MODEL, [message], None, None, TOKENS)
|
||||
assert same['input'] == output
|
||||
other = obj._body(SimpleNamespace(query_id='q'), MODEL, [message], None, None, TOKENS)
|
||||
assert 'opaque-secret' not in json.dumps(other)
|
||||
rotated = obj._body(query, MODEL, [message], None, None, {**TOKENS, 'connection_id': 'new'})
|
||||
assert 'opaque-secret' not in json.dumps(rotated)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
'events',
|
||||
[
|
||||
[{'type': 'response.failed', 'error': 'access-secret'}],
|
||||
[{'type': 'response.incomplete'}],
|
||||
[{'type': 'error'}],
|
||||
[{'type': 'response.output_text.delta', 'delta': 'partial'}],
|
||||
[],
|
||||
],
|
||||
)
|
||||
async def test_failure_and_truncated_stream_never_succeed(monkeypatch, events):
|
||||
obj = requester(monkeypatch, lambda request: stream(events))
|
||||
with pytest.raises(ValueError) as caught:
|
||||
await obj.invoke_llm(None, MODEL, [])
|
||||
assert 'access-secret' not in str(caught.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_terminal_only_text_stream_and_usage(monkeypatch):
|
||||
obj = requester(
|
||||
monkeypatch,
|
||||
lambda request: stream(
|
||||
[
|
||||
{
|
||||
'type': 'response.done',
|
||||
'response': {
|
||||
'output': [{'type': 'message', 'content': [{'type': 'output_text', 'text': 'done'}]}],
|
||||
'usage': {'input_tokens': 2, 'output_tokens': 1},
|
||||
},
|
||||
}
|
||||
]
|
||||
),
|
||||
)
|
||||
query = SimpleNamespace(query_id='q', variables={})
|
||||
chunks = [chunk async for chunk in obj.invoke_llm_stream(query, MODEL, [])]
|
||||
assert ''.join(chunk.content or '' for chunk in chunks) == 'done'
|
||||
assert chunks[-1].is_final
|
||||
assert query.variables['_stream_usage']['total_tokens'] == 3
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('status', [401, 403, 429, 500])
|
||||
async def test_http_error_secrecy_and_bounded_401_retry(monkeypatch, status):
|
||||
requests = []
|
||||
|
||||
def handler(request):
|
||||
requests.append(request)
|
||||
return httpx.Response(status, text='access-secret refresh-secret')
|
||||
|
||||
obj = requester(monkeypatch, handler)
|
||||
with pytest.raises(ValueError) as caught:
|
||||
await obj.invoke_llm(None, MODEL, [])
|
||||
assert 'secret' not in str(caught.value)
|
||||
assert len(requests) == (2 if status == 401 else 1)
|
||||
if status == 401:
|
||||
assert obj.auth.access.call_args.kwargs == {'rejected_token': 'access-secret'}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_catalog_mapping_filtering_and_deduplication(monkeypatch):
|
||||
requests = []
|
||||
|
||||
def handler(request):
|
||||
requests.append(request)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
'models': [
|
||||
{'slug': 'a', 'input_modalities': ['text', 'image'], 'supported_reasoning_levels': ['low']},
|
||||
{'slug': 'hidden', 'visibility': 'hide'},
|
||||
{'slug': 'a', 'input_modalities': ['text', 'image'], 'supported_reasoning_levels': ['low']},
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
obj = requester(monkeypatch, handler)
|
||||
catalog = await obj.scan_models()
|
||||
assert len(catalog['models']) == 1
|
||||
assert catalog['models'][0]['abilities'] == ['func_call', 'vision', 'reasoning']
|
||||
assert catalog['debug'] is None
|
||||
assert requests[0].url.path.endswith('/codex/models')
|
||||
assert 'client_version' in requests[0].url.params
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('payload', [{'models': ['secret']}, [], {'models': None}])
|
||||
async def test_catalog_malformed_safe(monkeypatch, payload):
|
||||
obj = requester(monkeypatch, lambda request: httpx.Response(200, json=payload))
|
||||
with pytest.raises(ValueError, match='invalid model catalog'):
|
||||
await obj.scan_models()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sse_multiline_crlf_comments_and_chunk_boundaries():
|
||||
class Bytes(httpx.AsyncByteStream):
|
||||
async def __aiter__(self):
|
||||
for value in b': comment\r\nevent: test\r\ndata: {"type":\r\ndata: "test"}\r\n\r\ndata: [DONE]\r\n\r\n':
|
||||
yield bytes([value])
|
||||
|
||||
response = httpx.Response(200, stream=Bytes())
|
||||
assert [event async for event in sse_events(response)] == [{'type': 'test'}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'key', ['base_url', 'headers', 'api_key', 'store', 'stream', 'previous_response_id', 'temperature']
|
||||
)
|
||||
def test_advanced_parameters_cannot_override_transport(monkeypatch, key):
|
||||
obj = requester(monkeypatch, lambda request: httpx.Response(200))
|
||||
with pytest.raises(ValueError, match='Unsupported Codex advanced'):
|
||||
obj._body(None, MODEL, [], None, {key: 'secret'}, TOKENS)
|
||||
@@ -0,0 +1,100 @@
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import AsyncMock, Mock
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from quart import Quart
|
||||
from sqlalchemy.exc import SQLAlchemyError
|
||||
|
||||
from langbot.pkg.api.http.controller.groups.provider.models import LLMModelsRouterGroup
|
||||
from langbot.pkg.api.http.authz import Permission
|
||||
from langbot.pkg.provider.modelmgr.requesters.codex import CodexRequester
|
||||
from tests.unit_tests.provider.test_codex import requester, MODEL, stream
|
||||
|
||||
|
||||
CASES = [
|
||||
(400, 400, 'codex_invalid_request'),
|
||||
(401, 400, 'codex_reauthentication_required'),
|
||||
(403, 403, 'codex_access_denied'),
|
||||
(429, 429, 'codex_rate_limited'),
|
||||
(500, 502, 'codex_upstream_failure'),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('upstream,status,code', CASES)
|
||||
async def test_requester_safe_error(monkeypatch, upstream, status, code):
|
||||
obj = requester(monkeypatch, lambda request: httpx.Response(upstream, text='credential-secret'))
|
||||
with pytest.raises(Exception) as caught:
|
||||
await obj.invoke_llm(None, MODEL, [])
|
||||
error = caught.value
|
||||
assert getattr(error, 'status_code', None) == status
|
||||
assert error.error_code == code
|
||||
assert 'secret' not in str(error)
|
||||
if upstream == 429:
|
||||
assert 'rate limit' in str(error).lower()
|
||||
assert 'usage limit reached' not in str(error).lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('events', [[{'type': 'response.failed', 'error': 'credential-secret'}], []])
|
||||
async def test_stream_safe_error(monkeypatch, events):
|
||||
obj = requester(monkeypatch, lambda request: stream(events))
|
||||
with pytest.raises(Exception) as caught:
|
||||
await obj.invoke_llm(None, MODEL, [])
|
||||
assert getattr(caught.value, 'status_code', None) == 502
|
||||
assert caught.value.error_code == 'codex_upstream_failure'
|
||||
assert 'secret' not in str(caught.value)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
'kind,code', [('usage_limit_reached', 'codex_usage_limit_reached'), ('unknown', 'codex_rate_limited')]
|
||||
)
|
||||
async def test_allowlisted_usage_error(monkeypatch, kind, code):
|
||||
obj = requester(
|
||||
monkeypatch,
|
||||
lambda request: httpx.Response(
|
||||
429, json={'error': {'type': kind, 'message': 'credential-secret', 'resets_at': 1789043289}}
|
||||
),
|
||||
)
|
||||
with pytest.raises(Exception) as caught:
|
||||
await obj.invoke_llm(None, MODEL, [])
|
||||
assert caught.value.error_code == code
|
||||
assert 'secret' not in str(caught.value)
|
||||
|
||||
|
||||
async def client_for(error):
|
||||
app = Quart(__name__)
|
||||
ap = SimpleNamespace(logger=Mock(), llm_model_service=SimpleNamespace(test_llm_model=AsyncMock(side_effect=error)))
|
||||
router = LLMModelsRouterGroup(ap, app)
|
||||
router._authenticate_api_key = AsyncMock(
|
||||
return_value=SimpleNamespace(
|
||||
workspace_uuid='w',
|
||||
workspace=SimpleNamespace(permissions=frozenset({Permission.PROVIDER_SECRET_MANAGE.value})),
|
||||
)
|
||||
)
|
||||
await router.initialize()
|
||||
return app.test_client(), ap
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('upstream,status,code', CASES)
|
||||
async def test_real_model_test_route_safe_error(upstream, status, code):
|
||||
error = CodexRequester._http_error(upstream)
|
||||
client, ap = await client_for(error)
|
||||
response = await client.post('/api/v1/provider/models/llm/model/test', json={}, headers={'X-API-Key': 'synthetic'})
|
||||
body = await response.get_json()
|
||||
assert response.status_code == status
|
||||
assert body['code'] == code
|
||||
assert body['msg'] == str(error)
|
||||
ap.logger.error.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('error', [ValueError('private-value-secret'), SQLAlchemyError('private-sql-secret')])
|
||||
async def test_real_model_test_route_unexpected_errors_hidden(error):
|
||||
client, _ = await client_for(error)
|
||||
response = await client.post('/api/v1/provider/models/llm/model/test', json={}, headers={'X-API-Key': 'synthetic'})
|
||||
assert response.status_code == 500
|
||||
assert 'secret' not in await response.get_data(as_text=True)
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Temporary Codex models use saved, tenant-scoped providers and synthetic SQLite credentials."""
|
||||
|
||||
import time
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import pytest_asyncio
|
||||
import sqlalchemy as sa
|
||||
from sqlalchemy.ext.asyncio import create_async_engine
|
||||
|
||||
from langbot.pkg.entity.persistence.model import CodexCredential, ModelProvider
|
||||
from langbot.pkg.provider.modelmgr.codex_auth import BASE_URL
|
||||
from langbot.pkg.provider.modelmgr.modelmgr import ModelManager
|
||||
from langbot.pkg.provider.modelmgr.requesters.codex import CodexRequester
|
||||
from langbot.pkg.workspace.errors import WorkspaceNotFoundError
|
||||
from tests.unit_tests.provider.conftest import (
|
||||
TEST_EXECUTION_CONTEXT,
|
||||
TEST_WORKSPACE_UUID,
|
||||
FakeProviderAPIRequester,
|
||||
)
|
||||
|
||||
|
||||
@pytest_asyncio.fixture
|
||||
async def manager(tmp_path, mock_app_for_modelmgr):
|
||||
engine = create_async_engine(f'sqlite+aiosqlite:///{tmp_path / "temporary-codex.db"}')
|
||||
async with engine.begin() as conn:
|
||||
await conn.run_sync(ModelProvider.__table__.create)
|
||||
await conn.run_sync(CodexCredential.__table__.create)
|
||||
for uuid, workspace, kind in (
|
||||
('saved', TEST_WORKSPACE_UUID, 'openai-codex'),
|
||||
('foreign', 'another-workspace', 'openai-codex'),
|
||||
('api', TEST_WORKSPACE_UUID, 'fake-requester'),
|
||||
):
|
||||
await conn.execute(
|
||||
sa.insert(ModelProvider).values(
|
||||
uuid=uuid,
|
||||
workspace_uuid=workspace,
|
||||
name='Saved provider',
|
||||
requester=kind,
|
||||
base_url=BASE_URL,
|
||||
api_keys=[],
|
||||
)
|
||||
)
|
||||
await conn.execute(
|
||||
sa.insert(CodexCredential).values(
|
||||
provider_uuid='saved',
|
||||
workspace_uuid=TEST_WORKSPACE_UUID,
|
||||
payload={
|
||||
'tokens': {
|
||||
'access_token': 'synthetic-access',
|
||||
'refresh_token': 'synthetic-refresh',
|
||||
'account_id': 'synthetic-account',
|
||||
'expires_at': time.time() + 3600,
|
||||
}
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
async def execute(statement):
|
||||
async with engine.begin() as conn:
|
||||
return await conn.execute(statement)
|
||||
|
||||
mock_app_for_modelmgr.persistence_mgr = SimpleNamespace(execute_async=execute)
|
||||
mgr = ModelManager(mock_app_for_modelmgr)
|
||||
mgr.requester_dict = {'openai-codex': CodexRequester, 'fake-requester': FakeProviderAPIRequester}
|
||||
try:
|
||||
yield mgr
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
|
||||
def info(provider_uuid='saved', **inline):
|
||||
result = {'name': 'codex-test', 'provider': {'requester': 'openai-codex', **inline}}
|
||||
if provider_uuid is not None:
|
||||
result['provider_uuid'] = provider_uuid
|
||||
return result
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize(
|
||||
'inline',
|
||||
[
|
||||
{},
|
||||
{'uuid': 'saved'},
|
||||
{
|
||||
'requester': 'fake-requester',
|
||||
'api_keys': ['untrusted'],
|
||||
'base_url': 'https://untrusted.invalid',
|
||||
'workspace_uuid': 'another-workspace',
|
||||
},
|
||||
],
|
||||
)
|
||||
async def test_codex_temporary_model_resolves_saved_provider_and_real_credentials(manager, inline):
|
||||
model = await manager.init_temporary_runtime_llm_model(TEST_EXECUTION_CONTEXT, info(**inline))
|
||||
provider = model.provider
|
||||
assert provider.provider_entity.uuid == 'saved'
|
||||
assert provider.provider_entity.requester == 'openai-codex'
|
||||
assert provider.provider_entity.api_keys == []
|
||||
assert provider.provider_entity.base_url == BASE_URL
|
||||
assert isinstance(provider.requester, CodexRequester)
|
||||
tokens = await provider.requester.auth.access(provider.requester.workspace, provider.requester.provider)
|
||||
assert tokens['access_token'] == 'synthetic-access'
|
||||
assert model.model_entity.provider_uuid == 'saved'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_temporary_model_accepts_inline_saved_identity(manager):
|
||||
model = await manager.init_temporary_runtime_llm_model(TEST_EXECUTION_CONTEXT, info(None, uuid='saved'))
|
||||
assert model.provider.provider_entity.name == 'Saved provider'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize('identity', ['missing', 'foreign', None])
|
||||
async def test_codex_temporary_model_rejects_unavailable_identity(manager, identity):
|
||||
with pytest.raises(WorkspaceNotFoundError):
|
||||
await manager.init_temporary_runtime_llm_model(
|
||||
TEST_EXECUTION_CONTEXT, info(identity, workspace_uuid='another-workspace')
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_temporary_model_rejects_non_codex_saved_provider(manager):
|
||||
with pytest.raises(ValueError):
|
||||
await manager.init_temporary_runtime_llm_model(TEST_EXECUTION_CONTEXT, info('api'))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_codex_temporary_model_rejects_conflicting_identities(manager):
|
||||
with pytest.raises(ValueError):
|
||||
await manager.init_temporary_runtime_llm_model(TEST_EXECUTION_CONTEXT, info('saved', uuid='foreign'))
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_key_temporary_model_preserves_inline_configuration(manager):
|
||||
model = await manager.init_temporary_runtime_llm_model(
|
||||
TEST_EXECUTION_CONTEXT,
|
||||
{
|
||||
'name': 'api-model',
|
||||
'provider': {
|
||||
'requester': 'fake-requester',
|
||||
'api_keys': ['synthetic-key'],
|
||||
'base_url': 'https://api.example.invalid',
|
||||
},
|
||||
},
|
||||
)
|
||||
assert model.provider.provider_entity.api_keys == ['synthetic-key']
|
||||
assert model.provider.provider_entity.base_url == 'https://api.example.invalid'
|
||||
Reference in New Issue
Block a user