Files
LangBot/tests/unit_tests/telemetry/test_telemetry.py
T
Hyu 4482e8d876 feat(telemetry)!: upload one record per execution node and one closure per chain
Core no longer assembles a whole trace locally. Every workflow node is uploaded
the moment it completes as its own `execution_node` record, and each execution
closes with exactly one `execution_chain` record; all records of one execution
carry the same `event_id`, so Space groups them back into a single chain.

- every execution starts at an inbound event: event-less Pipeline lanes and
  WebUI/API debug runs synthesize their virtual inbound event at the boundary
  and are marked debug=true, so a Runner is never a trace origin
- sampling is decided once per event from a stable hash of the event id (the
  previous `int(id[:8], 16)` raised on every `platform:`-prefixed id and so
  dropped every successful inbound-event chain) and is reported on each record
  as sample{mode,denominator}; the default mode is now `all`
- the local trace buffer, the 120s TTL, the per-process trace cap and the shared
  open-node mirror are gone, which also removes the truncated-chain upload and
  the re-buffered-closed-trace leak they caused
- a broken chain is always uploaded, and its reason falls back to the first
  failed node so the chain explains itself
- flush keeps records until the sender accepts them and counts what it drops;
  telemetry health replaces the unused send-task queue
- heartbeat reports bound Agent / plugin-processor / event-binding counts
- bound Pipeline and Agent resources (plugins, MCP servers, knowledge bases,
  tools with their Host source, skills, models and the Runner schema selectors
  that picked them) are reported on the chain record
- failure text is bounded and redacted instead of passing exception text through
- the dead per-query feature counters and their call sites are removed

Wire contract: schema 2 (`execution_node` / `execution_chain`).
2026-10-07 09:41:41 +08:00

631 lines
23 KiB
Python

"""Unit tests for telemetry module.
Tests cover:
- TelemetryManager initialization
- Payload sanitization logic (with real behavior verification)
- Early return conditions (disabled, empty config, no server)
- URL construction (with actual URL verification)
- HTTP request success/failure scenarios
"""
from __future__ import annotations
import pytest
import httpx
from unittest.mock import AsyncMock, Mock, patch
from importlib import import_module
def get_telemetry_module():
"""Lazy import to avoid circular import issues."""
return import_module('langbot.pkg.telemetry.telemetry')
class TestTelemetryManagerInit:
"""Tests for TelemetryManager initialization."""
def test_init_stores_app_reference(self):
"""Test that __init__ stores the Application reference."""
telemetry = get_telemetry_module()
mock_app = Mock()
manager = telemetry.TelemetryManager(mock_app)
assert manager.ap is mock_app
def test_init_empty_telemetry_config(self):
"""Test that telemetry_config starts empty."""
telemetry = get_telemetry_module()
mock_app = Mock()
manager = telemetry.TelemetryManager(mock_app)
assert manager.telemetry_config == {}
class TestTelemetryManagerInitialize:
"""Tests for initialize() method."""
@pytest.mark.asyncio
async def test_initialize_loads_space_config(self):
"""Test that initialize() loads space config from instance_config."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.instance_config = Mock()
mock_app.instance_config.data = {'space': {'url': 'https://example.com'}}
manager = telemetry.TelemetryManager(mock_app)
await manager.initialize()
assert manager.telemetry_config == {'url': 'https://example.com'}
@pytest.mark.asyncio
async def test_initialize_handles_empty_space_config(self):
"""Test that initialize() handles missing space config."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.instance_config = Mock()
mock_app.instance_config.data = {}
manager = telemetry.TelemetryManager(mock_app)
await manager.initialize()
assert manager.telemetry_config == {}
class TestTelemetrySendEarlyReturn:
"""Tests for early return conditions in send() method."""
@pytest.mark.asyncio
async def test_send_returns_when_config_empty(self):
"""Test that send() returns early when telemetry_config is empty."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {}
# Should return without making HTTP calls
await manager.send({'query_id': 'test'})
# No HTTP client should be created, no logs should be written
mock_app.logger.debug.assert_not_called()
mock_app.logger.warning.assert_not_called()
@pytest.mark.asyncio
async def test_send_returns_when_telemetry_disabled(self):
"""Test that send() returns early when disable_telemetry is True."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'disable_telemetry': True, 'url': 'https://example.com'}
await manager.send({'query_id': 'test'})
mock_app.logger.debug.assert_not_called()
@pytest.mark.asyncio
async def test_send_returns_when_server_empty(self):
"""Test that send() returns early when server URL is empty."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': ''}
await manager.send({'query_id': 'test'})
mock_app.logger.debug.assert_not_called()
class TestPayloadSanitization:
"""Tests for payload sanitization logic in send() method.
IMPORTANT: These tests verify actual behavior, not source code strings.
"""
@pytest.mark.asyncio
async def test_sanitize_null_query_id(self):
"""Test that null query_id is converted to empty string."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
captured_payloads = []
async def mock_post(url, json):
captured_payloads.append(json)
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': None})
assert len(captured_payloads) == 1
assert captured_payloads[0]['query_id'] == ''
@pytest.mark.asyncio
async def test_sanitize_query_id_string_value(self):
"""Test that query_id string value is preserved."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
captured_payloads = []
async def mock_post(url, json):
captured_payloads.append(json)
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'abc123'})
assert len(captured_payloads) == 1
assert captured_payloads[0]['query_id'] == 'abc123'
@pytest.mark.asyncio
async def test_sanitize_null_string_fields(self):
"""Test that null string fields are converted to empty strings."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
captured_payloads = []
async def mock_post(url, json):
captured_payloads.append(json)
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
payload = {
'query_id': 'test',
'adapter': None,
'runner': None,
'runner_category': None,
'model_name': None,
'version': None,
'edition': None,
'error': None,
'timestamp': None,
}
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send(payload)
assert len(captured_payloads) == 1
result = captured_payloads[0]
# All null string fields should be empty strings
for field in ['adapter', 'runner', 'runner_category', 'model_name', 'version', 'edition', 'error', 'timestamp']:
assert result[field] == '', f'Field {field} should be empty string, got {result[field]}'
@pytest.mark.asyncio
async def test_sanitize_string_fields_preserve_values(self):
"""Test that non-null string fields preserve their values."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
captured_payloads = []
async def mock_post(url, json):
captured_payloads.append(json)
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
payload = {
'query_id': 'test',
'adapter': 'gewechat',
'runner': 'local-agent',
'model_name': 'gpt-4',
'version': 'v1.0.0',
}
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send(payload)
assert len(captured_payloads) == 1
result = captured_payloads[0]
assert result['adapter'] == 'gewechat'
assert result['runner'] == 'local-agent'
assert result['model_name'] == 'gpt-4'
assert result['version'] == 'v1.0.0'
@pytest.mark.asyncio
async def test_sanitize_duration_ms_invalid_value(self):
"""Test that invalid duration_ms is converted to 0."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
captured_payloads = []
async def mock_post(url, json):
captured_payloads.append(json)
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'test', 'duration_ms': 'invalid'})
assert len(captured_payloads) == 1
assert captured_payloads[0]['duration_ms'] == 0
@pytest.mark.asyncio
async def test_sanitize_duration_ms_none_value(self):
"""Test that None duration_ms is converted to 0."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
captured_payloads = []
async def mock_post(url, json):
captured_payloads.append(json)
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'test', 'duration_ms': None})
assert len(captured_payloads) == 1
assert captured_payloads[0]['duration_ms'] == 0
@pytest.mark.asyncio
async def test_sanitize_duration_ms_valid_value(self):
"""Test that valid duration_ms is converted to int."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
captured_payloads = []
async def mock_post(url, json):
captured_payloads.append(json)
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'test', 'duration_ms': 123.45})
assert len(captured_payloads) == 1
assert captured_payloads[0]['duration_ms'] == 123
class TestURLConstruction:
"""Tests for URL construction in send() method.
IMPORTANT: These tests verify actual URLs sent, not source code strings.
"""
@pytest.mark.asyncio
async def test_url_strip_trailing_slash(self):
"""Test that trailing slash is stripped from server URL."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com/'}
captured_urls = []
async def mock_post(url, json):
captured_urls.append(url)
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'test'})
assert len(captured_urls) == 1
assert captured_urls[0] == 'https://example.com/api/v1/telemetry'
# No trailing slash before /api/v1/telemetry
@pytest.mark.asyncio
async def test_url_without_trailing_slash(self):
"""Test that URL without trailing slash works correctly."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
captured_urls = []
async def mock_post(url, json):
captured_urls.append(url)
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'test'})
assert len(captured_urls) == 1
assert captured_urls[0] == 'https://example.com/api/v1/telemetry'
class TestHTTPScenarios:
"""Tests for HTTP request success/failure scenarios."""
@pytest.mark.asyncio
async def test_send_http_success_logs_debug(self):
"""Test that HTTP 200 with code=0 logs debug message."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
mock_response = Mock(
status_code=200, text='{"code": 0, "msg": "success"}', json=Mock(return_value={'code': 0, 'msg': 'success'})
)
mock_client = Mock()
mock_client.post = AsyncMock(return_value=mock_response)
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'test'})
mock_app.logger.warning.assert_not_called()
mock_app.logger.debug.assert_called()
# Verify debug message contains URL and status
debug_call_args = mock_app.logger.debug.call_args[0][0]
assert 'Telemetry posted' in debug_call_args
assert 'https://example.com/api/v1/telemetry' in debug_call_args
@pytest.mark.asyncio
async def test_send_http_error_status_logs_warning(self):
"""Test that HTTP status >= 400 logs warning."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
mock_response = Mock(
status_code=500, text='Internal Server Error', json=Mock(return_value={'code': 500, 'msg': 'error'})
)
mock_client = Mock()
mock_client.post = AsyncMock(return_value=mock_response)
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'test'})
mock_app.logger.warning.assert_not_called()
mock_app.logger.debug.assert_called()
warning_call_args = mock_app.logger.debug.call_args[0][0]
assert 'status 500' in warning_call_args
@pytest.mark.asyncio
async def test_send_application_error_logs_warning(self):
"""Test that HTTP 200 with application code >= 400 logs warning."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
mock_response = Mock(
status_code=200,
text='{"code": 400, "msg": "Bad Request"}',
json=Mock(return_value={'code': 400, 'msg': 'Bad Request'}),
)
mock_client = Mock()
mock_client.post = AsyncMock(return_value=mock_response)
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'test'})
# Source code calls warning twice for application errors
assert mock_app.logger.debug.call_count >= 1
# Check that one of the calls contains application error info
all_warnings = [call[0][0] for call in mock_app.logger.debug.call_args_list]
assert any('400' in w for w in all_warnings), f'No warning contained error code 400: {all_warnings}'
@pytest.mark.asyncio
async def test_send_timeout_logs_warning(self):
"""Test that asyncio.TimeoutError logs warning."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
import asyncio
async def mock_post_timeout(url, json):
raise asyncio.TimeoutError()
mock_client = Mock()
mock_client.post = mock_post_timeout
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'test'})
mock_app.logger.warning.assert_not_called()
mock_app.logger.debug.assert_called()
warning_call_args = mock_app.logger.debug.call_args[0][0]
assert 'timed out' in warning_call_args
@pytest.mark.asyncio
async def test_send_network_error_logs_warning(self):
"""Test that network exceptions log warning without raising."""
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
async def mock_post_error(url, json):
raise httpx.ConnectError('Connection failed')
mock_client = Mock()
mock_client.post = mock_post_error
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
# Should not raise exception
await manager.send({'query_id': 'test'})
mock_app.logger.warning.assert_not_called()
mock_app.logger.debug.assert_called()
@pytest.mark.asyncio
async def test_send_never_raises_exception(self):
"""Test that send() never raises exceptions regardless of errors."""
telemetry = get_telemetry_module()
mock_app = Mock()
# Even logger may fail
mock_app.logger = Mock()
mock_app.logger.debug = Mock(side_effect=Exception('Logger failed'))
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
async def mock_post_error(url, json):
raise Exception('Unexpected error')
mock_client = Mock()
mock_client.post = mock_post_error
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
# Should never raise
await manager.send({'query_id': 'test'})
class TestTelemetryManagedRuntimeAuthentication:
@pytest.mark.asyncio
async def test_send_includes_managed_runtime_token_header(self):
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
captured = {}
async def mock_post(url, json, headers):
captured['headers'] = headers
return Mock(status_code=200, text='', json=Mock(return_value={'code': 0}))
mock_client = Mock()
mock_client.post = mock_post
mock_client.__aenter__ = AsyncMock(return_value=mock_client)
mock_client.__aexit__ = AsyncMock(return_value=None)
with (
patch.dict('os.environ', {'LANGBOT_TELEMETRY_INGEST_TOKEN': 'managed-runtime-secret'}),
patch.object(httpx, 'AsyncClient', return_value=mock_client),
):
await manager.send({'event_type': 'instance_heartbeat'})
assert captured['headers'] == {'X-LangBot-Telemetry-Token': 'managed-runtime-secret'}
class TestAuthenticatedWorkspaceReporter:
@pytest.mark.asyncio
async def test_workspace_owner_access_token_is_sent_as_bearer(self):
telemetry = get_telemetry_module()
mock_app = Mock()
mock_app.logger = Mock()
mock_app.user_service = Mock()
mock_app.user_service.get_workspace_owner = AsyncMock(
return_value=Mock(user='[email protected]', space_access_token='expired-token')
)
mock_app.space_service = Mock()
mock_app.space_service.get_valid_access_token = AsyncMock(return_value='refreshed-workspace-owner-token')
manager = telemetry.TelemetryManager(mock_app)
manager.telemetry_config = {'url': 'https://example.com'}
response = Mock(status_code=200, text='')
response.json = Mock(return_value={'code': 0})
mock_client = Mock()
mock_client.post = Mock(return_value=response)
with patch.object(httpx, 'AsyncClient', return_value=mock_client):
await manager.send({'query_id': 'q-1', 'workspace_uuid': 'workspace-1'})
mock_app.user_service.get_workspace_owner.assert_awaited_once_with('workspace-1')
mock_app.space_service.get_valid_access_token.assert_awaited_once_with('[email protected]')
assert mock_client.post.call_args.kwargs['headers'] == {
'Authorization': 'Bearer refreshed-workspace-owner-token'
}