xinyin025/LangBot
1
0
Fork 0
mirror of https://github.com/langbot-app/LangBot.git synced 2026-07-14 16:36:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add tool call observability

Browse Source
This commit is contained in:
Hyu
2026-07-02 15:52:46 +08:00
parent b1bc05d5d3
commit c809e3d14f
19 changed files with 1351 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files
web/tests/e2e/bot-session-tool-timeline.spec.ts
+179
View File
@@ -0,0 +1,179 @@
import { expect, test } from '@playwright/test';
import { installLangBotApiMocks } from './fixtures/langbot-api';
const botId = 'bot-tool-timeline';
const sessionId = 'person-tool-timeline-user';
const botName = 'Tool Timeline Bot';
const pipelineId = 'pipeline-tool-timeline';
const pipelineName = 'Tool Timeline Pipeline';
function at(minute: number, second = 0) {
return `2026-07-02T10:${String(minute).padStart(2, '0')}:${String(
second,
).padStart(2, '0')}Z`;
}
function sessionMessage(
id: string,
role: 'user' | 'assistant',
minute: number,
content: string,
) {
return {
id,
timestamp: at(minute),
bot_id: botId,
bot_name: botName,
pipeline_id: pipelineId,
pipeline_name: pipelineName,
message_content: content,
session_id: sessionId,
status: 'success',
level: 'info',
platform: role === 'user' ? 'person' : 'bot',
user_id: 'timeline-user',
user_name: 'Timeline User',
runner_name: role === 'assistant' ? 'local-agent' : null,
variables: '{}',
role,
};
}
function toolCall(
id: string,
minute: number,
toolName: string,
duration: number,
status: 'success' | 'error' = 'success',
) {
return {
id,
timestamp: at(minute, 30),
tool_name: toolName,
tool_source: 'native',
duration,
status,
bot_id: botId,
bot_name: botName,
pipeline_id: pipelineId,
pipeline_name: pipelineName,
session_id: sessionId,
message_id: 'user-message',
arguments: JSON.stringify({ target: toolName }),
result: status === 'success' ? JSON.stringify({ ok: true }) : null,
error_message: status === 'error' ? 'Tool execution failed' : null,
};
}
test.describe('bot session monitor tool timeline', () => {
test('renders tool calls as left-side agent events interleaved with messages', async ({
page,
}) => {
await installLangBotApiMocks(page, {
authenticated: true,
monitoringSessions: [
{
session_id: sessionId,
bot_id: botId,
bot_name: botName,
pipeline_id: pipelineId,
pipeline_name: pipelineName,
message_count: 3,
start_time: at(0),
last_activity: at(4),
is_active: true,
platform: 'person',
user_id: 'timeline-user',
user_name: 'Timeline User',
},
],
sessionMessages: {
[sessionId]: [
sessionMessage('user-message', 'user', 0, 'Need a timeline check'),
sessionMessage(
'assistant-step-1',
'assistant',
2,
'Agent step 1: inspected repository files',
),
sessionMessage(
'assistant-step-2',
'assistant',
4,
'Agent step 2: test suite finished',
),
],
},
sessionAnalyses: {
[sessionId]: {
session_id: sessionId,
found: true,
tool_calls: [
toolCall('tool-repo-read', 1, 'repo_file_read', 80),
toolCall('tool-test-run', 3, 'run_test_suite', 140),
],
},
},
});
await page.goto(`/home/bots?id=${botId}`);
await page.getByRole('tab', { name: /Sessions/ }).click();
await page.getByRole('button', { name: /Timeline User/ }).click();
await expect(page.getByText('Need a timeline check')).toBeVisible();
await expect(
page.getByText('repo_file_read', { exact: true }),
).toBeVisible();
await expect(
page.getByText('Agent step 1: inspected repository files'),
).toBeVisible();
await expect(
page.getByText('run_test_suite', { exact: true }),
).toBeVisible();
await expect(
page.getByText('Agent step 2: test suite finished'),
).toBeVisible();
await expect(page.getByText('{"target":"repo_file_read"}')).toHaveCount(0);
await expect(page.getByText('{"ok":true}')).toHaveCount(0);
await expect(
page.locator('div.flex.justify-start').filter({
hasText: 'repo_file_read',
}),
).toHaveCount(1);
await expect(
page.locator('div.flex.justify-start').filter({
hasText: 'run_test_suite',
}),
).toHaveCount(1);
await expect(
page.locator('div.flex.justify-end').filter({
hasText: 'repo_file_read',
}),
).toHaveCount(0);
await expect(
page.locator('div.flex.justify-end').filter({
hasText: 'run_test_suite',
}),
).toHaveCount(0);
const text = await page.locator('body').innerText();
expect(text.indexOf('Need a timeline check')).toBeLessThan(
text.indexOf('repo_file_read'),
);
expect(text.indexOf('repo_file_read')).toBeLessThan(
text.indexOf('Agent step 1: inspected repository files'),
);
expect(
text.indexOf('Agent step 1: inspected repository files'),
).toBeLessThan(text.indexOf('run_test_suite'));
expect(text.indexOf('run_test_suite')).toBeLessThan(
text.indexOf('Agent step 2: test suite finished'),
);
await page.getByText('repo_file_read', { exact: true }).click();
await expect(page.getByText('{"target":"repo_file_read"}')).toBeVisible();
await expect(page.getByText('{"ok":true}').first()).toBeVisible();
});
});
web/tests/e2e/fixtures/langbot-api.ts
+50 -1
View File
@@ -73,7 +73,10 @@ interface LangBotApiMockState {
knowledgeBases: KnowledgeBaseMock[];
mcpServers: MCPServerMock[];
monitoringData: unknown;
monitoringSessions: unknown[];
pipelines: PipelineMock[];
sessionAnalyses: Record<string, unknown>;
sessionMessages: Record<string, unknown[]>;
skills: SkillMock[];
}
@@ -123,12 +126,14 @@ function emptyMonitoringData() {
},
messages: [],
llmCalls: [],
toolCalls: [],
embeddingCalls: [],
sessions: [],
errors: [],
totalCount: {
messages: 0,
llmCalls: 0,
toolCalls: 0,
embeddingCalls: 0,
sessions: 0,
errors: 0,
@@ -693,6 +698,37 @@ async function handleBackendApi(route: Route, state: LangBotApiMockState) {
return fulfillJson(route, state.monitoringData);
}
if (path === '/api/v1/monitoring/sessions') {
return fulfillJson(route, {
sessions: state.monitoringSessions,
total: state.monitoringSessions.length,
});
}
if (path === '/api/v1/monitoring/messages') {
const sessionId = url.searchParams.get('sessionId') || '';
const messages = state.sessionMessages[sessionId] || [];
return fulfillJson(route, {
messages,
total: messages.length,
});
}
const sessionAnalysisMatch = path.match(
/^\/api\/v1\/monitoring\/sessions\/([^/]+)\/analysis$/,
);
if (sessionAnalysisMatch) {
const sessionId = decodeURIComponent(sessionAnalysisMatch[1]);
return fulfillJson(
route,
state.sessionAnalyses[sessionId] || {
session_id: sessionId,
found: true,
tool_calls: [],
},
);
}
if (path === '/api/v1/monitoring/overview') {
const data = state.monitoringData as { overview?: unknown };
return fulfillJson(route, data.overview || emptyMonitoringData().overview);
@@ -803,17 +839,30 @@ export async function installLangBotApiMocks(
options: {
authenticated?: boolean;
monitoringData?: unknown;
monitoringSessions?: unknown[];
sessionAnalyses?: Record<string, unknown>;
sessionMessages?: Record<string, unknown[]>;
storage?: JsonRecord;
} = {},
) {
const { authenticated = false, monitoringData, storage = {} } = options;
const {
authenticated = false,
monitoringData,
monitoringSessions,
sessionAnalyses,
sessionMessages,
storage = {},
} = options;
const state: LangBotApiMockState = {
bots: [],
counters: {},
knowledgeBases: [],
mcpServers: [],
monitoringData: monitoringData || emptyMonitoringData(),
monitoringSessions: monitoringSessions || [],
pipelines: [],
sessionAnalyses: sessionAnalyses || {},
sessionMessages: sessionMessages || {},
skills: [],
};
web/tests/e2e/monitoring-turns.spec.ts
+94
View File
@@ -6,6 +6,7 @@ import {
ErrorLog,
LLMCall,
MonitoringMessage,
ToolCall,
} from '../../src/app/home/monitoring/types/monitoring';
const bot = {
@@ -93,6 +94,34 @@ function errorLog(id: string, minute: number, messageId: string): ErrorLog {
};
}
function toolCall(
id: string,
minute: number,
messageId: string | undefined,
toolName: string,
duration: number,
sessionId = 'session-agent',
status: 'success' | 'error' = 'success',
): ToolCall {
return {
id,
timestamp: time(minute),
toolName,
toolSource: 'native',
duration,
status,
botId: bot.id,
botName: bot.name,
pipelineId: pipeline.id,
pipelineName: pipeline.name,
sessionId,
messageId,
arguments: JSON.stringify({ query: toolName }),
result: status === 'success' ? JSON.stringify({ ok: true }) : undefined,
errorMessage: status === 'error' ? 'Tool failed' : undefined,
};
}
function rawMessage(message: MonitoringMessage) {
return {
id: message.id,
@@ -151,6 +180,26 @@ function rawError(error: ErrorLog) {
};
}
function rawToolCall(call: ToolCall) {
return {
id: call.id,
timestamp: call.timestamp.toISOString(),
tool_name: call.toolName,
tool_source: call.toolSource,
duration: call.duration,
status: call.status,
bot_id: call.botId,
bot_name: call.botName,
pipeline_id: call.pipelineId,
pipeline_name: call.pipelineName,
session_id: call.sessionId,
message_id: call.messageId,
arguments: call.arguments,
result: call.result,
error_message: call.errorMessage,
};
}
function monitoringScenario() {
const messages = [
message(
@@ -179,10 +228,16 @@ function monitoringScenario() {
llmCall('agent-call-4', 20, 'agent-user-2', 50, 25, 80),
];
const errors = [errorLog('agent-error-1', 12, 'agent-user-1')];
const toolCalls = [
toolCall('agent-tool-1', 11, 'agent-user-1', 'repo_search', 90),
toolCall('agent-tool-2', 12, 'agent-user-1', 'run_tests', 150),
toolCall('agent-tool-3', 20, 'agent-user-2', 'rollback_lookup', 70),
];
return {
messages,
llmCalls,
toolCalls,
errors,
};
}
@@ -201,12 +256,14 @@ function rawMonitoringData() {
},
messages: scenario.messages.map(rawMessage),
llmCalls: scenario.llmCalls.map(rawLlmCall),
toolCalls: scenario.toolCalls.map(rawToolCall),
embeddingCalls: [],
sessions: [],
errors: scenario.errors.map(rawError),
totalCount: {
messages: scenario.messages.length,
llmCalls: scenario.llmCalls.length,
toolCalls: scenario.toolCalls.length,
embeddingCalls: 0,
sessions: 0,
errors: scenario.errors.length,
@@ -240,6 +297,7 @@ test.describe('monitoring conversation turn grouping', () => {
scenario.messages,
scenario.llmCalls,
scenario.errors,
scenario.toolCalls,
);
const agentTurn = turns.find((turn) => turn.id === 'agent-user-1');
@@ -254,9 +312,11 @@ test.describe('monitoring conversation turn grouping', () => {
'Final answer: deployment plan ready',
]);
expect(agentTurn?.llmCalls).toHaveLength(3);
expect(agentTurn?.toolCalls).toHaveLength(2);
expect(agentTurn?.errors).toHaveLength(1);
expect(agentTurn?.totalTokens).toBe(790);
expect(agentTurn?.totalDuration).toBe(600);
expect(agentTurn?.totalToolDuration).toBe(240);
});
test('starts a new turn for each later user message in the same session', () => {
@@ -314,6 +374,30 @@ test.describe('monitoring conversation turn grouping', () => {
expect(turns[0].totalTokens).toBe(30);
});
test('attaches tool calls without message ids by session time', () => {
const user = message('tool-fallback-user', 'user', 1, 'Use tool fallback');
const assistant = message(
'tool-fallback-assistant',
'assistant',
2,
'Tool fallback reply',
);
const call = toolCall(
'tool-fallback-call',
2,
undefined,
'memory_lookup',
45,
);
const turns = buildConversationTurns([user, assistant], [], [], [call]);
expect(turns).toHaveLength(1);
expect(turns[0].toolCalls).toHaveLength(1);
expect(turns[0].toolCalls[0].id).toBe(call.id);
expect(turns[0].totalToolDuration).toBe(45);
});
test('renders user-only, multi-agent, and multi-turn cases in the monitoring page', async ({
page,
}) => {
@@ -333,6 +417,7 @@ test.describe('monitoring conversation turn grouping', () => {
await expect(page.getByText('Agent step 1: inspect repo')).toBeVisible();
await expect(page.getByText('Assistant +2')).toBeVisible();
await expect(page.getByText('3 LLM')).toBeVisible();
await expect(page.getByText('2 tools')).toBeVisible();
await expect(page.getByText('790 tokens')).toBeVisible();
await expect(page.getByText('1 errors')).toBeVisible();
await expect(page.getByText('Continue with rollback plan')).toBeVisible();
@@ -354,6 +439,15 @@ test.describe('monitoring conversation turn grouping', () => {
await expect(page.getByText('In: 300')).toBeVisible();
await expect(page.getByText('Out: 90')).toBeVisible();
await expect(page.getByText('Total: 390')).toBeVisible();
await expect(page.getByText('Tool Calls (2)')).toBeVisible();
await expect(page.getByText('#1 repo_search')).toBeVisible();
await expect(page.getByText('#2 run_tests')).toBeVisible();
await expect(page.getByText('Arguments')).toHaveCount(0);
await expect(page.getByText('Result')).toHaveCount(0);
await page.getByText('#1 repo_search').click();
await expect(page.getByText('Arguments').first()).toBeVisible();
await expect(page.getByText('Result').first()).toBeVisible();
await expect(page.getByText('Tool retry failed')).toBeVisible();
});
});