mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-26 18:35:38 +08:00
feat(monitoring): unify execution views and trace tool calls
This commit is contained in:
@@ -227,6 +227,123 @@ function emptyMonitoringData() {
|
||||
};
|
||||
}
|
||||
|
||||
type RawMonitoringMessage = Record<string, unknown>;
|
||||
|
||||
/** Mirror of the backend execution status groups so fixtures stay realistic. */
|
||||
function executionStatusGroup(status: string, source: 'agent' | 'pipeline') {
|
||||
if (source === 'agent') {
|
||||
if (status === 'failed' || status === 'timeout') return 'failed';
|
||||
if (status === 'created' || status === 'queued') return 'queued';
|
||||
if (status === 'claimed' || status === 'running') return 'running';
|
||||
if (status === 'cancelled') return 'cancelled';
|
||||
return status;
|
||||
}
|
||||
if (status === 'success') return 'completed';
|
||||
if (status === 'error') return 'failed';
|
||||
if (status === 'pending') return 'running';
|
||||
if (status === 'discarded' || status === 'not_matched') return 'ignored';
|
||||
return status;
|
||||
}
|
||||
|
||||
/** Best-effort text extraction from a serialized message chain (mirrors backend). */
|
||||
function executionText(raw: unknown): string {
|
||||
const source = String(raw ?? '');
|
||||
try {
|
||||
const parts: string[] = [];
|
||||
const walk = (node: unknown, depth = 0): void => {
|
||||
if (depth > 6) return;
|
||||
if (Array.isArray(node)) {
|
||||
node.forEach((item) => walk(item, depth + 1));
|
||||
} else if (node && typeof node === 'object') {
|
||||
for (const [key, value] of Object.entries(node)) {
|
||||
if (key === 'text' && typeof value === 'string') parts.push(value);
|
||||
else walk(value, depth + 1);
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(JSON.parse(source));
|
||||
return parts.join(' ').trim() || source;
|
||||
} catch {
|
||||
return source;
|
||||
}
|
||||
}
|
||||
|
||||
/** Derive the unified execution list from a mocked monitoring payload. */
|
||||
function buildExecutionList(monitoringData: unknown) {
|
||||
const payload = monitoringData as { messages?: RawMonitoringMessage[] } | undefined;
|
||||
const items = (payload?.messages ?? [])
|
||||
.filter((message) => message.role !== 'assistant')
|
||||
.map((message) => {
|
||||
const status = String(message.status ?? 'success');
|
||||
return {
|
||||
source: 'pipeline' as const,
|
||||
id: String(message.id),
|
||||
event_id: null,
|
||||
status,
|
||||
status_group: executionStatusGroup(status, 'pipeline'),
|
||||
title: executionText(message.message_content),
|
||||
target_kind: 'pipeline' as const,
|
||||
target_id: (message.pipeline_id as string) ?? null,
|
||||
target_name: (message.pipeline_name as string) ?? null,
|
||||
bot_id: (message.bot_id as string) ?? null,
|
||||
bot_name: (message.bot_name as string) ?? null,
|
||||
pipeline_id: (message.pipeline_id as string) ?? null,
|
||||
pipeline_name: (message.pipeline_name as string) ?? null,
|
||||
runner_id: (message.runner_name as string) ?? null,
|
||||
conversation_id: null,
|
||||
session_id: (message.session_id as string) ?? null,
|
||||
created_at_ms: Date.parse(String(message.timestamp ?? '')) || null,
|
||||
started_at_ms: null,
|
||||
finished_at_ms: null,
|
||||
duration_ms: null,
|
||||
usage: null,
|
||||
cost: null,
|
||||
queue_name: null,
|
||||
debug: false,
|
||||
has_error: status === 'error',
|
||||
};
|
||||
});
|
||||
|
||||
const byStatus: Record<string, number> = {};
|
||||
for (const item of items) {
|
||||
byStatus[item.status_group] = (byStatus[item.status_group] ?? 0) + 1;
|
||||
}
|
||||
const completed = byStatus.completed ?? 0;
|
||||
const failed = byStatus.failed ?? 0;
|
||||
const denominator = completed + failed;
|
||||
|
||||
return {
|
||||
items,
|
||||
total: items.length,
|
||||
limit: items.length,
|
||||
offset: 0,
|
||||
has_more: false,
|
||||
summary: {
|
||||
executions: {
|
||||
total: items.length,
|
||||
by_source: { agent: 0, pipeline: items.length },
|
||||
by_status: byStatus,
|
||||
completed,
|
||||
failed,
|
||||
running: byStatus.running ?? 0,
|
||||
queued: byStatus.queued ?? 0,
|
||||
cancelled: byStatus.cancelled ?? 0,
|
||||
ignored: byStatus.ignored ?? 0,
|
||||
waiting: 0,
|
||||
success_rate:
|
||||
denominator > 0 ? Math.round((completed / denominator) * 10000) / 100 : null,
|
||||
denominator,
|
||||
p50_duration_ms: null,
|
||||
p95_duration_ms: null,
|
||||
duration_sample: 0,
|
||||
debug: 0,
|
||||
real: items.length,
|
||||
},
|
||||
tokens: { total_tokens: 0, calls: 0, calls_with_usage: 0 },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function emptyTokenStatistics() {
|
||||
return {
|
||||
summary: {
|
||||
@@ -1163,6 +1280,46 @@ async function handleBackendApi(route: Route, state: LangBotApiMockState) {
|
||||
return fulfillJson(route, data.overview || emptyMonitoringData().overview);
|
||||
}
|
||||
|
||||
if (path === '/api/v1/monitoring/executions') {
|
||||
return fulfillJson(route, buildExecutionList(state.monitoringData));
|
||||
}
|
||||
|
||||
const executionDetail = path.match(
|
||||
/^\/api\/v1\/monitoring\/executions\/(agent|pipeline)\/(.+)$/,
|
||||
);
|
||||
if (executionDetail) {
|
||||
const [, source, executionId] = executionDetail;
|
||||
if (source === 'agent') {
|
||||
return fulfillJson(route, {
|
||||
source: 'agent',
|
||||
run: { run_id: executionId, status: 'completed', created_at: 0 },
|
||||
events: [],
|
||||
has_more: false,
|
||||
next_cursor: null,
|
||||
});
|
||||
}
|
||||
const payload = state.monitoringData as {
|
||||
messages?: RawMonitoringMessage[];
|
||||
llmCalls?: RawMonitoringMessage[];
|
||||
toolCalls?: RawMonitoringMessage[];
|
||||
errors?: RawMonitoringMessage[];
|
||||
};
|
||||
const relatedToExecution = (rows: RawMonitoringMessage[] | undefined) =>
|
||||
(rows ?? []).filter((row) => row.message_id === executionId);
|
||||
return fulfillJson(route, {
|
||||
source: 'pipeline',
|
||||
message:
|
||||
(payload.messages ?? []).find((row) => row.id === executionId) ?? {
|
||||
id: executionId,
|
||||
status: 'success',
|
||||
message_content: '',
|
||||
},
|
||||
llm_calls: relatedToExecution(payload.llmCalls),
|
||||
tool_calls: relatedToExecution(payload.toolCalls),
|
||||
errors: relatedToExecution(payload.errors),
|
||||
});
|
||||
}
|
||||
|
||||
if (path === '/api/v1/monitoring/token-statistics') {
|
||||
return fulfillJson(route, emptyTokenStatistics());
|
||||
}
|
||||
|
||||
@@ -59,7 +59,14 @@ test.describe('authenticated app shell', () => {
|
||||
await page.goto('/home/monitoring');
|
||||
|
||||
await expect(page).toHaveURL(/\/home\/monitoring$/);
|
||||
await expect(page.getByText('Total Messages').first()).toBeVisible();
|
||||
// The unified execution view is the default entry point.
|
||||
await expect(page.getByRole('tab', { name: 'Executions' })).toHaveAttribute(
|
||||
'data-state',
|
||||
'active',
|
||||
);
|
||||
await expect(
|
||||
page.getByText('Execution Success Rate').first(),
|
||||
).toBeVisible();
|
||||
await expect(
|
||||
page.getByRole('tab', { name: 'Message Records' }),
|
||||
).toBeVisible();
|
||||
|
||||
@@ -304,6 +304,7 @@ test.describe('monitoring request contracts', () => {
|
||||
await expect(page.getByText('No message records')).toHaveCount(0);
|
||||
failing = false;
|
||||
await page.getByRole('button', { name: 'Retry', exact: true }).click();
|
||||
await page.getByRole('tab', { name: 'Message Records' }).click();
|
||||
await expect(
|
||||
page.getByText('Recovered monitoring', { exact: true }),
|
||||
).toBeVisible();
|
||||
@@ -324,6 +325,8 @@ test.describe('monitoring request contracts', () => {
|
||||
await page.getByRole('option', { name: /Last 7 days/i }).click();
|
||||
await expect.poll(() => pending.length).toBe(3);
|
||||
await respond(pending[2], 'Latest filter data');
|
||||
// The message-turn view now lives behind its own tab.
|
||||
await page.getByRole('tab', { name: 'Message Records' }).click();
|
||||
await expect(
|
||||
page.getByText('Latest filter data', { exact: true }),
|
||||
).toBeVisible();
|
||||
@@ -598,6 +601,7 @@ test.describe('monitoring conversation turn grouping', () => {
|
||||
});
|
||||
|
||||
await page.goto('/home/monitoring');
|
||||
await page.getByRole('tab', { name: 'Message Records' }).click();
|
||||
|
||||
await expect(page.getByText('3 conversation turns')).toBeVisible();
|
||||
await expect(
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
import assert from 'node:assert/strict';
|
||||
import fs from 'node:fs';
|
||||
import test from 'node:test';
|
||||
import ts from 'typescript';
|
||||
|
||||
// The resolver is pure: recorded calls carry a time and a session, the execution
|
||||
// list carries the runs' windows, so the link is decided without any re-query.
|
||||
const source = fs.readFileSync(
|
||||
new URL('../../src/app/home/monitoring/utils/callLinks.ts', import.meta.url),
|
||||
'utf8',
|
||||
);
|
||||
const module = { exports: {} };
|
||||
new Function(
|
||||
'exports',
|
||||
ts.transpileModule(source, {
|
||||
compilerOptions: {
|
||||
module: ts.ModuleKind.CommonJS,
|
||||
target: ts.ScriptTarget.ES2022,
|
||||
},
|
||||
}).outputText,
|
||||
)(module.exports);
|
||||
const { findCallExecution, resolveCallTarget } = module.exports;
|
||||
|
||||
const at = (iso) => new Date(iso);
|
||||
|
||||
function run(overrides) {
|
||||
return {
|
||||
source: 'agent',
|
||||
id: 'run-1',
|
||||
session_id: 'session-a',
|
||||
started_at_ms: at('2026-10-07T10:00:00Z').getTime(),
|
||||
finished_at_ms: at('2026-10-07T10:00:30Z').getTime(),
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function call(overrides) {
|
||||
return {
|
||||
id: 'call-1',
|
||||
timestamp: at('2026-10-07T10:00:10Z'),
|
||||
sessionId: 'session-a',
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
test('links a pipeline call to the execution carrying its message id', () => {
|
||||
const execution = run({
|
||||
source: 'pipeline',
|
||||
id: 'message-9',
|
||||
session_id: null,
|
||||
});
|
||||
assert.equal(
|
||||
findCallExecution(call({ messageId: 'message-9', sessionId: undefined }), [
|
||||
execution,
|
||||
]),
|
||||
execution,
|
||||
);
|
||||
});
|
||||
|
||||
test('links an agent call to the run whose window contains it', () => {
|
||||
const execution = run({});
|
||||
assert.equal(findCallExecution(call({}), [execution]), execution);
|
||||
});
|
||||
|
||||
test('prefers the run that was live when a session has consecutive runs', () => {
|
||||
const first = run({ id: 'run-1' });
|
||||
const second = run({
|
||||
id: 'run-2',
|
||||
started_at_ms: at('2026-10-07T10:05:00Z').getTime(),
|
||||
finished_at_ms: at('2026-10-07T10:05:30Z').getTime(),
|
||||
});
|
||||
assert.equal(
|
||||
findCallExecution(call({ timestamp: at('2026-10-07T10:05:10Z') }), [
|
||||
first,
|
||||
second,
|
||||
]).id,
|
||||
'run-2',
|
||||
);
|
||||
});
|
||||
|
||||
test('an open run owns every later call of its session', () => {
|
||||
const execution = run({ finished_at_ms: null });
|
||||
assert.equal(
|
||||
findCallExecution(call({ timestamp: at('2026-10-07T10:20:00Z') }), [
|
||||
execution,
|
||||
]),
|
||||
execution,
|
||||
);
|
||||
});
|
||||
|
||||
test('a call outside every window links to nothing', () => {
|
||||
const execution = run({});
|
||||
assert.equal(
|
||||
findCallExecution(call({ timestamp: at('2026-10-07T11:00:00Z') }), [
|
||||
execution,
|
||||
]),
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
findCallExecution(call({ timestamp: at('2026-10-07T09:00:00Z') }), [
|
||||
execution,
|
||||
]),
|
||||
null,
|
||||
);
|
||||
assert.equal(
|
||||
findCallExecution(call({ sessionId: 'other' }), [execution]),
|
||||
null,
|
||||
);
|
||||
});
|
||||
|
||||
test('a call keeps its message target when the execution list cannot resolve it', () => {
|
||||
assert.deepEqual(
|
||||
resolveCallTarget(
|
||||
call({ messageId: 'message-9', sessionId: 'other' }),
|
||||
[run({})],
|
||||
new Set(['message-9']),
|
||||
),
|
||||
{ kind: 'message', id: 'message-9' },
|
||||
);
|
||||
// An Agent call carries its run id in the same field: an id the message list
|
||||
// does not hold must not become a jump to the message tab.
|
||||
assert.equal(
|
||||
resolveCallTarget(
|
||||
call({ messageId: 'run-9', sessionId: 'other' }),
|
||||
[run({})],
|
||||
new Set(['message-9']),
|
||||
),
|
||||
null,
|
||||
);
|
||||
assert.equal(resolveCallTarget(call({}), []), null);
|
||||
});
|
||||
Reference in New Issue
Block a user