feat(monitoring): unify execution views and trace tool calls

This commit is contained in:
Hyu
2026-10-07 22:36:10 +08:00
parent de886edc1b
commit 9f76037e64
42 changed files with 4181 additions and 889 deletions
+157
View File
@@ -227,6 +227,123 @@ function emptyMonitoringData() {
};
}
type RawMonitoringMessage = Record<string, unknown>;
/** Mirror of the backend execution status groups so fixtures stay realistic. */
function executionStatusGroup(status: string, source: 'agent' | 'pipeline') {
if (source === 'agent') {
if (status === 'failed' || status === 'timeout') return 'failed';
if (status === 'created' || status === 'queued') return 'queued';
if (status === 'claimed' || status === 'running') return 'running';
if (status === 'cancelled') return 'cancelled';
return status;
}
if (status === 'success') return 'completed';
if (status === 'error') return 'failed';
if (status === 'pending') return 'running';
if (status === 'discarded' || status === 'not_matched') return 'ignored';
return status;
}
/** Best-effort text extraction from a serialized message chain (mirrors backend). */
function executionText(raw: unknown): string {
const source = String(raw ?? '');
try {
const parts: string[] = [];
const walk = (node: unknown, depth = 0): void => {
if (depth > 6) return;
if (Array.isArray(node)) {
node.forEach((item) => walk(item, depth + 1));
} else if (node && typeof node === 'object') {
for (const [key, value] of Object.entries(node)) {
if (key === 'text' && typeof value === 'string') parts.push(value);
else walk(value, depth + 1);
}
}
};
walk(JSON.parse(source));
return parts.join(' ').trim() || source;
} catch {
return source;
}
}
/** Derive the unified execution list from a mocked monitoring payload. */
function buildExecutionList(monitoringData: unknown) {
const payload = monitoringData as { messages?: RawMonitoringMessage[] } | undefined;
const items = (payload?.messages ?? [])
.filter((message) => message.role !== 'assistant')
.map((message) => {
const status = String(message.status ?? 'success');
return {
source: 'pipeline' as const,
id: String(message.id),
event_id: null,
status,
status_group: executionStatusGroup(status, 'pipeline'),
title: executionText(message.message_content),
target_kind: 'pipeline' as const,
target_id: (message.pipeline_id as string) ?? null,
target_name: (message.pipeline_name as string) ?? null,
bot_id: (message.bot_id as string) ?? null,
bot_name: (message.bot_name as string) ?? null,
pipeline_id: (message.pipeline_id as string) ?? null,
pipeline_name: (message.pipeline_name as string) ?? null,
runner_id: (message.runner_name as string) ?? null,
conversation_id: null,
session_id: (message.session_id as string) ?? null,
created_at_ms: Date.parse(String(message.timestamp ?? '')) || null,
started_at_ms: null,
finished_at_ms: null,
duration_ms: null,
usage: null,
cost: null,
queue_name: null,
debug: false,
has_error: status === 'error',
};
});
const byStatus: Record<string, number> = {};
for (const item of items) {
byStatus[item.status_group] = (byStatus[item.status_group] ?? 0) + 1;
}
const completed = byStatus.completed ?? 0;
const failed = byStatus.failed ?? 0;
const denominator = completed + failed;
return {
items,
total: items.length,
limit: items.length,
offset: 0,
has_more: false,
summary: {
executions: {
total: items.length,
by_source: { agent: 0, pipeline: items.length },
by_status: byStatus,
completed,
failed,
running: byStatus.running ?? 0,
queued: byStatus.queued ?? 0,
cancelled: byStatus.cancelled ?? 0,
ignored: byStatus.ignored ?? 0,
waiting: 0,
success_rate:
denominator > 0 ? Math.round((completed / denominator) * 10000) / 100 : null,
denominator,
p50_duration_ms: null,
p95_duration_ms: null,
duration_sample: 0,
debug: 0,
real: items.length,
},
tokens: { total_tokens: 0, calls: 0, calls_with_usage: 0 },
},
};
}
function emptyTokenStatistics() {
return {
summary: {
@@ -1163,6 +1280,46 @@ async function handleBackendApi(route: Route, state: LangBotApiMockState) {
return fulfillJson(route, data.overview || emptyMonitoringData().overview);
}
if (path === '/api/v1/monitoring/executions') {
return fulfillJson(route, buildExecutionList(state.monitoringData));
}
const executionDetail = path.match(
/^\/api\/v1\/monitoring\/executions\/(agent|pipeline)\/(.+)$/,
);
if (executionDetail) {
const [, source, executionId] = executionDetail;
if (source === 'agent') {
return fulfillJson(route, {
source: 'agent',
run: { run_id: executionId, status: 'completed', created_at: 0 },
events: [],
has_more: false,
next_cursor: null,
});
}
const payload = state.monitoringData as {
messages?: RawMonitoringMessage[];
llmCalls?: RawMonitoringMessage[];
toolCalls?: RawMonitoringMessage[];
errors?: RawMonitoringMessage[];
};
const relatedToExecution = (rows: RawMonitoringMessage[] | undefined) =>
(rows ?? []).filter((row) => row.message_id === executionId);
return fulfillJson(route, {
source: 'pipeline',
message:
(payload.messages ?? []).find((row) => row.id === executionId) ?? {
id: executionId,
status: 'success',
message_content: '',
},
llm_calls: relatedToExecution(payload.llmCalls),
tool_calls: relatedToExecution(payload.toolCalls),
errors: relatedToExecution(payload.errors),
});
}
if (path === '/api/v1/monitoring/token-statistics') {
return fulfillJson(route, emptyTokenStatistics());
}
+8 -1
View File
@@ -59,7 +59,14 @@ test.describe('authenticated app shell', () => {
await page.goto('/home/monitoring');
await expect(page).toHaveURL(/\/home\/monitoring$/);
await expect(page.getByText('Total Messages').first()).toBeVisible();
// The unified execution view is the default entry point.
await expect(page.getByRole('tab', { name: 'Executions' })).toHaveAttribute(
'data-state',
'active',
);
await expect(
page.getByText('Execution Success Rate').first(),
).toBeVisible();
await expect(
page.getByRole('tab', { name: 'Message Records' }),
).toBeVisible();
+4
View File
@@ -304,6 +304,7 @@ test.describe('monitoring request contracts', () => {
await expect(page.getByText('No message records')).toHaveCount(0);
failing = false;
await page.getByRole('button', { name: 'Retry', exact: true }).click();
await page.getByRole('tab', { name: 'Message Records' }).click();
await expect(
page.getByText('Recovered monitoring', { exact: true }),
).toBeVisible();
@@ -324,6 +325,8 @@ test.describe('monitoring request contracts', () => {
await page.getByRole('option', { name: /Last 7 days/i }).click();
await expect.poll(() => pending.length).toBe(3);
await respond(pending[2], 'Latest filter data');
// The message-turn view now lives behind its own tab.
await page.getByRole('tab', { name: 'Message Records' }).click();
await expect(
page.getByText('Latest filter data', { exact: true }),
).toBeVisible();
@@ -598,6 +601,7 @@ test.describe('monitoring conversation turn grouping', () => {
});
await page.goto('/home/monitoring');
await page.getByRole('tab', { name: 'Message Records' }).click();
await expect(page.getByText('3 conversation turns')).toBeVisible();
await expect(
@@ -0,0 +1,131 @@
import assert from 'node:assert/strict';
import fs from 'node:fs';
import test from 'node:test';
import ts from 'typescript';
// The resolver is pure: recorded calls carry a time and a session, the execution
// list carries the runs' windows, so the link is decided without any re-query.
const source = fs.readFileSync(
new URL('../../src/app/home/monitoring/utils/callLinks.ts', import.meta.url),
'utf8',
);
const module = { exports: {} };
new Function(
'exports',
ts.transpileModule(source, {
compilerOptions: {
module: ts.ModuleKind.CommonJS,
target: ts.ScriptTarget.ES2022,
},
}).outputText,
)(module.exports);
const { findCallExecution, resolveCallTarget } = module.exports;
const at = (iso) => new Date(iso);
function run(overrides) {
return {
source: 'agent',
id: 'run-1',
session_id: 'session-a',
started_at_ms: at('2026-10-07T10:00:00Z').getTime(),
finished_at_ms: at('2026-10-07T10:00:30Z').getTime(),
...overrides,
};
}
function call(overrides) {
return {
id: 'call-1',
timestamp: at('2026-10-07T10:00:10Z'),
sessionId: 'session-a',
...overrides,
};
}
test('links a pipeline call to the execution carrying its message id', () => {
const execution = run({
source: 'pipeline',
id: 'message-9',
session_id: null,
});
assert.equal(
findCallExecution(call({ messageId: 'message-9', sessionId: undefined }), [
execution,
]),
execution,
);
});
test('links an agent call to the run whose window contains it', () => {
const execution = run({});
assert.equal(findCallExecution(call({}), [execution]), execution);
});
test('prefers the run that was live when a session has consecutive runs', () => {
const first = run({ id: 'run-1' });
const second = run({
id: 'run-2',
started_at_ms: at('2026-10-07T10:05:00Z').getTime(),
finished_at_ms: at('2026-10-07T10:05:30Z').getTime(),
});
assert.equal(
findCallExecution(call({ timestamp: at('2026-10-07T10:05:10Z') }), [
first,
second,
]).id,
'run-2',
);
});
test('an open run owns every later call of its session', () => {
const execution = run({ finished_at_ms: null });
assert.equal(
findCallExecution(call({ timestamp: at('2026-10-07T10:20:00Z') }), [
execution,
]),
execution,
);
});
test('a call outside every window links to nothing', () => {
const execution = run({});
assert.equal(
findCallExecution(call({ timestamp: at('2026-10-07T11:00:00Z') }), [
execution,
]),
null,
);
assert.equal(
findCallExecution(call({ timestamp: at('2026-10-07T09:00:00Z') }), [
execution,
]),
null,
);
assert.equal(
findCallExecution(call({ sessionId: 'other' }), [execution]),
null,
);
});
test('a call keeps its message target when the execution list cannot resolve it', () => {
assert.deepEqual(
resolveCallTarget(
call({ messageId: 'message-9', sessionId: 'other' }),
[run({})],
new Set(['message-9']),
),
{ kind: 'message', id: 'message-9' },
);
// An Agent call carries its run id in the same field: an id the message list
// does not hold must not become a jump to the message tab.
assert.equal(
resolveCallTarget(
call({ messageId: 'run-9', sessionId: 'other' }),
[run({})],
new Set(['message-9']),
),
null,
);
assert.equal(resolveCallTarget(call({}), []), null);
});