Improve monitoring conversation turns

2026-07-14 16:36:07 +00:00 · 2026-07-02 14:50:38 +08:00
parent 13dba887d5
commit b1bc05d5d3
15 changed files with 1259 additions and 268 deletions
@@ -72,6 +72,7 @@ interface LangBotApiMockState {
  counters: Record<string, number>;
  knowledgeBases: KnowledgeBaseMock[];
  mcpServers: MCPServerMock[];
+  monitoringData: unknown;
  pipelines: PipelineMock[];
  skills: SkillMock[];
 }
@@ -689,11 +690,12 @@ async function handleBackendApi(route: Route, state: LangBotApiMockState) {
  }

  if (path === '/api/v1/monitoring/data') {
-    return fulfillJson(route, emptyMonitoringData());
+    return fulfillJson(route, state.monitoringData);
  }

  if (path === '/api/v1/monitoring/overview') {
-    return fulfillJson(route, emptyMonitoringData().overview);
+    const data = state.monitoringData as { overview?: unknown };
+    return fulfillJson(route, data.overview || emptyMonitoringData().overview);
  }

  if (path === '/api/v1/monitoring/token-statistics') {
@@ -798,14 +800,19 @@ async function handleCloudApi(route: Route) {

 export async function installLangBotApiMocks(
  page: Page,
-  options: { authenticated?: boolean; storage?: JsonRecord } = {},
+  options: {
+    authenticated?: boolean;
+    monitoringData?: unknown;
+    storage?: JsonRecord;
+  } = {},
 ) {
-  const { authenticated = false, storage = {} } = options;
+  const { authenticated = false, monitoringData, storage = {} } = options;
  const state: LangBotApiMockState = {
    bots: [],
    counters: {},
    knowledgeBases: [],
    mcpServers: [],
+    monitoringData: monitoringData || emptyMonitoringData(),
    pipelines: [],
    skills: [],
  };
@@ -0,0 +1,359 @@
+import { expect, test } from '@playwright/test';
+
+import { installLangBotApiMocks } from './fixtures/langbot-api';
+import { buildConversationTurns } from '../../src/app/home/monitoring/utils/conversationTurns';
+import {
+  ErrorLog,
+  LLMCall,
+  MonitoringMessage,
+} from '../../src/app/home/monitoring/types/monitoring';
+
+const bot = {
+  id: 'bot-monitoring',
+  name: 'Monitoring Bot',
+};
+
+const pipeline = {
+  id: 'pipeline-monitoring',
+  name: 'Monitoring Pipeline',
+};
+
+function time(minute: number) {
+  return new Date(`2026-07-02T10:${String(minute).padStart(2, '0')}:00Z`);
+}
+
+function message(
+  id: string,
+  role: 'user' | 'assistant',
+  minute: number,
+  content: string,
+  sessionId = 'session-agent',
+): MonitoringMessage {
+  return {
+    id,
+    timestamp: time(minute),
+    botId: bot.id,
+    botName: bot.name,
+    pipelineId: pipeline.id,
+    pipelineName: pipeline.name,
+    messageContent: content,
+    sessionId,
+    status: 'success',
+    level: 'info',
+    platform: role === 'user' ? 'person' : 'bot',
+    userId: 'user-1',
+    userName: 'Playwright User',
+    runnerName: 'local-agent',
+    variables: '{}',
+    role,
+  };
+}
+
+function llmCall(
+  id: string,
+  minute: number,
+  messageId: string | undefined,
+  input: number,
+  output: number,
+  duration: number,
+  sessionId = 'session-agent',
+): LLMCall {
+  return {
+    id,
+    timestamp: time(minute),
+    modelName: 'gpt-5.5',
+    tokens: {
+      input,
+      output,
+      total: input + output,
+    },
+    duration,
+    status: 'success',
+    botId: bot.id,
+    botName: bot.name,
+    pipelineId: pipeline.id,
+    pipelineName: pipeline.name,
+    sessionId,
+    messageId,
+  };
+}
+
+function errorLog(id: string, minute: number, messageId: string): ErrorLog {
+  return {
+    id,
+    timestamp: time(minute),
+    errorType: 'ToolExecutionError',
+    errorMessage: 'Tool retry failed',
+    botId: bot.id,
+    botName: bot.name,
+    pipelineId: pipeline.id,
+    pipelineName: pipeline.name,
+    sessionId: 'session-agent',
+    messageId,
+  };
+}
+
+function rawMessage(message: MonitoringMessage) {
+  return {
+    id: message.id,
+    timestamp: message.timestamp.toISOString(),
+    bot_id: message.botId,
+    bot_name: message.botName,
+    pipeline_id: message.pipelineId,
+    pipeline_name: message.pipelineName,
+    message_content: message.messageContent,
+    session_id: message.sessionId,
+    status: message.status,
+    level: message.level,
+    platform: message.platform,
+    user_id: message.userId,
+    user_name: message.userName,
+    runner_name: message.runnerName,
+    variables: message.variables,
+    role: message.role,
+  };
+}
+
+function rawLlmCall(call: LLMCall) {
+  return {
+    id: call.id,
+    timestamp: call.timestamp.toISOString(),
+    model_name: call.modelName,
+    input_tokens: call.tokens.input,
+    output_tokens: call.tokens.output,
+    total_tokens: call.tokens.total,
+    duration: call.duration,
+    cost: call.cost,
+    status: call.status,
+    bot_id: call.botId,
+    bot_name: call.botName,
+    pipeline_id: call.pipelineId,
+    pipeline_name: call.pipelineName,
+    session_id: call.sessionId,
+    error_message: call.errorMessage,
+    message_id: call.messageId,
+  };
+}
+
+function rawError(error: ErrorLog) {
+  return {
+    id: error.id,
+    timestamp: error.timestamp.toISOString(),
+    error_type: error.errorType,
+    error_message: error.errorMessage,
+    bot_id: error.botId,
+    bot_name: error.botName,
+    pipeline_id: error.pipelineId,
+    pipeline_name: error.pipelineName,
+    session_id: error.sessionId,
+    stack_trace: error.stackTrace,
+    message_id: error.messageId,
+  };
+}
+
+function monitoringScenario() {
+  const messages = [
+    message(
+      'single-user',
+      'user',
+      1,
+      'Standalone question with no reply',
+      'session-single',
+    ),
+    message('agent-user-1', 'user', 10, 'Need deployment plan'),
+    message('agent-assistant-1', 'assistant', 11, 'Agent step 1: inspect repo'),
+    message('agent-assistant-2', 'assistant', 12, 'Agent step 2: run tests'),
+    message(
+      'agent-assistant-3',
+      'assistant',
+      13,
+      'Final answer: deployment plan ready',
+    ),
+    message('agent-user-2', 'user', 20, 'Continue with rollback plan'),
+    message('agent-assistant-4', 'assistant', 21, 'Rollback plan ready'),
+  ];
+  const llmCalls = [
+    llmCall('agent-call-1', 10, 'agent-user-1', 100, 40, 120),
+    llmCall('agent-call-2', 11, 'agent-user-1', 200, 60, 220),
+    llmCall('agent-call-3', 12, 'agent-user-1', 300, 90, 260),
+    llmCall('agent-call-4', 20, 'agent-user-2', 50, 25, 80),
+  ];
+  const errors = [errorLog('agent-error-1', 12, 'agent-user-1')];
+
+  return {
+    messages,
+    llmCalls,
+    errors,
+  };
+}
+
+function rawMonitoringData() {
+  const scenario = monitoringScenario();
+
+  return {
+    overview: {
+      total_messages: scenario.messages.length,
+      llm_calls: scenario.llmCalls.length,
+      embedding_calls: 0,
+      model_calls: scenario.llmCalls.length,
+      success_rate: 100,
+      active_sessions: 2,
+    },
+    messages: scenario.messages.map(rawMessage),
+    llmCalls: scenario.llmCalls.map(rawLlmCall),
+    embeddingCalls: [],
+    sessions: [],
+    errors: scenario.errors.map(rawError),
+    totalCount: {
+      messages: scenario.messages.length,
+      llmCalls: scenario.llmCalls.length,
+      embeddingCalls: 0,
+      sessions: 0,
+      errors: scenario.errors.length,
+    },
+  };
+}
+
+test.describe('monitoring conversation turn grouping', () => {
+  test('keeps a single user message as one observable turn', () => {
+    const userOnly = message(
+      'single-user-only',
+      'user',
+      1,
+      'No answer yet',
+      'session-user-only',
+    );
+
+    const turns = buildConversationTurns([userOnly], [], []);
+
+    expect(turns).toHaveLength(1);
+    expect(turns[0].id).toBe(userOnly.id);
+    expect(turns[0].userMessage?.messageContent).toBe('No answer yet');
+    expect(turns[0].assistantMessages).toHaveLength(0);
+    expect(turns[0].llmCalls).toHaveLength(0);
+    expect(turns[0].totalTokens).toBe(0);
+  });
+
+  test('groups multi-step agent execution and multiple replies into one user turn', () => {
+    const scenario = monitoringScenario();
+    const turns = buildConversationTurns(
+      scenario.messages,
+      scenario.llmCalls,
+      scenario.errors,
+    );
+
+    const agentTurn = turns.find((turn) => turn.id === 'agent-user-1');
+
+    expect(agentTurn).toBeTruthy();
+    expect(agentTurn?.userMessage?.messageContent).toBe('Need deployment plan');
+    expect(
+      agentTurn?.assistantMessages.map((item) => item.messageContent),
+    ).toEqual([
+      'Agent step 1: inspect repo',
+      'Agent step 2: run tests',
+      'Final answer: deployment plan ready',
+    ]);
+    expect(agentTurn?.llmCalls).toHaveLength(3);
+    expect(agentTurn?.errors).toHaveLength(1);
+    expect(agentTurn?.totalTokens).toBe(790);
+    expect(agentTurn?.totalDuration).toBe(600);
+  });
+
+  test('starts a new turn for each later user message in the same session', () => {
+    const firstUser = message('same-session-user-1', 'user', 1, 'First');
+    const firstReply = message(
+      'same-session-reply-1',
+      'assistant',
+      2,
+      'First reply',
+    );
+    const secondUser = message('same-session-user-2', 'user', 3, 'Second');
+    const secondReply = message(
+      'same-session-reply-2',
+      'assistant',
+      4,
+      'Second reply',
+    );
+
+    const turns = buildConversationTurns(
+      [firstUser, firstReply, secondUser, secondReply],
+      [
+        llmCall('same-session-call-1', 1, firstUser.id, 10, 5, 40),
+        llmCall('same-session-call-2', 3, secondUser.id, 20, 10, 50),
+      ],
+      [],
+    );
+
+    expect(turns.map((turn) => turn.id)).toEqual([
+      'same-session-user-2',
+      'same-session-user-1',
+    ]);
+    expect(
+      turns[0].assistantMessages.map((item) => item.messageContent),
+    ).toEqual(['Second reply']);
+    expect(
+      turns[1].assistantMessages.map((item) => item.messageContent),
+    ).toEqual(['First reply']);
+  });
+
+  test('attaches calls without message ids by session time', () => {
+    const user = message('fallback-user', 'user', 1, 'Use session fallback');
+    const assistant = message(
+      'fallback-assistant',
+      'assistant',
+      2,
+      'Fallback reply',
+    );
+    const call = llmCall('fallback-call', 2, undefined, 25, 5, 70);
+
+    const turns = buildConversationTurns([user, assistant], [call], []);
+
+    expect(turns).toHaveLength(1);
+    expect(turns[0].llmCalls).toHaveLength(1);
+    expect(turns[0].llmCalls[0].id).toBe(call.id);
+    expect(turns[0].totalTokens).toBe(30);
+  });
+
+  test('renders user-only, multi-agent, and multi-turn cases in the monitoring page', async ({
+    page,
+  }) => {
+    await installLangBotApiMocks(page, {
+      authenticated: true,
+      monitoringData: rawMonitoringData(),
+    });
+
+    await page.goto('/home/monitoring');
+
+    await expect(page.getByText('3 conversation turns')).toBeVisible();
+    await expect(
+      page.getByText('Standalone question with no reply'),
+    ).toBeVisible();
+    await expect(page.getByText('No assistant reply recorded')).toBeVisible();
+    await expect(page.getByText('Need deployment plan')).toBeVisible();
+    await expect(page.getByText('Agent step 1: inspect repo')).toBeVisible();
+    await expect(page.getByText('Assistant +2')).toBeVisible();
+    await expect(page.getByText('3 LLM')).toBeVisible();
+    await expect(page.getByText('790 tokens')).toBeVisible();
+    await expect(page.getByText('1 errors')).toBeVisible();
+    await expect(page.getByText('Continue with rollback plan')).toBeVisible();
+    await expect(page.getByText('Rollback plan ready')).toBeVisible();
+
+    const agentTurn = page
+      .locator('div[role="button"]')
+      .filter({ hasText: 'Need deployment plan' });
+    await expect(agentTurn).toHaveCount(1);
+    await agentTurn.click();
+
+    await expect(page.getByText('Conversation Trace')).toBeVisible();
+    await expect(page.getByText('Agent step 2: run tests')).toBeVisible();
+    await expect(
+      page.getByText('Final answer: deployment plan ready'),
+    ).toBeVisible();
+    await expect(page.getByText('LLM Calls (3)')).toBeVisible();
+    await expect(page.getByText('#3 gpt-5.5')).toBeVisible();
+    await expect(page.getByText('In: 300')).toBeVisible();
+    await expect(page.getByText('Out: 90')).toBeVisible();
+    await expect(page.getByText('Total: 390')).toBeVisible();
+    await expect(page.getByText('Tool retry failed')).toBeVisible();
+  });
+});