test: add frontend smoke and backend e2e CI (#2251)

web/.gitignore

@@ -12,6 +12,8 @@
 
 # testing
 /coverage
+/playwright-report
+/test-results
 
 # next.js
 /dist/

web/README.md

@@ -1,3 +1,13 @@
 # Debug LangBot Frontend
 
 Please refer to the [Development Guide](https://link.langbot.app/en/docs/dev-config) for more information.
+
+## Tests
+
+Run the frontend smoke tests without a backend process:
+
+```bash
+pnpm test:e2e
+```
+
+The Playwright suite starts Vite and mocks the LangBot backend and Space APIs.

web/package.json

@@ -6,6 +6,7 @@
     "dev": "vite",
     "build": "tsc && vite build",
     "preview": "vite preview",
+    "test:e2e": "playwright test",
     "lint": "eslint .",
     "format": "prettier --write ."
   },
@@ -86,6 +87,7 @@
     "zod": "^3.24.4"
   },
   "devDependencies": {
+    "@playwright/test": "^1.61.0",
     "@types/debug": "^4.1.12",
     "@types/estree": "^1.0.8",
     "@types/estree-jsx": "^1.0.5",

web/playwright.config.ts

@@ -0,0 +1,25 @@
+import { defineConfig, devices } from '@playwright/test';
+
+export default defineConfig({
+  testDir: './tests/e2e',
+  fullyParallel: true,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 1 : 0,
+  reporter: process.env.CI ? [['github'], ['list']] : 'list',
+  use: {
+    baseURL: 'http://127.0.0.1:4173',
+    trace: 'on-first-retry',
+  },
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+  ],
+  webServer: {
+    command: 'pnpm exec vite --host 127.0.0.1 --port 4173',
+    url: 'http://127.0.0.1:4173',
+    reuseExistingServer: !process.env.CI,
+    timeout: 120_000,
+  },
+});

web/pnpm-lock.yaml

@@ -192,6 +192,9 @@ dependencies:
     version: 3.25.76
 
 devDependencies:
+  '@playwright/test':
+    specifier: ^1.61.0
+    version: 1.61.0
   '@types/debug':
     specifier: ^4.1.12
     version: 4.1.12
@@ -529,6 +532,14 @@ packages:
     engines: {node: ^12.20.0 || ^14.18.0 || >=16.0.0}
     dev: true
 
+  /@playwright/test@1.61.0:
+    resolution: {integrity: sha512-cKA5B6lpFEMyMGjxF54QihfYpB4FkEGH+qZhtArDEG+wezQAJY8Pq6C7T1SjWz+FFzt3TbyoXBQYk/0292TdJA==}
+    engines: {node: '>=18'}
+    hasBin: true
+    dependencies:
+      playwright: 1.61.0
+    dev: true
+
   /@radix-ui/number@1.1.1:
     resolution: {integrity: sha512-MkKCwxlXTgz6CFoJx3pCwn07GKp36+aZyu/u2Ln2VrA5DcdyCZkASEDBTd8x5whTQQL5CiYf4prXKLcgQdv29g==}
     dev: false
@@ -3204,6 +3215,14 @@ packages:
     engines: {node: '>=0.4.x'}
     dev: false
 
+  /fsevents@2.3.2:
+    resolution: {integrity: sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==}
+    engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
+    os: [darwin]
+    requiresBuild: true
+    dev: true
+    optional: true
+
   /fsevents@2.3.3:
     resolution: {integrity: sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==}
     engines: {node: ^8.16.0 || ^10.6.0 || >=11.0.0}
@@ -4940,6 +4959,22 @@ packages:
     hasBin: true
     dev: true
 
+  /playwright-core@1.61.0:
+    resolution: {integrity: sha512-caX7TrY3Ml6egyDX0WUcTHDxodl/b51y5wJOdCEA36QviK/s2g081hvmGs8eaE3DWb6NYZQ6BjO/QkNRPenoPA==}
+    engines: {node: '>=18'}
+    hasBin: true
+    dev: true
+
+  /playwright@1.61.0:
+    resolution: {integrity: sha512-Z+7BeeqQPRRzklHsVFP4KTGIyMxKUmfeRA4WisM6G3/XW6nwGeX6fX9qYaDa+CiUqpOkb2f6X3nar05R3kSuJQ==}
+    engines: {node: '>=18'}
+    hasBin: true
+    dependencies:
+      playwright-core: 1.61.0
+    optionalDependencies:
+      fsevents: 2.3.2
+    dev: true
+
   /pngjs@5.0.0:
     resolution: {integrity: sha512-40QW5YalBNfQo5yRYmiw7Yz6TKKVr3h6970B2YE+3fQpsWcrbj1PzJgxeJ19DRQjhMbKPIuMY8rFaXc8moolVw==}
     engines: {node: '>=10.13.0'}

web/tests/e2e/fixtures/langbot-api.ts

@@ -0,0 +1,417 @@
+import { Page, Route } from '@playwright/test';
+
+type JsonRecord = Record<string, unknown>;
+
+interface SkillMock {
+  name: string;
+  display_name: string;
+  description: string;
+  instructions: string;
+  package_root: string;
+  updated_at: string;
+}
+
+interface LangBotApiMockState {
+  skills: SkillMock[];
+}
+
+function ok(data: unknown) {
+  return {
+    code: 0,
+    message: 'ok',
+    data,
+    timestamp: Date.now(),
+  };
+}
+
+async function fulfillJson(route: Route, data: unknown) {
+  await route.fulfill({
+    status: 200,
+    contentType: 'application/json',
+    body: JSON.stringify(ok(data)),
+  });
+}
+
+function routePath(route: Route) {
+  return new URL(route.request().url()).pathname;
+}
+
+function emptyMonitoringData() {
+  return {
+    overview: {
+      total_messages: 0,
+      llm_calls: 0,
+      embedding_calls: 0,
+      model_calls: 0,
+      success_rate: 0,
+      active_sessions: 0,
+    },
+    messages: [],
+    llmCalls: [],
+    embeddingCalls: [],
+    sessions: [],
+    errors: [],
+    totalCount: {
+      messages: 0,
+      llmCalls: 0,
+      embeddingCalls: 0,
+      sessions: 0,
+      errors: 0,
+    },
+  };
+}
+
+function emptyTokenStatistics() {
+  return {
+    summary: {
+      total_calls: 0,
+      success_calls: 0,
+      error_calls: 0,
+      total_input_tokens: 0,
+      total_output_tokens: 0,
+      total_tokens: 0,
+      total_cost: 0,
+      avg_tokens_per_call: 0,
+      avg_duration_ms: 0,
+      avg_tokens_per_second: 0,
+      zero_token_success_calls: 0,
+    },
+    by_model: [],
+    timeseries: [],
+    bucket: 'day',
+  };
+}
+
+function makeSkill(data: JsonRecord): SkillMock {
+  return {
+    name: String(data.name || ''),
+    display_name: String(data.display_name || ''),
+    description: String(data.description || ''),
+    instructions: String(data.instructions || ''),
+    package_root: String(data.package_root || ''),
+    updated_at: new Date().toISOString(),
+  };
+}
+
+async function handleBackendApi(route: Route, state: LangBotApiMockState) {
+  const request = route.request();
+  const url = new URL(request.url());
+  const path = url.pathname;
+  const method = request.method();
+
+  if (path === '/api/v1/system/info') {
+    return fulfillJson(route, {
+      debug: false,
+      version: 'frontend-smoke',
+      edition: 'community',
+      cloud_service_url: 'https://space.langbot.app',
+      enable_marketplace: true,
+      allow_modify_login_info: true,
+      disable_models_service: false,
+      limitation: {
+        max_bots: -1,
+        max_pipelines: -1,
+        max_extensions: -1,
+      },
+      outbound_ips: [],
+      wizard_status: 'completed',
+      wizard_progress: null,
+    });
+  }
+
+  if (path === '/api/v1/user/account-info') {
+    return fulfillJson(route, {
+      initialized: true,
+      account_type: 'local',
+      has_password: true,
+    });
+  }
+
+  if (path === '/api/v1/user/check-token') {
+    return fulfillJson(route, { token: '' });
+  }
+
+  if (path === '/api/v1/user/auth') {
+    return fulfillJson(route, { token: 'playwright-token' });
+  }
+
+  if (path === '/api/v1/user/info') {
+    return fulfillJson(route, {
+      user: 'admin@example.com',
+      account_type: 'local',
+      has_password: true,
+    });
+  }
+
+  if (path === '/api/v1/user/space-credits') {
+    return fulfillJson(route, { credits: null });
+  }
+
+  if (path === '/api/v1/platform/bots') {
+    return fulfillJson(route, { bots: [] });
+  }
+
+  if (path === '/api/v1/pipelines') {
+    return fulfillJson(route, { pipelines: [] });
+  }
+
+  if (path === '/api/v1/knowledge/bases') {
+    return fulfillJson(route, { bases: [] });
+  }
+
+  if (path === '/api/v1/knowledge/migration/status') {
+    return fulfillJson(route, {
+      needed: false,
+      internal_kb_count: 0,
+      external_kb_count: 0,
+    });
+  }
+
+  if (path === '/api/v1/plugins') {
+    return fulfillJson(route, { plugins: [] });
+  }
+
+  if (path === '/api/v1/extensions') {
+    return fulfillJson(route, { extensions: [] });
+  }
+
+  if (path === '/api/v1/mcp/servers') {
+    return fulfillJson(route, { servers: [] });
+  }
+
+  if (path === '/api/v1/skills') {
+    if (method === 'POST') {
+      const skill = makeSkill(
+        JSON.parse(request.postData() || '{}') as JsonRecord,
+      );
+      state.skills = [
+        ...state.skills.filter((item) => item.name !== skill.name),
+        skill,
+      ];
+      return fulfillJson(route, { skill });
+    }
+
+    return fulfillJson(route, { skills: state.skills });
+  }
+
+  const skillFileMatch = path.match(
+    /^\/api\/v1\/skills\/([^/]+)\/files\/(.+)$/,
+  );
+  if (skillFileMatch) {
+    const skillName = decodeURIComponent(skillFileMatch[1]);
+    const filePath = decodeURIComponent(skillFileMatch[2]);
+    const skill = state.skills.find((item) => item.name === skillName);
+    return fulfillJson(route, {
+      skill: { name: skillName },
+      path: filePath,
+      content: skill?.instructions || '',
+    });
+  }
+
+  const skillFilesMatch = path.match(/^\/api\/v1\/skills\/([^/]+)\/files$/);
+  if (skillFilesMatch) {
+    const skillName = decodeURIComponent(skillFilesMatch[1]);
+    return fulfillJson(route, {
+      skill: { name: skillName },
+      base_path: '.',
+      entries: [
+        {
+          path: 'SKILL.md',
+          name: 'SKILL.md',
+          is_dir: false,
+          size: null,
+        },
+      ],
+      truncated: false,
+    });
+  }
+
+  const skillMatch = path.match(/^\/api\/v1\/skills\/([^/]+)$/);
+  if (skillMatch) {
+    const skillName = decodeURIComponent(skillMatch[1]);
+    const skill = state.skills.find((item) => item.name === skillName) || {
+      name: skillName,
+      display_name: '',
+      description: '',
+      instructions: '',
+      package_root: '',
+      updated_at: new Date().toISOString(),
+    };
+    return fulfillJson(route, { skill });
+  }
+
+  if (path === '/api/v1/system/status/plugin-system') {
+    return fulfillJson(route, {
+      is_enable: true,
+      is_connected: true,
+      plugin_connector_error: '',
+    });
+  }
+
+  if (path === '/api/v1/plugins/debug-info') {
+    return fulfillJson(route, {
+      debug_url: 'ws://127.0.0.1:5300/plugin/debug',
+      plugin_debug_key: 'test-debug-key',
+    });
+  }
+
+  if (path === '/api/v1/box/status') {
+    return fulfillJson(route, {
+      available: true,
+      enabled: true,
+      profile: 'playwright',
+      recent_error_count: 0,
+      active_sessions: 0,
+      managed_processes: 0,
+      session_ttl_sec: 3600,
+      backend: {
+        name: 'playwright',
+        available: true,
+      },
+    });
+  }
+
+  if (path === '/api/v1/box/sessions') {
+    return fulfillJson(route, []);
+  }
+
+  if (path === '/api/v1/monitoring/data') {
+    return fulfillJson(route, emptyMonitoringData());
+  }
+
+  if (path === '/api/v1/monitoring/overview') {
+    return fulfillJson(route, emptyMonitoringData().overview);
+  }
+
+  if (path === '/api/v1/monitoring/token-statistics') {
+    return fulfillJson(route, emptyTokenStatistics());
+  }
+
+  if (path === '/api/v1/monitoring/feedback/stats') {
+    return fulfillJson(route, {
+      total_feedback: 0,
+      total_likes: 0,
+      total_dislikes: 0,
+      satisfaction_rate: 0,
+    });
+  }
+
+  if (path === '/api/v1/monitoring/feedback') {
+    return fulfillJson(route, { feedback: [], total: 0 });
+  }
+
+  if (path === '/api/v1/survey/pending') {
+    return fulfillJson(route, { survey: null });
+  }
+
+  if (path === '/api/v1/system/tasks') {
+    return fulfillJson(route, { tasks: [] });
+  }
+
+  if (
+    path === '/api/v1/marketplace/plugins' ||
+    path === '/api/v1/marketplace/plugins/search' ||
+    path === '/api/v1/marketplace/extensions/search' ||
+    path === '/api/v1/marketplace/mcps/search' ||
+    path === '/api/v1/marketplace/skills/search'
+  ) {
+    return fulfillJson(route, { plugins: [], total: 0 });
+  }
+
+  if (path === '/api/v1/marketplace/tags') {
+    return fulfillJson(route, { tags: [] });
+  }
+
+  if (path === '/api/v1/marketplace/recommendation-lists') {
+    return fulfillJson(route, { lists: [] });
+  }
+
+  if (path === '/api/v1/dist/info/releases') {
+    return fulfillJson(route, []);
+  }
+
+  if (path === '/api/v1/dist/info/repo') {
+    return fulfillJson(route, {
+      repo: {
+        stargazers_count: 0,
+        forks_count: 0,
+        open_issues_count: 0,
+      },
+      contributors: [],
+    });
+  }
+
+  await fulfillJson(route, {});
+}
+
+async function handleCloudApi(route: Route) {
+  const path = routePath(route);
+
+  if (
+    path === '/api/v1/marketplace/plugins' ||
+    path === '/api/v1/marketplace/plugins/search' ||
+    path === '/api/v1/marketplace/extensions/search' ||
+    path === '/api/v1/marketplace/mcps/search' ||
+    path === '/api/v1/marketplace/skills/search'
+  ) {
+    return fulfillJson(route, { plugins: [], total: 0 });
+  }
+
+  if (path === '/api/v1/marketplace/tags') {
+    return fulfillJson(route, { tags: [] });
+  }
+
+  if (path === '/api/v1/marketplace/recommendation-lists') {
+    return fulfillJson(route, { lists: [] });
+  }
+
+  if (path === '/api/v1/dist/info/releases') {
+    return fulfillJson(route, []);
+  }
+
+  if (path === '/api/v1/dist/info/repo') {
+    return fulfillJson(route, {
+      repo: {
+        stargazers_count: 0,
+        forks_count: 0,
+        open_issues_count: 0,
+      },
+      contributors: [],
+    });
+  }
+
+  await fulfillJson(route, {});
+}
+
+export async function installLangBotApiMocks(
+  page: Page,
+  options: { authenticated?: boolean; storage?: JsonRecord } = {},
+) {
+  const { authenticated = false, storage = {} } = options;
+  const state: LangBotApiMockState = {
+    skills: [],
+  };
+
+  await page.addInitScript(
+    ({ authenticated, storage }) => {
+      localStorage.setItem('langbot_language', 'en-US');
+      localStorage.setItem('extensions_group_by_type', 'false');
+
+      if (authenticated) {
+        localStorage.setItem('token', 'playwright-token');
+        localStorage.setItem('userEmail', 'admin@example.com');
+      } else {
+        localStorage.removeItem('token');
+        localStorage.removeItem('userEmail');
+      }
+
+      for (const [key, value] of Object.entries(storage)) {
+        localStorage.setItem(key, String(value));
+      }
+    },
+    { authenticated, storage },
+  );
+
+  await page.route('**/api/v1/**', (route) => handleBackendApi(route, state));
+  await page.route('https://space.langbot.app/**', handleCloudApi);
+}

web/tests/e2e/home-smoke.spec.ts

@@ -0,0 +1,133 @@
+import { expect, test } from '@playwright/test';
+
+import { installLangBotApiMocks } from './fixtures/langbot-api';
+
+const appRoutes = [
+  {
+    path: '/home/bots',
+    heading: 'Bots',
+    bodyText: 'Select a bot from the sidebar',
+  },
+  {
+    path: '/home/pipelines',
+    heading: 'Pipelines',
+    bodyText: 'Select a pipeline from the sidebar',
+  },
+  {
+    path: '/home/extensions',
+    heading: 'Extensions',
+    bodyText: 'No extensions installed',
+  },
+  {
+    path: '/home/mcp',
+    heading: 'MCP',
+    bodyText: 'Select an MCP server from the sidebar',
+  },
+  {
+    path: '/home/knowledge',
+    heading: 'Knowledge',
+    bodyText: 'Select a knowledge base from the sidebar',
+  },
+];
+
+test.describe('authenticated app shell', () => {
+  for (const route of appRoutes) {
+    test(`${route.path} renders without a backend process`, async ({
+      page,
+    }) => {
+      await installLangBotApiMocks(page, { authenticated: true });
+
+      await page.goto(route.path);
+
+      await expect(page).toHaveURL(new RegExp(`${route.path}$`));
+      await expect(page.getByText('Home').first()).toBeVisible();
+      await expect(
+        page.getByRole('button', { name: 'Dashboard' }),
+      ).toBeVisible();
+      await expect(page.getByText('Extensions').first()).toBeVisible();
+      await expect(page.getByText(route.heading).first()).toBeVisible();
+      await expect(page.getByText(route.bodyText)).toBeVisible();
+      await expect(page.getByText('Backend unavailable')).toHaveCount(0);
+    });
+  }
+
+  test('/home/monitoring loads dashboard data from mocked APIs', async ({
+    page,
+  }) => {
+    await installLangBotApiMocks(page, { authenticated: true });
+
+    await page.goto('/home/monitoring');
+
+    await expect(page).toHaveURL(/\/home\/monitoring$/);
+    await expect(page.getByText('Total Messages').first()).toBeVisible();
+    await expect(
+      page.getByRole('tab', { name: 'Message Records' }),
+    ).toBeVisible();
+    await expect(
+      page.getByRole('tab', { name: 'Token Monitoring' }),
+    ).toBeVisible();
+
+    await page.getByRole('tab', { name: 'Token Monitoring' }).click();
+    await expect(
+      page.getByText('No token usage in the selected time range'),
+    ).toBeVisible();
+    await expect(page.getByText('Unable to connect to server')).toHaveCount(0);
+  });
+
+  test('/home/extensions shows plugin debug information from the backend', async ({
+    page,
+  }) => {
+    await installLangBotApiMocks(page, { authenticated: true });
+
+    await page.goto('/home/extensions');
+
+    await page.getByRole('button', { name: 'Debug Info' }).click();
+
+    await expect(page.getByText('Plugin Debug Information')).toBeVisible();
+    await expect(page.getByRole('textbox').nth(0)).toHaveValue(
+      'ws://127.0.0.1:5300/plugin/debug',
+    );
+    await expect(page.getByRole('textbox').nth(1)).toHaveValue(
+      'test-debug-key',
+    );
+  });
+
+  test('/home/skills?action=create creates a manual skill', async ({
+    page,
+  }) => {
+    await installLangBotApiMocks(page, { authenticated: true });
+
+    await page.goto('/home/skills?action=create');
+
+    await expect(page).toHaveURL(/\/home\/skills\?action=create$/);
+    await expect(page.getByText('Create Skill').first()).toBeVisible();
+    await expect(page.getByText('Import Local Skill Directory')).toBeVisible();
+
+    const saveButton = page.getByRole('button', { name: 'Save' });
+    await expect(saveButton).toBeEnabled();
+    await saveButton.click();
+    await expect(page.getByText('Skill name cannot be empty')).toBeVisible();
+
+    await page.locator('#display_name').fill('Daily Summary');
+    await page.locator('#name').fill('daily_summary');
+    await page
+      .locator('#description')
+      .fill('Summarizes the current conversation for handoff.');
+    await page
+      .locator('#instructions')
+      .fill('Summarize the conversation in five concise bullet points.');
+    await saveButton.click();
+
+    await expect(page).toHaveURL(/\/home\/skills\?id=daily_summary$/);
+    await expect(
+      page.getByRole('heading', { name: 'Daily Summary' }),
+    ).toBeVisible();
+    await expect(page.locator('#name')).toHaveValue('daily_summary');
+    await expect(page.locator('#description')).toHaveValue(
+      'Summarizes the current conversation for handoff.',
+    );
+    await expect(page.locator('#instructions')).toHaveValue(
+      'Summarize the conversation in five concise bullet points.',
+    );
+  });
+});

web/tests/e2e/login.spec.ts

@@ -0,0 +1,22 @@
+import { expect, test } from '@playwright/test';
+
+import { installLangBotApiMocks } from './fixtures/langbot-api';
+
+test('local account login reaches the authenticated home shell', async ({
+  page,
+}) => {
+  await installLangBotApiMocks(page);
+
+  await page.goto('/login');
+
+  await expect(page.getByText('Welcome')).toBeVisible();
+  await page.getByPlaceholder('Enter email address').fill('admin@example.com');
+  await page.getByPlaceholder('Enter password').fill('password');
+  await page.getByRole('button', { name: 'Login with password' }).click();
+
+  await expect(page).toHaveURL(/\/home$/);
+  await expect(page.getByText('Home').first()).toBeVisible();
+  await expect(page.getByRole('button', { name: 'Dashboard' })).toBeVisible();
+  await expect(page.getByText('Total Messages').first()).toBeVisible();
+  await expect(page.getByText('Unable to connect to server')).toHaveCount(0);
+});