feat(telemetry): add isolated beta quality diagnostics and release identity

src/langbot/pkg/api/http/controller/group.py

@@ -26,6 +26,7 @@ from ..authz import (
 )
 from ..context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
 from ....cloud.support_admin import SupportAdminSessionError
+from ... import management_diagnostics as diagnostics
 
 if typing.TYPE_CHECKING:
     from ....core.app import Application
@@ -223,6 +224,7 @@ class RouterGroup(abc.ABC):
 
                 try:
                     if request_context is not None:
+                        diagnostics.workspace(request_context)
                         with bounded_executor.blocking_work_scope(request_context.workspace_uuid):
                             persistence_mgr = getattr(
                                 self.ap,
@@ -274,7 +276,14 @@ class RouterGroup(abc.ABC):
                     )
                     return self.internal_error_response(request_id)
 
-            new_f = handler_error
+            # Observe outside authentication, using the registered Core handler
+            # identity rather than the URL (which can contain user identifiers).
+            new_f = diagnostics.observe(
+                diagnostics.operation_id('http', f, rule=rule, methods=options.get('methods')),
+                source='http',
+                ap=self.ap,
+                http=True,
+            )(handler_error)
             # Quart/Flask requires a unique endpoint name even when the same URL
             # intentionally has separate handlers for different HTTP methods.
             # Include the method set so CRUD routes can declare distinct
@@ -561,6 +570,7 @@ class RouterGroup(abc.ABC):
     def fail(self, code: int | str, msg: str) -> quart.Response:
         """Return an error response"""
 
+        diagnostics.outcome('failed')
         return quart.jsonify(
             {
                 'code': code,

src/langbot/pkg/api/http/controller/groups/agent_debug_stream.py

@@ -7,6 +7,7 @@ import contextlib
 import json
 
 import quart
+from .... import management_diagnostics as diagnostics
 
 from .....agent.runner.errors import (
     RunnerError,
@@ -29,6 +30,9 @@ def debug_stream_response(service, context, agent_uuid: str, payload: dict) -> q
                 result = await service.debug_agent(context, agent_uuid, payload, on_result=on_result)
                 await queue.put({'kind': 'completed', 'data': result})
             except Exception as exc:
+                # The stream still uses HTTP 200 when execution returns an
+                # error frame. Mark the inherited request, not its contents.
+                diagnostics.outcome('failed')
                 if isinstance(exc, RunnerExecutionError):
                     code, message = exc.error_code or 'runner_execution_failed', exc.message
                 elif isinstance(exc, RunnerNotFoundError):

src/langbot/pkg/api/http/controller/groups/pipelines/embed.py

@@ -21,6 +21,7 @@ import httpx
 import quart
 
 from ... import group
+from ..... import management_diagnostics as diagnostics
 from ......utils import httpclient, paths
 from ......platform.sources.websocket_manager import WebSocketScope, is_valid_session_id, ws_connection_manager
 from .websocket_chat import create_scoped_duplex_tasks, wait_for_duplex_tasks
@@ -327,20 +328,24 @@ class EmbedRouterGroup(group.RouterGroup):
         # -- Embed WebSocket endpoint ----------------------------------------
 
         @self.quart_app.websocket(self.path + '/<bot_uuid>/ws/connect')
+        @diagnostics.observe('websocket.embed.session', source='websocket', ap=self.ap)
         async def embed_websocket_connect(bot_uuid: str):
             """WebSocket connection for embed widget, keyed by bot_uuid."""
             await quart.websocket.accept()
             if not _is_valid_uuid(bot_uuid):
                 await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Invalid bot_uuid format'}))
+                diagnostics.outcome('rejected')
                 return
 
             runtime_bot, pipeline_uuid = await self._resolve_bot(bot_uuid)
             if runtime_bot is None:
                 await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Bot not found or not available'}))
+                diagnostics.outcome('rejected')
                 return
 
             session_type = quart.websocket.args.get('session_type', 'person')
             if session_type not in ['person', 'group']:
+                diagnostics.outcome('rejected')
                 await quart.websocket.send(
                     json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
                 )
@@ -349,6 +354,7 @@ class EmbedRouterGroup(group.RouterGroup):
             session_id = quart.websocket.args.get('session_id', '')
             if not is_valid_session_id(session_id):
                 await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Valid session_id is required'}))
+                diagnostics.outcome('rejected')
                 return
 
             try:
@@ -356,13 +362,16 @@ class EmbedRouterGroup(group.RouterGroup):
                 await self._assert_execution_active(runtime_bot)
             except Exception:
                 await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
+                diagnostics.outcome('rejected')
                 return
 
+            diagnostics.workspace(runtime_bot.execution_context)
             try:
                 proxy_bot = await self.ap.platform_mgr.get_websocket_proxy_bot(runtime_bot.execution_context)
                 websocket_adapter = proxy_bot.adapter
                 if not websocket_adapter:
                     await quart.websocket.send(json.dumps({'type': 'error', 'message': 'WebSocket adapter not found'}))
+                    diagnostics.outcome('rejected')
                     return
 
                 connection = await ws_connection_manager.add_connection(
@@ -420,11 +429,13 @@ class EmbedRouterGroup(group.RouterGroup):
                 try:
                     await wait_for_duplex_tasks(receive_task, send_task)
                 except Exception as e:
+                    diagnostics.outcome('failed')
                     logger.error(f'Embed WebSocket task error: {e}')
                 finally:
                     await ws_connection_manager.remove_connection(connection.connection_id)
 
             except Exception as e:
+                diagnostics.outcome('failed')
                 logger.error(f'Embed WebSocket connection error: {e}', exc_info=True)
                 try:
                     await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'}))
@@ -439,28 +450,35 @@ class EmbedRouterGroup(group.RouterGroup):
                 message = await quart.websocket.receive()
                 await ws_connection_manager.update_activity(connection.connection_id)
 
-                try:
-                    data = await asyncio.to_thread(json.loads, message)
-                    message_type = data.get('type', 'message')
+                with diagnostics.scope(self.ap, 'websocket.embed.message', source='websocket'):
+                    diagnostics.workspace(getattr(owner_bot, 'execution_context', None))
+                    try:
+                        data = await asyncio.to_thread(json.loads, message)
+                        message_type = data.get('type', 'message')
 
-                    if message_type == 'ping':
-                        await connection.send_queue.put(
-                            {'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
-                        )
-                    elif message_type == 'message':
-                        try:
-                            current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid)
-                        except Exception:
-                            await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'})
+                        if message_type == 'ping':
+                            await connection.send_queue.put(
+                                {'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
+                            )
+                        elif message_type == 'message':
+                            try:
+                                current_bot = await self._resolve_connected_bot(owner_bot, pipeline_uuid)
+                            except Exception:
+                                diagnostics.outcome('rejected')
+                                await connection.send_queue.put({'type': 'error', 'message': 'Bot is unavailable'})
+                                break
+                            await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot)
+                        elif message_type == 'disconnect':
                             break
-                        await websocket_adapter.handle_websocket_message(connection, data, owner_bot=current_bot)
-                    elif message_type == 'disconnect':
-                        break
+                        else:
+                            diagnostics.outcome('skipped')
 
-                except json.JSONDecodeError:
-                    await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
+                    except json.JSONDecodeError:
+                        diagnostics.outcome('rejected')
+                        await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
 
         except Exception as e:
+            diagnostics.outcome('failed')
             logger.error(f'Embed receive error: {e}', exc_info=True)
         finally:
             connection.is_active = False
@@ -476,11 +494,13 @@ class EmbedRouterGroup(group.RouterGroup):
                     message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
                     if message is None:
                         break
-                    encoded = await asyncio.to_thread(json.dumps, message)
-                    await quart.websocket.send(encoded)
+                    with diagnostics.scope(self.ap, 'websocket.embed.send', source='websocket'):
+                        encoded = await asyncio.to_thread(json.dumps, message)
+                        await quart.websocket.send(encoded)
                 except asyncio.TimeoutError:
                     continue
         except Exception as e:
+            diagnostics.outcome('failed')
             logger.error(f'Embed send error: {e}', exc_info=True)
         finally:
             connection.is_active = False

src/langbot/pkg/api/http/controller/groups/pipelines/websocket_chat.py

@@ -14,6 +14,7 @@ import quart
 from ....authz import Permission, permissions_for_role, require_permission
 from ....context import PrincipalContext, PrincipalType, RequestContext, WorkspaceContext
 from ... import group
+from ..... import management_diagnostics as diagnostics
 from ......core.task_boundary import run_in_workspace_uow
 from ......platform.sources.websocket_manager import WebSocketScope, ws_connection_manager
 from ......utils import bounded_executor
@@ -210,6 +211,7 @@ class WebSocketChatRouterGroup(group.RouterGroup):
 
     async def initialize(self) -> None:
         @self.quart_app.websocket(self.path + '/connect')
+        @diagnostics.observe('websocket.dashboard.session', source='websocket', ap=self.ap)
         async def websocket_connect(pipeline_uuid: str):
             """Open one authenticated dashboard debug connection."""
 
@@ -217,11 +219,14 @@ class WebSocketChatRouterGroup(group.RouterGroup):
             try:
                 request_context, token = await self._authenticate_websocket()
             except Exception:
+                diagnostics.outcome('rejected')
                 await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Unauthorized'}))
                 return
 
+            diagnostics.workspace(request_context)
             session_type = quart.websocket.args.get('session_type', 'person')
             if session_type not in ['person', 'group']:
+                diagnostics.outcome('rejected')
                 await quart.websocket.send(
                     json.dumps({'type': 'error', 'message': 'session_type must be person or group'})
                 )
@@ -230,6 +235,7 @@ class WebSocketChatRouterGroup(group.RouterGroup):
             try:
                 websocket_adapter = await self._get_scoped_adapter(request_context, pipeline_uuid)
                 if websocket_adapter is None:
+                    diagnostics.outcome('rejected')
                     await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Pipeline not found'}))
                     return
 
@@ -288,11 +294,13 @@ class WebSocketChatRouterGroup(group.RouterGroup):
                 try:
                     await wait_for_duplex_tasks(receive_task, send_task)
                 except Exception as exc:
+                    diagnostics.outcome('failed')
                     logger.error(f'WebSocket task execution error: {exc}')
                 finally:
                     await ws_connection_manager.remove_connection(connection.connection_id)
 
             except Exception:
+                diagnostics.outcome('failed')
                 logger.error('Dashboard WebSocket connection error', exc_info=True)
                 try:
                     await quart.websocket.send(json.dumps({'type': 'error', 'message': 'Internal server error'}))
@@ -410,28 +418,34 @@ class WebSocketChatRouterGroup(group.RouterGroup):
                 message = await quart.websocket.receive()
                 await ws_connection_manager.update_activity(connection.connection_id)
 
-                try:
-                    data = await asyncio.to_thread(json.loads, message)
-                    message_type = data.get('type', 'message')
-                    if message_type == 'ping':
-                        await connection.send_queue.put(
-                            {'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
-                        )
-                    elif message_type == 'message':
-                        try:
-                            request_context = await self._revalidate_websocket_authorization(request_context, token)
-                        except Exception:
-                            await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'})
+                with diagnostics.scope(self.ap, 'websocket.dashboard.message', source='websocket'):
+                    diagnostics.workspace(request_context)
+                    try:
+                        data = await asyncio.to_thread(json.loads, message)
+                        message_type = data.get('type', 'message')
+                        if message_type == 'ping':
+                            await connection.send_queue.put(
+                                {'type': 'pong', 'timestamp': datetime.datetime.now().isoformat()}
+                            )
+                        elif message_type == 'message':
+                            try:
+                                request_context = await self._revalidate_websocket_authorization(request_context, token)
+                            except Exception:
+                                diagnostics.outcome('rejected')
+                                await connection.send_queue.put({'type': 'error', 'message': 'Unauthorized'})
+                                break
+                            await websocket_adapter.handle_websocket_message(connection, data)
+                        elif message_type == 'disconnect':
                             break
-                        await websocket_adapter.handle_websocket_message(connection, data)
-                    elif message_type == 'disconnect':
-                        break
-                    else:
-                        logger.warning(f'Unknown WebSocket message type: {message_type}')
-                except json.JSONDecodeError:
-                    await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
+                        else:
+                            diagnostics.outcome('skipped')
+                            logger.warning(f'Unknown WebSocket message type: {message_type}')
+                    except json.JSONDecodeError:
+                        diagnostics.outcome('rejected')
+                        await connection.send_queue.put({'type': 'error', 'message': 'Invalid JSON format'})
 
         except Exception:
+            diagnostics.outcome('failed')
             logger.error('Dashboard WebSocket receive error', exc_info=True)
         finally:
             connection.is_active = False
@@ -447,11 +461,13 @@ class WebSocketChatRouterGroup(group.RouterGroup):
                     message = await asyncio.wait_for(connection.send_queue.get(), timeout=1.0)
                     if message is None:
                         break
-                    encoded = await asyncio.to_thread(json.dumps, message)
-                    await quart.websocket.send(encoded)
+                    with diagnostics.scope(self.ap, 'websocket.dashboard.send', source='websocket'):
+                        encoded = await asyncio.to_thread(json.dumps, message)
+                        await quart.websocket.send(encoded)
                 except asyncio.TimeoutError:
                     continue
         except Exception:
+            diagnostics.outcome('failed')
             logger.error('Dashboard WebSocket send error', exc_info=True)
         finally:
             connection.is_active = False

src/langbot/pkg/api/http/service/agent.py

@@ -6,6 +6,7 @@ import fnmatch
 import time
 import uuid
 import typing
+from ... import management_diagnostics as diagnostics
 
 import sqlalchemy
 from langbot_plugin.api.entities.builtin.runner.delivery import DeliveryContext
@@ -130,6 +131,7 @@ class AgentService:
 
         return None
 
+    @diagnostics.observe('http.agent.debug_agent', source='webui_debug')
     async def debug_agent(
         self,
         context: RequestContext,
@@ -144,6 +146,7 @@ class AgentService:
         delivers outputs to a real platform, and supports both message and
         non-message event envelopes.
         """
+        diagnostics.workspace(context)
         agent = await self.get_agent(context, agent_uuid)
         if agent is None or agent.get('kind') not in {AGENT_KIND_AGENT, AGENT_KIND_EVENT_PROCESSOR}:
             raise ValueError('Agent not found')