mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-08-09 12:40:59 +00:00
fix(cloud): bound tenant maintenance and monitoring work
This commit is contained in:
@@ -21,6 +21,8 @@ from .tenant import TenantContext, require_workspace_uuid
|
||||
LOG_FILE_PATTERN = re.compile(r'^langbot-(\d{4}-\d{2}-\d{2})\.log(?:\.\d+)?$')
|
||||
DEFAULT_UPLOAD_FILE_RETENTION_DAYS = 7
|
||||
DEFAULT_LOG_RETENTION_DAYS = 3
|
||||
DEFAULT_MAX_FILES_PER_RUN = 1000
|
||||
HARD_MAX_FILES_PER_RUN = 10000
|
||||
UPLOAD_OWNER_TYPES = ('upload_image', 'upload_document', 'upload')
|
||||
|
||||
|
||||
@@ -51,6 +53,17 @@ class MaintenanceService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
def _max_files_per_run(self) -> int:
|
||||
cleanup_cfg = (
|
||||
getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('storage', {}).get('cleanup', {})
|
||||
)
|
||||
value = self._positive_int(
|
||||
cleanup_cfg.get('max_files_per_run', DEFAULT_MAX_FILES_PER_RUN),
|
||||
DEFAULT_MAX_FILES_PER_RUN,
|
||||
'storage.cleanup.max_files_per_run',
|
||||
)
|
||||
return min(value, HARD_MAX_FILES_PER_RUN)
|
||||
|
||||
@_workspace_scope
|
||||
async def cleanup_expired_files(self, context: ExecutionContext) -> dict[str, int]:
|
||||
if not isinstance(context, ExecutionContext):
|
||||
@@ -252,6 +265,7 @@ class MaintenanceService:
|
||||
provider = self.ap.storage_mgr.storage_provider
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(days=retention_days)
|
||||
candidates = []
|
||||
max_candidates = self._max_files_per_run()
|
||||
paginator = provider.s3_client.get_paginator('list_objects_v2')
|
||||
|
||||
seen_prefixes: set[str] = set()
|
||||
@@ -274,6 +288,8 @@ class MaintenanceService:
|
||||
'modified_at': last_modified.isoformat(),
|
||||
}
|
||||
)
|
||||
if len(candidates) >= max_candidates:
|
||||
return candidates
|
||||
|
||||
return candidates
|
||||
|
||||
@@ -310,6 +326,7 @@ class MaintenanceService:
|
||||
storage_root = Path('data/storage')
|
||||
cutoff = datetime.datetime.now().timestamp() - retention_days * 86400
|
||||
candidates = []
|
||||
max_candidates = self._max_files_per_run()
|
||||
seen_roots: set[Path] = set()
|
||||
for owner_type in UPLOAD_OWNER_TYPES:
|
||||
scoped_root = storage_root / self.ap.storage_mgr.scoped_prefix(context, owner_type=owner_type)
|
||||
@@ -335,6 +352,8 @@ class MaintenanceService:
|
||||
if include_paths:
|
||||
item['path'] = str(entry)
|
||||
candidates.append(item)
|
||||
if len(candidates) >= max_candidates:
|
||||
return candidates
|
||||
return candidates
|
||||
|
||||
def _expired_log_candidates(self, retention_days: int, include_paths: bool = False) -> list[dict[str, Any]]:
|
||||
|
||||
@@ -15,6 +15,20 @@ from ..context import ExecutionContext
|
||||
from .tenant import TenantContext, require_workspace_uuid
|
||||
|
||||
|
||||
_DEFAULT_MONITORING_PAGE_ROWS = 1000
|
||||
_DEFAULT_MONITORING_EXPORT_ROWS = 10000
|
||||
_DEFAULT_MONITORING_DETAIL_ROWS = 2000
|
||||
_DEFAULT_MONITORING_TIMESERIES_BUCKETS = 1000
|
||||
_DEFAULT_MONITORING_MAX_OFFSET = 1000000
|
||||
_HARD_MAX_MONITORING_PAGE_ROWS = 5000
|
||||
_HARD_MAX_MONITORING_EXPORT_ROWS = 50000
|
||||
_HARD_MAX_MONITORING_DETAIL_ROWS = 10000
|
||||
_HARD_MAX_MONITORING_TIMESERIES_BUCKETS = 10000
|
||||
_HARD_MAX_MONITORING_OFFSET = 10000000
|
||||
_DEFAULT_CLEANUP_BATCHES_PER_TABLE = 4
|
||||
_HARD_MAX_CLEANUP_BATCHES_PER_TABLE = 100
|
||||
|
||||
|
||||
def _workspace_transaction(method):
|
||||
"""Run an explicit service entrypoint in one Workspace transaction."""
|
||||
|
||||
@@ -38,6 +52,88 @@ class MonitoringService:
|
||||
def __init__(self, ap: app.Application) -> None:
|
||||
self.ap = ap
|
||||
|
||||
def _configured_query_limit(self, name: str, default: int, hard_max: int) -> int:
|
||||
config = (
|
||||
getattr(getattr(self.ap, 'instance_config', None), 'data', {}).get('monitoring', {}).get('query_limits', {})
|
||||
)
|
||||
try:
|
||||
value = int(config.get(name, default))
|
||||
except (TypeError, ValueError):
|
||||
value = default
|
||||
return min(max(value, 1), hard_max)
|
||||
|
||||
def normalize_page_window(self, limit: int, offset: int = 0) -> tuple[int, int]:
|
||||
"""Clamp tenant-controlled pagination before constructing a DB query."""
|
||||
|
||||
page_cap = self._configured_query_limit(
|
||||
'page_rows',
|
||||
_DEFAULT_MONITORING_PAGE_ROWS,
|
||||
_HARD_MAX_MONITORING_PAGE_ROWS,
|
||||
)
|
||||
offset_cap = self._configured_query_limit(
|
||||
'max_offset',
|
||||
_DEFAULT_MONITORING_MAX_OFFSET,
|
||||
_HARD_MAX_MONITORING_OFFSET,
|
||||
)
|
||||
try:
|
||||
normalized_limit = int(limit)
|
||||
except (TypeError, ValueError):
|
||||
normalized_limit = 100
|
||||
try:
|
||||
normalized_offset = int(offset)
|
||||
except (TypeError, ValueError):
|
||||
normalized_offset = 0
|
||||
return (
|
||||
min(max(normalized_limit, 1), page_cap),
|
||||
min(max(normalized_offset, 0), offset_cap),
|
||||
)
|
||||
|
||||
def normalize_export_limit(self, limit: int) -> int:
|
||||
"""Clamp exports that are currently materialized as an in-memory list."""
|
||||
|
||||
export_cap = self._configured_query_limit(
|
||||
'export_rows',
|
||||
_DEFAULT_MONITORING_EXPORT_ROWS,
|
||||
_HARD_MAX_MONITORING_EXPORT_ROWS,
|
||||
)
|
||||
try:
|
||||
normalized = int(limit)
|
||||
except (TypeError, ValueError):
|
||||
normalized = _DEFAULT_MONITORING_EXPORT_ROWS
|
||||
return min(max(normalized, 1), export_cap)
|
||||
|
||||
def _detail_limit(self) -> int:
|
||||
return self._configured_query_limit(
|
||||
'detail_rows',
|
||||
_DEFAULT_MONITORING_DETAIL_ROWS,
|
||||
_HARD_MAX_MONITORING_DETAIL_ROWS,
|
||||
)
|
||||
|
||||
def _timeseries_bucket_limit(self) -> int:
|
||||
return self._configured_query_limit(
|
||||
'timeseries_buckets',
|
||||
_DEFAULT_MONITORING_TIMESERIES_BUCKETS,
|
||||
_HARD_MAX_MONITORING_TIMESERIES_BUCKETS,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _token_bucket_expression(
|
||||
timestamp_column: sqlalchemy.Column,
|
||||
*,
|
||||
bucket: str,
|
||||
dialect_name: str,
|
||||
):
|
||||
"""Build a server-side hour/day bucket for supported business databases."""
|
||||
|
||||
if bucket not in {'hour', 'day'}:
|
||||
bucket = 'hour'
|
||||
if dialect_name == 'postgresql':
|
||||
return sqlalchemy.func.date_trunc(bucket, timestamp_column)
|
||||
if dialect_name == 'sqlite':
|
||||
bucket_format = '%Y-%m-%d %H:00' if bucket == 'hour' else '%Y-%m-%d'
|
||||
return sqlalchemy.func.strftime(bucket_format, timestamp_column)
|
||||
raise RuntimeError(f'Unsupported monitoring database dialect: {dialect_name}')
|
||||
|
||||
@staticmethod
|
||||
def _require_write_context(context: ExecutionContext | None) -> str:
|
||||
"""Reject background/runtime writes that lost their execution fence."""
|
||||
@@ -57,6 +153,7 @@ class MonitoringService:
|
||||
context: ExecutionContext,
|
||||
retention_days: int,
|
||||
batch_size: int = 1000,
|
||||
max_batches_per_table: int | None = None,
|
||||
) -> dict[str, int]:
|
||||
"""Delete monitoring records older than the specified retention period.
|
||||
|
||||
@@ -72,6 +169,24 @@ class MonitoringService:
|
||||
raise ValueError('retention_days must be >= 1')
|
||||
if batch_size < 1:
|
||||
raise ValueError('batch_size must be >= 1')
|
||||
if max_batches_per_table is None:
|
||||
cleanup_config = (
|
||||
getattr(getattr(self.ap, 'instance_config', None), 'data', {})
|
||||
.get('monitoring', {})
|
||||
.get('auto_cleanup', {})
|
||||
)
|
||||
max_batches_per_table = cleanup_config.get(
|
||||
'max_batches_per_table_per_run',
|
||||
_DEFAULT_CLEANUP_BATCHES_PER_TABLE,
|
||||
)
|
||||
try:
|
||||
max_batches_per_table = int(max_batches_per_table)
|
||||
except (TypeError, ValueError):
|
||||
max_batches_per_table = _DEFAULT_CLEANUP_BATCHES_PER_TABLE
|
||||
max_batches_per_table = min(
|
||||
max(max_batches_per_table, 1),
|
||||
_HARD_MAX_CLEANUP_BATCHES_PER_TABLE,
|
||||
)
|
||||
|
||||
cutoff = datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None) - datetime.timedelta(
|
||||
days=retention_days
|
||||
@@ -132,6 +247,7 @@ class MonitoringService:
|
||||
pk_column=pk_column,
|
||||
cutoff=cutoff,
|
||||
batch_size=batch_size,
|
||||
max_batches=max_batches_per_table,
|
||||
)
|
||||
return deleted_counts
|
||||
|
||||
@@ -157,11 +273,12 @@ class MonitoringService:
|
||||
pk_column: sqlalchemy.Column,
|
||||
cutoff: datetime.datetime,
|
||||
batch_size: int,
|
||||
max_batches: int,
|
||||
) -> int:
|
||||
workspace_uuid = self._require_write_context(context)
|
||||
deleted_total = 0
|
||||
|
||||
while True:
|
||||
for _batch_number in range(max_batches):
|
||||
|
||||
async def delete_batch() -> tuple[int, int]:
|
||||
select_result = await self.ap.persistence_mgr.execute_async(
|
||||
@@ -739,6 +856,8 @@ class MonitoringService:
|
||||
"""
|
||||
LLMCall = persistence_monitoring.MonitoringLLMCall
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
if bucket not in {'hour', 'day'}:
|
||||
bucket = 'hour'
|
||||
|
||||
conditions = [LLMCall.workspace_uuid == workspace_uuid]
|
||||
if bot_ids:
|
||||
@@ -812,21 +931,29 @@ class MonitoringService:
|
||||
}
|
||||
|
||||
# ---- Per-model breakdown ----
|
||||
by_model_query = _apply(
|
||||
sqlalchemy.select(
|
||||
LLMCall.model_name,
|
||||
sqlalchemy.func.count(LLMCall.id),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.input_tokens), 0),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.output_tokens), 0),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.total_tokens), 0),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.duration), 0),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.cost), 0.0),
|
||||
sqlalchemy.func.sum(sqlalchemy.case((LLMCall.status == 'error', 1), else_=0)),
|
||||
).group_by(LLMCall.model_name)
|
||||
model_total_tokens = sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.total_tokens), 0)
|
||||
model_limit, _unused_offset = self.normalize_page_window(_HARD_MAX_MONITORING_PAGE_ROWS)
|
||||
by_model_query = (
|
||||
_apply(
|
||||
sqlalchemy.select(
|
||||
LLMCall.model_name,
|
||||
sqlalchemy.func.count(LLMCall.id),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.input_tokens), 0),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.output_tokens), 0),
|
||||
model_total_tokens,
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.duration), 0),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.cost), 0.0),
|
||||
sqlalchemy.func.sum(sqlalchemy.case((LLMCall.status == 'error', 1), else_=0)),
|
||||
).group_by(LLMCall.model_name)
|
||||
)
|
||||
.order_by(model_total_tokens.desc())
|
||||
.limit(model_limit + 1)
|
||||
)
|
||||
by_model_result = await self.ap.persistence_mgr.execute_async(by_model_query)
|
||||
by_model_rows = by_model_result.all()
|
||||
by_model_truncated = len(by_model_rows) > model_limit
|
||||
by_model = []
|
||||
for mrow in by_model_result.all():
|
||||
for mrow in by_model_rows[:model_limit]:
|
||||
(
|
||||
model_name,
|
||||
m_calls,
|
||||
@@ -851,44 +978,59 @@ class MonitoringService:
|
||||
'avg_duration_ms': int((m_duration or 0) / m_calls) if m_calls > 0 else 0,
|
||||
}
|
||||
)
|
||||
by_model.sort(key=lambda x: x['total_tokens'], reverse=True)
|
||||
|
||||
# ---- Time-bucketed series ----
|
||||
# Use a DB-agnostic bucketing approach: fetch (timestamp, tokens) rows and
|
||||
# aggregate in Python. The window is bounded by the time filter, so this is
|
||||
# cheap for typical dashboard ranges (hours/days).
|
||||
series_query = _apply(
|
||||
sqlalchemy.select(
|
||||
LLMCall.timestamp,
|
||||
LLMCall.input_tokens,
|
||||
LLMCall.output_tokens,
|
||||
LLMCall.total_tokens,
|
||||
).order_by(LLMCall.timestamp.asc())
|
||||
# Aggregate before materialization. Requests may omit their time window,
|
||||
# so fetching every historical call and bucketing in Python is unsafe.
|
||||
engine = self.ap.persistence_mgr.get_db_engine()
|
||||
bucket_expression = self._token_bucket_expression(
|
||||
LLMCall.timestamp,
|
||||
bucket=bucket,
|
||||
dialect_name=engine.dialect.name,
|
||||
)
|
||||
bucket_limit = self._timeseries_bucket_limit()
|
||||
series_query = (
|
||||
_apply(
|
||||
sqlalchemy.select(
|
||||
bucket_expression.label('bucket'),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.input_tokens), 0),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.output_tokens), 0),
|
||||
sqlalchemy.func.coalesce(sqlalchemy.func.sum(LLMCall.total_tokens), 0),
|
||||
sqlalchemy.func.count(LLMCall.id),
|
||||
).group_by(bucket_expression)
|
||||
)
|
||||
.order_by(bucket_expression.desc())
|
||||
.limit(bucket_limit + 1)
|
||||
)
|
||||
series_result = await self.ap.persistence_mgr.execute_async(series_query)
|
||||
|
||||
bucket_fmt = '%Y-%m-%d %H:00' if bucket == 'hour' else '%Y-%m-%d'
|
||||
buckets: dict[str, dict] = {}
|
||||
for srow in series_result.all():
|
||||
ts, s_in, s_out, s_total = srow
|
||||
if ts is None:
|
||||
series_rows = series_result.all()
|
||||
timeseries_truncated = len(series_rows) > bucket_limit
|
||||
timeseries = []
|
||||
for bucket_value, s_in, s_out, s_total, calls in reversed(series_rows[:bucket_limit]):
|
||||
if bucket_value is None:
|
||||
continue
|
||||
key = ts.strftime(bucket_fmt)
|
||||
b = buckets.setdefault(
|
||||
key,
|
||||
{'bucket': key, 'input_tokens': 0, 'output_tokens': 0, 'total_tokens': 0, 'calls': 0},
|
||||
bucket_key = (
|
||||
bucket_value.strftime(bucket_fmt)
|
||||
if isinstance(bucket_value, (datetime.datetime, datetime.date))
|
||||
else str(bucket_value)
|
||||
)
|
||||
timeseries.append(
|
||||
{
|
||||
'bucket': bucket_key,
|
||||
'input_tokens': int(s_in or 0),
|
||||
'output_tokens': int(s_out or 0),
|
||||
'total_tokens': int(s_total or 0),
|
||||
'calls': int(calls or 0),
|
||||
}
|
||||
)
|
||||
b['input_tokens'] += int(s_in or 0)
|
||||
b['output_tokens'] += int(s_out or 0)
|
||||
b['total_tokens'] += int(s_total or 0)
|
||||
b['calls'] += 1
|
||||
|
||||
timeseries = [buckets[k] for k in sorted(buckets.keys())]
|
||||
|
||||
return {
|
||||
'summary': summary,
|
||||
'by_model': by_model,
|
||||
'by_model_truncated': by_model_truncated,
|
||||
'timeseries': timeseries,
|
||||
'timeseries_truncated': timeseries_truncated,
|
||||
'bucket': bucket,
|
||||
}
|
||||
|
||||
@@ -904,6 +1046,7 @@ class MonitoringService:
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Get messages with filters"""
|
||||
limit, offset = self.normalize_page_window(limit, offset)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -958,6 +1101,7 @@ class MonitoringService:
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Get LLM calls with filters"""
|
||||
limit, offset = self.normalize_page_window(limit, offset)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringLLMCall.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -1012,6 +1156,7 @@ class MonitoringService:
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Get tool calls with filters"""
|
||||
limit, offset = self.normalize_page_window(limit, offset)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringToolCall.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -1064,6 +1209,7 @@ class MonitoringService:
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Get embedding calls with filters"""
|
||||
limit, offset = self.normalize_page_window(limit, offset)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringEmbeddingCall.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -1116,6 +1262,7 @@ class MonitoringService:
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Get sessions with filters"""
|
||||
limit, offset = self.normalize_page_window(limit, offset)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringSession.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -1171,6 +1318,7 @@ class MonitoringService:
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Get errors with filters"""
|
||||
limit, offset = self.normalize_page_window(limit, offset)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringError.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -1218,8 +1366,9 @@ class MonitoringService:
|
||||
context: TenantContext,
|
||||
session_id: str,
|
||||
) -> dict:
|
||||
"""Get detailed analysis for a specific session"""
|
||||
"""Get bounded session details with full statistics computed in SQL."""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
detail_limit = self._detail_limit()
|
||||
# Get session info
|
||||
session_query = sqlalchemy.select(persistence_monitoring.MonitoringSession).where(
|
||||
persistence_monitoring.MonitoringSession.workspace_uuid == workspace_uuid,
|
||||
@@ -1236,60 +1385,99 @@ class MonitoringService:
|
||||
|
||||
session = session_row[0] if isinstance(session_row, tuple) else session_row
|
||||
|
||||
# Get messages for this session
|
||||
messages_query = (
|
||||
sqlalchemy.select(persistence_monitoring.MonitoringMessage)
|
||||
.where(
|
||||
message_stats_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
sqlalchemy.func.count(persistence_monitoring.MonitoringMessage.id).label('total'),
|
||||
sqlalchemy.func.sum(
|
||||
sqlalchemy.case(
|
||||
(persistence_monitoring.MonitoringMessage.status == 'success', 1),
|
||||
else_=0,
|
||||
)
|
||||
).label('success'),
|
||||
sqlalchemy.func.sum(
|
||||
sqlalchemy.case(
|
||||
(persistence_monitoring.MonitoringMessage.status == 'error', 1),
|
||||
else_=0,
|
||||
)
|
||||
).label('error'),
|
||||
sqlalchemy.func.sum(
|
||||
sqlalchemy.case(
|
||||
(persistence_monitoring.MonitoringMessage.status == 'pending', 1),
|
||||
else_=0,
|
||||
)
|
||||
).label('pending'),
|
||||
sqlalchemy.func.min(persistence_monitoring.MonitoringMessage.timestamp).label('first_timestamp'),
|
||||
sqlalchemy.func.max(persistence_monitoring.MonitoringMessage.timestamp).label('last_timestamp'),
|
||||
).where(
|
||||
persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid,
|
||||
persistence_monitoring.MonitoringMessage.session_id == session_id,
|
||||
)
|
||||
.order_by(persistence_monitoring.MonitoringMessage.timestamp.asc())
|
||||
)
|
||||
messages_result = await self.ap.persistence_mgr.execute_async(messages_query)
|
||||
messages_rows = messages_result.all()
|
||||
message_stats = message_stats_result.one()
|
||||
|
||||
# Count messages by status
|
||||
success_messages = 0
|
||||
error_messages = 0
|
||||
pending_messages = 0
|
||||
for row in messages_rows:
|
||||
msg = row[0] if isinstance(row, tuple) else row
|
||||
if msg.status == 'success':
|
||||
success_messages += 1
|
||||
elif msg.status == 'error':
|
||||
error_messages += 1
|
||||
elif msg.status == 'pending':
|
||||
pending_messages += 1
|
||||
|
||||
# Get LLM calls for this session
|
||||
llm_query = sqlalchemy.select(persistence_monitoring.MonitoringLLMCall).where(
|
||||
persistence_monitoring.MonitoringLLMCall.workspace_uuid == workspace_uuid,
|
||||
persistence_monitoring.MonitoringLLMCall.session_id == session_id,
|
||||
llm_stats_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
sqlalchemy.func.count(persistence_monitoring.MonitoringLLMCall.id).label('total_calls'),
|
||||
sqlalchemy.func.coalesce(
|
||||
sqlalchemy.func.sum(persistence_monitoring.MonitoringLLMCall.input_tokens),
|
||||
0,
|
||||
).label('total_input_tokens'),
|
||||
sqlalchemy.func.coalesce(
|
||||
sqlalchemy.func.sum(persistence_monitoring.MonitoringLLMCall.output_tokens),
|
||||
0,
|
||||
).label('total_output_tokens'),
|
||||
sqlalchemy.func.coalesce(
|
||||
sqlalchemy.func.sum(persistence_monitoring.MonitoringLLMCall.total_tokens),
|
||||
0,
|
||||
).label('total_tokens'),
|
||||
sqlalchemy.func.coalesce(
|
||||
sqlalchemy.func.sum(persistence_monitoring.MonitoringLLMCall.duration),
|
||||
0,
|
||||
).label('total_duration'),
|
||||
sqlalchemy.func.sum(
|
||||
sqlalchemy.case(
|
||||
(persistence_monitoring.MonitoringLLMCall.status == 'success', 1),
|
||||
else_=0,
|
||||
)
|
||||
).label('success_calls'),
|
||||
sqlalchemy.func.sum(
|
||||
sqlalchemy.case(
|
||||
(persistence_monitoring.MonitoringLLMCall.status != 'success', 1),
|
||||
else_=0,
|
||||
)
|
||||
).label('error_calls'),
|
||||
).where(
|
||||
persistence_monitoring.MonitoringLLMCall.workspace_uuid == workspace_uuid,
|
||||
persistence_monitoring.MonitoringLLMCall.session_id == session_id,
|
||||
)
|
||||
)
|
||||
llm_result = await self.ap.persistence_mgr.execute_async(llm_query)
|
||||
llm_rows = llm_result.all()
|
||||
llm_stats = llm_stats_result.one()
|
||||
|
||||
# Calculate LLM statistics
|
||||
total_llm_calls = len(llm_rows)
|
||||
total_input_tokens = 0
|
||||
total_output_tokens = 0
|
||||
total_tokens = 0
|
||||
total_duration = 0
|
||||
success_llm_calls = 0
|
||||
error_llm_calls = 0
|
||||
|
||||
for row in llm_rows:
|
||||
llm_call = row[0] if isinstance(row, tuple) else row
|
||||
total_input_tokens += llm_call.input_tokens
|
||||
total_output_tokens += llm_call.output_tokens
|
||||
total_tokens += llm_call.total_tokens
|
||||
total_duration += llm_call.duration
|
||||
if llm_call.status == 'success':
|
||||
success_llm_calls += 1
|
||||
else:
|
||||
error_llm_calls += 1
|
||||
|
||||
# Get tool calls for this session
|
||||
tool_stats_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
sqlalchemy.func.count(persistence_monitoring.MonitoringToolCall.id).label('total_calls'),
|
||||
sqlalchemy.func.coalesce(
|
||||
sqlalchemy.func.sum(persistence_monitoring.MonitoringToolCall.duration),
|
||||
0,
|
||||
).label('total_duration'),
|
||||
sqlalchemy.func.sum(
|
||||
sqlalchemy.case(
|
||||
(persistence_monitoring.MonitoringToolCall.status == 'success', 1),
|
||||
else_=0,
|
||||
)
|
||||
).label('success_calls'),
|
||||
sqlalchemy.func.sum(
|
||||
sqlalchemy.case(
|
||||
(persistence_monitoring.MonitoringToolCall.status != 'success', 1),
|
||||
else_=0,
|
||||
)
|
||||
).label('error_calls'),
|
||||
).where(
|
||||
persistence_monitoring.MonitoringToolCall.workspace_uuid == workspace_uuid,
|
||||
persistence_monitoring.MonitoringToolCall.session_id == session_id,
|
||||
)
|
||||
)
|
||||
tool_stats = tool_stats_result.one()
|
||||
tool_query = (
|
||||
sqlalchemy.select(persistence_monitoring.MonitoringToolCall)
|
||||
.where(
|
||||
@@ -1297,9 +1485,12 @@ class MonitoringService:
|
||||
persistence_monitoring.MonitoringToolCall.session_id == session_id,
|
||||
)
|
||||
.order_by(persistence_monitoring.MonitoringToolCall.timestamp.asc())
|
||||
.limit(detail_limit + 1)
|
||||
)
|
||||
tool_result = await self.ap.persistence_mgr.execute_async(tool_query)
|
||||
tool_rows = tool_result.all()
|
||||
tool_calls_truncated = len(tool_rows) > detail_limit
|
||||
tool_rows = tool_rows[:detail_limit]
|
||||
|
||||
tool_calls = [
|
||||
self.ap.persistence_mgr.serialize_model(
|
||||
@@ -1308,19 +1499,6 @@ class MonitoringService:
|
||||
for row in tool_rows
|
||||
]
|
||||
|
||||
total_tool_calls = len(tool_rows)
|
||||
success_tool_calls = 0
|
||||
error_tool_calls = 0
|
||||
total_tool_duration = 0
|
||||
for row in tool_rows:
|
||||
tool_call = row[0] if isinstance(row, tuple) else row
|
||||
total_tool_duration += tool_call.duration
|
||||
if tool_call.status == 'success':
|
||||
success_tool_calls += 1
|
||||
else:
|
||||
error_tool_calls += 1
|
||||
|
||||
# Get errors for this session
|
||||
error_query = (
|
||||
sqlalchemy.select(persistence_monitoring.MonitoringError)
|
||||
.where(
|
||||
@@ -1328,9 +1506,12 @@ class MonitoringService:
|
||||
persistence_monitoring.MonitoringError.session_id == session_id,
|
||||
)
|
||||
.order_by(persistence_monitoring.MonitoringError.timestamp.desc())
|
||||
.limit(detail_limit + 1)
|
||||
)
|
||||
error_result = await self.ap.persistence_mgr.execute_async(error_query)
|
||||
error_rows = error_result.all()
|
||||
errors_truncated = len(error_rows) > detail_limit
|
||||
error_rows = error_rows[:detail_limit]
|
||||
|
||||
errors = [
|
||||
self.ap.persistence_mgr.serialize_model(
|
||||
@@ -1339,42 +1520,49 @@ class MonitoringService:
|
||||
for row in error_rows
|
||||
]
|
||||
|
||||
# Calculate session duration
|
||||
if messages_rows:
|
||||
first_msg = messages_rows[0][0] if isinstance(messages_rows[0], tuple) else messages_rows[0]
|
||||
last_msg = messages_rows[-1][0] if isinstance(messages_rows[-1], tuple) else messages_rows[-1]
|
||||
session_duration_seconds = int((last_msg.timestamp - first_msg.timestamp).total_seconds())
|
||||
if message_stats.first_timestamp is not None and message_stats.last_timestamp is not None:
|
||||
session_duration_seconds = int(
|
||||
(message_stats.last_timestamp - message_stats.first_timestamp).total_seconds()
|
||||
)
|
||||
else:
|
||||
session_duration_seconds = 0
|
||||
total_llm_calls = int(llm_stats.total_calls or 0)
|
||||
total_tool_calls = int(tool_stats.total_calls or 0)
|
||||
|
||||
return {
|
||||
'session_id': session_id,
|
||||
'found': True,
|
||||
'session': self.ap.persistence_mgr.serialize_model(persistence_monitoring.MonitoringSession, session),
|
||||
'message_stats': {
|
||||
'total': len(messages_rows),
|
||||
'success': success_messages,
|
||||
'error': error_messages,
|
||||
'pending': pending_messages,
|
||||
'total': int(message_stats.total or 0),
|
||||
'success': int(message_stats.success or 0),
|
||||
'error': int(message_stats.error or 0),
|
||||
'pending': int(message_stats.pending or 0),
|
||||
},
|
||||
'llm_stats': {
|
||||
'total_calls': total_llm_calls,
|
||||
'success_calls': success_llm_calls,
|
||||
'error_calls': error_llm_calls,
|
||||
'total_input_tokens': total_input_tokens,
|
||||
'total_output_tokens': total_output_tokens,
|
||||
'total_tokens': total_tokens,
|
||||
'average_duration_ms': int(total_duration / total_llm_calls) if total_llm_calls > 0 else 0,
|
||||
'success_calls': int(llm_stats.success_calls or 0),
|
||||
'error_calls': int(llm_stats.error_calls or 0),
|
||||
'total_input_tokens': int(llm_stats.total_input_tokens or 0),
|
||||
'total_output_tokens': int(llm_stats.total_output_tokens or 0),
|
||||
'total_tokens': int(llm_stats.total_tokens or 0),
|
||||
'average_duration_ms': (int(llm_stats.total_duration / total_llm_calls) if total_llm_calls > 0 else 0),
|
||||
},
|
||||
'tool_calls': tool_calls,
|
||||
'tool_stats': {
|
||||
'total_calls': total_tool_calls,
|
||||
'success_calls': success_tool_calls,
|
||||
'error_calls': error_tool_calls,
|
||||
'total_duration_ms': total_tool_duration,
|
||||
'average_duration_ms': int(total_tool_duration / total_tool_calls) if total_tool_calls > 0 else 0,
|
||||
'success_calls': int(tool_stats.success_calls or 0),
|
||||
'error_calls': int(tool_stats.error_calls or 0),
|
||||
'total_duration_ms': int(tool_stats.total_duration or 0),
|
||||
'average_duration_ms': (
|
||||
int(tool_stats.total_duration / total_tool_calls) if total_tool_calls > 0 else 0
|
||||
),
|
||||
},
|
||||
'errors': errors,
|
||||
'detail_truncated': {
|
||||
'tool_calls': tool_calls_truncated,
|
||||
'errors': errors_truncated,
|
||||
},
|
||||
'session_duration_seconds': session_duration_seconds,
|
||||
}
|
||||
|
||||
@@ -1383,8 +1571,9 @@ class MonitoringService:
|
||||
context: TenantContext,
|
||||
message_id: str,
|
||||
) -> dict:
|
||||
"""Get detailed information for a specific message including associated LLM calls and errors"""
|
||||
"""Get bounded message details with full statistics computed in SQL."""
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
detail_limit = self._detail_limit()
|
||||
# Get message info
|
||||
message_query = sqlalchemy.select(persistence_monitoring.MonitoringMessage).where(
|
||||
persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid,
|
||||
@@ -1401,7 +1590,31 @@ class MonitoringService:
|
||||
|
||||
message = message_row[0] if isinstance(message_row, tuple) else message_row
|
||||
|
||||
# Get LLM calls for this message
|
||||
llm_stats_result = await self.ap.persistence_mgr.execute_async(
|
||||
sqlalchemy.select(
|
||||
sqlalchemy.func.count(persistence_monitoring.MonitoringLLMCall.id).label('total_calls'),
|
||||
sqlalchemy.func.coalesce(
|
||||
sqlalchemy.func.sum(persistence_monitoring.MonitoringLLMCall.input_tokens),
|
||||
0,
|
||||
).label('total_input_tokens'),
|
||||
sqlalchemy.func.coalesce(
|
||||
sqlalchemy.func.sum(persistence_monitoring.MonitoringLLMCall.output_tokens),
|
||||
0,
|
||||
).label('total_output_tokens'),
|
||||
sqlalchemy.func.coalesce(
|
||||
sqlalchemy.func.sum(persistence_monitoring.MonitoringLLMCall.total_tokens),
|
||||
0,
|
||||
).label('total_tokens'),
|
||||
sqlalchemy.func.coalesce(
|
||||
sqlalchemy.func.sum(persistence_monitoring.MonitoringLLMCall.duration),
|
||||
0,
|
||||
).label('total_duration'),
|
||||
).where(
|
||||
persistence_monitoring.MonitoringLLMCall.workspace_uuid == workspace_uuid,
|
||||
persistence_monitoring.MonitoringLLMCall.message_id == message_id,
|
||||
)
|
||||
)
|
||||
llm_stats = llm_stats_result.one()
|
||||
llm_query = (
|
||||
sqlalchemy.select(persistence_monitoring.MonitoringLLMCall)
|
||||
.where(
|
||||
@@ -1409,9 +1622,12 @@ class MonitoringService:
|
||||
persistence_monitoring.MonitoringLLMCall.message_id == message_id,
|
||||
)
|
||||
.order_by(persistence_monitoring.MonitoringLLMCall.timestamp.asc())
|
||||
.limit(detail_limit + 1)
|
||||
)
|
||||
llm_result = await self.ap.persistence_mgr.execute_async(llm_query)
|
||||
llm_rows = llm_result.all()
|
||||
llm_calls_truncated = len(llm_rows) > detail_limit
|
||||
llm_rows = llm_rows[:detail_limit]
|
||||
|
||||
llm_calls = [
|
||||
self.ap.persistence_mgr.serialize_model(
|
||||
@@ -1420,13 +1636,6 @@ class MonitoringService:
|
||||
for row in llm_rows
|
||||
]
|
||||
|
||||
# Calculate LLM statistics
|
||||
total_input_tokens = sum(call.input_tokens for call in llm_rows)
|
||||
total_output_tokens = sum(call.output_tokens for call in llm_rows)
|
||||
total_tokens = sum(call.total_tokens for call in llm_rows)
|
||||
total_duration = sum(call.duration for call in llm_rows)
|
||||
|
||||
# Get errors for this message
|
||||
error_query = (
|
||||
sqlalchemy.select(persistence_monitoring.MonitoringError)
|
||||
.where(
|
||||
@@ -1434,9 +1643,12 @@ class MonitoringService:
|
||||
persistence_monitoring.MonitoringError.message_id == message_id,
|
||||
)
|
||||
.order_by(persistence_monitoring.MonitoringError.timestamp.asc())
|
||||
.limit(detail_limit + 1)
|
||||
)
|
||||
error_result = await self.ap.persistence_mgr.execute_async(error_query)
|
||||
error_rows = error_result.all()
|
||||
errors_truncated = len(error_rows) > detail_limit
|
||||
error_rows = error_rows[:detail_limit]
|
||||
|
||||
errors = [
|
||||
self.ap.persistence_mgr.serialize_model(
|
||||
@@ -1444,6 +1656,7 @@ class MonitoringService:
|
||||
)
|
||||
for row in error_rows
|
||||
]
|
||||
total_llm_calls = int(llm_stats.total_calls or 0)
|
||||
|
||||
return {
|
||||
'message_id': message_id,
|
||||
@@ -1451,14 +1664,18 @@ class MonitoringService:
|
||||
'message': self.ap.persistence_mgr.serialize_model(persistence_monitoring.MonitoringMessage, message),
|
||||
'llm_calls': llm_calls,
|
||||
'llm_stats': {
|
||||
'total_calls': len(llm_rows),
|
||||
'total_input_tokens': total_input_tokens,
|
||||
'total_output_tokens': total_output_tokens,
|
||||
'total_tokens': total_tokens,
|
||||
'total_duration_ms': total_duration,
|
||||
'average_duration_ms': int(total_duration / len(llm_rows)) if len(llm_rows) > 0 else 0,
|
||||
'total_calls': total_llm_calls,
|
||||
'total_input_tokens': int(llm_stats.total_input_tokens or 0),
|
||||
'total_output_tokens': int(llm_stats.total_output_tokens or 0),
|
||||
'total_tokens': int(llm_stats.total_tokens or 0),
|
||||
'total_duration_ms': int(llm_stats.total_duration or 0),
|
||||
'average_duration_ms': (int(llm_stats.total_duration / total_llm_calls) if total_llm_calls > 0 else 0),
|
||||
},
|
||||
'errors': errors,
|
||||
'detail_truncated': {
|
||||
'llm_calls': llm_calls_truncated,
|
||||
'errors': errors_truncated,
|
||||
},
|
||||
}
|
||||
|
||||
# ========== Export Methods ==========
|
||||
@@ -1548,6 +1765,7 @@ class MonitoringService:
|
||||
limit: int = 100000,
|
||||
) -> list[dict]:
|
||||
"""Export messages as list of dictionaries for CSV conversion"""
|
||||
limit = self.normalize_export_limit(limit)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringMessage.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -1603,6 +1821,7 @@ class MonitoringService:
|
||||
limit: int = 100000,
|
||||
) -> list[dict]:
|
||||
"""Export LLM calls as list of dictionaries for CSV conversion"""
|
||||
limit = self.normalize_export_limit(limit)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringLLMCall.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -1657,6 +1876,7 @@ class MonitoringService:
|
||||
limit: int = 100000,
|
||||
) -> list[dict]:
|
||||
"""Export embedding calls as list of dictionaries for CSV conversion"""
|
||||
limit = self.normalize_export_limit(limit)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringEmbeddingCall.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -1708,6 +1928,7 @@ class MonitoringService:
|
||||
limit: int = 100000,
|
||||
) -> list[dict]:
|
||||
"""Export errors as list of dictionaries for CSV conversion"""
|
||||
limit = self.normalize_export_limit(limit)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringError.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -1758,6 +1979,7 @@ class MonitoringService:
|
||||
limit: int = 100000,
|
||||
) -> list[dict]:
|
||||
"""Export sessions as list of dictionaries for CSV conversion"""
|
||||
limit = self.normalize_export_limit(limit)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringSession.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -2011,6 +2233,7 @@ class MonitoringService:
|
||||
offset: int = 0,
|
||||
) -> tuple[list[dict], int]:
|
||||
"""Get feedback list with filters."""
|
||||
limit, offset = self.normalize_page_window(limit, offset)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringFeedback.workspace_uuid == workspace_uuid]
|
||||
|
||||
@@ -2063,6 +2286,7 @@ class MonitoringService:
|
||||
limit: int = 100000,
|
||||
) -> list[dict]:
|
||||
"""Export feedback as list of dictionaries for CSV conversion."""
|
||||
limit = self.normalize_export_limit(limit)
|
||||
workspace_uuid = require_workspace_uuid(context)
|
||||
conditions = [persistence_monitoring.MonitoringFeedback.workspace_uuid == workspace_uuid]
|
||||
|
||||
|
||||
Reference in New Issue
Block a user