fix(monitoring): persist session summaries to avoid history scans

- Maintain counts and display snapshots in source transactions with replay-safe deduplication.
- Read session lists exclusively from summaries and remove execution-only filters.
- Automatically initialize historical summaries in bounded, resumable batches during OSS startup.
- Add behavioral contracts covering accounting, retention, query cost and PostgreSQL concurrency.

Verified with the required session_summary suite and real container upgrade startup.
This commit is contained in:
BiFangKNT
2026-10-20 06:12:00 +08:00
parent 62b647761f
commit 810d053df9
33 changed files with 4227 additions and 466 deletions
+117 -174
View File
@@ -5,6 +5,7 @@ import uuid
import datetime
import functools
import json
from contextlib import asynccontextmanager
import sqlalchemy
from sqlalchemy.dialects import postgresql as postgresql_dialect
from sqlalchemy.dialects import sqlite as sqlite_dialect
@@ -20,6 +21,12 @@ from ..authz import WorkspaceRequiredError
from ..context import ExecutionContext
from .tenant import TenantContext, require_workspace_uuid
from .monitoring_execution_details import ExecutionDetailsMixin
from ....persistence.session_summary_contract import (
SessionListFilter,
SessionSummaryReader,
)
from ....persistence.session_summary_reader import SqlSessionSummaryReader, SUMMARY_FIELDS
from ....persistence.session_summary_writer import SessionSummaryWriter, message_summary_change
_DEFAULT_MONITORING_PAGE_ROWS = 1000
@@ -147,6 +154,28 @@ class MonitoringService(ExecutionDetailsMixin):
def __init__(self, ap: app.Application) -> None:
self.ap = ap
self._session_list: SessionSummaryReader = SqlSessionSummaryReader(self._summary_session)
@asynccontextmanager
async def _summary_session(self, workspace: str):
"""Join the owning Workspace transaction, including source writes."""
async with self.ap.persistence_mgr.tenant_uow(workspace) as uow:
yield uow.session
def _summary_writer(self):
return SessionSummaryWriter(self.ap.persistence_mgr.get_db_engine().dialect.name)
@asynccontextmanager
async def _summary_write_session(self, workspace: str):
from ....persistence.session_summary_backfill import require_initialized
async with self._summary_session(workspace) as session:
await require_initialized(session, workspace)
yield session
def _serialize_session(self, row):
values = self.ap.persistence_mgr.serialize_model(persistence_monitoring.MonitoringSession, row)
return {field: values[field] for field in SUMMARY_FIELDS}
def _configured_query_limit(self, name: str, default: int, hard_max: int) -> int:
config = (
@@ -292,6 +321,12 @@ class MonitoringService(ExecutionDetailsMixin):
)
tables_and_columns: list[tuple[str, type, sqlalchemy.Column, sqlalchemy.Column]] = [
(
'monitoring_session_receipts',
persistence_monitoring.MonitoringSessionReceipt,
persistence_monitoring.MonitoringSessionReceipt.timestamp,
persistence_monitoring.MonitoringSessionReceipt.source_key,
),
(
'monitoring_messages',
persistence_monitoring.MonitoringMessage,
@@ -359,7 +394,8 @@ class MonitoringService(ExecutionDetailsMixin):
else:
deleted_counts = await delete_records()
if sum(deleted_counts.values()) > 0:
# Receipt expiry alone must not trigger database-wide VACUUM work.
if any(count for table, count in deleted_counts.items() if table != 'monitoring_session_receipts'):
await self._release_sqlite_space()
return deleted_counts
@@ -375,15 +411,28 @@ class MonitoringService(ExecutionDetailsMixin):
max_batches: int,
) -> int:
workspace_uuid = self._require_write_context(context)
if model_cls is persistence_monitoring.MonitoringSessionReceipt:
return await self._expire_session_receipts(workspace_uuid, cutoff, batch_size, max_batches)
deleted_total = 0
for _batch_number in range(max_batches):
async def delete_batch() -> tuple[int, int]:
key_columns = list(model_cls.__table__.primary_key.columns)
conditions = [model_cls.workspace_uuid == workspace_uuid, ts_column < cutoff]
if model_cls is persistence_monitoring.MonitoringSession:
receipt = persistence_monitoring.MonitoringSessionReceipt
conditions.append(
~sqlalchemy.exists().where(
receipt.workspace_uuid == workspace_uuid,
receipt.bot_id == model_cls.bot_id,
receipt.session_id == model_cls.session_id,
)
)
select_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.select(*key_columns)
.where(model_cls.workspace_uuid == workspace_uuid, ts_column < cutoff)
.where(*conditions)
.order_by(ts_column, *key_columns)
.limit(batch_size)
)
pk_values = [tuple(row) for row in select_result.all()]
@@ -392,9 +441,8 @@ class MonitoringService(ExecutionDetailsMixin):
delete_result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.delete(model_cls).where(
model_cls.workspace_uuid == workspace_uuid,
*conditions,
sqlalchemy.tuple_(*key_columns).in_(pk_values),
ts_column < cutoff,
)
)
return len(pk_values), int(delete_result.rowcount or 0)
@@ -414,6 +462,47 @@ class MonitoringService(ExecutionDetailsMixin):
return deleted_total
async def _expire_session_receipts(self, workspace, cutoff, batch_size, max_batches):
receipt = persistence_monitoring.MonitoringSessionReceipt
expired = [receipt.workspace_uuid == workspace, receipt.timestamp < cutoff]
deleted = 0
for _ in range(max_batches):
async with self._summary_session(workspace) as session:
rows = (
(
await session.execute(
sqlalchemy.select(receipt.bot_id, receipt.source_key)
.where(*expired)
.order_by(receipt.timestamp, receipt.bot_id, receipt.source_key)
.limit(batch_size)
)
)
.mappings()
.all()
)
event_keys = [
(row['bot_id'], row['source_key']) for row in rows if row['source_key'].startswith('event:')
]
deleted += await self._summary_writer().expire_ingress(session, workspace, cutoff, event_keys)
# A message receipt can be locked by late linking, which next locks
# summaries. Never hold a message lock while sealing ingress.
message_keys = [
(row['bot_id'], row['source_key']) for row in rows if not row['source_key'].startswith('event:')
]
if message_keys:
async with self._summary_session(workspace) as session:
result = await session.execute(
sqlalchemy.delete(receipt).where(
*expired,
~receipt.source_key.startswith('event:'),
sqlalchemy.tuple_(receipt.bot_id, receipt.source_key).in_(message_keys),
)
)
deleted += result.rowcount
if len(rows) < batch_size:
break
return deleted
async def _release_sqlite_space(self) -> None:
database_type = self.ap.instance_config.data.get('database', {}).get('use', 'sqlite')
if database_type != 'sqlite':
@@ -587,9 +676,12 @@ class MonitoringService(ExecutionDetailsMixin):
'parent_message_id': parent_message_id,
}
await self.ap.persistence_mgr.execute_async(
sqlalchemy.insert(persistence_monitoring.MonitoringMessage).values(message_data)
)
async with self._summary_write_session(workspace_uuid) as session:
await session.execute(sqlalchemy.insert(persistence_monitoring.MonitoringMessage).values(message_data))
await self._summary_writer().apply_once(
session,
message_summary_change(workspace_uuid, message_data),
)
return message_id
@@ -745,96 +837,6 @@ class MonitoringService(ExecutionDetailsMixin):
return call_id
@_workspace_transaction
async def record_session_start(
self,
context: ExecutionContext,
session_id: str,
bot_id: str,
bot_name: str,
pipeline_id: str,
pipeline_name: str,
platform: str | None = None,
user_id: str | int | None = None,
user_name: str | None = None,
) -> None:
"""Record a new session"""
workspace_uuid = self._require_write_context(context)
session_data = {
'workspace_uuid': workspace_uuid,
'session_id': session_id,
'bot_id': bot_id,
'bot_name': bot_name,
'pipeline_id': pipeline_id,
'pipeline_name': pipeline_name,
'message_count': 1,
'start_time': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
'last_activity': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
'is_active': True,
'platform': platform,
'user_id': _normalize_user_id(user_id),
'user_name': user_name,
}
model = persistence_monitoring.MonitoringSession
dialect = self.ap.persistence_mgr.get_db_engine().dialect.name
insert = postgresql_dialect.insert if dialect == 'postgresql' else sqlite_dialect.insert
statement = insert(model).values(session_data)
await self.ap.persistence_mgr.execute_async(
statement.on_conflict_do_update(
index_elements=['workspace_uuid', 'bot_id', 'session_id'],
set_={
'message_count': model.message_count + 1,
'last_activity': statement.excluded.last_activity,
'pipeline_id': statement.excluded.pipeline_id,
'pipeline_name': statement.excluded.pipeline_name,
},
)
)
@_workspace_transaction
async def update_session_activity(
self,
context: ExecutionContext,
session_id: str,
pipeline_id: str | None = None,
pipeline_name: str | None = None,
bot_id: str | None = None,
) -> bool:
"""Update session last activity time and increment message count.
Also updates pipeline info if the bot's pipeline has changed.
Returns:
True if session was found and updated, False if session doesn't exist.
"""
workspace_uuid = self._require_write_context(context)
bot_id = bot_id if bot_id is not None else context.bot_uuid
if not bot_id:
raise ValueError('Session activity requires a bot_id')
update_values = {
'last_activity': datetime.datetime.now(datetime.timezone.utc).replace(tzinfo=None),
'message_count': persistence_monitoring.MonitoringSession.message_count + 1,
}
# Update pipeline info if provided (handles pipeline switch)
if pipeline_id is not None:
update_values['pipeline_id'] = pipeline_id
if pipeline_name is not None:
update_values['pipeline_name'] = pipeline_name
result = await self.ap.persistence_mgr.execute_async(
sqlalchemy.update(persistence_monitoring.MonitoringSession)
.where(
persistence_monitoring.MonitoringSession.workspace_uuid == workspace_uuid,
persistence_monitoring.MonitoringSession.session_id == session_id,
persistence_monitoring.MonitoringSession.bot_id == bot_id,
)
.values(update_values)
)
# Check if any rows were updated
return result.rowcount > 0
@_workspace_transaction
async def record_error(
self,
@@ -2050,13 +2052,13 @@ class MonitoringService(ExecutionDetailsMixin):
limit, offset = self.normalize_page_window(limit, offset)
workspace_uuid = require_workspace_uuid(context)
if bot_ids and session_ids and not pipeline_ids and not execution_statuses and mode == 'all':
source = self._session_message_source(workspace_uuid, bot_ids)
source = self._conversation_message_source(workspace_uuid, bot_ids)
query = sqlalchemy.select(source).where(source.c.session_id.in_(session_ids))
if start_time:
query = query.where(source.c.timestamp >= start_time)
if end_time:
query = query.where(source.c.timestamp <= end_time)
return await self._session_projection_page(
return await self._conversation_projection_page(
workspace_uuid,
query.order_by(source.c.timestamp.desc(), source.c.id.desc()),
limit,
@@ -2311,7 +2313,7 @@ class MonitoringService(ExecutionDetailsMixin):
raise ValueError('bot_id and session_id are required')
workspace = require_workspace_uuid(context)
execute = self.ap.persistence_mgr.execute_async
source = self._session_message_source(workspace, [bot_id])
source = self._conversation_message_source(workspace, [bot_id])
exists = (
await execute(
sqlalchemy.select(source.c.session_id)
@@ -2396,81 +2398,22 @@ class MonitoringService(ExecutionDetailsMixin):
is_active: bool | None = None,
limit: int = 100,
offset: int = 0,
execution_statuses: list[str] | None = None,
mode: str = 'all',
) -> tuple[list[dict], int]:
"""Get sessions with filters"""
"""Interactive lists may only read the materialized summary contract."""
workspace = require_workspace_uuid(context)
limit, offset = self.normalize_page_window(limit, offset)
workspace_uuid = require_workspace_uuid(context)
if bot_ids and not pipeline_ids and not execution_statuses and mode == 'all':
return await self.get_bot_conversation_sessions(
workspace_uuid,
bot_ids,
start_time,
end_time,
user_query,
is_active,
limit,
offset,
)
conditions = [persistence_monitoring.MonitoringSession.workspace_uuid == workspace_uuid]
conditions.append(
self._monitoring_execution_condition(
persistence_monitoring.MonitoringSession, workspace_uuid, execution_statuses, mode
)
)
if bot_ids:
conditions.append(persistence_monitoring.MonitoringSession.bot_id.in_(bot_ids))
if pipeline_ids:
conditions.append(
self._monitoring_processor_condition(
persistence_monitoring.MonitoringSession, workspace_uuid, pipeline_ids
)
)
if start_time:
conditions.append(persistence_monitoring.MonitoringSession.last_activity >= start_time)
if end_time:
conditions.append(persistence_monitoring.MonitoringSession.last_activity <= end_time)
if user_query and user_query.strip():
user_pattern = f'%{user_query.strip()}%'
conditions.append(
sqlalchemy.or_(
persistence_monitoring.MonitoringSession.user_id.ilike(user_pattern),
persistence_monitoring.MonitoringSession.user_name.ilike(user_pattern),
)
)
if is_active is not None:
conditions.append(persistence_monitoring.MonitoringSession.is_active == is_active)
# Get total count
count_query = sqlalchemy.select(sqlalchemy.func.count(persistence_monitoring.MonitoringSession.session_id))
if conditions:
count_query = count_query.where(sqlalchemy.and_(*conditions))
count_result = await self.ap.persistence_mgr.execute_async(count_query)
total = count_result.scalar() or 0
# Get sessions
query = sqlalchemy.select(persistence_monitoring.MonitoringSession).order_by(
persistence_monitoring.MonitoringSession.last_activity.desc()
)
if conditions:
query = query.where(sqlalchemy.and_(*conditions))
query = query.limit(limit).offset(offset)
result = await self.ap.persistence_mgr.execute_async(query)
sessions_rows = result.all()
return (
[
self.ap.persistence_mgr.serialize_model(
persistence_monitoring.MonitoringSession, row[0] if isinstance(row, tuple) else row
)
for row in sessions_rows
],
total,
return await self._session_list.list_page(
workspace,
SessionListFilter(
bot_ids=tuple(bot_ids or ()),
pipeline_ids=tuple(pipeline_ids or ()),
start_time=start_time,
end_time=end_time,
user_query=user_query,
is_active=is_active,
),
limit,
offset,
)
async def get_errors(
@@ -2728,7 +2671,7 @@ class MonitoringService(ExecutionDetailsMixin):
return {
'session_id': session_id,
'found': True,
'session': self.ap.persistence_mgr.serialize_model(persistence_monitoring.MonitoringSession, session),
'session': self._serialize_session(session),
'message_stats': {
'total': int(message_stats.total or 0),
'success': int(message_stats.success or 0),
@@ -100,7 +100,7 @@ def transcript_item(row):
class ExecutionDetailsMixin:
def _session_message_source(self, workspace, bot_ids, *, summary_only=False):
def _conversation_message_source(self, workspace, bot_ids):
"""Combine delivered messages with ingress missing from the legacy ledger."""
message = models.MonitoringMessage
conversation = sa.func.coalesce(
@@ -148,11 +148,7 @@ class ExecutionDetailsMixin:
'event_id': EventLog.event_id,
'run_id': EventLog.run_id,
}
columns = [
column
for column in message.__table__.columns
if not summary_only or column.name not in {'message_content', 'variables'}
]
columns = list(message.__table__.columns)
stored = sa.select(*columns).where(message.workspace_uuid == workspace, message.bot_id.in_(bot_ids))
incoming = sa.select(
*[values.get(column.name, sa.literal(None)).label(column.name) for column in columns]
@@ -168,13 +164,21 @@ class ExecutionDetailsMixin:
message.workspace_uuid == workspace,
message.bot_id == EventLog.bot_id,
sa.func.coalesce(message.role, 'user') != 'assistant',
sa.or_(message.event_id == EventLog.event_id, message.id == EventLog.event_id),
message.event_id == EventLog.event_id,
)
),
~sa.exists(
sa.select(message.id).where(
message.workspace_uuid == workspace,
message.bot_id == EventLog.bot_id,
sa.func.coalesce(message.role, 'user') != 'assistant',
message.id == EventLog.event_id,
)
),
)
return sa.union_all(stored, incoming).subquery()
async def _session_projection_page(self, workspace, statement, limit, offset):
async def _conversation_projection_page(self, workspace, statement, limit, offset):
count = (
await self._execution_query(workspace, sa.select(sa.func.count()).select_from(statement.subquery()))
).scalar_one()
@@ -184,79 +188,6 @@ class ExecutionDetailsMixin:
for row in rows
], count
async def get_bot_conversation_sessions(
self, workspace, bot_ids, start_time, end_time, user_query, is_active, limit, offset
):
"""Read session summaries across processor types without mutating history."""
# Window functions sort and retain their input rows. Carrying historical
# inline media through that sort can consume gigabytes for a summary.
messages = self._session_message_source(workspace, bot_ids, summary_only=True)
key = [messages.c.workspace_uuid, messages.c.bot_id, messages.c.session_id]
ranked = (
sa.select(
*messages.c,
sa.func.row_number()
.over(
partition_by=key,
order_by=[
sa.case((messages.c.role == 'user', 0), else_=1),
messages.c.timestamp.desc(),
messages.c.id.desc(),
],
)
.label('position'),
sa.func.min(messages.c.timestamp).over(partition_by=key).label('first_seen'),
sa.func.max(messages.c.timestamp).over(partition_by=key).label('last_seen'),
sa.func.count().over(partition_by=key).label('count'),
)
.where(messages.c.session_id != '')
.subquery()
)
session = models.MonitoringSession
fields = list(session.__table__.columns)
projected = {
'start_time': ranked.c.first_seen,
'last_activity': ranked.c.last_seen,
'message_count': ranked.c['count'],
'is_active': sa.literal(True),
}
derived = sa.select(
*[
(projected[field.name] if field.name in projected else ranked.c[field.name]).label(field.name)
for field in fields
]
).where(ranked.c.position == 1)
existing = sa.select(*fields).where(session.workspace_uuid == workspace, session.bot_id.in_(bot_ids))
combined = sa.union_all(existing, derived).subquery()
group = [combined.c.workspace_uuid, combined.c.bot_id, combined.c.session_id]
merged = sa.select(
*[
combined.c[field.name]
for field in fields
if field.name not in {'message_count', 'start_time', 'last_activity'}
],
sa.func.max(combined.c.message_count).over(partition_by=group).label('message_count'),
sa.func.min(combined.c.start_time).over(partition_by=group).label('start_time'),
sa.func.max(combined.c.last_activity).over(partition_by=group).label('last_activity'),
sa.func.row_number().over(partition_by=group, order_by=combined.c.last_activity.desc()).label('position'),
).subquery()
query = sa.select(*[merged.c[field.name] for field in fields]).where(merged.c.position == 1)
if start_time:
query = query.where(merged.c.last_activity >= start_time)
if end_time:
query = query.where(merged.c.last_activity <= end_time)
if user_query and user_query.strip():
pattern = f'%{user_query.strip()}%'
query = query.where(sa.or_(merged.c.user_id.ilike(pattern), merged.c.user_name.ilike(pattern)))
if is_active is not None:
query = query.where(merged.c.is_active == is_active)
return await self._session_projection_page(
workspace,
query.order_by(merged.c.last_activity.desc(), merged.c.session_id),
limit,
offset,
)
def _execution_json_text(self, column, key):
# SQLite JSON functions accept TEXT directly; PostgreSQL operators
# require an explicit JSON cast for the existing text-backed journals.
@@ -276,12 +207,21 @@ class ExecutionDetailsMixin:
)
.values(run_id=run_id, event_id=sa.func.coalesce(models.MonitoringMessage.event_id, event_id))
)
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)
if callable(tenant_uow):
async with tenant_uow(workspace) as uow:
await uow.session.execute(statement)
else:
await self.ap.persistence_mgr.execute_async(statement)
async with self._summary_write_session(workspace) as session:
message = (
await session.execute(
sa.select(
models.MonitoringMessage.bot_id,
models.MonitoringMessage.event_id,
models.MonitoringMessage.role,
).where(
models.MonitoringMessage.workspace_uuid == workspace, models.MonitoringMessage.id == message_id
)
)
).first()
await session.execute(statement)
if message is not None and not message.event_id and message.role != 'assistant':
await self._summary_writer().link_event(session, workspace, message.bot_id, message_id, event_id)
async def record_ingress_event(
self,
@@ -308,41 +248,44 @@ class ExecutionDetailsMixin:
# of storing an unbounded adapter object in the event journal.
content = parsed(self._sanitize_message_content(json.dumps(payload, ensure_ascii=False, default=str)), {})
summary = event_preview(content, event_type=event_type)
await store.append_event(
event_id=event_id,
event_type=event_type,
source='platform',
workspace_id=workspace,
bot_id=bot_id,
actor_id=actor_id,
actor_name=actor_name,
conversation_id=conversation_id,
actor_type='user' if actor_id else 'system',
input_summary=summary,
input_json=content,
metadata={'status': status, 'bot_name': bot_name, 'platform': platform},
)
safe_routes = []
for route in routes or ():
if isinstance(route, BaseException):
safe_routes.append({'status': 'failed', 'reason': str(route) or type(route).__name__})
elif isinstance(route, dict):
safe_routes.append(
{
key: route.get(key)
for key in (
'status',
'reason',
'target_type',
'target_uuid',
'run_id',
'failure_code',
)
}
)
metadata = {'status': status, 'bot_name': bot_name, 'platform': platform}
if routes is not None:
safe_routes = []
for route in routes:
if isinstance(route, BaseException):
safe_routes.append({'status': 'failed', 'reason': str(route) or type(route).__name__})
elif isinstance(route, dict):
safe_routes.append(
{
key: route.get(key)
for key in (
'status',
'reason',
'target_type',
'target_uuid',
'run_id',
'failure_code',
)
}
)
metadata = {'status': status, 'bot_name': bot_name, 'platform': platform, 'routes': safe_routes}
async with AsyncSession(self.ap.persistence_mgr.get_db_engine()) as session:
metadata['routes'] = safe_routes
async with self._summary_write_session(workspace) as session:
await store.append_event(
event_id=event_id,
event_type=event_type,
source='platform',
workspace_id=workspace,
bot_id=bot_id,
actor_id=actor_id,
actor_name=actor_name,
conversation_id=conversation_id,
actor_type='user' if actor_id else 'system',
input_summary=summary,
input_json=content,
metadata=metadata,
session=session,
)
if routes is not None:
await session.execute(
sa.update(EventLog)
.where(
@@ -351,7 +294,31 @@ class ExecutionDetailsMixin:
)
.values(metadata_json=json.dumps(metadata, ensure_ascii=False))
)
await session.commit()
if event_type == 'message.received':
from ....persistence.session_summary_writer import ingress_summary_change
event = (
(
await session.execute(
sa.select(
EventLog.bot_id,
EventLog.event_id,
EventLog.conversation_id,
EventLog.created_at,
EventLog.event_time,
EventLog.actor_id,
EventLog.actor_name,
EventLog.metadata_json,
).where(EventLog.workspace_id == workspace, EventLog.event_id == event_id)
)
)
.mappings()
.one()
)
await self._summary_writer().apply_once(
session,
ingress_summary_change(workspace, event),
)
async def _execution_query(self, workspace, statement):
tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None)