fix(storage): expire historical inline images with bounded TTL cleanup

- Apply media retention to inline images while preserving text and URLs
- Persist per-workspace cursors atomically with payload updates
- Bound candidate scans and verify event ownership before loading payloads
- Add cleanup indexes and regression tests, and fix Windows path assertions

src/langbot/pkg/api/http/service/maintenance.py

@@ -114,6 +114,10 @@ class MaintenanceService:
         }
         if media_cache is not None and is_singleton:
             result['media_files'] = media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0)
+        if media_cache is not None:
+            from ....storage.inline_media_cleanup import cleanup_inline_media
+
+            result['inline_media_records'] = await cleanup_inline_media(self.ap, context, media_retention_days)
         return result
 
     async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]:

src/langbot/pkg/core/app.py

@@ -545,7 +545,7 @@ class Application:
                                         if total_deleted > 0:
                                             self.logger.info(
                                                 f'Storage maintenance for Workspace {context.workspace_uuid}: '
-                                                f'deleted expired files: {deleted}'
+                                                f'cleaned expired storage data: {deleted}'
                                             )
                                     except asyncio.CancelledError:
                                         raise

src/langbot/pkg/entity/persistence/agent_run.py

@@ -111,6 +111,7 @@ class AgentRun(Base):
     """Additional metadata JSON."""
 
     __table_args__ = (
+        sqlalchemy.Index('ix_agent_run_media_ttl', 'workspace_id', 'created_at', 'id'),
         sqlalchemy.Index(
             'ix_agent_run_scope_status', 'bot_id', 'workspace_id', 'conversation_id', 'thread_id', 'status'
         ),
@@ -202,4 +203,5 @@ class AgentRunEvent(Base):
     __table_args__ = (
         sqlalchemy.UniqueConstraint('run_id', 'sequence', name='uq_agent_run_event_run_sequence'),
         sqlalchemy.Index('ix_agent_run_event_run_sequence', 'run_id', 'sequence'),
+        sqlalchemy.Index('ix_agent_run_event_media_ttl', 'created_at', 'id', 'run_id'),
     )

src/langbot/pkg/entity/persistence/event_log.py

@@ -84,3 +84,5 @@ class EventLog(Base):
 
     metadata_json = sqlalchemy.Column(sqlalchemy.Text, nullable=True)
     """Additional metadata as JSON string."""
+
+    __table_args__ = (sqlalchemy.Index('ix_event_log_media_ttl', 'workspace_id', 'created_at', 'id'),)

src/langbot/pkg/entity/persistence/monitoring.py

@@ -39,6 +39,7 @@ class MonitoringMessage(Base):
 
     __table_args__ = (
         sqlalchemy.Index('ix_monitoring_messages_workspace_timestamp', 'workspace_uuid', 'timestamp'),
+        sqlalchemy.Index('ix_monitoring_messages_media_ttl', 'workspace_uuid', 'timestamp', 'id'),
         sqlalchemy.Index('ix_monitoring_messages_workspace_session', 'workspace_uuid', 'session_id'),
         sqlalchemy.Index(
             'ix_monitoring_messages_conversation_page', 'workspace_uuid', 'bot_id', 'session_id', 'timestamp', 'id'

src/langbot/pkg/entity/persistence/transcript.py

@@ -74,6 +74,7 @@ class Transcript(Base):
 
     # Indexes
     __table_args__ = (
+        sqlalchemy.Index('ix_transcript_media_ttl', 'workspace_id', 'created_at', 'id'),
         sqlalchemy.Index('ix_transcript_conversation_seq', 'conversation_id', 'seq'),
         sqlalchemy.Index('ix_transcript_conversation_created', 'conversation_id', 'created_at'),
         sqlalchemy.Index('ix_transcript_scope_seq', 'bot_id', 'workspace_id', 'conversation_id', 'thread_id', 'seq'),

src/langbot/pkg/persistence/alembic/versions/0039_execution_lease_indexes.py

@@ -15,6 +15,8 @@ def upgrade():
         ('agent_run', 'workspace_id'),
         ('monitoring_messages', 'workspace_uuid'),
     ):
+        if not inspector.has_table(table):
+            continue
         index = f'ix_{table}_workspace_status_lease'
         # Fresh installations create these indexes from ORM metadata before migrations.
         if index not in {item['name'] for item in inspector.get_indexes(table)}:
@@ -24,6 +26,8 @@ def upgrade():
 def downgrade():
     inspector = sa.inspect(op.get_bind())
     for table in ('agent_run', 'monitoring_messages'):
+        if not inspector.has_table(table):
+            continue
         index = f'ix_{table}_workspace_status_lease'
         if index in {item['name'] for item in inspector.get_indexes(table)}:
             op.drop_index(index, table_name=table)

src/langbot/pkg/persistence/alembic/versions/0040_conversation_page_index.py

@@ -13,11 +13,13 @@ INDEX = 'ix_monitoring_messages_conversation_page'
 
 
 def upgrade():
+    inspector = sa.inspect(op.get_bind())
     # Fresh databases already receive this index from ORM metadata.
-    if INDEX not in {index['name'] for index in sa.inspect(op.get_bind()).get_indexes(TABLE)}:
+    if inspector.has_table(TABLE) and INDEX not in {index['name'] for index in inspector.get_indexes(TABLE)}:
         op.create_index(INDEX, TABLE, ['workspace_uuid', 'bot_id', 'session_id', 'timestamp', 'id'])
 
 
 def downgrade():
-    if INDEX in {index['name'] for index in sa.inspect(op.get_bind()).get_indexes(TABLE)}:
+    inspector = sa.inspect(op.get_bind())
+    if inspector.has_table(TABLE) and INDEX in {index['name'] for index in inspector.get_indexes(TABLE)}:
         op.drop_index(INDEX, table_name=TABLE)

src/langbot/pkg/persistence/alembic/versions/0041_inline_media_ttl_indexes.py

@@ -0,0 +1,37 @@
+"""Index incremental expiry of historical inline images."""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = '0041_inline_media_ttl_indexes'
+down_revision = '0040_conversation_page_index'
+branch_labels = None
+depends_on = None
+
+INDEXES = (
+    ('monitoring_messages', ['workspace_uuid', 'timestamp', 'id']),
+    ('event_log', ['workspace_id', 'created_at', 'id']),
+    ('agent_run', ['workspace_id', 'created_at', 'id']),
+    ('agent_run_event', ['created_at', 'id', 'run_id']),
+    ('transcript', ['workspace_id', 'created_at', 'id']),
+)
+
+
+def upgrade():
+    inspector = sa.inspect(op.get_bind())
+    for table, columns in INDEXES:
+        if not inspector.has_table(table):
+            continue
+        name = f'ix_{table}_media_ttl'
+        if name not in {index['name'] for index in inspector.get_indexes(table)}:
+            op.create_index(name, table, columns)
+
+
+def downgrade():
+    inspector = sa.inspect(op.get_bind())
+    for table, _ in reversed(INDEXES):
+        if not inspector.has_table(table):
+            continue
+        name = f'ix_{table}_media_ttl'
+        if name in {index['name'] for index in inspector.get_indexes(table)}:
+            op.drop_index(name, table_name=table)

src/langbot/pkg/storage/inline_media_cleanup.py

@@ -0,0 +1,147 @@
+"""Incrementally expire historical inline images without rewriting record history."""
+
+from __future__ import annotations
+
+import asyncio
+import datetime
+import json
+from typing import TYPE_CHECKING
+
+import sqlalchemy as sa
+
+from ..api.http.authz import WorkspaceRequiredError
+from ..api.http.context import ExecutionContext
+from ..entity.persistence.agent_run import AgentRun, AgentRunEvent
+from ..entity.persistence.event_log import EventLog
+from ..entity.persistence.metadata import WorkspaceMetadata
+from ..entity.persistence.monitoring import MonitoringMessage
+from ..entity.persistence.transcript import Transcript
+from ..persistence.datetime_utils import as_naive_utc
+from .media import MediaCache
+
+if TYPE_CHECKING:
+    from ..core.app import Application
+
+
+ROWS_PER_TABLE = 100
+CHARACTERS_PER_TABLE = 8 * 1024 * 1024
+CURSOR_PREFIX = 'inline_media_ttl_v1:'
+
+
+def _redact_columns(values: dict) -> dict:
+    updates = {}
+    for name, raw in values.items():
+        if not raw or ('base64' not in raw and 'data:image/' not in raw):
+            continue
+        try:
+            payload = json.loads(raw)
+        except (ValueError, TypeError, RecursionError):
+            # Preserve malformed historical text rather than deleting an entire record.
+            continue
+        if MediaCache.has_inline_images(payload):
+            updates[name] = json.dumps(MediaCache.redact_inline_images(payload), ensure_ascii=False)
+    return updates
+
+
+async def cleanup_inline_media(ap: Application, context: ExecutionContext, retention_days: int) -> int:
+    """Use the media TTL for immutable history, with durable per-Workspace cursors.
+
+    Bound each table by both candidate count and serialized payload characters. One
+    oversized row is processed on its own so it cannot permanently block the cursor.
+    """
+    if not isinstance(context, ExecutionContext) or not context.workspace_uuid:
+        raise WorkspaceRequiredError('Inline media cleanup requires an ExecutionContext')
+    if retention_days < 1:
+        raise ValueError('retention_days must be >= 1')
+    workspace = context.workspace_uuid
+    cutoff = as_naive_utc(datetime.datetime.now(datetime.timezone.utc)) - datetime.timedelta(days=retention_days)
+    sources = (
+        (
+            MonitoringMessage,
+            MonitoringMessage.timestamp,
+            MonitoringMessage.workspace_uuid == workspace,
+            ('message_content', 'variables'),
+        ),
+        (EventLog, EventLog.created_at, EventLog.workspace_id == workspace, ('input_json', 'metadata_json')),
+        (AgentRun, AgentRun.created_at, AgentRun.workspace_id == workspace, ('metadata_json',)),
+        (
+            AgentRunEvent,
+            AgentRunEvent.created_at,
+            sa.exists(
+                sa.select(AgentRun.id).where(
+                    AgentRun.run_id == AgentRunEvent.run_id, AgentRun.workspace_id == workspace
+                )
+            ),
+            ('data_json', 'metadata_json'),
+        ),
+        (
+            Transcript,
+            Transcript.created_at,
+            Transcript.workspace_id == workspace,
+            ('content_json', 'attachment_refs_json', 'metadata_json'),
+        ),
+    )
+    manager = ap.persistence_mgr
+    cleaned = 0
+    for model, timestamp, scope, fields in sources:
+        consumed = 0
+        cursor_key = CURSOR_PREFIX + model.__tablename__
+        cursor_filter = (WorkspaceMetadata.workspace_uuid == workspace, WorkspaceMetadata.key == cursor_key)
+        for _ in range(ROWS_PER_TABLE):
+            async with manager.tenant_uow(workspace):
+                cursor_result = await manager.execute_async(
+                    sa.select(WorkspaceMetadata.value).where(*cursor_filter).with_for_update()
+                )
+                cursor_raw = cursor_result.scalar_one_or_none()
+                # Events have no Workspace column. Limit indexed candidates before
+                # checking ownership so other Workspaces cannot cause unbounded scans.
+                query = sa.select(model.id, timestamp).where(timestamp < cutoff)
+                if model is not AgentRunEvent:
+                    query = query.where(scope)
+                if cursor_raw is not None:
+                    after_time, after_id = json.loads(cursor_raw)
+                    query = query.where(
+                        sa.tuple_(timestamp, model.id)
+                        > sa.tuple_(datetime.datetime.fromisoformat(after_time), after_id)
+                    )
+                result = await manager.execute_async(query.order_by(timestamp, model.id).limit(1))
+                row = result.first()
+                if row is None:
+                    break
+                payload = (
+                    await manager.execute_async(
+                        sa.select(*(getattr(model, field) for field in fields)).where(model.id == row[0], scope)
+                    )
+                ).first()
+                # Missing and foreign candidates still consume budget and advance the
+                # cursor. Their payloads are never returned to the cleanup worker.
+                values = dict(zip(fields, payload)) if payload is not None else {}
+                consumed += sum(len(value) for value in values.values() if value)
+                updates = await asyncio.to_thread(_redact_columns, values) if values else {}
+                if updates:
+                    if model is AgentRun:
+                        # Media expiry must not appear as new execution activity.
+                        updates['updated_at'] = AgentRun.updated_at
+                    changed = await manager.execute_async(
+                        sa.update(model)
+                        .where(
+                            scope,
+                            model.id == row[0],
+                            *(getattr(model, name) == values[name] for name in fields if name in updates),
+                        )
+                        .values(**updates)
+                    )
+                    if not changed.rowcount:
+                        # A concurrent writer changed this payload; retry it next time.
+                        break
+                    cleaned += 1
+                cursor = json.dumps([row[1].isoformat(), row[0]])
+                if cursor_raw is None:
+                    await manager.execute_async(
+                        sa.insert(WorkspaceMetadata).values(workspace_uuid=workspace, key=cursor_key, value=cursor)
+                    )
+                else:
+                    await manager.execute_async(sa.update(WorkspaceMetadata).where(*cursor_filter).values(value=cursor))
+            if consumed >= CHARACTERS_PER_TABLE:
+                break
+    return cleaned

src/langbot/templates/config.yaml

@@ -224,7 +224,8 @@ storage:
     max_object_read_bytes: 10485760
     # Media content cache (images & attachments externalized from monitoring and pipelines)
     media_cache:
-        # Retention period in days for cached media (defaults to 30 days)
+        # Retention for CAS files and historical inline images (defaults to 30 days).
+        # Inline images expire by record creation time; message text is retained.
         retention_days: 30
         # Maximum disk storage for cached media in MB (0 means unlimited, defaults to 0)
         max_size_mb: 0