mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-09-16 23:07:14 +00:00
feat(storage): media content-addressable cache and monitoring base64 externalization
- Add MediaCache using xxHash3-128 (with sha256 fallback) content-addressable storage - Externalize message chain image payloads before recording monitoring and discarded messages - Strip base64 payloads to null in SQLite monitoring_messages, dropping row size from megabytes to hundreds of bytes - Add GET /api/v1/files/media/<filename> route with immutable HTTP cache headers to serve cached media - Integrate age-based retention (default 30 days) and configurable disk quota with MaintenanceService cleanup loop - Add defensive sanitizer in MonitoringService.record_message against oversized raw base64 payloads - Add comprehensive unit tests and end-to-end verification covering CAS deduplication, route serving, and LRU pruning
This commit is contained in:
@@ -47,6 +47,21 @@ class FilesRouterGroup(group.RouterGroup):
|
||||
|
||||
return quart.Response(image_bytes, mimetype=mime_type)
|
||||
|
||||
@self.route(
|
||||
'/media/<filename>',
|
||||
methods=['GET'],
|
||||
auth_type=group.AuthType.NONE,
|
||||
)
|
||||
async def get_media_file(filename: str) -> quart.Response:
|
||||
media = await self.ap.storage_mgr.media_cache.get_media(filename)
|
||||
if media is None:
|
||||
return quart.Response('Media not found or expired', status=404)
|
||||
media_bytes, mime_type = media
|
||||
headers = {
|
||||
'Cache-Control': 'public, max-age=2592000, immutable',
|
||||
}
|
||||
return quart.Response(media_bytes, mimetype=mime_type, headers=headers)
|
||||
|
||||
@self.route(
|
||||
'/images',
|
||||
methods=['POST'],
|
||||
|
||||
@@ -80,6 +80,25 @@ class MaintenanceService:
|
||||
DEFAULT_LOG_RETENTION_DAYS,
|
||||
'storage.cleanup.log_retention_days',
|
||||
)
|
||||
media_cfg = self.ap.instance_config.data.get('storage', {}).get('media_cache', {})
|
||||
media_retention_days = self._positive_int(
|
||||
media_cfg.get('retention_days'),
|
||||
30,
|
||||
'storage.media_cache.retention_days',
|
||||
)
|
||||
media_max_size_mb = self._non_negative_int(
|
||||
media_cfg.get('max_size_mb'),
|
||||
0,
|
||||
'storage.media_cache.max_size_mb',
|
||||
)
|
||||
media_cleanup = (
|
||||
await self.ap.storage_mgr.media_cache.cleanup(
|
||||
media_retention_days,
|
||||
media_max_size_mb,
|
||||
)
|
||||
if hasattr(self.ap.storage_mgr, 'media_cache') and await self._is_oss_singleton(context)
|
||||
else {}
|
||||
)
|
||||
|
||||
return {
|
||||
'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days),
|
||||
@@ -89,6 +108,7 @@ class MaintenanceService:
|
||||
)
|
||||
if await self._is_oss_singleton(context)
|
||||
else 0,
|
||||
'media_files': media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0),
|
||||
}
|
||||
|
||||
async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]:
|
||||
@@ -466,6 +486,17 @@ class MaintenanceService:
|
||||
count += len(files)
|
||||
return count
|
||||
|
||||
def _non_negative_int(self, value: Any, default: int, name: str) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
except (TypeError, ValueError):
|
||||
self.ap.logger.warning(f'Invalid {name}: {value!r}, using {default}')
|
||||
return default
|
||||
if parsed < 0:
|
||||
self.ap.logger.warning(f'{name} must be non-negative: {value!r}, using {default}')
|
||||
return default
|
||||
return parsed
|
||||
|
||||
def _positive_int(self, value: Any, default: int, name: str) -> int:
|
||||
try:
|
||||
parsed = int(value)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import uuid
|
||||
import datetime
|
||||
import functools
|
||||
@@ -417,6 +418,32 @@ class MonitoringService:
|
||||
|
||||
# ========== Recording Methods ==========
|
||||
|
||||
def _sanitize_message_content(self, content: str) -> str:
|
||||
"""Strip raw base64 data to protect database storage from unbounded bloating."""
|
||||
if not content or len(content) < 10000 or (';base64,' not in content and 'data:image/' not in content):
|
||||
return content
|
||||
try:
|
||||
data = json.loads(content)
|
||||
|
||||
def _strip_node(node):
|
||||
if isinstance(node, list):
|
||||
return [_strip_node(x) for x in node]
|
||||
if isinstance(node, dict):
|
||||
res = dict(node)
|
||||
if res.get('type') == 'Image' and res.get('base64'):
|
||||
res['base64'] = None
|
||||
for k, v in list(res.items()):
|
||||
if isinstance(v, (list, dict)):
|
||||
res[k] = _strip_node(v)
|
||||
return res
|
||||
return node
|
||||
|
||||
return json.dumps(_strip_node(data), ensure_ascii=False)
|
||||
except Exception:
|
||||
return re.sub(
|
||||
r'data:image/[a-zA-Z0-9.+_-]+;base64,[\sA-Za-z0-9+/=]{1000,}', '[base64 image omitted]', content
|
||||
)
|
||||
|
||||
@_workspace_transaction
|
||||
async def record_message(
|
||||
self,
|
||||
@@ -439,6 +466,7 @@ class MonitoringService:
|
||||
"""Record a message"""
|
||||
workspace_uuid = self._require_write_context(context)
|
||||
message_id = str(uuid.uuid4())
|
||||
message_content = self._sanitize_message_content(message_content)
|
||||
message_data = {
|
||||
'id': message_id,
|
||||
'workspace_uuid': workspace_uuid,
|
||||
|
||||
Reference in New Issue
Block a user