feat(storage): media content-addressable cache and monitoring base64 externalization

- Add MediaCache using xxHash3-128 (with sha256 fallback) content-addressable storage
- Externalize message chain image payloads before recording monitoring and discarded messages
- Strip base64 payloads to null in SQLite monitoring_messages, dropping row size from megabytes to hundreds of bytes
- Add GET /api/v1/files/media/<filename> route with immutable HTTP cache headers to serve cached media
- Integrate age-based retention (default 30 days) and configurable disk quota with MaintenanceService cleanup loop
- Add defensive sanitizer in MonitoringService.record_message against oversized raw base64 payloads
- Add comprehensive unit tests and end-to-end verification covering CAS deduplication, route serving, and LRU pruning
This commit is contained in:
BiFangKNT
2026-09-15 17:58:23 +08:00
parent 38ff4766ef
commit 1143d6a5ae
11 changed files with 607 additions and 62 deletions
@@ -47,6 +47,21 @@ class FilesRouterGroup(group.RouterGroup):
return quart.Response(image_bytes, mimetype=mime_type)
@self.route(
'/media/<filename>',
methods=['GET'],
auth_type=group.AuthType.NONE,
)
async def get_media_file(filename: str) -> quart.Response:
media = await self.ap.storage_mgr.media_cache.get_media(filename)
if media is None:
return quart.Response('Media not found or expired', status=404)
media_bytes, mime_type = media
headers = {
'Cache-Control': 'public, max-age=2592000, immutable',
}
return quart.Response(media_bytes, mimetype=mime_type, headers=headers)
@self.route(
'/images',
methods=['POST'],
@@ -80,6 +80,25 @@ class MaintenanceService:
DEFAULT_LOG_RETENTION_DAYS,
'storage.cleanup.log_retention_days',
)
media_cfg = self.ap.instance_config.data.get('storage', {}).get('media_cache', {})
media_retention_days = self._positive_int(
media_cfg.get('retention_days'),
30,
'storage.media_cache.retention_days',
)
media_max_size_mb = self._non_negative_int(
media_cfg.get('max_size_mb'),
0,
'storage.media_cache.max_size_mb',
)
media_cleanup = (
await self.ap.storage_mgr.media_cache.cleanup(
media_retention_days,
media_max_size_mb,
)
if hasattr(self.ap.storage_mgr, 'media_cache') and await self._is_oss_singleton(context)
else {}
)
return {
'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days),
@@ -89,6 +108,7 @@ class MaintenanceService:
)
if await self._is_oss_singleton(context)
else 0,
'media_files': media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0),
}
async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]:
@@ -466,6 +486,17 @@ class MaintenanceService:
count += len(files)
return count
def _non_negative_int(self, value: Any, default: int, name: str) -> int:
try:
parsed = int(value)
except (TypeError, ValueError):
self.ap.logger.warning(f'Invalid {name}: {value!r}, using {default}')
return default
if parsed < 0:
self.ap.logger.warning(f'{name} must be non-negative: {value!r}, using {default}')
return default
return parsed
def _positive_int(self, value: Any, default: int, name: str) -> int:
try:
parsed = int(value)
@@ -1,5 +1,6 @@
from __future__ import annotations
import re
import uuid
import datetime
import functools
@@ -417,6 +418,32 @@ class MonitoringService:
# ========== Recording Methods ==========
def _sanitize_message_content(self, content: str) -> str:
"""Strip raw base64 data to protect database storage from unbounded bloating."""
if not content or len(content) < 10000 or (';base64,' not in content and 'data:image/' not in content):
return content
try:
data = json.loads(content)
def _strip_node(node):
if isinstance(node, list):
return [_strip_node(x) for x in node]
if isinstance(node, dict):
res = dict(node)
if res.get('type') == 'Image' and res.get('base64'):
res['base64'] = None
for k, v in list(res.items()):
if isinstance(v, (list, dict)):
res[k] = _strip_node(v)
return res
return node
return json.dumps(_strip_node(data), ensure_ascii=False)
except Exception:
return re.sub(
r'data:image/[a-zA-Z0-9.+_-]+;base64,[\sA-Za-z0-9+/=]{1000,}', '[base64 image omitted]', content
)
@_workspace_transaction
async def record_message(
self,
@@ -439,6 +466,7 @@ class MonitoringService:
"""Record a message"""
workspace_uuid = self._require_write_context(context)
message_id = str(uuid.uuid4())
message_content = self._sanitize_message_content(message_content)
message_data = {
'id': message_id,
'workspace_uuid': workspace_uuid,