feat(storage): group usage by application and runtimes

This commit is contained in:
Hyu
2026-09-03 00:11:05 +08:00
parent 380bdfeca6
commit be0214cf4d
8 changed files with 543 additions and 41 deletions
+109 -2
View File
@@ -117,8 +117,6 @@ class MaintenanceService:
('logs', Path('data/logs')),
('storage', Path('data/storage')),
('vector_store', Path('data/chroma')),
('plugins', Path('data/plugins')),
('mcp', Path('data/mcp')),
('temp', Path('data/temp')),
]
else:
@@ -126,6 +124,11 @@ class MaintenanceService:
roots = [('storage', scoped_storage_path)]
sections = await asyncio.to_thread(self._collect_sections, roots)
runtime_processes = await self._runtime_storage_processes(
context,
core_sections=sections,
include_instance_storage=is_oss_singleton,
)
monitoring_counts = await self._monitoring_counts(context)
binary_storage = await self._binary_storage_stats(context)
@@ -146,6 +149,12 @@ class MaintenanceService:
'log_retention_days': log_retention_days,
},
'sections': sections,
'processes': runtime_processes,
'total_size_bytes': sum(
int(process.get('size_bytes') or 0)
for process in runtime_processes
if process.get('status') == 'available'
),
'database': {
'type': database_type,
'monitoring_counts': monitoring_counts,
@@ -158,6 +167,104 @@ class MaintenanceService:
'tasks': self.ap.task_mgr.get_stats() if is_oss_singleton and self.ap.task_mgr else {},
}
async def _runtime_storage_processes(
self,
context: TenantContext,
*,
core_sections: list[dict[str, Any]],
include_instance_storage: bool,
) -> list[dict[str, Any]]:
core_process = {
'key': 'langbot',
'status': 'available',
'source': 'local_process',
'size_bytes': sum(int(item.get('size_bytes') or 0) for item in core_sections),
'directories': [
{
**item,
'kind': 'root',
'error_count': 0,
}
for item in core_sections
],
}
plugin_task = self._plugin_runtime_storage_process(include_instance_storage)
box_task = self._box_runtime_storage_process(context)
plugin_process, box_process = await asyncio.gather(plugin_task, box_task)
return [core_process, plugin_process, box_process]
async def _plugin_runtime_storage_process(self, include_instance_storage: bool) -> dict[str, Any]:
if not include_instance_storage:
return self._unavailable_process(
'plugin_runtime',
'not_applicable',
'Instance-wide Plugin Runtime storage is hidden in multi-Workspace deployments',
)
connector = getattr(self.ap, 'plugin_connector', None)
if connector is None or not getattr(connector, 'is_enable_plugin', True):
return self._unavailable_process('plugin_runtime', 'disabled', 'Plugin Runtime is disabled')
runtime_handler = getattr(connector, 'handler', None)
get_analysis = getattr(runtime_handler, 'get_storage_analysis', None)
if not callable(get_analysis):
return self._unavailable_process(
'plugin_runtime',
'unavailable',
'Plugin Runtime does not support storage analysis',
)
try:
result = await get_analysis()
except Exception as exc:
self.ap.logger.warning(f'Failed to collect Plugin Runtime storage analysis: {exc}')
return self._unavailable_process('plugin_runtime', 'unavailable', str(exc))
return self._available_process('plugin_runtime', result, source='runtime_rpc')
async def _box_runtime_storage_process(self, context: TenantContext) -> dict[str, Any]:
box_service = getattr(self.ap, 'box_service', None)
if box_service is None or not getattr(box_service, 'enabled', True):
return self._unavailable_process('box_runtime', 'disabled', 'Box Runtime is disabled')
get_analysis = getattr(box_service, 'get_storage_analysis', None)
if not callable(get_analysis):
return self._unavailable_process(
'box_runtime',
'unavailable',
'Box Runtime does not support storage analysis',
)
try:
result = await get_analysis(context)
except Exception as exc:
self.ap.logger.warning(f'Failed to collect Box Runtime storage analysis: {exc}')
return self._unavailable_process('box_runtime', 'unavailable', str(exc))
return self._available_process('box_runtime', result, source='runtime_rpc')
@staticmethod
def _available_process(key: str, result: Any, *, source: str) -> dict[str, Any]:
payload = result if isinstance(result, dict) else {}
directories = payload.get('directories')
if not isinstance(directories, list):
directories = []
return {
'key': key,
'status': 'available',
'source': source,
'size_bytes': int(payload.get('size_bytes') or 0),
'directories': directories,
'active_sessions': payload.get('active_sessions'),
'managed_processes': payload.get('managed_processes'),
}
@staticmethod
def _unavailable_process(key: str, status: str, error: str) -> dict[str, Any]:
return {
'key': key,
'status': status,
'source': 'runtime_rpc',
'size_bytes': None,
'directories': [],
'error': error,
}
def _collect_sections(
self,
roots: list[tuple[str, Path | None]],
+10
View File
@@ -1409,6 +1409,16 @@ class BoxService:
except Exception:
return []
async def get_storage_analysis(self, context: TenantContext) -> dict:
"""Return Workspace-scoped storage measured by the Box Runtime."""
if not self._enabled:
raise BoxError('Box runtime is disabled')
if not self._available:
raise BoxError(self._connector_error or 'Box runtime is not available')
execution_context = await self._validated_execution_context(context)
return await self.client.get_storage_analysis(action_context=self._action_context(execution_context))
def build_spec(self, spec_payload: dict, skip_host_mount_validation: bool = False) -> BoxSpec:
spec_payload = dict(spec_payload)
spec_payload.setdefault('env', {})
+10
View File
@@ -2578,6 +2578,16 @@ class RuntimeConnectionHandler(handler.Handler):
timeout=10,
)
async def get_storage_analysis(self) -> dict[str, Any]:
"""Return instance-scoped filesystem usage measured by Plugin Runtime."""
with self.installation_scope(None):
return await self.call_action(
LangBotToRuntimeAction.GET_STORAGE_ANALYSIS,
{},
timeout=60,
)
async def reconcile_plugin_installations(
self,
installations: tuple[PluginInstallationDesiredState, ...],
@@ -308,15 +308,98 @@ class TestMaintenanceServiceGetStorageAnalysis:
# Execute
result = await service.get_storage_analysis(TEST_CONTEXT)
# Verify - all sections present
# Verify - only LangBot-owned sections are scanned by this process.
sections = {s['key'] for s in result['sections']}
assert 'database' in sections
assert 'logs' in sections
assert 'storage' in sections
assert 'vector_store' in sections
assert 'plugins' in sections
assert 'mcp' in sections
assert 'temp' in sections
assert 'plugins' not in sections
assert 'mcp' not in sections
assert [process['key'] for process in result['processes']] == [
'langbot',
'plugin_runtime',
'box_runtime',
]
async def test_get_storage_analysis_aggregates_runtime_owned_directories(self):
plugin_handler = SimpleNamespace(
get_storage_analysis=AsyncMock(
return_value={
'size_bytes': 25,
'directories': [
{
'key': 'artifacts',
'path': 'runtime/data/plugin-runtime/artifacts',
'kind': 'root',
'exists': True,
'size_bytes': 25,
'file_count': 2,
'error_count': 0,
}
],
}
)
)
box_service = SimpleNamespace(
enabled=True,
get_storage_analysis=AsyncMock(
return_value={
'size_bytes': 40,
'directories': [
{
'key': 'workspace',
'path': 'box/data/box/default/tenants/test',
'kind': 'root',
'exists': True,
'size_bytes': 40,
'file_count': 3,
'error_count': 0,
},
{
'key': 'mcp',
'path': 'box/data/box/default/tenants/test/.mcp',
'kind': 'detail',
'parent_key': 'workspace',
'exists': True,
'size_bytes': 30,
'file_count': 2,
'error_count': 0,
},
],
'active_sessions': 1,
'managed_processes': 2,
}
),
)
ap = SimpleNamespace(
instance_config=SimpleNamespace(data={}),
persistence_mgr=SimpleNamespace(execute_async=AsyncMock(return_value=_create_mock_result(scalar_value=0))),
logger=SimpleNamespace(warning=Mock()),
task_mgr=None,
plugin_connector=SimpleNamespace(
is_enable_plugin=True,
handler=plugin_handler,
),
box_service=box_service,
)
service = MaintenanceService(ap)
service._path_size = Mock(return_value=10)
service._file_count = Mock(return_value=1)
service._monitoring_counts = AsyncMock(return_value={})
service._binary_storage_stats = AsyncMock(return_value={'count': 0, 'size_bytes': 0})
service._expired_uploaded_candidates = AsyncMock(return_value=[])
service._expired_log_candidates = Mock(return_value=[])
result = await service.get_storage_analysis(TEST_CONTEXT)
assert result['total_size_bytes'] == 115
assert result['processes'][1]['directories'][0]['key'] == 'artifacts'
assert result['processes'][2]['directories'][1]['key'] == 'mcp'
assert result['processes'][2]['managed_processes'] == 2
plugin_handler.get_storage_analysis.assert_awaited_once_with()
box_service.get_storage_analysis.assert_awaited_once_with(TEST_CONTEXT)
async def test_get_storage_analysis_postgresql(self):
"""Handles PostgreSQL database type."""
+14
View File
@@ -430,6 +430,20 @@ async def test_box_service_get_sessions_delegates_to_client():
client.get_sessions.assert_awaited_once()
@pytest.mark.asyncio
async def test_box_service_get_storage_analysis_is_binding_checked_and_scoped():
client = Mock()
client.get_storage_analysis = AsyncMock(return_value={'total_size_bytes': 42, 'directories': []})
service = BoxService(make_app(Mock()), client=client)
service._available = True
result = await service.get_storage_analysis(_CONTEXT)
assert result == {'total_size_bytes': 42, 'directories': []}
client.get_storage_analysis.assert_awaited_once_with(action_context=_ACTION_CONTEXT)
@pytest.mark.asyncio
async def test_box_service_relay_connection_is_binding_checked_and_scoped():
app = make_app(Mock())
@@ -20,6 +20,7 @@ import {
import { Button } from '@/components/ui/button';
import { Badge } from '@/components/ui/badge';
import { ScrollArea } from '@/components/ui/scroll-area';
import { Skeleton } from '@/components/ui/skeleton';
import { backendClient } from '@/app/infra/http';
import { PanelToolbar } from '../settings-dialog/panel-layout';
@@ -31,6 +32,24 @@ interface StorageSection {
file_count: number;
}
interface RuntimeStorageDirectory extends StorageSection {
kind: 'root' | 'detail';
parent_key?: string;
error_count?: number;
scope?: 'runtime_host' | 'sandbox_sessions';
}
interface RuntimeStorageProcess {
key: 'langbot' | 'plugin_runtime' | 'box_runtime';
status: 'available' | 'unavailable' | 'disabled' | 'not_applicable';
source: 'local_process' | 'runtime_rpc';
size_bytes: number | null;
directories: RuntimeStorageDirectory[];
error?: string;
active_sessions?: number | null;
managed_processes?: number | null;
}
interface CleanupCandidate {
key?: string;
name?: string;
@@ -46,6 +65,8 @@ interface StorageAnalysis {
log_retention_days: number;
};
sections: StorageSection[];
processes?: RuntimeStorageProcess[];
total_size_bytes?: number;
database: {
type: string;
monitoring_counts: Record<string, number>;
@@ -114,7 +135,9 @@ export default function StorageAnalysisPanel({
const totalBytes = useMemo(() => {
return (
analysis?.sections.reduce((sum, item) => sum + item.size_bytes, 0) ?? 0
analysis?.total_size_bytes ??
analysis?.sections.reduce((sum, item) => sum + item.size_bytes, 0) ??
0
);
}, [analysis]);
@@ -168,6 +191,8 @@ export default function StorageAnalysisPanel({
</div>
)}
{loading && !analysis && <StorageAnalysisSkeleton />}
{analysis && (
<>
<div className="grid grid-cols-1 gap-3 md:grid-cols-4">
@@ -219,41 +244,28 @@ export default function StorageAnalysisPanel({
</div>
</section>
<section>
<h2 className="mb-2 text-sm font-medium">
{t('storageAnalysis.sections')}
</h2>
<div className="overflow-hidden rounded-md border">
{analysis.sections.map((section) => (
<div
key={section.key}
className="grid grid-cols-[1fr_auto_auto_auto] gap-3 border-b px-3 py-2 text-sm last:border-b-0"
>
<div className="min-w-0">
<div className="font-medium">
{t(`storageAnalysis.sectionNames.${section.key}`)}
</div>
<div className="break-all text-xs text-muted-foreground">
{section.path || '-'}
</div>
</div>
{section.exists ? (
<span />
) : (
<Badge variant="outline" className="self-center">
{t('storageAnalysis.missing')}
</Badge>
)}
<div className="self-center tabular-nums">
{formatBytes(section.size_bytes)}
</div>
<div className="self-center text-muted-foreground tabular-nums">
{section.file_count}
</div>
</div>
))}
</div>
</section>
{analysis.processes?.length ? (
<section>
<div className="mb-3">
<h2 className="text-sm font-medium">
{t('storageAnalysis.processStorage')}
</h2>
<p className="mt-1 text-xs text-muted-foreground">
{t('storageAnalysis.processStorageDescription')}
</p>
</div>
<div className="space-y-3">
{analysis.processes.map((process) => (
<RuntimeStorageGroup
key={process.key}
process={process}
/>
))}
</div>
</section>
) : (
<LegacyStorageSections sections={analysis.sections} />
)}
<section className="grid grid-cols-1 gap-4 md:grid-cols-2">
<MetricPanel
@@ -286,6 +298,171 @@ export default function StorageAnalysisPanel({
);
}
function StorageAnalysisSkeleton() {
return (
<div className="space-y-4" aria-hidden="true">
<div className="grid grid-cols-1 gap-3 md:grid-cols-4">
{Array.from({ length: 4 }).map((_, index) => (
<Skeleton key={index} className="h-24 rounded-md" />
))}
</div>
{Array.from({ length: 3 }).map((_, index) => (
<div key={index} className="space-y-3 rounded-md border p-4">
<Skeleton className="h-5 w-40" />
<Skeleton className="h-12 w-full" />
<Skeleton className="h-12 w-full" />
</div>
))}
</div>
);
}
function RuntimeStorageGroup({ process }: { process: RuntimeStorageProcess }) {
const { t } = useTranslation();
const available = process.status === 'available';
return (
<div className="overflow-hidden rounded-md border bg-card">
<div className="flex flex-wrap items-start justify-between gap-3 border-b bg-muted/20 px-4 py-3">
<div className="min-w-0">
<div className="flex flex-wrap items-center gap-2">
<h3 className="text-sm font-semibold">
{t(`storageAnalysis.processNames.${process.key}`)}
</h3>
<Badge variant={available ? 'secondary' : 'outline'}>
{t(`storageAnalysis.statusLabels.${process.status}`)}
</Badge>
</div>
<p className="mt-1 text-xs text-muted-foreground">
{t(`storageAnalysis.processDescriptions.${process.key}`)}
</p>
<p className="mt-1 text-xs text-muted-foreground">
{t(`storageAnalysis.sourceLabels.${process.source}`)}
</p>
</div>
<div className="text-right">
<div className="text-base font-semibold tabular-nums">
{formatBytes(process.size_bytes)}
</div>
{process.key === 'box_runtime' && available && (
<div className="mt-1 text-xs text-muted-foreground">
{t('storageAnalysis.boxActivity', {
sessions: process.active_sessions ?? 0,
processes: process.managed_processes ?? 0,
})}
</div>
)}
</div>
</div>
{!available ? (
<div className="px-4 py-4 text-sm text-muted-foreground">
{process.error || t('storageAnalysis.runtimeUnavailable')}
</div>
) : process.directories.length === 0 ? (
<div className="px-4 py-6 text-center text-sm text-muted-foreground">
{t('storageAnalysis.noManagedDirectories')}
</div>
) : (
<div>
<div className="hidden grid-cols-[minmax(0,1fr)_100px_90px] gap-4 border-b px-4 py-2 text-xs text-muted-foreground sm:grid">
<span>{t('storageAnalysis.directory')}</span>
<span className="text-right">{t('storageAnalysis.size')}</span>
<span className="text-right">{t('storageAnalysis.files')}</span>
</div>
{process.directories.map((directory) => (
<div
key={directory.key}
className="grid grid-cols-[minmax(0,1fr)_auto] gap-3 border-b px-4 py-3 text-sm last:border-b-0 sm:grid-cols-[minmax(0,1fr)_100px_90px] sm:gap-4"
>
<div
className={`min-w-0 ${
directory.kind === 'detail'
? 'ml-3 border-l-2 border-muted pl-3'
: ''
}`}
>
<div className="flex flex-wrap items-center gap-2">
<span className="font-medium">
{t(`storageAnalysis.sectionNames.${directory.key}`)}
</span>
{!directory.exists && (
<Badge variant="outline" className="font-normal">
{t('storageAnalysis.notCreated')}
</Badge>
)}
{!!directory.error_count && (
<Badge variant="outline" className="font-normal">
{t('storageAnalysis.scanWarnings', {
count: directory.error_count,
})}
</Badge>
)}
{directory.scope && (
<Badge variant="outline" className="font-normal">
{t(`storageAnalysis.scopeLabels.${directory.scope}`)}
</Badge>
)}
</div>
<div className="mt-1 break-all font-mono text-xs text-muted-foreground">
{directory.path || '-'}
</div>
</div>
<div className="self-center text-right tabular-nums">
{formatBytes(directory.size_bytes)}
</div>
<div className="col-span-2 self-center text-right text-xs text-muted-foreground tabular-nums sm:col-span-1 sm:text-sm">
{directory.file_count}
</div>
</div>
))}
</div>
)}
</div>
);
}
function LegacyStorageSections({ sections }: { sections: StorageSection[] }) {
const { t } = useTranslation();
return (
<section>
<h2 className="mb-2 text-sm font-medium">
{t('storageAnalysis.sections')}
</h2>
<div className="overflow-hidden rounded-md border">
{sections.map((section) => (
<div
key={section.key}
className="grid grid-cols-[1fr_auto_auto_auto] gap-3 border-b px-3 py-2 text-sm last:border-b-0"
>
<div className="min-w-0">
<div className="font-medium">
{t(`storageAnalysis.sectionNames.${section.key}`)}
</div>
<div className="break-all text-xs text-muted-foreground">
{section.path || '-'}
</div>
</div>
{section.exists ? (
<span />
) : (
<Badge variant="outline" className="self-center">
{t('storageAnalysis.missing')}
</Badge>
)}
<div className="self-center tabular-nums">
{formatBytes(section.size_bytes)}
</div>
<div className="self-center text-muted-foreground tabular-nums">
{section.file_count}
</div>
</div>
))}
</div>
</section>
);
}
function SummaryItem({
label,
value,
+52
View File
@@ -2050,6 +2050,44 @@ const enUS = {
databaseType: 'Database type',
days: 'days',
missing: 'Missing',
notCreated: 'Not created yet',
processStorage: 'Storage by process',
processStorageDescription:
'Each runtime measures the directories it owns. Detail rows are included in their parent directory total.',
directory: 'Directory',
size: 'Size',
files: 'Files',
runtimeUnavailable: 'Runtime storage statistics are unavailable.',
noManagedDirectories: 'No managed directories were reported.',
scanWarnings: '{{count}} unreadable entries',
boxActivity: '{{sessions}} sessions · {{processes}} managed processes',
statusLabels: {
available: 'Available',
unavailable: 'Unavailable',
disabled: 'Disabled',
not_applicable: 'Not applicable',
},
sourceLabels: {
local_process: 'Measured by the LangBot process',
runtime_rpc: 'Measured by the runtime process over authenticated RPC',
},
scopeLabels: {
runtime_host: 'Runtime host',
sandbox_sessions: 'Sandbox sessions',
},
processNames: {
langbot: 'LangBot main process',
plugin_runtime: 'Plugin Runtime',
box_runtime: 'Box Runtime',
},
processDescriptions: {
langbot:
'Application database, logs, uploaded files, vector data and temporary files.',
plugin_runtime:
'Plugin packages, verified artifacts, dependency environments and private installation data.',
box_runtime:
'Sandbox workspaces, MCP process workspaces, attachment exchange directories and skills.',
},
expiredUploads: 'Expired uploads',
expiredLogs: 'Expired logs',
noExpiredUploads: 'No expired uploaded files',
@@ -2062,6 +2100,20 @@ const enUS = {
plugins: 'Plugins',
mcp: 'MCP',
temp: 'Temporary files',
legacy_plugins: 'Legacy plugin packages',
artifacts: 'Verified plugin artifacts',
dependency_environments: 'Plugin dependency environments',
installations: 'Plugin installation data',
staging: 'Plugin staging files',
rpc_transfer: 'Runtime RPC transfer files',
workspace: 'Sandbox workspace',
inbox: 'Inbound attachments',
outbox: 'Outbound attachments',
skills: 'Skills',
session_workspaces: 'Sandbox session workspaces',
session_caches: 'Sandbox runtime caches',
session_temp: 'Sandbox temporary files',
managed_process_workspaces: 'Managed-process workspaces (including MCP)',
},
},
limitation: {
+49
View File
@@ -1958,6 +1958,41 @@ const zhHans = {
databaseType: '数据库类型',
days: '天',
missing: '不存在',
notCreated: '尚未创建',
processStorage: '各进程管理的存储',
processStorageDescription:
'由每个运行时直接统计其负责的目录;明细目录已包含在上级目录的总占用中。',
directory: '目录',
size: '占用',
files: '文件数',
runtimeUnavailable: '暂时无法获取该运行时的存储统计。',
noManagedDirectories: '该进程未报告任何受管目录。',
scanWarnings: '{{count}} 项无法读取',
boxActivity: '{{sessions}} 个会话 · {{processes}} 个受管进程',
statusLabels: {
available: '统计正常',
unavailable: '运行时未连接',
disabled: '未启用',
not_applicable: '不适用',
},
sourceLabels: {
local_process: '由 LangBot 主进程直接统计',
runtime_rpc: '由运行时进程通过鉴权 RPC 直接统计',
},
scopeLabels: {
runtime_host: '运行时宿主目录',
sandbox_sessions: '沙箱会话内部',
},
processNames: {
langbot: 'LangBot 主程序',
plugin_runtime: 'Plugin Runtime',
box_runtime: 'Box Runtime',
},
processDescriptions: {
langbot: '应用数据库、日志、上传文件、向量数据和主程序临时文件。',
plugin_runtime: '插件包、校验后的制品、依赖环境和安装实例私有数据。',
box_runtime: '沙箱工作区、MCP 进程工作区、附件交换目录和技能目录。',
},
expiredUploads: '过期上传文件',
expiredLogs: '过期日志',
noExpiredUploads: '暂无过期上传文件',
@@ -1970,6 +2005,20 @@ const zhHans = {
plugins: '插件',
mcp: 'MCP',
temp: '临时文件',
legacy_plugins: '旧版插件包',
artifacts: '已校验插件制品',
dependency_environments: '插件依赖环境',
installations: '插件安装实例数据',
staging: '插件安装暂存区',
rpc_transfer: '运行时 RPC 传输文件',
workspace: '沙箱工作区',
inbox: '输入附件',
outbox: '输出附件',
skills: '技能目录',
session_workspaces: '沙箱会话工作区',
session_caches: '沙箱运行时缓存',
session_temp: '沙箱临时文件',
managed_process_workspaces: '受管进程工作区(包括 MCP',
},
},
limitation: {