mirror of
https://github.com/langbot-app/LangBot.git
synced 2026-10-23 08:55:44 +08:00
fix(box): validate and export explicitly selected files (#2641)
* fix(box): retain outbox files when export fails * test(box): retain ownership evidence in CI logs * test(box): exercise LocalAgent export repair through real runtime and Docker Co-authored-by: multica-agent <[email protected]> * fix(box): complete bounded outbox recovery contract * feat(box): export explicitly selected files Co-authored-by: multica-agent <[email protected]> * test(box): exercise explicit export on real Docker Co-authored-by: multica-agent <[email protected]> * test(box): verify explicit exports with live model and Docker Co-authored-by: multica-agent <[email protected]> * fix(box): align explicit export SDK and integration contracts * docs(box): describe selected-file export contract --------- Co-authored-by: HaiYangBG1 <[email protected]> Co-authored-by: Chin Rock <[email protected]> Co-authored-by: multica-agent <[email protected]>
This commit is contained in:
co-authored by
multica-agent
HaiYangBG1
Chin Rock
parent
bce94bd0d9
commit
947d8addea
@@ -0,0 +1,123 @@
|
||||
"""Bounded export diagnostics preserved by the SDK's text-only error relay."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
|
||||
from langbot_plugin.api.entities.builtin.runner.errors import AgentAPIError
|
||||
from langbot_plugin.box.errors import BoxValidationError
|
||||
|
||||
|
||||
EXPORT_ERROR_PREFIX = 'LANGBOT_EXPORT_V1:'
|
||||
SCRIPT_ERROR_PREFIX = 'LANGBOT_OUTBOX_LIMIT:'
|
||||
SCRIPT_ERROR_EXIT_CODE = 73
|
||||
LIMIT_LABELS = {
|
||||
'file_bytes': 'per-file byte',
|
||||
'collection_bytes': 'total-byte',
|
||||
'file_count': 'file-count',
|
||||
'depth': 'directory-depth',
|
||||
'entries': 'directory-entry',
|
||||
'directories': 'directory',
|
||||
'run_files': 'per-run file-count',
|
||||
'run_bytes': 'per-run byte',
|
||||
}
|
||||
|
||||
|
||||
class BoxExportError(BoxValidationError):
|
||||
def __init__(self, error: AgentAPIError):
|
||||
self.error = error
|
||||
super().__init__(EXPORT_ERROR_PREFIX + error.model_dump_json())
|
||||
|
||||
|
||||
def export_error(
|
||||
kind: str,
|
||||
*,
|
||||
limit: int | None = None,
|
||||
actual: int | None = None,
|
||||
path: str = '',
|
||||
lower_bound: bool = False,
|
||||
remaining: int | None = None,
|
||||
absolute_path: bool = False,
|
||||
) -> BoxExportError:
|
||||
if kind in LIMIT_LABELS:
|
||||
code = 'box.export.limit_exceeded'
|
||||
message = f'Attachment export failed: Sandbox outbox exceeds the {LIMIT_LABELS[kind]} limit'
|
||||
action = 'Reduce or split the requested output within the indicated limit, then retry export.'
|
||||
if kind in {'depth', 'entries', 'directories'}:
|
||||
action = 'Flatten the outbox or remove unnecessary entries, preserving the requested deliverables.'
|
||||
elif kind == 'transport_bytes':
|
||||
code = 'box.export.transport_truncated'
|
||||
message = 'Attachment export failed: the SDK explicitly reported truncated export output'
|
||||
action = 'Reduce or split attachments so their base64/JSON response fits the transport limit.'
|
||||
elif kind == 'invalid_response':
|
||||
code = 'box.export.invalid_response'
|
||||
message = 'Attachment export failed: Sandbox outbox returned invalid attachment data'
|
||||
action = 'Inspect the sandbox/runtime response; do not assume a size limit caused this failure.'
|
||||
else:
|
||||
raise ValueError(f'Unknown export failure kind: {kind}')
|
||||
return BoxExportError(
|
||||
AgentAPIError(
|
||||
code=code,
|
||||
message=message,
|
||||
retryable=kind != 'invalid_response',
|
||||
details={
|
||||
'version': 1,
|
||||
'phase': 'export',
|
||||
'limit_type': kind,
|
||||
'limit': limit,
|
||||
'actual': actual,
|
||||
'actual_is_lower_bound': lower_bound,
|
||||
'remaining': remaining,
|
||||
'relative_paths': [path] if path and not absolute_path else [],
|
||||
'sandbox_paths': [path] if path and absolute_path else [],
|
||||
'files_retained': True,
|
||||
'suggested_action': action,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def script_export_error(
|
||||
stderr: str, exit_code: int, limits: dict[str, int], *, selected_paths: list[str] | None = None
|
||||
) -> BoxExportError | None:
|
||||
if exit_code != SCRIPT_ERROR_EXIT_CODE or len(stderr) > 8192:
|
||||
return None
|
||||
line = stderr.strip().splitlines()[-1] if stderr.strip() else ''
|
||||
if not line.startswith(SCRIPT_ERROR_PREFIX):
|
||||
return None
|
||||
try:
|
||||
value = json.loads(line.removeprefix(SCRIPT_ERROR_PREFIX))
|
||||
kind, limit, actual, path = (value[k] for k in ('kind', 'limit', 'actual', 'path'))
|
||||
if (
|
||||
kind not in limits
|
||||
or type(limit) is not int
|
||||
or limit != limits[kind]
|
||||
or type(actual) is not int
|
||||
or actual <= limit
|
||||
or not isinstance(path, str)
|
||||
or len(path) > 2400
|
||||
or (selected_paths is None and path.startswith('/'))
|
||||
or (selected_paths is not None and path not in selected_paths)
|
||||
or '\\' in path
|
||||
or '\x00' in path
|
||||
or '..' in path.split('/')
|
||||
):
|
||||
return None
|
||||
return export_error(
|
||||
kind, limit=limit, actual=actual, path=path, lower_bound=True, absolute_path=selected_paths is not None
|
||||
)
|
||||
except (ValueError, TypeError, KeyError):
|
||||
return None
|
||||
|
||||
|
||||
def transport_export_error(stdout: str) -> BoxExportError | None:
|
||||
clipped = re.search(r'\n\.\.\. \[raw output clipped at (\d+) bytes, (\d+) bytes discarded\]$', stdout)
|
||||
if clipped:
|
||||
limit, discarded = map(int, clipped.groups())
|
||||
if limit > 0 and discarded > 0:
|
||||
return export_error('transport_bytes', limit=limit, actual=limit + discarded)
|
||||
clipped = re.search(r'\n\.\.\. \[output clipped at (\d+) bytes\]$', stdout)
|
||||
if clipped and int(clipped[1]) > 0:
|
||||
return export_error('transport_bytes', limit=int(clipped[1]))
|
||||
return None
|
||||
@@ -12,6 +12,7 @@ from dataclasses import dataclass, field, replace
|
||||
from langbot_plugin.api.entities.builtin.platform import message as pm
|
||||
from langbot_plugin.api.entities.builtin.runner.box import BoxAcquireRequest
|
||||
from langbot_plugin.box.errors import BoxValidationError
|
||||
from .export_errors import export_error
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -25,6 +26,7 @@ class RunBoxBinding:
|
||||
submitted: set = field(default_factory=set)
|
||||
lock: asyncio.Lock = field(default_factory=asyncio.Lock)
|
||||
read_only_mounts: list[dict] = field(default_factory=list)
|
||||
exported_fingerprints: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
def prepare_input_files(query, agent_input):
|
||||
@@ -213,26 +215,72 @@ class RunnerBoxService:
|
||||
binding.imported[key] = {'id': key, **items[0]}
|
||||
return {'items': [binding.imported[key] for key in ids]}
|
||||
|
||||
async def export_files(self, query):
|
||||
async def export_files(self, query, paths):
|
||||
binding = binding_for(query)
|
||||
if not isinstance(paths, list) or not paths or len(paths) > self.box._ATTACHMENT_MAX_FILES:
|
||||
raise BoxValidationError(f'paths must contain 1 to {self.box._ATTACHMENT_MAX_FILES} file paths')
|
||||
if any(
|
||||
not isinstance(p, str)
|
||||
or not p.startswith('/')
|
||||
or '\\' in p
|
||||
or '\x00' in p
|
||||
or any(c in {'', '.', '..'} for c in p.split('/')[1:])
|
||||
for p in paths
|
||||
):
|
||||
raise BoxValidationError('paths must be normalized absolute sandbox file paths')
|
||||
if len(set(paths)) != len(paths):
|
||||
raise BoxValidationError('Duplicate export file paths')
|
||||
async with binding.lock:
|
||||
items = await self.box.collect_outbound_attachments(query)
|
||||
items = await self.box.collect_selected_files(query, paths)
|
||||
unseen = {}
|
||||
for item in items:
|
||||
if len(binding.exported) >= 100:
|
||||
raise BoxValidationError('At most 100 output files may be exported per run')
|
||||
size = len(base64.b64decode(item['base64'].split(';base64,')[-1]))
|
||||
if sum(x['size'] for x in binding.exported.values()) + size > self.box._ATTACHMENT_MAX_TOTAL_BYTES:
|
||||
raise BoxValidationError('Output files exceed the per-run byte limit')
|
||||
data = base64.b64decode(item['base64'].split(';base64,')[-1])
|
||||
fingerprint = (item.get('_outbox_path', item['name']), item['type'], hashlib.sha256(data).hexdigest())
|
||||
if fingerprint not in binding.exported_fingerprints:
|
||||
unseen[fingerprint] = (item, len(data))
|
||||
if len(binding.exported) + len(unseen) > 100:
|
||||
first = next(iter(unseen.values()))[0]
|
||||
raise export_error(
|
||||
'run_files',
|
||||
limit=100,
|
||||
actual=len(binding.exported) + len(unseen),
|
||||
remaining=max(0, 100 - len(binding.exported)),
|
||||
path=first.get('_outbox_path', first['name']),
|
||||
absolute_path=True,
|
||||
)
|
||||
total_bytes = sum(x['size'] for x in binding.exported.values())
|
||||
batch_bytes = sum(size for _, size in unseen.values())
|
||||
if unseen and total_bytes + batch_bytes > self.box._ATTACHMENT_MAX_TOTAL_BYTES:
|
||||
first = next(iter(unseen.values()))[0]
|
||||
raise export_error(
|
||||
'run_bytes',
|
||||
limit=self.box._ATTACHMENT_MAX_TOTAL_BYTES,
|
||||
actual=total_bytes + batch_bytes,
|
||||
remaining=max(0, self.box._ATTACHMENT_MAX_TOTAL_BYTES - total_bytes),
|
||||
path=first.get('_outbox_path', first['name']),
|
||||
absolute_path=True,
|
||||
)
|
||||
pending = {}
|
||||
fingerprints = {}
|
||||
for fingerprint, (item, size) in unseen.items():
|
||||
key = str(uuid.uuid4())
|
||||
binding.exported[key] = {
|
||||
pending[key] = {
|
||||
**item,
|
||||
'id': key,
|
||||
'size': size,
|
||||
}
|
||||
fingerprints[fingerprint] = key
|
||||
binding.exported.update(pending)
|
||||
binding.exported_fingerprints.update(fingerprints)
|
||||
try:
|
||||
warning = await self.box.clear_selected_files(query, items)
|
||||
except Exception:
|
||||
warning = 'Files accepted, but cleanup failed; do not repeat delivery.'
|
||||
return {
|
||||
'warning': warning,
|
||||
'items': [
|
||||
{k: v for k, v in item.items() if k != 'base64'}
|
||||
{k: v for k, v in item.items() if k not in {'base64', '_outbox_path'}}
|
||||
for key, item in binding.exported.items()
|
||||
if key not in binding.submitted
|
||||
]
|
||||
],
|
||||
}
|
||||
|
||||
@@ -23,12 +23,18 @@ def outbox_script(
|
||||
'max_total_bytes': max_total_bytes,
|
||||
}
|
||||
return f'settings = {settings!r}\n' + textwrap.dedent("""\
|
||||
import base64, errno, json, os, stat
|
||||
import base64, errno, json, os, stat, sys
|
||||
directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
|
||||
file_flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
|
||||
out = []
|
||||
budget = {'entries': 512, 'directories': 64, 'bytes': 0}
|
||||
|
||||
def reject(kind, limit, actual, path):
|
||||
failure = {'kind': kind, 'limit': limit, 'actual': actual, 'path': path}
|
||||
print('Sandbox outbox exceeds the ' + kind.replace('_', '-') + ' limit', file=sys.stderr)
|
||||
print('LANGBOT_OUTBOX_LIMIT:' + json.dumps(failure), file=sys.stderr)
|
||||
raise SystemExit(73)
|
||||
|
||||
def component(value):
|
||||
if (not value or value in {'.', '..'} or len(os.fsencode(value)) > 240
|
||||
or any(c in value for c in ('/', '\\\\', '\\x00'))):
|
||||
@@ -37,13 +43,11 @@ def outbox_script(
|
||||
|
||||
def read_directory(fd, prefix='', depth=0):
|
||||
if depth > 8:
|
||||
return
|
||||
reject('depth', 8, depth, prefix)
|
||||
with os.scandir(fd) as entries:
|
||||
for entry in entries:
|
||||
if len(out) >= settings['max_files']:
|
||||
return
|
||||
if budget['entries'] <= 0:
|
||||
raise RuntimeError('Sandbox outbox exceeds the directory-entry limit')
|
||||
reject('entries', 512, 513, prefix)
|
||||
budget['entries'] -= 1
|
||||
name = component(entry.name)
|
||||
relative = prefix + '/' + name if prefix else name
|
||||
@@ -51,7 +55,7 @@ def outbox_script(
|
||||
continue
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
if budget['directories'] <= 0:
|
||||
raise RuntimeError('Sandbox outbox exceeds the directory limit')
|
||||
reject('directories', 64, 65, relative)
|
||||
budget['directories'] -= 1
|
||||
flags = directory_flags
|
||||
elif entry.is_file(follow_symlinks=False):
|
||||
@@ -71,13 +75,19 @@ def outbox_script(
|
||||
continue
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
continue
|
||||
if len(out) >= settings['max_files']:
|
||||
reject('file_count', settings['max_files'], len(out) + 1, relative)
|
||||
remaining = settings['max_total_bytes'] - budget['bytes']
|
||||
if metadata.st_size > settings['max_file_bytes'] or metadata.st_size > remaining:
|
||||
raise RuntimeError('Sandbox output attachment exceeds the export size limit')
|
||||
if metadata.st_size > settings['max_file_bytes']:
|
||||
reject('file_bytes', settings['max_file_bytes'], metadata.st_size, relative)
|
||||
if metadata.st_size > remaining:
|
||||
reject('collection_bytes', settings['max_total_bytes'], budget['bytes'] + metadata.st_size, relative)
|
||||
with os.fdopen(child_fd, 'rb', closefd=False) as source:
|
||||
data = source.read(settings['max_file_bytes'] + 1)
|
||||
if len(data) > settings['max_file_bytes'] or len(data) > remaining:
|
||||
raise RuntimeError('Sandbox output attachment exceeds the export size limit')
|
||||
if len(data) > settings['max_file_bytes']:
|
||||
reject('file_bytes', settings['max_file_bytes'], len(data), relative)
|
||||
if len(data) > remaining:
|
||||
reject('collection_bytes', settings['max_total_bytes'], budget['bytes'] + len(data), relative)
|
||||
out.append({'name': relative, 'b64': base64.b64encode(data).decode('ascii')})
|
||||
budget['bytes'] += len(data)
|
||||
finally:
|
||||
|
||||
@@ -11,6 +11,15 @@ class UnsafeWorkspacePathError(OSError):
|
||||
"""A tenant-controlled path could not be opened without following links."""
|
||||
|
||||
|
||||
class WorkspaceReadLimitError(ValueError):
|
||||
"""The outbox cannot be read completely within the export limits."""
|
||||
|
||||
def __init__(self, kind: str, limit: int, actual: int, path: str = '', *, lower_bound: bool = False):
|
||||
self.kind, self.limit, self.actual, self.path = kind, limit, actual, path
|
||||
self.lower_bound = lower_bound
|
||||
super().__init__(f'Sandbox outbox exceeds the {kind} limit')
|
||||
|
||||
|
||||
_DIRECTORY_FLAGS = (
|
||||
os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
|
||||
)
|
||||
@@ -185,16 +194,16 @@ def _read_directory(
|
||||
total: list[int],
|
||||
remaining_entries: list[int],
|
||||
remaining_directories: list[int],
|
||||
max_entries: int,
|
||||
max_directories: int,
|
||||
depth: int,
|
||||
) -> None:
|
||||
if depth > 8:
|
||||
return
|
||||
raise WorkspaceReadLimitError('depth', 8, depth, prefix, lower_bound=True)
|
||||
with os.scandir(directory_fd) as iterator:
|
||||
for entry in iterator:
|
||||
if len(output) >= max_files or total[0] >= max_total_bytes:
|
||||
return
|
||||
if remaining_entries[0] <= 0:
|
||||
raise UnsafeWorkspacePathError('Sandbox outbox exceeds the directory-entry limit')
|
||||
raise WorkspaceReadLimitError('entries', max_entries, max_entries + 1, prefix, lower_bound=True)
|
||||
remaining_entries[0] -= 1
|
||||
name = _component(entry.name)
|
||||
relative = f'{prefix}/{name}' if prefix else name
|
||||
@@ -202,14 +211,11 @@ def _read_directory(
|
||||
continue
|
||||
if entry.is_dir(follow_symlinks=False):
|
||||
if remaining_directories[0] <= 0:
|
||||
raise UnsafeWorkspacePathError('Sandbox outbox exceeds the directory limit')
|
||||
raise WorkspaceReadLimitError(
|
||||
'directories', max_directories, max_directories + 1, relative, lower_bound=True
|
||||
)
|
||||
remaining_directories[0] -= 1
|
||||
try:
|
||||
child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=directory_fd)
|
||||
except PermissionError:
|
||||
raise
|
||||
except OSError:
|
||||
continue
|
||||
child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=directory_fd)
|
||||
try:
|
||||
_read_directory(
|
||||
child_fd,
|
||||
@@ -221,28 +227,37 @@ def _read_directory(
|
||||
total=total,
|
||||
remaining_entries=remaining_entries,
|
||||
remaining_directories=remaining_directories,
|
||||
max_entries=max_entries,
|
||||
max_directories=max_directories,
|
||||
depth=depth + 1,
|
||||
)
|
||||
finally:
|
||||
os.close(child_fd)
|
||||
continue
|
||||
try:
|
||||
file_fd = os.open(name, _FILE_READ_FLAGS, dir_fd=directory_fd)
|
||||
except PermissionError:
|
||||
raise
|
||||
except OSError:
|
||||
if not entry.is_file(follow_symlinks=False):
|
||||
continue
|
||||
file_fd = os.open(name, _FILE_READ_FLAGS, dir_fd=directory_fd)
|
||||
try:
|
||||
metadata = os.fstat(file_fd)
|
||||
if not stat.S_ISREG(metadata.st_mode) or metadata.st_size > max_file_bytes:
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
continue
|
||||
if len(output) >= max_files:
|
||||
raise WorkspaceReadLimitError('file_count', max_files, len(output) + 1, relative, lower_bound=True)
|
||||
if metadata.st_size > max_file_bytes:
|
||||
raise WorkspaceReadLimitError('file_bytes', max_file_bytes, metadata.st_size, relative)
|
||||
remaining = max_total_bytes - total[0]
|
||||
if metadata.st_size > remaining:
|
||||
continue
|
||||
raise WorkspaceReadLimitError(
|
||||
'collection_bytes', max_total_bytes, total[0] + metadata.st_size, relative, lower_bound=True
|
||||
)
|
||||
with os.fdopen(file_fd, 'rb', closefd=False) as file_obj:
|
||||
data = file_obj.read(max_file_bytes + 1)
|
||||
if len(data) > max_file_bytes or len(data) > remaining:
|
||||
continue
|
||||
if len(data) > max_file_bytes:
|
||||
raise WorkspaceReadLimitError('file_bytes', max_file_bytes, len(data), relative, lower_bound=True)
|
||||
if len(data) > remaining:
|
||||
raise WorkspaceReadLimitError(
|
||||
'collection_bytes', max_total_bytes, total[0] + len(data), relative, lower_bound=True
|
||||
)
|
||||
output.append((relative, data))
|
||||
total[0] += len(data)
|
||||
finally:
|
||||
@@ -260,7 +275,11 @@ def read_regular_files(
|
||||
max_entries: int = 512,
|
||||
max_directories: int = 64,
|
||||
) -> list[tuple[str, bytes]]:
|
||||
"""Read bounded regular files without following tenant-created links."""
|
||||
"""Read regular files completely within bounds, without following links.
|
||||
|
||||
Missing outboxes are empty. Unreadable files/directories and exhausted
|
||||
limits raise, so a caller cannot mistake a partial read for a complete export.
|
||||
"""
|
||||
|
||||
output: list[tuple[str, bytes]] = []
|
||||
try:
|
||||
@@ -275,13 +294,18 @@ def read_regular_files(
|
||||
total=[0],
|
||||
remaining_entries=[max_entries],
|
||||
remaining_directories=[max_directories],
|
||||
max_entries=max_entries,
|
||||
max_directories=max_directories,
|
||||
depth=0,
|
||||
)
|
||||
except (FileNotFoundError, UnsafeWorkspacePathError):
|
||||
# A missing directory is an empty outbox. An unsafe existing path is
|
||||
# deliberately surfaced to the caller rather than followed.
|
||||
if os.path.lexists(os.path.join(root, subdir, query_key)):
|
||||
raise
|
||||
except UnsafeWorkspacePathError as exc:
|
||||
# Only failure to open a missing outbox is empty. A file disappearing
|
||||
# during traversal, or a permission failure, is an incomplete read.
|
||||
if isinstance(exc.__cause__, FileNotFoundError):
|
||||
return []
|
||||
if isinstance(exc.__cause__, PermissionError):
|
||||
raise exc.__cause__ from None
|
||||
raise
|
||||
return output
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Explicit file selection inside a sandbox, with no symlink traversal."""
|
||||
|
||||
import textwrap
|
||||
|
||||
|
||||
def selected_files_script(paths, *, max_file_bytes, max_total_bytes, cleanup=None):
|
||||
settings = dict(paths=paths, max_file_bytes=max_file_bytes, max_total_bytes=max_total_bytes, cleanup=cleanup)
|
||||
return f'settings = {settings!r}\n' + textwrap.dedent("""\
|
||||
import os, stat, json, base64, hashlib, sys
|
||||
flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
|
||||
def reject(kind, limit, actual, path):
|
||||
print('LANGBOT_OUTBOX_LIMIT:' + json.dumps(dict(kind=kind, limit=limit, actual=actual, path=path)), file=sys.stderr)
|
||||
raise SystemExit(73)
|
||||
|
||||
result = []
|
||||
total = 0
|
||||
for path in settings['paths']:
|
||||
parts = path.split('/')
|
||||
if not path.startswith('/') or any(p in ('', '.', '..') for p in parts[1:]):
|
||||
raise ValueError('Expected a normalized absolute sandbox file path: ' + path)
|
||||
fd = os.open('/', flags)
|
||||
try:
|
||||
for part in parts[1:-1]:
|
||||
child = os.open(part, flags, dir_fd=fd)
|
||||
os.close(fd)
|
||||
fd = child
|
||||
file_fd = os.open(parts[-1], os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=fd)
|
||||
with os.fdopen(file_fd, 'rb') as source:
|
||||
metadata = os.fstat(source.fileno())
|
||||
if not stat.S_ISREG(metadata.st_mode):
|
||||
raise ValueError('Not a regular file: ' + path)
|
||||
if metadata.st_size > settings['max_file_bytes']:
|
||||
reject('file_bytes', settings['max_file_bytes'], metadata.st_size, path)
|
||||
data = source.read(settings['max_file_bytes'] + 1)
|
||||
total += len(data)
|
||||
if len(data) > settings['max_file_bytes']:
|
||||
reject('file_bytes', settings['max_file_bytes'], len(data), path)
|
||||
if total > settings['max_total_bytes']:
|
||||
reject('collection_bytes', settings['max_total_bytes'], total, path)
|
||||
digest = hashlib.sha256(data).hexdigest()
|
||||
if settings['cleanup'] is not None:
|
||||
current = os.stat(parts[-1], dir_fd=fd, follow_symlinks=False)
|
||||
if (current.st_dev, current.st_ino) == (metadata.st_dev, metadata.st_ino) and digest == settings['cleanup'].get(path):
|
||||
os.unlink(parts[-1], dir_fd=fd)
|
||||
else:
|
||||
raise ValueError('File changed after acceptance; retained: ' + path)
|
||||
else:
|
||||
result.append({'name': path, 'b64': base64.b64encode(data).decode('ascii')})
|
||||
finally:
|
||||
os.close(fd)
|
||||
print(json.dumps(result))
|
||||
""")
|
||||
+132
-41
@@ -20,6 +20,7 @@ from langbot_plugin.box.security import BOX_SHARED_WORKSPACE_PROBE_PREFIX
|
||||
from .admission import SandboxAdmissionController, require_cloud_admission_policy
|
||||
from .connector import BoxRuntimeConnector, _get_box_config
|
||||
from . import secure_fs
|
||||
from .export_errors import export_error, script_export_error, transport_export_error
|
||||
from ..utils import httpclient
|
||||
from ..api.http.context import ExecutionContext
|
||||
from ..api.http.service.tenant import TenantContext, require_workspace_uuid
|
||||
@@ -1126,41 +1127,53 @@ class BoxService:
|
||||
)
|
||||
return result
|
||||
|
||||
async def collect_outbound_attachments(self, query: pipeline_query.Query) -> list[dict]:
|
||||
async def collect_outbound_attachments(self, query: pipeline_query.Query, *, clear: bool = True) -> list[dict]:
|
||||
"""Collect files the agent produced in the sandbox outbox.
|
||||
|
||||
Read regular files within the configured attachment limits from the
|
||||
current run's outbox. Private container output uses the bound sandbox
|
||||
channel; invalid or truncated responses retain the files. Clear a
|
||||
successful collection and return ``{type, name, base64}`` descriptors.
|
||||
Read and traversal limits fail without clearing output. Runner export
|
||||
defers cleanup until its cumulative run limits have also been checked.
|
||||
"""
|
||||
if not self._available:
|
||||
if not clear:
|
||||
raise BoxError('Box runtime is not available; output files were retained')
|
||||
return []
|
||||
if self._cloud_managed:
|
||||
await self.require_workspace_sandbox(self._query_execution_context(query))
|
||||
|
||||
host_dir = self._host_query_dir(self.OUTBOX_SUBDIR, query)
|
||||
if host_dir is not None:
|
||||
try:
|
||||
entries = await asyncio.to_thread(self._read_outbox_host, host_dir)
|
||||
except PermissionError:
|
||||
# Root-owned container output can be private to its sandbox UID.
|
||||
try:
|
||||
if host_dir is not None:
|
||||
try:
|
||||
entries = await asyncio.to_thread(self._read_outbox_host, host_dir)
|
||||
except PermissionError:
|
||||
# Root-owned container output can be unreadable to Core.
|
||||
# Retry the complete read inside the already-bound Box.
|
||||
entries = await self._read_outbox_via_exec(query)
|
||||
else:
|
||||
entries = await self._read_outbox_via_exec(query)
|
||||
else:
|
||||
entries = await self._read_outbox_via_exec(query)
|
||||
except OSError as exc:
|
||||
raise BoxError('Failed to read sandbox outbox; output files were retained') from exc
|
||||
|
||||
attachments = self._classify_outbound_entries(entries)
|
||||
attachments = self._classify_outbound_entries(entries, include_source_path=not clear)
|
||||
|
||||
# Always clear the per-query outbox after reading — even when nothing
|
||||
# was collected — so a later turn that reuses the same query_id (the
|
||||
# counter resets across restarts) never inherits stale files.
|
||||
await self._clear_outbox(query, host_dir)
|
||||
# A successful empty read still clears stale state, but an incomplete
|
||||
# read must never reach cleanup. Export can be retried after the failure.
|
||||
if clear:
|
||||
await self._clear_outbox(query, host_dir)
|
||||
if attachments:
|
||||
self.ap.logger.info(
|
||||
f'Collected {len(attachments)} outbound attachment(s) from sandbox: query_id={query.query_id}'
|
||||
)
|
||||
return attachments
|
||||
|
||||
async def clear_outbound_attachments(self, query: pipeline_query.Query) -> None:
|
||||
"""Clear output after a complete read has been accepted by the Runner."""
|
||||
await self._clear_outbox(query, self._host_query_dir(self.OUTBOX_SUBDIR, query))
|
||||
|
||||
def _read_outbox_host(self, host_dir: str) -> list[dict]:
|
||||
"""Read outbox files straight off the bind-mounted host directory."""
|
||||
import base64 as _b64
|
||||
@@ -1179,53 +1192,128 @@ class BoxService:
|
||||
)
|
||||
except secure_fs.UnsafeWorkspacePathError as exc:
|
||||
raise BoxValidationError('Sandbox outbox contains an unsafe symbolic link') from exc
|
||||
except secure_fs.WorkspaceReadLimitError as exc:
|
||||
raise export_error(
|
||||
exc.kind, limit=exc.limit, actual=exc.actual, path=exc.path, lower_bound=exc.lower_bound
|
||||
) from exc
|
||||
return [{'name': name, 'b64': _b64.b64encode(data).decode('ascii')} for name, data in files]
|
||||
|
||||
async def _read_outbox_via_exec(self, query: pipeline_query.Query) -> list[dict]:
|
||||
async def _read_outbox_via_exec(self, query: pipeline_query.Query, paths=None) -> list[dict]:
|
||||
"""Fallback: read the outbox over the exec channel (E2B / remote).
|
||||
|
||||
Use the same Workspace-scoped execution path as tools, retaining raw
|
||||
stdout because base64 payloads can exceed the display truncation limit.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
import base64
|
||||
import binascii
|
||||
from .sandbox_files import outbox_script
|
||||
|
||||
max_file_bytes = self._ATTACHMENT_MAX_BYTES
|
||||
max_files = self._ATTACHMENT_MAX_FILES
|
||||
max_total_bytes = self._ATTACHMENT_MAX_TOTAL_BYTES
|
||||
script = outbox_script(
|
||||
self.OUTBOX_MOUNT_DIR,
|
||||
self._attachment_query_key(query),
|
||||
operation='read',
|
||||
max_file_bytes=max_file_bytes,
|
||||
max_files=self._ATTACHMENT_MAX_FILES,
|
||||
max_files=max_files,
|
||||
max_total_bytes=max_total_bytes,
|
||||
)
|
||||
if paths is not None:
|
||||
from .selected_files import selected_files_script
|
||||
|
||||
script = selected_files_script(paths, max_file_bytes=max_file_bytes, max_total_bytes=max_total_bytes)
|
||||
spec_payload: dict = {
|
||||
'cmd': f"python3 - <<'LBPY'\n{script}\nLBPY",
|
||||
'timeout_sec': 120,
|
||||
'session_id': self.resolve_box_session_id(query),
|
||||
}
|
||||
result = await self._execute_spec_payload_result(spec_payload, query)
|
||||
if not result.ok:
|
||||
if 'Sandbox output attachment exceeds the export size limit' in (result.stderr or ''):
|
||||
raise BoxValidationError(
|
||||
f'Sandbox output attachments exceed the export limit '
|
||||
f'({max_file_bytes} bytes per file, {max_total_bytes} bytes total); files were retained'
|
||||
)
|
||||
raise BoxValidationError(f'Failed to read sandbox outbox (exit code {result.exit_code})')
|
||||
try:
|
||||
entries = _json.loads(str(result.stdout or '').strip().splitlines()[-1])
|
||||
if not isinstance(entries, list) or any(
|
||||
not isinstance(entry, dict)
|
||||
or not isinstance(entry.get('name'), str)
|
||||
or not isinstance(entry.get('b64'), str)
|
||||
for entry in entries
|
||||
):
|
||||
raise ValueError('Invalid attachment list')
|
||||
return entries
|
||||
except (ValueError, IndexError) as exc:
|
||||
raise BoxValidationError('Sandbox outbox returned invalid attachment data') from exc
|
||||
result = await self._execute_spec_payload_result(spec_payload, query)
|
||||
except BoxError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
raise BoxError('Failed to read sandbox outbox; output files were retained') from exc
|
||||
if not result.ok:
|
||||
failure = script_export_error(
|
||||
result.stderr or '',
|
||||
result.exit_code,
|
||||
{
|
||||
'file_bytes': max_file_bytes,
|
||||
'collection_bytes': max_total_bytes,
|
||||
'file_count': max_files,
|
||||
'depth': 8,
|
||||
'entries': 512,
|
||||
'directories': 64,
|
||||
},
|
||||
selected_paths=paths,
|
||||
)
|
||||
if failure is not None:
|
||||
raise failure
|
||||
raise BoxValidationError(
|
||||
f'Failed to read sandbox outbox or selected files (exit code {result.exit_code}); '
|
||||
f'output files were retained: {(result.stderr or "")[-4000:]}'
|
||||
)
|
||||
truncated = transport_export_error(result.stdout)
|
||||
if truncated is not None:
|
||||
raise truncated
|
||||
try:
|
||||
entries = json.loads(result.stdout)
|
||||
if not isinstance(entries, list) or len(entries) > max_files:
|
||||
raise ValueError('Invalid file list')
|
||||
total_bytes = 0
|
||||
for entry in entries:
|
||||
if not isinstance(entry, dict) or not isinstance(entry.get('name'), str):
|
||||
raise ValueError('Invalid file entry')
|
||||
if paths is None and entry['name'].startswith('/'):
|
||||
raise ValueError('Expected a relative outbox path')
|
||||
if paths is not None and entry['name'] not in paths:
|
||||
raise ValueError('Unexpected exported path')
|
||||
for component in entry['name'].lstrip('/').split('/'):
|
||||
secure_fs._component(component)
|
||||
encoded = entry.get('b64')
|
||||
if not isinstance(encoded, str) or len(encoded) > 4 * ((max_file_bytes + 2) // 3):
|
||||
raise ValueError('Invalid file payload')
|
||||
size = len(base64.b64decode(encoded, validate=True))
|
||||
total_bytes += size
|
||||
if size > max_file_bytes or total_bytes > max_total_bytes:
|
||||
raise ValueError('File payload exceeds export limits')
|
||||
except (TypeError, ValueError, OSError, binascii.Error) as exc:
|
||||
raise export_error('invalid_response') from exc
|
||||
return entries
|
||||
|
||||
async def collect_selected_files(self, query, paths):
|
||||
if not self._available:
|
||||
raise BoxError('Box runtime is not available')
|
||||
entries = await self._read_outbox_via_exec(query, paths)
|
||||
if len(entries) != len(paths) or {e['name'] for e in entries} != set(paths):
|
||||
raise BoxValidationError('Incomplete selected file export; files were retained')
|
||||
return self._classify_outbound_entries(entries, include_source_path=True)
|
||||
|
||||
async def clear_selected_files(self, query, items):
|
||||
import base64
|
||||
import hashlib
|
||||
from .selected_files import selected_files_script
|
||||
|
||||
digests = {
|
||||
item['_outbox_path']: hashlib.sha256(base64.b64decode(item['base64'].split(';base64,')[-1])).hexdigest()
|
||||
for item in items
|
||||
}
|
||||
script = selected_files_script(
|
||||
list(digests),
|
||||
max_file_bytes=self._ATTACHMENT_MAX_BYTES,
|
||||
max_total_bytes=self._ATTACHMENT_MAX_TOTAL_BYTES,
|
||||
cleanup=digests,
|
||||
)
|
||||
result = await self._execute_spec_payload_result(
|
||||
{
|
||||
'cmd': f"python3 - <<'LBPY'\n{script}\nLBPY",
|
||||
'timeout_sec': 120,
|
||||
'session_id': self.resolve_box_session_id(query),
|
||||
},
|
||||
query,
|
||||
)
|
||||
return None if result.ok else 'Files accepted, but cleanup failed; do not repeat delivery.'
|
||||
|
||||
async def _clear_outbox(self, query: pipeline_query.Query, host_dir: str | None) -> None:
|
||||
"""Empty the per-query outbox after collection.
|
||||
@@ -1285,7 +1373,7 @@ class BoxService:
|
||||
self.ap.logger.warning(f'Failed to clear sandbox outbox {target_dir}: {exc}')
|
||||
|
||||
@staticmethod
|
||||
def _classify_outbound_entries(entries: list[dict]) -> list[dict]:
|
||||
def _classify_outbound_entries(entries: list[dict], *, include_source_path: bool = False) -> list[dict]:
|
||||
"""Classify outbox files into Image/Voice/File component descriptors."""
|
||||
image_exts = {'png', 'jpg', 'jpeg', 'gif', 'webp', 'bmp'}
|
||||
voice_exts = {'wav', 'mp3', 'silk', 'amr', 'ogg', 'm4a', 'aac'}
|
||||
@@ -1301,17 +1389,20 @@ class BoxService:
|
||||
for entry in entries or []:
|
||||
name = str(entry.get('name', '') or '')
|
||||
b64 = entry.get('b64')
|
||||
if not name or not b64:
|
||||
if not name or not isinstance(b64, str):
|
||||
continue
|
||||
ext = name.rsplit('.', 1)[-1].lower() if '.' in name else ''
|
||||
base_name = os.path.basename(name)
|
||||
if ext in image_exts:
|
||||
mime = mime_by_ext.get(ext, 'image/png')
|
||||
attachments.append({'type': 'Image', 'name': base_name, 'base64': f'data:{mime};base64,{b64}'})
|
||||
item = {'type': 'Image', 'name': base_name, 'base64': f'data:{mime};base64,{b64}'}
|
||||
elif ext in voice_exts:
|
||||
attachments.append({'type': 'Voice', 'name': base_name, 'base64': f'data:audio/{ext};base64,{b64}'})
|
||||
item = {'type': 'Voice', 'name': base_name, 'base64': f'data:audio/{ext};base64,{b64}'}
|
||||
else:
|
||||
attachments.append({'type': 'File', 'name': base_name, 'base64': b64})
|
||||
item = {'type': 'File', 'name': base_name, 'base64': b64}
|
||||
if include_source_path:
|
||||
item['_outbox_path'] = name
|
||||
attachments.append(item)
|
||||
return attachments
|
||||
|
||||
async def shutdown(self):
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
from langbot_plugin.entities.io.actions.enums import PluginToRuntimeAction as Action
|
||||
from langbot_plugin.runtime.io.handler import ActionResponse
|
||||
from ..box.runner import RunnerBoxService
|
||||
from ..box.export_errors import BoxExportError
|
||||
from .agent_run_support import _validate_agent_run_session
|
||||
|
||||
|
||||
@@ -45,7 +46,7 @@ def register(h):
|
||||
if action == Action.IMPORT_BOX_ATTACHMENTS:
|
||||
return ActionResponse.success(await service.import_attachments(query, data.get('attachment_ids')))
|
||||
if action == Action.EXPORT_BOX_FILES:
|
||||
return ActionResponse.success(await service.export_files(query))
|
||||
return ActionResponse.success(await service.export_files(query, data.get('paths')))
|
||||
raise ValueError('Unsupported Box action')
|
||||
|
||||
def register_action(action):
|
||||
@@ -53,6 +54,8 @@ def register(h):
|
||||
async def invoke(data):
|
||||
try:
|
||||
return await dispatch(action, data)
|
||||
except BoxExportError as exc:
|
||||
return ActionResponse.error(str(exc), data={'error': exc.error.model_dump(mode='json')})
|
||||
except Exception as exc:
|
||||
return ActionResponse.error(f'{type(exc).__name__}: {exc}')
|
||||
|
||||
|
||||
Reference in New Issue
Block a user