fix(box): recover private outbox files without following links

src/langbot/pkg/box/sandbox_files.py

@@ -0,0 +1,139 @@
+"""Bounded, link-free outbox operations executed inside the bound sandbox."""
+
+import os
+import textwrap
+
+
+def outbox_script(
+    outbox_root: str,
+    query_key: str,
+    *,
+    operation: str,
+    max_file_bytes: int,
+    max_files: int,
+    max_total_bytes: int,
+) -> str:
+    settings = {
+        'workspace': os.path.dirname(outbox_root),
+        'outbox': os.path.basename(outbox_root),
+        'query': query_key,
+        'operation': operation,
+        'max_file_bytes': max_file_bytes,
+        'max_files': max_files,
+        'max_total_bytes': max_total_bytes,
+    }
+    return f'settings = {settings!r}\n' + textwrap.dedent("""\
+        import base64, errno, json, os, stat
+        directory_flags = os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
+        file_flags = os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK
+        out = []
+        budget = {'entries': 512, 'directories': 64, 'bytes': 0}
+
+        def component(value):
+            if (not value or value in {'.', '..'} or len(os.fsencode(value)) > 240
+                    or any(c in value for c in ('/', '\\\\', '\\x00'))):
+                raise RuntimeError('Unsafe sandbox outbox path component')
+            return value
+
+        def read_directory(fd, prefix='', depth=0):
+            if depth > 8:
+                return
+            with os.scandir(fd) as entries:
+                for entry in entries:
+                    if len(out) >= settings['max_files']:
+                        return
+                    if budget['entries'] <= 0:
+                        raise RuntimeError('Sandbox outbox exceeds the directory-entry limit')
+                    budget['entries'] -= 1
+                    name = component(entry.name)
+                    relative = prefix + '/' + name if prefix else name
+                    if entry.is_symlink():
+                        continue
+                    if entry.is_dir(follow_symlinks=False):
+                        if budget['directories'] <= 0:
+                            raise RuntimeError('Sandbox outbox exceeds the directory limit')
+                        budget['directories'] -= 1
+                        flags = directory_flags
+                    elif entry.is_file(follow_symlinks=False):
+                        flags = file_flags
+                    else:
+                        continue
+                    try:
+                        child_fd = os.open(name, flags, dir_fd=fd)
+                    except OSError as exc:
+                        if exc.errno in {errno.ENOENT, errno.ELOOP, errno.ENOTDIR}:
+                            continue
+                        raise
+                    try:
+                        metadata = os.fstat(child_fd)
+                        if flags == directory_flags and stat.S_ISDIR(metadata.st_mode):
+                            read_directory(child_fd, relative, depth + 1)
+                            continue
+                        if not stat.S_ISREG(metadata.st_mode):
+                            continue
+                        remaining = settings['max_total_bytes'] - budget['bytes']
+                        if metadata.st_size > settings['max_file_bytes'] or metadata.st_size > remaining:
+                            raise RuntimeError('Sandbox output attachment exceeds the export size limit')
+                        with os.fdopen(child_fd, 'rb', closefd=False) as source:
+                            data = source.read(settings['max_file_bytes'] + 1)
+                        if len(data) > settings['max_file_bytes'] or len(data) > remaining:
+                            raise RuntimeError('Sandbox output attachment exceeds the export size limit')
+                        out.append({'name': relative, 'b64': base64.b64encode(data).decode('ascii')})
+                        budget['bytes'] += len(data)
+                    finally:
+                        os.close(child_fd)
+
+        def clear_directory(fd, depth=0):
+            if depth > 16:
+                raise RuntimeError('Sandbox cleanup exceeded its depth limit')
+            with os.scandir(fd) as entries:
+                for entry in entries:
+                    if budget['entries'] <= 0:
+                        raise RuntimeError('Sandbox cleanup exceeded its inode limit')
+                    budget['entries'] -= 1
+                    name = component(entry.name)
+                    try:
+                        child_fd = os.open(name, directory_flags, dir_fd=fd)
+                    except FileNotFoundError:
+                        continue
+                    except OSError as exc:
+                        if exc.errno not in {errno.ELOOP, errno.ENOTDIR}:
+                            raise
+                        try:
+                            os.unlink(name, dir_fd=fd)
+                        except FileNotFoundError:
+                            pass
+                    else:
+                        try:
+                            clear_directory(child_fd, depth + 1)
+                        finally:
+                            os.close(child_fd)
+                        try:
+                            os.rmdir(name, dir_fd=fd)
+                        except FileNotFoundError:
+                            pass
+
+        try:
+            workspace_fd = os.open(settings['workspace'], directory_flags)
+            try:
+                outbox_fd = os.open(component(settings['outbox']), directory_flags, dir_fd=workspace_fd)
+                try:
+                    query_fd = os.open(component(settings['query']), directory_flags, dir_fd=outbox_fd)
+                    try:
+                        if settings['operation'] == 'read':
+                            read_directory(query_fd)
+                        elif settings['operation'] == 'clear':
+                            budget['entries'] = 4096
+                            clear_directory(query_fd)
+                        else:
+                            raise RuntimeError('Unknown sandbox outbox operation')
+                    finally:
+                        os.close(query_fd)
+                finally:
+                    os.close(outbox_fd)
+            finally:
+                os.close(workspace_fd)
+        except FileNotFoundError:
+            pass
+        print(json.dumps(out))
+        """)

src/langbot/pkg/box/secure_fs.py

@@ -14,7 +14,9 @@ class UnsafeWorkspacePathError(OSError):
 _DIRECTORY_FLAGS = (
     os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
 )
-_FILE_READ_FLAGS = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
+_FILE_READ_FLAGS = (
+    os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0) | getattr(os, 'O_NONBLOCK', 0)
+)
 _FILE_WRITE_FLAGS = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_NOFOLLOW', 0) | getattr(os, 'O_CLOEXEC', 0)
 _MAX_REMOVAL_ENTRIES = 4096
 _MAX_REMOVAL_DEPTH = 16
@@ -45,6 +47,8 @@ def _unsafe(path: str, exc: BaseException | None = None) -> UnsafeWorkspacePathE
 def _root_fd(root: str):
     try:
         fd = os.open(root, _DIRECTORY_FLAGS)
+    except PermissionError:
+        raise
     except OSError as exc:
         raise _unsafe(root, exc)
     try:
@@ -64,6 +68,8 @@ def _open_dir_at(parent_fd: int, name: str, *, create: bool) -> int:
             pass
     try:
         fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
+    except PermissionError:
+        raise
     except OSError as exc:
         raise _unsafe(name, exc)
     if not stat.S_ISDIR(os.fstat(fd).st_mode):
@@ -91,6 +97,8 @@ def _remove_entry(
             child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=parent_fd)
         except FileNotFoundError:
             return
+        except PermissionError:
+            raise
         except OSError as exc:
             if exc.errno not in {errno.ELOOP, errno.ENOTDIR, errno.EACCES}:
                 raise
@@ -198,6 +206,8 @@ def _read_directory(
                 remaining_directories[0] -= 1
                 try:
                     child_fd = os.open(name, _DIRECTORY_FLAGS, dir_fd=directory_fd)
+                except PermissionError:
+                    raise
                 except OSError:
                     continue
                 try:
@@ -218,6 +228,8 @@ def _read_directory(
                 continue
             try:
                 file_fd = os.open(name, _FILE_READ_FLAGS, dir_fd=directory_fd)
+            except PermissionError:
+                raise
             except OSError:
                 continue
             try:

src/langbot/pkg/box/service.py

@@ -1129,12 +1129,10 @@ class BoxService:
     async def collect_outbound_attachments(self, query: pipeline_query.Query) -> list[dict]:
         """Collect files the agent produced in the sandbox outbox.
 
-        Reads ``/workspace/outbox/<query_id>/`` (recursively) — directly from
-        the bind-mounted host directory when available (no size limit), else
-        via the exec channel — returns a list of ``{type, name, base64}``
-        ready to become platform message components, then clears the outbox so
-        a later turn in the same session does not re-send stale files. Returns
-        ``[]`` when nothing was produced.
+        Read regular files within the configured attachment limits from the
+        current run's outbox. Private container output uses the bound sandbox
+        channel; invalid or truncated responses retain the files. Clear a
+        successful collection and return ``{type, name, base64}`` descriptors.
         """
         if not self._available:
             return []
@@ -1143,7 +1141,11 @@ class BoxService:
 
         host_dir = self._host_query_dir(self.OUTBOX_SUBDIR, query)
         if host_dir is not None:
-            entries = await asyncio.to_thread(self._read_outbox_host, host_dir)
+            try:
+                entries = await asyncio.to_thread(self._read_outbox_host, host_dir)
+            except PermissionError:
+                # Root-owned container output can be private to its sandbox UID.
+                entries = await self._read_outbox_via_exec(query)
         else:
             entries = await self._read_outbox_via_exec(query)
 
@@ -1187,57 +1189,17 @@ class BoxService:
         """
         import json as _json
 
-        target_dir = f'{self.OUTBOX_MOUNT_DIR}/{self._attachment_query_key(query)}'
-        # Outbound bytes travel in raw stdout, not argv. Use the regular
-        # attachment limits rather than the smaller inbound command limit.
+        from .sandbox_files import outbox_script
+
         max_file_bytes = self._ATTACHMENT_MAX_BYTES
-        max_files = self._ATTACHMENT_MAX_FILES
         max_total_bytes = self._ATTACHMENT_MAX_TOTAL_BYTES
-        max_scan_entries = 1000
-        script = (
-            'import base64, json, os\n'
-            f'target = {target_dir!r}\n'
-            f'max_file_bytes = {max_file_bytes}\n'
-            f'max_files = {max_files}\n'
-            f'max_total_bytes = {max_total_bytes}\n'
-            f'max_scan_entries = {max_scan_entries}\n'
-            'out = []\n'
-            'total_bytes = 0\n'
-            'scanned_entries = 0\n'
-            'stack = [target]\n'
-            'if os.path.isdir(target):\n'
-            '    while stack and len(out) < max_files and scanned_entries < max_scan_entries:\n'
-            '        current = stack.pop()\n'
-            '        try:\n'
-            '            with os.scandir(current) as iterator:\n'
-            '                entries = sorted(iterator, key=lambda item: item.name, reverse=True)\n'
-            '        except OSError:\n'
-            '            continue\n'
-            '        for entry in entries:\n'
-            '            scanned_entries += 1\n'
-            '            if scanned_entries > max_scan_entries:\n'
-            '                break\n'
-            '            try:\n'
-            '                if entry.is_dir(follow_symlinks=False):\n'
-            '                    stack.append(entry.path)\n'
-            '                    continue\n'
-            '                if not entry.is_file(follow_symlinks=False):\n'
-            '                    continue\n'
-            '                size = entry.stat(follow_symlinks=False).st_size\n'
-            '                if size > max_file_bytes or total_bytes + size > max_total_bytes:\n'
-            "                    raise RuntimeError('Sandbox output attachment exceeds the export size limit')\n"
-            "                with open(entry.path, 'rb') as f:\n"
-            '                    data = f.read(max_file_bytes + 1)\n'
-            '                if len(data) > max_file_bytes or total_bytes + len(data) > max_total_bytes:\n'
-            "                    raise RuntimeError('Sandbox output attachment exceeds the export size limit')\n"
-            '            except OSError:\n'
-            '                continue\n'
-            '            rel = os.path.relpath(entry.path, target)\n'
-            "            out.append({'name': rel, 'b64': base64.b64encode(data).decode('ascii')})\n"
-            '            total_bytes += len(data)\n'
-            '            if len(out) >= max_files:\n'
-            '                break\n'
-            'print(json.dumps(out))\n'
+        script = outbox_script(
+            self.OUTBOX_MOUNT_DIR,
+            self._attachment_query_key(query),
+            operation='read',
+            max_file_bytes=max_file_bytes,
+            max_files=self._ATTACHMENT_MAX_FILES,
+            max_total_bytes=max_total_bytes,
         )
         spec_payload: dict = {
             'cmd': f"python3 - <<'LBPY'\n{script}\nLBPY",
@@ -1288,19 +1250,37 @@ class BoxService:
                 try:
                     secure_fs.reset_directory(tenant_root, subdir, query_key)
                     return False
+                except secure_fs.UnsafeWorkspacePathError:
+                    raise
                 except OSError:
                     return True
 
-            survived = await asyncio.to_thread(_clear)
+            try:
+                survived = await asyncio.to_thread(_clear)
+            except secure_fs.UnsafeWorkspacePathError as exc:
+                self.ap.logger.warning(f'Failed to clear unsafe sandbox outbox {target_dir}: {exc}')
+                return
             if not survived:
                 return
             # Root-owned container files survived the host delete — fall through.
 
         try:
-            await self.execute_tool(
-                {'command': f'rm -rf {target_dir} && mkdir -p {target_dir}', 'timeout_sec': 30},
+            from .sandbox_files import outbox_script
+
+            script = outbox_script(
+                self.OUTBOX_MOUNT_DIR,
+                self._attachment_query_key(query),
+                operation='clear',
+                max_file_bytes=self._ATTACHMENT_MAX_BYTES,
+                max_files=self._ATTACHMENT_MAX_FILES,
+                max_total_bytes=self._ATTACHMENT_MAX_TOTAL_BYTES,
+            )
+            result = await self.execute_tool(
+                {'command': f"python3 - <<'LBPY'\n{script}\nLBPY", 'timeout_sec': 30},
                 query,
             )
+            if not result.get('ok'):
+                raise BoxValidationError('Sandbox outbox cleanup failed')
         except Exception as exc:
             self.ap.logger.warning(f'Failed to clear sandbox outbox {target_dir}: {exc}')