From 9b130680ca6d60b43ce592c6ee90a6bff568b965 Mon Sep 17 00:00:00 2001 From: Hyu Date: Sun, 13 Sep 2026 00:39:21 +0800 Subject: [PATCH 1/9] ci(discord): announce published stable releases via webhook (#2539) Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com> --- .github/discord-release/README.md | 111 ++++++ .github/discord-release/announce.py | 162 +++++++++ .github/discord-release/test_announce.py | 427 +++++++++++++++++++++++ .github/workflows/discord-release.yml | 64 ++++ 4 files changed, 764 insertions(+) create mode 100644 .github/discord-release/README.md create mode 100644 .github/discord-release/announce.py create mode 100644 .github/discord-release/test_announce.py create mode 100644 .github/workflows/discord-release.yml diff --git a/.github/discord-release/README.md b/.github/discord-release/README.md new file mode 100644 index 000000000..21deee164 --- /dev/null +++ b/.github/discord-release/README.md @@ -0,0 +1,111 @@ +# Discord release announcements + +This independent workflow announces new stable LangBot releases in the channel +selected by a dedicated Discord incoming webhook. It does not change the existing +release/build workflows, edit releases, run a persistent service, poll, or backfill. +Announcements run on publication, independently of artifact builds finishing. + +## Setup and read-only validation + +1. In the intended community **announcement channel**, create a dedicated incoming + webhook (Channel Settings → Integrations → Webhooks). Copy its URL; do not reuse + a webhook belonging to another automation. +2. In `langbot-app/LangBot` → Settings → Secrets and variables → Actions, create the + **repository secret** `DISCORD_RELEASE_WEBHOOK_URL`. Its value must be exactly + `https://discord.com/api/webhooks//` — no query, trailing slash, + API-version segment, or alternate domain. Treat the entire URL as a password. +3. Once this workflow is on `master`, open Actions → **Discord Release Announcement** + → Run workflow, choosing `master`. Alternatively: + + ```sh + gh workflow run discord-release.yml --repo langbot-app/LangBot --ref master + ``` + +4. Inspect **Validate webhook (GET only, no message)**. It checks webhook type `1` + and reports `guild_id` and `channel_id`; compare both with the intended server + and channel using Discord Developer Mode → Copy ID. The secret determines the + destination; no channel ID is guessed or overridden. The URL/token is never + logged. Dispatch cannot send a test message or announce an old release, even + when run again. Missing/invalid secrets fail validation clearly; offline tests + do not need secrets. + +GET validation confirms the webhook's identity, not delivery or notification +permissions. Verify those on the first genuine release. `mention_everyone=true` +confirms Discord parsed the mention; it cannot prove every member received a push +notification (member/server notification settings still apply). + +## Activation and message + +The workflow and `.github/discord-release/` helper **must be in the commit targeted +by each new release tag**. Merging to `master` does not enable announcements for +old tags whose commits lack these files. Manual dispatch becomes available when +the workflow is on the default branch. Only publish release tags from trusted, +reviewed commits: release workflows execute that tag's code with the secret. + +Only `release` events with action `published`, `draft=false`, and +`prerelease=false` can send. Drafts and prereleases are skipped; release edits do +not trigger announcements. The helper requires the repository to be exactly +`langbot-app/LangBot`, a stable `vX.Y.Z` tag (ASCII digits, at most 64 characters), +and its exact canonical GitHub release URL. Other naming schemes fail closed. + +Example message (the version and URL come from the validated event file): + +```text +@everyone LangBot v4.10.11 is now available! +Release notes: https://github.com/langbot-app/LangBot/releases/tag/v4.10.11 +``` + +The release title/body is never copied. There is one literal `@everyone`, explicit +`allowed_mentions.parse=["everyone"]`, empty user/role allowlists, and no reply +mention. TTS and notification-suppressing flags are disabled. Requests use HTTPS +only to `discord.com`, an explicit User-Agent, and no redirects or automatic +retries. After a webhook identity GET, one `POST ?wait=true` obtains a message ID; +an exact `/messages/` GET verifies its ID, webhook/channel, content, +`mention_everyone=true`, and empty user/role mention arrays before success. + +## Repeat guard and manual recovery + +Production sending requires **`GITHUB_RUN_ATTEMPT == "1"`**. Any Actions rerun +(including “Re-run failed jobs”) refuses to POST and requires manual reconciliation, +even if the first attempt failed before sending. Read-only dispatch may be rerun. + +This is a practical repeat guard, **not durable exactly-once delivery**. It cannot +prevent duplicates from a separate new run/event (for example deleting/recreating +a release), separate automation, or manual posting. It stores no durable dedupe +state and never modifies the release to mark delivery. + +If a POST times out, returns an error, or readback fails, the message may already +exist. The workflow fails rather than blindly sending again. A returned message ID +is included in the safe error when available. A runner termination can also leave +an ambiguous send without that log line. + +1. Inspect the announcement channel and the failed run logs. Locate the canonical + release link and, if available, the returned message ID. A failed verification + does **not** mean the message was absent. +2. If present, reconcile the existing message/mention problem manually; do not + rerun, create another release event, or send a duplicate ping. +3. If an operator has positively confirmed no message exists, fix the secret or + permission issue and use read-only dispatch to validate configuration. A + maintainer may then post the announcement manually once in Discord and record + the message link in the incident/run notes. Do not override the attempt guard + or delete/recreate a release to force recovery. +4. If absence cannot be established, pause and reconcile rather than resending. + +To stop future sends, disable **Discord Release Announcement** in Actions. Rotate +or delete the dedicated Discord webhook if the URL is exposed, and update the +secret before validation. No rollback of release artifacts is involved. + +## Local checks + +Requires Python 3.11+ and the standard library only: + +```sh +python3 -m unittest discover -s .github/discord-release -p 'test_*.py' -v +python3 -m py_compile .github/discord-release/announce.py .github/discord-release/test_announce.py +``` + +Tests exercise policy, CLI/event-file handling, mention payloads, hostile inputs, +HTTP failures, exact message readback, and refusal to retry. Only the HTTPS +transport is mocked for Discord tests; no live Discord requests or messages are +made. Changes to this directory or its workflow run the offline tests on push and +pull request; tests also gate release sending and read-only dispatch validation. diff --git a/.github/discord-release/announce.py b/.github/discord-release/announce.py new file mode 100644 index 000000000..c8548d380 --- /dev/null +++ b/.github/discord-release/announce.py @@ -0,0 +1,162 @@ +"""Announce only first-attempt stable releases; dispatch is read-only validation.""" + +import http.client +import json +import os +from pathlib import Path +import re +import sys + +REPOSITORY = 'langbot-app/LangBot' +RELEASE_PREFIX = f'https://github.com/{REPOSITORY}/releases/tag/' +RECONCILE = ( + 'Do not resend or bypass the run-attempt guard; manual reconciliation is required. ' + 'Inspect the announcement channel and workflow logs before any manual recovery ' + '(see .github/discord-release/README.md).' +) + + +class AnnouncementError(Exception): + """A safe, operator-facing error containing no webhook URL or response body.""" + + +def release_payload(event, attempt): + """Return a bounded, mention-safe payload, or None for draft/preview releases.""" + if not isinstance(event, dict) or event.get('action') != 'published': + raise AnnouncementError('Only release.published events are accepted.') + repository = event.get('repository') + if not isinstance(repository, dict) or repository.get('full_name') != REPOSITORY: + raise AnnouncementError('Unexpected release repository.') + release = event.get('release') + if not isinstance(release, dict) or any(type(release.get(key)) is not bool for key in ('draft', 'prerelease')): + raise AnnouncementError('Invalid release flags.') + if release['draft'] or release['prerelease']: + return None + if attempt != '1': + raise AnnouncementError(f'Release reruns or missing run attempts are refused. {RECONCILE}') + tag = release.get('tag_name') + if not isinstance(tag, str) or len(tag) > 64 or not re.fullmatch(r'v[0-9]+\.[0-9]+\.[0-9]+', tag): + raise AnnouncementError('Expected a stable release tag in vX.Y.Z format (at most 64 characters).') + url = RELEASE_PREFIX + tag + if release.get('html_url') != url: + raise AnnouncementError('Release URL must be the canonical LangBot release URL matching its tag.') + return { + 'content': f'@everyone LangBot {tag} is now available!\nRelease notes: {url}', + 'allowed_mentions': {'parse': ['everyone'], 'users': [], 'roles': [], 'replied_user': False}, + 'tts': False, + 'flags': 0, + } + + +def is_snowflake(value): + return isinstance(value, str) and re.fullmatch(r'[0-9]{1,20}', value) is not None + + +class DiscordWebhook: + def __init__(self, url): + if not url: + raise AnnouncementError('DISCORD_RELEASE_WEBHOOK_URL is missing. Set the repository Actions secret.') + match = re.fullmatch(r'https://discord\.com(/api/webhooks/([0-9]{1,20})/[A-Za-z0-9_-]+)', url) + if not match: + raise AnnouncementError('Invalid webhook URL; expected https://discord.com/api/webhooks//.') + self.path, self.id = match.groups() + + def _request(self, method, suffix='', payload=None): + # Direct HTTPS, default certificate verification, no proxies or redirect/retry machinery. + connection = http.client.HTTPSConnection('discord.com', timeout=20) + try: + body = json.dumps(payload).encode('utf-8') if payload is not None else None + connection.request( + method, + self.path + suffix, + body=body, + headers={'Content-Type': 'application/json', 'User-Agent': 'LangBot-Release-Announcements/1.0'}, + ) + response = connection.getresponse() + if response.status != 200: + raise AnnouncementError(f'Discord {method} returned HTTP {response.status}; no retry was attempted.') + raw = response.read(1_048_577) + if len(raw) > 1_048_576: + raise AnnouncementError('Discord response exceeded the size limit.') + return json.loads(raw) + except (OSError, http.client.HTTPException, ValueError, UnicodeError): + # Exceptions and bodies can contain the token; never print them or chain them. + raise AnnouncementError( + f'Discord {method} failed or returned invalid JSON; no retry was attempted.' + ) from None + finally: + connection.close() + + def validate(self): + """GET only: verify an incoming webhook and return safe identifying fields.""" + webhook = self._request('GET') + if ( + not isinstance(webhook, dict) + or type(webhook.get('type')) is not int + or webhook['type'] != 1 + or webhook.get('id') != self.id + or not is_snowflake(webhook.get('guild_id')) + or not is_snowflake(webhook.get('channel_id')) + ): + raise AnnouncementError('Expected an incoming (type 1) webhook with matching ID and guild/channel IDs.') + return {key: webhook[key] for key in ('id', 'type', 'guild_id', 'channel_id')} + + def send(self, payload): + """One POST, followed by exact message GET; never automatically retry a send.""" + webhook = self.validate() + message_id = None + try: + sent = self._request('POST', '?wait=true', payload) + if not isinstance(sent, dict) or not is_snowflake(sent.get('id')): + raise AnnouncementError('Discord did not return a valid message ID.') + message_id = sent['id'] + saved = self._request('GET', f'/messages/{message_id}') + if ( + not isinstance(saved, dict) + or saved.get('id') != message_id + or saved.get('webhook_id') != self.id + or saved.get('channel_id') != webhook['channel_id'] + or saved.get('content') != payload['content'] + or saved.get('mention_everyone') is not True + or saved.get('mentions') != [] + or saved.get('mention_roles') != [] + ): + raise AnnouncementError('Discord message readback did not match content, identity, or mentions.') + except AnnouncementError as error: + reference = f' Returned message ID: {message_id}.' if message_id else '' + raise AnnouncementError(f'Delivery not confirmed. {error}{reference} {RECONCILE}') from None + return message_id + + +def main(env=None): + env = os.environ if env is None else env + try: + if env.get('GITHUB_REPOSITORY') != REPOSITORY: + raise AnnouncementError('This workflow is restricted to langbot-app/LangBot.') + name = env.get('GITHUB_EVENT_NAME') + if name == 'workflow_dispatch': + webhook = DiscordWebhook(env.get('DISCORD_RELEASE_WEBHOOK_URL')).validate() + print( + f'Validated incoming webhook: guild_id={webhook["guild_id"]} channel_id={webhook["channel_id"]}. No message sent.' + ) + return 0 + if name != 'release': + raise AnnouncementError('Only release and workflow_dispatch events are accepted by this helper.') + try: + event = json.loads(Path(env.get('GITHUB_EVENT_PATH', '')).read_text(encoding='utf-8')) + except (OSError, ValueError, UnicodeError): + raise AnnouncementError('Cannot read a valid JSON release event from GITHUB_EVENT_PATH.') from None + payload = release_payload(event, env.get('GITHUB_RUN_ATTEMPT')) + if payload is None: + print('Skipped draft or prerelease; no message sent.') + return 0 + message_id = DiscordWebhook(env.get('DISCORD_RELEASE_WEBHOOK_URL')).send(payload) + print(f'Announcement verified by exact message readback: message_id={message_id}.') + return 0 + except AnnouncementError as error: + print(f'Error: {error}', file=sys.stderr) + return 1 + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/.github/discord-release/test_announce.py b/.github/discord-release/test_announce.py new file mode 100644 index 000000000..f77bae8b0 --- /dev/null +++ b/.github/discord-release/test_announce.py @@ -0,0 +1,427 @@ +"""Offline contract tests; no Discord credentials or network required.""" + +import contextlib +import io +import json +import os +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest +from unittest.mock import MagicMock, patch + +try: + import announce +except ModuleNotFoundError: + announce = None + +WEBHOOK = 'https://discord.com/api/webhooks/123456789012345678/fixture_token-ONLY' +WEBHOOK_ID = '123456789012345678' +GUILD_ID = '234567890123456789' +CHANNEL_ID = '345678901234567890' +MESSAGE_ID = '456789012345678901' +REPO = 'langbot-app/LangBot' +URL = f'https://github.com/{REPO}/releases/tag/v4.10.11' +CONTENT = f'@everyone LangBot v4.10.11 is now available!\nRelease notes: {URL}' + + +def event(): + return { + 'action': 'published', + 'repository': {'full_name': REPO}, + 'release': { + 'draft': False, + 'prerelease': False, + 'tag_name': 'v4.10.11', + 'html_url': URL, + 'name': 'Hostile @everyone <@123> $(touch /tmp/unsafe)', + 'body': '@everyone @here <@123> <@&456> `hostile`', + }, + } + + +def metadata(): + return {'id': WEBHOOK_ID, 'type': 1, 'guild_id': GUILD_ID, 'channel_id': CHANNEL_ID} + + +def message(): + return { + 'id': MESSAGE_ID, + 'webhook_id': WEBHOOK_ID, + 'channel_id': CHANNEL_ID, + 'content': CONTENT, + 'mention_everyone': True, + 'mentions': [], + 'mention_roles': [], + } + + +class BaseTest(unittest.TestCase): + def setUp(self): + self.assertIsNotNone(announce, 'The release announcement helper must exist') + + +class PolicyTests(BaseTest): + def test_payload_has_one_literal_everyone_and_no_untrusted_body(self): + payload = announce.release_payload(event(), '1') + self.assertEqual(payload['content'], CONTENT) + self.assertEqual(json.dumps(payload).count('@everyone'), 1) + self.assertEqual( + payload['allowed_mentions'], + { + 'parse': ['everyone'], + 'users': [], + 'roles': [], + 'replied_user': False, + }, + ) + self.assertIs(payload['tts'], False) + self.assertEqual(payload['flags'], 0) + + def test_drafts_and_prereleases_are_skipped(self): + for flag in ('draft', 'prerelease'): + with self.subTest(flag=flag): + value = event() + value['release'][flag] = True + self.assertIsNone(announce.release_payload(value, '1')) + + def test_only_published_action_is_accepted(self): + for action in ('edited', 'created', 'released', 'deleted', '', None): + with self.subTest(action=action): + value = event() + value['action'] = action + with self.assertRaises(announce.AnnouncementError): + announce.release_payload(value, '1') + + def test_reruns_and_missing_attempt_refuse_manual_reconciliation(self): + for attempt in ('2', '3', '', None, '01', '0', '1\n'): + with self.subTest(attempt=attempt): + with self.assertRaisesRegex(announce.AnnouncementError, 'manual reconciliation'): + announce.release_payload(event(), attempt) + + def test_repository_must_match_exactly(self): + for repo in ('evil/LangBot', 'langbot-app/langbot', None): + value = event() + value['repository']['full_name'] = repo + with self.assertRaises(announce.AnnouncementError): + announce.release_payload(value, '1') + + def test_hostile_and_noncanonical_tags_are_rejected(self): + for tag in ( + 'v1.2.3 @everyone', + 'v1.2.3\n', + 'v1.2.3/../../x', + 'v1.2.3?x=y', + '$(id)', + 'v1.2.3-rc.1', + 'v１.2.3', + 'v1.2.3%0a', + '<@123>', + 'v1.2.' + '3' * 100, + '', + None, + 123, + ): + with self.subTest(tag=tag): + value = event() + value['release']['tag_name'] = tag + value['release']['html_url'] = f'https://github.com/{REPO}/releases/tag/{tag}' + with self.assertRaises(announce.AnnouncementError): + announce.release_payload(value, '1') + + def test_release_url_must_be_canonical_and_match_tag(self): + for url in ( + 'https://evil.example/tag/v4.10.11', + URL + '?x=y', + URL + '#anchor', + URL + '/', + URL.replace('v4.10.11', 'v4.10.12'), + URL.replace('github.com', 'github.com@evil.example'), + URL.replace('https:', 'http:'), + URL + '\n', + None, + ): + with self.subTest(url=url): + value = event() + value['release']['html_url'] = url + with self.assertRaises(announce.AnnouncementError): + announce.release_payload(value, '1') + + def test_malformed_events_fail_closed(self): + for value in (None, [], {}, {'release': []}, {'repository': None}): + with self.subTest(value=value): + with self.assertRaises(announce.AnnouncementError): + announce.release_payload(value, '1') + for flag in ('draft', 'prerelease'): + for bad in (None, 'false', 0, 1): + value = event() + value['release'][flag] = bad + with self.assertRaises(announce.AnnouncementError): + announce.release_payload(value, '1') + + +class DiscordTests(BaseTest): + def setUp(self): + super().setUp() + self.patch = patch('announce.http.client.HTTPSConnection') + self.connection_class = self.patch.start() + self.addCleanup(self.patch.stop) + self.connection = self.connection_class.return_value + + def respond(self, *values): + responses = [] + for value in values: + response = MagicMock() + response.status = 200 + response.read.return_value = json.dumps(value).encode() + responses.append(response) + self.connection.getresponse.side_effect = responses + + def methods(self): + return [call.args[0] for call in self.connection.request.call_args_list] + + def test_webhook_validation_is_get_only_and_reports_ids(self): + self.respond(metadata()) + result = announce.DiscordWebhook(WEBHOOK).validate() + self.assertEqual(result, metadata()) + self.assertEqual(self.methods(), ['GET']) + self.assertEqual( + self.connection.request.call_args.args[:2], ('GET', f'/api/webhooks/{WEBHOOK_ID}/fixture_token-ONLY') + ) + self.connection_class.assert_called_with('discord.com', timeout=20) + self.connection.close.assert_called_once() + + def test_invalid_webhook_urls_are_rejected_before_network(self): + for url in ( + '', + None, + WEBHOOK + '/', + WEBHOOK + '?wait=true', + WEBHOOK + '#x', + WEBHOOK + '\n', + ' ' + WEBHOOK, + WEBHOOK.replace('https:', 'http:'), + WEBHOOK.replace('discord.com', 'discord.com.evil.example'), + WEBHOOK.replace('discord.com', 'discord.com@evil.example'), + WEBHOOK.replace('discord.com', 'discord.com:443'), + WEBHOOK.replace('/api/', '/api/v10/'), + WEBHOOK.replace(WEBHOOK_ID, 'abc'), + WEBHOOK + '/../../x', + WEBHOOK.replace('fixture_token-ONLY', 'a%2Fb'), + ): + with self.subTest(url=url): + with self.assertRaises(announce.AnnouncementError): + announce.DiscordWebhook(url) + self.connection_class.assert_not_called() + + def test_webhook_metadata_requires_incoming_type_and_ids(self): + invalid = [ + None, + [], + {}, + dict(metadata(), type=2), + dict(metadata(), type=True), + dict(metadata(), id='999'), + dict(metadata(), channel_id=None), + dict(metadata(), guild_id='::error::hostile'), + ] + for value in invalid: + with self.subTest(value=value): + self.respond(value) + with self.assertRaises(announce.AnnouncementError): + announce.DiscordWebhook(WEBHOOK).validate() + self.assertNotIn('POST', self.methods()) + + def test_send_waits_and_reads_back_exact_returned_message(self): + self.respond(metadata(), message(), message()) + result = announce.DiscordWebhook(WEBHOOK).send(announce.release_payload(event(), '1')) + self.assertEqual(result, MESSAGE_ID) + self.assertEqual(self.methods(), ['GET', 'POST', 'GET']) + calls = self.connection.request.call_args_list + self.assertEqual(calls[1].args[:2], ('POST', f'/api/webhooks/{WEBHOOK_ID}/fixture_token-ONLY?wait=true')) + self.assertEqual(json.loads(calls[1].kwargs['body']), announce.release_payload(event(), '1')) + self.assertEqual( + calls[2].args[:2], ('GET', f'/api/webhooks/{WEBHOOK_ID}/fixture_token-ONLY/messages/{MESSAGE_ID}') + ) + + def test_readback_must_match_content_mentions_and_identity(self): + for field, bad in ( + ('content', 'wrong'), + ('mention_everyone', False), + ('mention_everyone', 1), + ('mentions', [{'id': '123'}]), + ('mention_roles', ['123']), + ('id', '999'), + ('channel_id', '999'), + ('webhook_id', '999'), + ): + with self.subTest(field=field, bad=bad): + self.connection.reset_mock() + self.respond(metadata(), message(), dict(message(), **{field: bad})) + with self.assertRaisesRegex(announce.AnnouncementError, 'manual reconciliation'): + announce.DiscordWebhook(WEBHOOK).send(announce.release_payload(event(), '1')) + self.assertEqual(self.methods().count('POST'), 1) + + def test_missing_readback_fields_fail_closed(self): + for field in message(): + value = message() + del value[field] + self.respond(metadata(), message(), value) + with self.assertRaises(announce.AnnouncementError): + announce.DiscordWebhook(WEBHOOK).send(announce.release_payload(event(), '1')) + + def test_unsafe_post_message_id_never_becomes_get_path(self): + for value in (None, {}, dict(message(), id='../evil'), dict(message(), id='123?x=y')): + self.connection.reset_mock() + self.respond(metadata(), value) + with self.assertRaisesRegex(announce.AnnouncementError, 'manual reconciliation'): + announce.DiscordWebhook(WEBHOOK).send(announce.release_payload(event(), '1')) + self.assertEqual(self.methods(), ['GET', 'POST']) + + def test_post_failure_never_retries_and_never_logs_secret(self): + for status in (301, 302, 307, 308, 400, 401, 403, 429, 500, 204): + with self.subTest(status=status): + self.connection.reset_mock() + self.respond(metadata(), message()) + responses = list(self.connection.getresponse.side_effect) + responses[1].status = status + self.connection.getresponse.side_effect = responses + with self.assertRaisesRegex(announce.AnnouncementError, 'manual reconciliation') as caught: + announce.DiscordWebhook(WEBHOOK).send(announce.release_payload(event(), '1')) + self.assertNotIn('fixture_token', str(caught.exception)) + self.assertEqual(self.methods(), ['GET', 'POST']) + + def test_ambiguous_timeout_never_retries_or_echoes_exception(self): + self.respond(metadata()) + first = next(self.connection.getresponse.side_effect) + self.connection.getresponse.side_effect = [first, TimeoutError(WEBHOOK)] + with self.assertRaisesRegex(announce.AnnouncementError, 'manual reconciliation') as caught: + announce.DiscordWebhook(WEBHOOK).send(announce.release_payload(event(), '1')) + self.assertNotIn('fixture_token', str(caught.exception)) + self.assertEqual(self.methods(), ['GET', 'POST']) + + def test_malformed_json_response_is_sanitized(self): + self.respond(metadata()) + response = next(self.connection.getresponse.side_effect) + response.read.return_value = WEBHOOK.encode() + self.connection.getresponse.side_effect = [response] + with self.assertRaises(announce.AnnouncementError) as caught: + announce.DiscordWebhook(WEBHOOK).validate() + self.assertNotIn('fixture_token', str(caught.exception)) + + def test_get_redirect_is_not_followed(self): + self.respond(metadata()) + response = next(self.connection.getresponse.side_effect) + response.status = 302 + response.getheader.return_value = 'https://evil.example/' + self.connection.getresponse.side_effect = [response] + with self.assertRaises(announce.AnnouncementError): + announce.DiscordWebhook(WEBHOOK).validate() + self.assertEqual(self.methods(), ['GET']) + + +class EntrypointTests(BaseTest): + def run_main(self, data=None, **overrides): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'event.json' + path.write_text(json.dumps(event() if data is None else data)) + env = { + 'GITHUB_EVENT_NAME': 'release', + 'GITHUB_EVENT_PATH': str(path), + 'GITHUB_REPOSITORY': REPO, + 'GITHUB_RUN_ATTEMPT': '1', + 'DISCORD_RELEASE_WEBHOOK_URL': WEBHOOK, + } + env.update(overrides) + output = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output): + result = announce.main(env) + return result, output.getvalue() + + def test_dispatch_only_validates_even_if_event_contains_release(self): + with patch('announce.DiscordWebhook') as client: + client.return_value.validate.return_value = metadata() + result, output = self.run_main(GITHUB_EVENT_NAME='workflow_dispatch') + self.assertEqual(result, 0) + client.return_value.validate.assert_called_once() + client.return_value.send.assert_not_called() + self.assertIn(GUILD_ID, output) + self.assertIn(CHANNEL_ID, output) + self.assertNotIn('fixture_token', output) + + def test_production_release_sends_once(self): + with patch('announce.DiscordWebhook') as client: + client.return_value.send.return_value = MESSAGE_ID + result, output = self.run_main() + self.assertEqual(result, 0) + client.return_value.send.assert_called_once_with(announce.release_payload(event(), '1')) + self.assertIn(MESSAGE_ID, output) + + def test_skipped_releases_need_no_secret_or_network(self): + for flag in ('draft', 'prerelease'): + value = event() + value['release'][flag] = True + with patch('announce.DiscordWebhook') as client: + result, _ = self.run_main(value, DISCORD_RELEASE_WEBHOOK_URL='') + self.assertEqual(result, 0) + client.assert_not_called() + + def test_rerun_never_constructs_client(self): + with patch('announce.DiscordWebhook') as client: + result, output = self.run_main(GITHUB_RUN_ATTEMPT='2') + self.assertEqual(result, 1) + self.assertIn('manual reconciliation', output) + client.assert_not_called() + + def test_unexpected_event_or_repository_cannot_send(self): + for overrides in ( + {'GITHUB_EVENT_NAME': 'push'}, + {'GITHUB_EVENT_NAME': 'pull_request'}, + {'GITHUB_REPOSITORY': 'evil/LangBot'}, + ): + with patch('announce.DiscordWebhook') as client: + result, _ = self.run_main(**overrides) + self.assertEqual(result, 1) + client.assert_not_called() + + def test_missing_secret_fails_clearly_for_send_and_validation(self): + for name in ('release', 'workflow_dispatch'): + result, output = self.run_main(GITHUB_EVENT_NAME=name, DISCORD_RELEASE_WEBHOOK_URL='') + self.assertEqual(result, 1) + self.assertIn('DISCORD_RELEASE_WEBHOOK_URL is missing', output) + + def test_cli_reads_event_file_and_redacts_invalid_input(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / 'event.json' + value = event() + value['release']['tag_name'] = '::error::hostile @everyone' + path.write_text(json.dumps(value)) + env = dict( + os.environ, + GITHUB_EVENT_NAME='release', + GITHUB_EVENT_PATH=str(path), + GITHUB_REPOSITORY=REPO, + GITHUB_RUN_ATTEMPT='1', + DISCORD_RELEASE_WEBHOOK_URL=WEBHOOK, + ) + result = subprocess.run( + [sys.executable, str(Path(__file__).with_name('announce.py'))], + env=env, + text=True, + capture_output=True, + check=False, + ) + self.assertEqual(result.returncode, 1) + self.assertNotIn('hostile', result.stderr) + self.assertNotIn('fixture_token', result.stderr) + self.assertNotIn('Traceback', result.stderr) + + def test_unreadable_event_fails_safely(self): + result, output = self.run_main(GITHUB_EVENT_PATH='/nonexistent/event.json') + self.assertEqual(result, 1) + self.assertNotIn('Traceback', output) + + +if __name__ == '__main__': + unittest.main() diff --git a/.github/workflows/discord-release.yml b/.github/workflows/discord-release.yml new file mode 100644 index 000000000..23e5d8033 --- /dev/null +++ b/.github/workflows/discord-release.yml @@ -0,0 +1,64 @@ +name: Discord Release Announcement + +on: + release: + types: [published] + workflow_dispatch: + push: + paths: + - '.github/workflows/discord-release.yml' + - '.github/discord-release/**' + pull_request: + paths: + - '.github/workflows/discord-release.yml' + - '.github/discord-release/**' + +permissions: + contents: read + +jobs: + tests: + name: Offline announcement tests + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - name: Test helper without secrets or network + run: python3 -m unittest discover -s .github/discord-release -p 'test_*.py' -v + + validate: + name: Validate webhook (GET only, no message) + if: github.repository == 'langbot-app/LangBot' && github.event_name == 'workflow_dispatch' + needs: tests + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + - name: Validate incoming webhook and report guild/channel IDs + env: + DISCORD_RELEASE_WEBHOOK_URL: ${{ secrets.DISCORD_RELEASE_WEBHOOK_URL }} + run: python3 .github/discord-release/announce.py + + announce: + name: Announce published stable release + if: >- + github.repository == 'langbot-app/LangBot' && + github.event_name == 'release' && github.event.action == 'published' && + github.event.release.draft == false && github.event.release.prerelease == false + needs: tests + runs-on: ubuntu-24.04 + timeout-minutes: 5 + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4 + with: + persist-credentials: false + # The helper refuses GITHUB_RUN_ATTEMPT != 1 with recovery guidance. + # Never interpolate release data into a shell command. + - name: Send once and verify the exact Discord message + env: + DISCORD_RELEASE_WEBHOOK_URL: ${{ secrets.DISCORD_RELEASE_WEBHOOK_URL }} + run: python3 .github/discord-release/announce.py From 940895c541e03d8febdfff8877d769a46e0645ca Mon Sep 17 00:00:00 2001 From: hedging8563 Date: Sun, 13 Sep 2026 21:24:38 +0800 Subject: [PATCH 2/9] chore(brand): refresh TokenLab logo --- .../pkg/provider/modelmgr/requesters/tokenlab.svg | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/langbot/pkg/provider/modelmgr/requesters/tokenlab.svg b/src/langbot/pkg/provider/modelmgr/requesters/tokenlab.svg index 2308dca3b..6c193a921 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/tokenlab.svg +++ b/src/langbot/pkg/provider/modelmgr/requesters/tokenlab.svg @@ -1,5 +1,8 @@ - - - - + + TokenLab + Specimen Split symbol, positive master for sizes from 32 to 96 pixels. + + + + From 60bb67f0256d10320f2ce8900bf704eb2a8d3146 Mon Sep 17 00:00:00 2001 From: sheetung <30528385+sheetung@users.noreply.github.com> Date: Mon, 14 Sep 2026 00:31:19 +0800 Subject: [PATCH 3/9] Fnos packaging (#2524) * feat(packaging): add fnOS FPK packaging and CI workflow Add packaging/fnos/ shell (manifest, lifecycle cmd scripts, install/ upgrade/uninstall wizards, desktop entry, EULA) plus in-repo build script. A release now auto-builds langbot--fnos.fpk via .github/workflows/build-fnos-fpk.yaml, uploaded to the release assets. * ci(fnos): skip release upload on manual dispatch github.event.release.tag_name is empty when triggered via workflow_dispatch, causing 'gh release upload' to fail with 'requires at least 2 arg(s)'. Restrict the step to release events. * ci(fnos): normalize release asset name Strip a trailing -fnos from the tag-derived version before appending the suffix, avoiding langbot--fnos-fnos.fpk when the tag itself already carries -fnos. * ci(fnos): use release tag version for auto build, manifest for manual Auto build (release/tag) reads version from tag like other release workflows; build.sh strips the v prefix when injecting into manifest. Manual dispatch falls back to the version maintained in manifest. * feat(fnos): add post-install deployment notice to install wizard Last wizard step now informs users that first startup takes about 5-10 minutes for dependency setup before the web UI is ready. * docs(fnos): add packaging directory README * refactor(fnos): rename app from ai.langbot to langbot Rename appname, desktop entry, data share, build artifacts, and all references from ai.langbot to langbot across packaging files. --------- Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com> --- .github/workflows/build-fnos-fpk.yaml | 78 ++++++++ .gitignore | 9 + packaging/fnos/LICENSE | 145 +++++++++++++++ packaging/fnos/README.md | 78 ++++++++ packaging/fnos/app/desktop/langbot.main.url | 9 + packaging/fnos/app/ui/config | 13 ++ packaging/fnos/build.sh | 180 ++++++++++++++++++ packaging/fnos/cmd/config_callback | 6 + packaging/fnos/cmd/config_init | 4 + packaging/fnos/cmd/install_callback | 149 +++++++++++++++ packaging/fnos/cmd/install_init | 5 + packaging/fnos/cmd/main | 194 ++++++++++++++++++++ packaging/fnos/cmd/uninstall_callback | 21 +++ packaging/fnos/cmd/uninstall_init | 20 ++ packaging/fnos/cmd/upgrade_callback | 77 ++++++++ packaging/fnos/cmd/upgrade_init | 20 ++ packaging/fnos/config/privilege | 5 + packaging/fnos/config/resource | 9 + packaging/fnos/manifest | 16 ++ packaging/fnos/wizard/install | 47 +++++ packaging/fnos/wizard/uninstall | 17 ++ packaging/fnos/wizard/upgrade | 38 ++++ 22 files changed, 1140 insertions(+) create mode 100644 .github/workflows/build-fnos-fpk.yaml create mode 100644 packaging/fnos/LICENSE create mode 100644 packaging/fnos/README.md create mode 100644 packaging/fnos/app/desktop/langbot.main.url create mode 100644 packaging/fnos/app/ui/config create mode 100644 packaging/fnos/build.sh create mode 100755 packaging/fnos/cmd/config_callback create mode 100755 packaging/fnos/cmd/config_init create mode 100755 packaging/fnos/cmd/install_callback create mode 100755 packaging/fnos/cmd/install_init create mode 100755 packaging/fnos/cmd/main create mode 100755 packaging/fnos/cmd/uninstall_callback create mode 100755 packaging/fnos/cmd/uninstall_init create mode 100755 packaging/fnos/cmd/upgrade_callback create mode 100755 packaging/fnos/cmd/upgrade_init create mode 100644 packaging/fnos/config/privilege create mode 100644 packaging/fnos/config/resource create mode 100644 packaging/fnos/manifest create mode 100644 packaging/fnos/wizard/install create mode 100644 packaging/fnos/wizard/uninstall create mode 100644 packaging/fnos/wizard/upgrade diff --git a/.github/workflows/build-fnos-fpk.yaml b/.github/workflows/build-fnos-fpk.yaml new file mode 100644 index 000000000..e24906e39 --- /dev/null +++ b/.github/workflows/build-fnos-fpk.yaml @@ -0,0 +1,78 @@ +name: Build fnOS FPK + +on: + workflow_dispatch: + ## 发布release的时候会自动构建 + release: + types: [published] + +permissions: + contents: write + +jobs: + build-fnos-fpk: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + persist-credentials: false + + - name: Check version + id: check_version + run: | + echo $GITHUB_REF + # 如果是tag，则去掉refs/tags/前缀（与其他 release workflow 一致，版本号取 tag 名） + if [[ $GITHUB_REF == refs/tags/* ]]; then + echo "It's a tag" + echo "version=$(echo $GITHUB_REF | awk -F '/' '{print $3}')" >> $GITHUB_OUTPUT + else + # 手动触发（workflow_dispatch）：读不到 tag，使用 manifest 内维护的版本 + echo "It's not a tag" + echo "version=$(grep '^version=' packaging/fnos/manifest | cut -d= -f2)" >> $GITHUB_OUTPUT + fi + + - name: Setup Node + uses: actions/setup-node@v2 + with: + node-version: '22' + + - name: Setup Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install build tools + run: | + pip install pillow + # fnpack：飞牛官方打包 CLI（静态二进制） + curl -fsSL -o /usr/local/bin/fnpack \ + https://static2.fnnas.com/fnpack/fnpack-1.2.3-linux-amd64 + chmod +x /usr/local/bin/fnpack + + - name: Build FPK + env: + FPK_VERSION: ${{ steps.check_version.outputs.version }} + run: | + bash packaging/fnos/build.sh + test -f packaging/fnos/langbot.fpk + + - name: Upload Artifact + uses: actions/upload-artifact@v4 + with: + name: langbot-${{ steps.check_version.outputs.version }}-fnos + path: packaging/fnos/langbot.fpk + + - name: Upload To Release + # 仅 release 触发时执行；手动/workflow_dispatch 触发时没有 release， + # 且 github.event.release.tag_name 为空（否则 gh release upload 缺参数报错） + if: github.event_name == 'release' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # tag 可能已带 -fnos 后缀（如 v4.10.9-fnos），产物名统一为 + # langbot-<基础版本>-fnos.fpk，避免出现 -fnos-fnos + VER="${{ steps.check_version.outputs.version }}" + BASE="${VER#v}"; BASE="${BASE%-fnos}" + cp packaging/fnos/langbot.fpk "langbot-${BASE}-fnos.fpk" + gh release upload ${{ github.event.release.tag_name }} "langbot-${BASE}-fnos.fpk" diff --git a/.gitignore b/.gitignore index db632fb19..459a22c82 100644 --- a/.gitignore +++ b/.gitignore @@ -60,3 +60,12 @@ web/.next/ web/.pnpm-home .tmp Caddyfile + +# fnOS packaging build artifacts (packaging/fnos/build.sh) +packaging/fnos/app/langbot/ +packaging/fnos/app/bin/ +packaging/fnos/ICON.PNG +packaging/fnos/ICON_256.PNG +packaging/fnos/app/ui/images/ +packaging/fnos/app/desktop/images/ +packaging/fnos/*.fpk diff --git a/packaging/fnos/LICENSE b/packaging/fnos/LICENSE new file mode 100644 index 000000000..e297f1789 --- /dev/null +++ b/packaging/fnos/LICENSE @@ -0,0 +1,145 @@ + + LangBot 用户许可协议 + User License Agreement + + 飞牛 fnOS 平台发行版 | 最后更新：2026 年 8 月 + +感谢您使用 LangBot！本协议是您（用户）与 LangBot 开源项目（以下简称「LangBot」「我们」）之间，就您在飞牛 fnOS 平台（含飞牛 NAS 设备、飞牛 OS 及其应用中心，以下简称「平台」）上安装、运行、使用 LangBot 应用所订立的合法协议。 + +您一旦在飞牛应用中心勾选「我接受许可协议的条款」并继续安装、或以其他方式运行 LangBot，即表示您已阅读、理解并同意本协议的全部内容。如您不同意本协议，请不要安装或使用本软件。 + +--- + +一、软件性质 + +1.1 LangBot 是一款基于 LLM 的多平台智能对话机器人开源软件。主程序源代码按照 Apache License, Version 2.0 公开，您可在遵守开源协议的前提下自由使用、修改与再分发。 + +1.2 本发行版系 LangBot 社区为飞牛 fnOS 平台打包构建的自托管移植版本，与飞牛官方、飞牛硬件厂商不存在从属或关联关系。飞牛应用中心提供的分发渠道不构成对软件功能、可用性的任何担保。 + +--- + +二、使用授权 + +2.1 授予您一份有限的、非排他的、不可转让的个人使用许可：您可在一台或多台您合法拥有或管理的 fnOS 设备上安装、运行本软件，用于个人、家庭或组织内部合法用途。 + +2.2 您不得： + (a) 将本软件用于违反中国大陆地区法律法规或您所在司法管辖区法律的用途； + (b) 对机器人账号进行骚扰、诈骗、批量营销、发布违法违规内容等滥用行为； + (c) 逆向工程、反编译本软件所包含的第三方二进制（uv 等），但适用法律明确允许或对应开源许可证另作规定的除外； + (d) 试图干扰、过载或损害任何由本软件对外提供的服务或其基础设施。 + +--- + +三、服务可用性与「现状」提供 + +3.1 我们努力提供稳定可靠的软件，但**不保证运行无中断或无错误**。软件可能因维护、升级、不可抗力或其他原因暂时不可用。 + +3.2 本软件按「现状」「按可用」提供。我们不作任何明示或默示担保，包括但不限于对适销性、特定用途适用性与非侵权性的默示担保。我们不保证： + (a) 软件将满足您的具体需求； + (b) 运行不间断、及时、安全或无错误； + (c) 使用获得的结果准确或可靠； + (d) 任何错误都会被修复。 + +--- + +四、责任限制 + +在适用法律允许的最大范围内： +(a) 我们不对任何**间接、附带、特殊、惩罚性或后果性损失**承担责任，包括但不限于利润损失、数据丢失、商誉损失、业务中断或其他无形损失，无论是否已被告知该等损害发生的可能性； +(b) 无论基于合同、侵权、严格责任或其他任何理论，我们就本软件所引起的所有索赔，向您承担的**累计赔偿总额**不超过 15 美元（或等值当地货币）。 + +--- + +五、用户责任 + +5.1 您对通过本软件发送的所有内容和消息（包括您所配置并运行的机器人发出的消息）承担全部责任。 + +5.2 您必须遵守所有适用的法律法规，包括但不限于数据保护、隐私、消费者保护和反垃圾邮件法律。 + +5.3 您有责任保管好自己的账号凭据和各类 API Key，并**自行对重要数据进行备份**。我们对因服务中断、账号终止或系统故障等任何原因造成的数据丢失不承担责任。 + +5.4 您不得将本软件用于任何非法活动、发送垃圾信息、骚扰他人或侵害他人合法权益。 + +5.5 您使用机器人接入任何即时通讯平台（QQ、微信、飞书、钉钉、Telegram 等）前，须自行确认已获得该平台授权并遵守其开发者协议与社区规范；因违规接入导致的账号封禁、平台处罚，由您自行承担。 + +--- + +六、数据与隐私 + +6.1 本自托管版本默认情况下，所有配置、对话记录、知识库与插件数据均保存在您所安装的 fnOS 设备本地共享目录（langbot/data）中，不会被自动上传至除您显式配置的模型/服务提供商以外的任何第三方。您对自己的数据及备份负责。 + +6.2 LangBot 默认启用最少量的匿名遥测，用于帮助改进产品。详细政策见官方文档： + https://docs.langbot.app/zh/insight/data-collection-policy + +6.3 启用遥测时仅可能发送：查询事件（适配器类型、运行器类型、模型名称、处理耗时、版本号、匿名工作区 UUID、插件/功能使用计数、不含用户内容的错误追踪）、每日一次的工作区心跳（部署概况、资源对象数量）、完全自愿的问卷回答。 + +6.4 我们绝对不收集：消息内容、用户名/手机号/平台账号 ID、API 密钥或凭据、IP 地址、文件或媒体内容。 + +6.5 关闭方式：进入 LangBot Web 管理界面 → 设置 → Space 遥测，关闭开关；或在配置文件 data/config.yaml 中设置 `space.disable_telemetry: true`。关闭后所有功能照常运行。 + +--- + +七、不可抗力 + +因不可抗力事件导致的履约失败或延迟，我们不承担责任。不可抗力包括但不限于：自然灾害（地震、洪水、飓风等）、战争、恐怖主义或内乱、政府行为或法规、网络攻击（DDoS、勒索软件等）、第三方服务故障（AI 模型提供商、即时通讯平台、飞牛平台运行时等）、电力故障或互联网连接中断、流行病或公共卫生紧急事件。 + +--- + +八、第三方服务 + +本软件可能依赖或集成第三方服务，包括但不限于：即时通讯平台（Telegram、Discord、微信、QQ、Slack 等）、AI 模型提供商（OpenAI、Anthropic、Google、深度求索等）、飞牛平台的应用中心运行时与依赖应用（如 Node.js）。我们不对第三方服务的可用性、准确性、可靠性或安全性负责；使用第三方服务须受其各自条款约束，第三方服务的变更可能不经通知即影响本软件功能。 + +--- + +九、软件修改与终止 + +9.1 我们保留随时修改、暂停或终止软件或其中任何部分的权利，无论是否事先通知。 +9.2 我们保留随时修改本协议的权利。对重大变更我们会尽力在项目主页公告，继续使用软件即视为接受修订后的协议。 +9.3 您可以在飞牛应用中心卸载本软件。卸载时向导会询问是否保留数据，您可自主选择。终止后您继续使用软件的权利立即终止，我们无义务保留您的数据。 + +--- + +十、赔偿 + +您同意赔偿、抗辩并使 LangBot 团队及其关联贡献者、管理人员、代理人、员工免受任何及所有因以下事项引起或与之相关的索赔、损失、损害、责任、成本和费用（包括合理的律师费）： +(a) 您对软件的使用； +(b) 您违反本协议； +(c) 您违反任何适用的法律或法规； +(d) 您侵犯任何第三方权利； +(e) 您或您所配置的机器人通过本软件传输的内容。 + +--- + +十一、知识产权 + +11.1 本软件及其设计、代码、文档和品牌归 LangBot 项目所有并受知识产权法保护。 +11.2 使用本软件并不授予您对软件的任何所有权。 +11.3 您保留通过本软件创建和传输内容的所有权。 + +--- + +十二、争议解决 + +因本协议引起或与之相关的任何争议，应首先通过友好协商解决。协商在 30 日内未达成一致的，任何一方均可依适用法律规定向有管辖权的法院提起诉讼。 + +--- + +十三、可分割性 + +如本协议的任何条款被认定为不可执行或无效，该条款应在最小必要范围内予以限制或剔除，其余条款继续完全有效。 + +--- + +十四、完整协议 + +本协议连同我们的数据收集政策（https://docs.langbot.app/zh/insight/data-collection-policy）构成您与我们之间关于本软件的完整协议，并取代所有先前的协议与谅解。 + +--- + +附录：开源许可证声明 +LangBot 主程序代码依照 Apache License 2.0 发布。详细条款见： +https://github.com/langbot-app/LangBot/blob/master/LICENSE +或 LangBot 源码包内的 LICENSE 文件。 + +数据收集政策：https://docs.langbot.app/zh/insight/data-collection-policy + diff --git a/packaging/fnos/README.md b/packaging/fnos/README.md new file mode 100644 index 000000000..1bd672b67 --- /dev/null +++ b/packaging/fnos/README.md @@ -0,0 +1,78 @@ +# LangBot fnOS Packaging + +This directory packages LangBot as a `.fpk` app for the fnOS App Store. It is a native deployment: no Docker involved — uv creates a Python virtual environment directly on the NAS, and Node.js v22 from the fnOS App Store provides the Box sandbox and npx MCP capabilities. + +## Directory Structure + +``` +packaging/fnos/ +├── manifest # App metadata (appname/version/port/dependency declarations) +├── build.sh # One-shot build script (shared by local and CI) +├── LICENSE +├── config/ +│ ├── privilege # Privilege config (run-as: root) +│ └── resource # Persistent data share declaration (langbot/data) +├── cmd/ # Lifecycle scripts (fnOS invokes them with TRIM_* env vars) +│ ├── main # Service start/stop manager (start/stop/status, owns PID/log) +│ ├── install_init # Pre-install hook +│ ├── install_callback # Post-install hook: create venv, uv sync deps, seed config.yaml port +│ ├── upgrade_init # Pre-upgrade hook +│ ├── upgrade_callback # Post-upgrade hook +│ ├── uninstall_init # Pre-uninstall hook +│ ├── uninstall_callback # Post-uninstall hook (keeps data per wizard choice) +│ ├── config_init # Pre-config-change hook +│ └── config_callback # Post-config-change hook (apply new port etc.) +├── wizard/ # Install wizards (JSON forms; values passed as wizard_* env vars) +│ ├── install # On install: Node version, web port, deployment-time notice +│ ├── upgrade # On upgrade: Node version confirmation +│ └── uninstall # On uninstall: whether to keep data +├── app/ +│ ├── ui/config # Desktop entry declaration (${wizard_port} placeholder, substituted by fnOS at install) +│ ├── desktop/langbot.main.url +│ ├── langbot/ # [generated] repo source synced via rsync (includes web/dist) +│ └── bin/ # [generated] offline uv binaries (x86_64/aarch64) +├── ICON.PNG / ICON_256.PNG # [generated] derived from res/logo-blue.png +└── langbot.fpk # [generated] final artifact +``` + +Paths marked `[generated]` are produced by `build.sh`, ignored via `.gitignore`; everything else is a git-tracked source file. + +## Building + +### Locally + +```bash +bash packaging/fnos/build.sh +``` + +Dependencies: python3 + Pillow, node + npm (or pnpm), fnpack (official fnOS packaging CLI, download from https://developer.fnnas.com/docs/cli/fnpack). + +The version comes from `version=` maintained in the manifest; it can also be injected: `FPK_VERSION=4.10.10-1 bash packaging/fnos/build.sh`. + +### CI + +[`.github/workflows/build-fnos-fpk.yaml`](../../.github/workflows/build-fnos-fpk.yaml) triggers automatically on Release publication and uploads `langbot--fnos.fpk` to the Release; it can also be triggered manually via workflow_dispatch. + +Version sources (consistent with the other release workflows): + +| Trigger | Version source | +|---|---| +| Release/tag auto build | Tag name (`v4.10.10-1` → in-package `4.10.10-1`; build.sh strips the `v` prefix on injection) | +| Manual workflow_dispatch / local build | Version maintained in manifest | + +## Final Artifact + +`langbot.fpk` (gzip + tar archive), containing: + +- `manifest` — realigned and appended with a `checksum` field by fnpack +- `app.tgz` — app payload (source, web/dist, uv binaries, entry configs) +- `cmd/`, `config/`, `wizard/` — lifecycle scripts and wizards +- `ICON.PNG`, `ICON_256.PNG`, `LICENSE` + +The first startup after installation takes about 5-10 minutes to finish dependency deployment (uv venv + sync); after that the web admin UI is reachable via the desktop icon or `http://:` (default port 5300). + +## References + +- fnOS developer docs: https://developer.fnnas.com/ +- fnOS app wizard: https://developer.fnnas.com/docs/core-concepts/wizard/ +- fnpack CLI: https://developer.fnnas.com/docs/cli/fnpack diff --git a/packaging/fnos/app/desktop/langbot.main.url b/packaging/fnos/app/desktop/langbot.main.url new file mode 100644 index 000000000..513870fe8 --- /dev/null +++ b/packaging/fnos/app/desktop/langbot.main.url @@ -0,0 +1,9 @@ +{ + "title": "LangBot", + "icon": "images/icon-256.png", + "type": "url", + "protocol": "http", + "port": "${wizard_port}", + "url": "/", + "allUsers": true +} diff --git a/packaging/fnos/app/ui/config b/packaging/fnos/app/ui/config new file mode 100644 index 000000000..d45765dd4 --- /dev/null +++ b/packaging/fnos/app/ui/config @@ -0,0 +1,13 @@ +{ + ".url": { + "langbot.main": { + "title": "LangBot", + "icon": "images/icon-{0}.png", + "type": "url", + "protocol": "http", + "port": "${wizard_port}", + "url": "/", + "allUsers": true + } + } +} diff --git a/packaging/fnos/build.sh b/packaging/fnos/build.sh new file mode 100644 index 000000000..74675e2a3 --- /dev/null +++ b/packaging/fnos/build.sh @@ -0,0 +1,180 @@ +#!/usr/bin/env bash +# packaging/fnos/build.sh - Build LangBot fnOS FPK package (in-repo version) +# 在 LangBot 仓库内直接打包飞牛 fnOS 应用 +# Usage: +# bash packaging/fnos/build.sh # 版本取 manifest 中 version= +# FPK_VERSION=4.10.9 bash packaging/fnos/build.sh # 注入版本（CI 用 release tag） +# Dependencies: python3+Pillow, node+npm, fnpack +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SRC_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" # LangBot 仓库根 +FPK_DIR="${SCRIPT_DIR}" + +echo "==> LangBot fnOS FPK builder (in-repo)" +echo " Source root: ${SRC_ROOT}" +echo " FPK dir: ${FPK_DIR}" + +# --- 0. Inject version (release tag) --- +if [ -n "${FPK_VERSION:-}" ]; then + sed -i "s/^version=.*/version=${FPK_VERSION#v}/" "${FPK_DIR}/manifest" +else + # 无注入版本时自动跟进仓库主版本（pyproject.toml） + PY_VER=$(grep -m1 '^version = ' "${SRC_ROOT}/pyproject.toml" | cut -d'"' -f2) + if [ -n "${PY_VER}" ]; then + sed -i "s/^version=.*/version=${PY_VER}/" "${FPK_DIR}/manifest" + fi +fi +MANIFEST_VER=$(grep '^version=' "${FPK_DIR}/manifest" | cut -d= -f2) +echo " FPK version: ${MANIFEST_VER}" + +# --- 1. Build frontend --- +echo "[1/5] Building frontend (web/dist)..." +cd "${SRC_ROOT}/web" +if command -v pnpm >/dev/null 2>&1; then + pnpm install --frozen-lockfile 2>/dev/null || pnpm install + pnpm build +else + npm install + npx vite build +fi +[ -d dist ] || { echo "ERROR: web/dist missing" >&2; exit 1; } +echo " Frontend built" + +# --- 2. Sync source into packaging/fnos/app/langbot/ --- +echo "[2/5] Syncing source to app/langbot/..." +rm -rf "${FPK_DIR}/app/langbot" +mkdir -p "${FPK_DIR}/app/langbot" +cd "${SRC_ROOT}" +rsync -a \ + --exclude='.git' \ + --exclude='.venv' \ + --exclude='__pycache__' \ + --exclude='*.pyc' \ + --exclude='web/node_modules' \ + --exclude='web/.vite' \ + --exclude='tests' \ + --exclude='packaging' \ + --exclude='.pytest_cache' \ + --exclude='.mypy_cache' \ + --exclude='.ruff_cache' \ + --exclude='data' \ + --exclude='*.log' \ + --exclude='.dockerignore' \ + --exclude='Dockerfile' \ + --exclude='docker/' \ + --exclude='kubernetes.yaml' \ + --exclude='.github/' \ + --exclude='docs/' \ + --exclude='examples/' \ + --exclude='res/' \ + ./ "${FPK_DIR}/app/langbot/" + +[ -d "${FPK_DIR}/app/langbot/web/dist" ] || { echo "ERROR: web/dist missing after rsync!" >&2; exit 1; } +echo " Source synced ($(du -sh "${FPK_DIR}/app/langbot" | cut -f1))" + +# --- 2.5 Download bundled uv binaries (offline install on NAS) --- +echo "[2.5/5] Downloading bundled uv binaries..." +UV_VERSION="0.12.9" +mkdir -p "${FPK_DIR}/app/bin" +for arch in x86_64 aarch64; do + out="${FPK_DIR}/app/bin/uv-${arch}" + if [ -x "${out}" ]; then + echo " uv-${arch} already present, skip" + continue + fi + tmp="$(mktemp -d)" + if curl -sSL -o "${tmp}/uv.tar.gz" \ + "https://github.com/astral-sh/uv/releases/download/${UV_VERSION}/uv-${arch}-unknown-linux-gnu.tar.gz" \ + && tar xzf "${tmp}/uv.tar.gz" -C "${tmp}" \ + && cp "${tmp}/uv-${arch}-unknown-linux-gnu/uv" "${out}"; then + chmod +x "${out}" + echo " uv-${arch} downloaded (${UV_VERSION})" + else + echo " WARNING: failed to download uv for ${arch}, install will fall back to online install" >&2 + fi + rm -rf "${tmp}" +done + +# --- 3. Regenerate icons from res/logo-blue.png --- +echo "[3/5] Generating icons from res/logo-blue.png..." +export LOGO_SRC="${SRC_ROOT}/res/logo-blue.png" +export OUT_DIR="${FPK_DIR}" +python3 << 'PYEOF' +from PIL import Image +import os, sys + +src = os.environ.get("LOGO_SRC") +out_dir = os.environ.get("OUT_DIR") +if not src or not out_dir: + print("ERROR: LOGO_SRC or OUT_DIR not set", file=sys.stderr) + sys.exit(1) + +if not os.path.isfile(src): + print(f"ERROR: logo source not found: {src}", file=sys.stderr) + sys.exit(1) + +img = Image.open(src).convert("RGBA") + +for size, name in [(64, "ICON.PNG"), (256, "ICON_256.PNG")]: + img.resize((size, size), Image.LANCZOS).save(os.path.join(out_dir, name)) + +ui_dir = os.path.join(out_dir, "app/ui/images") +os.makedirs(ui_dir, exist_ok=True) +for size in [64, 256]: + img.resize((size, size), Image.LANCZOS).save(os.path.join(ui_dir, f"icon-{size}.png")) + +desktop_dir = os.path.join(out_dir, "app/desktop/images") +os.makedirs(desktop_dir, exist_ok=True) +for size in [64, 256]: + img.resize((size, size), Image.LANCZOS).save(os.path.join(desktop_dir, f"icon-{size}.png")) + +print(" Icons generated") +PYEOF + +# --- 4. Validate structure --- +echo "[4/5] Validating FPK structure..." +ERRORS=0 +[ -f "${FPK_DIR}/manifest" ] || { echo " MISSING: manifest"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/config/privilege" ] || { echo " MISSING: config/privilege"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/config/resource" ] || { echo " MISSING: config/resource"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/ICON.PNG" ] || { echo " MISSING: ICON.PNG"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/ICON_256.PNG" ] || { echo " MISSING: ICON_256.PNG"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/app/ui/config" ] || { echo " MISSING: app/ui/config"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/app/ui/images/icon-64.png" ] || { echo " MISSING: app/ui/images/icon-64.png"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/app/ui/images/icon-256.png" ] || { echo " MISSING: app/ui/images/icon-256.png"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/app/desktop/langbot.main.url" ] || { echo " MISSING: app/desktop/langbot.main.url"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/app/desktop/images/icon-64.png" ] || { echo " MISSING: app/desktop/images/icon-64.png"; ERRORS=$((ERRORS+1)); } +[ -f "${FPK_DIR}/app/desktop/images/icon-256.png" ] || { echo " MISSING: app/desktop/images/icon-256.png"; ERRORS=$((ERRORS+1)); } +[ -d "${FPK_DIR}/cmd" ] || { echo " MISSING: cmd/"; ERRORS=$((ERRORS+1)); } +[ -d "${FPK_DIR}/wizard" ] || { echo " MISSING: wizard/"; ERRORS=$((ERRORS+1)); } + +for script in "${FPK_DIR}/cmd/"*; do + [ -x "${script}" ] || { echo " NOT EXECUTABLE: cmd/$(basename "$script")"; ERRORS=$((ERRORS+1)); } +done + +if [ "${ERRORS}" -gt 0 ]; then + echo "FAILED: ${ERRORS} validation errors" >&2 + exit 1 +fi +echo " Structure OK" + +# --- 5. Build FPK --- +echo "[5/5] Building .fpk..." +if ! command -v fnpack >/dev/null 2>&1; then + echo "ERROR: fnpack not found in PATH." >&2 + echo " Download from https://developer.fnnas.com/docs/cli/fnpack" >&2 + exit 1 +fi + +cd "${FPK_DIR}" +fnpack build +FPK_FILE=$(ls -t *.fpk 2>/dev/null | head -1) +if [ -n "${FPK_FILE}" ]; then + echo "" + echo "==> Done! FPK: ${FPK_DIR}/${FPK_FILE}" + echo " Size: $(du -sh "${FPK_FILE}" | cut -f1)" +else + echo "WARNING: fnpack finished but no .fpk found in ${FPK_DIR}" >&2 + exit 1 +fi diff --git a/packaging/fnos/cmd/config_callback b/packaging/fnos/cmd/config_callback new file mode 100755 index 000000000..ccecdcdb8 --- /dev/null +++ b/packaging/fnos/cmd/config_callback @@ -0,0 +1,6 @@ +#!/bin/bash +# cmd/config_callback - post-config hook +# Applies wizard-provided settings (e.g. Node.js version) that cmd/main reads. +# Nothing to persist currently; cmd/main reads wizard_node_version at runtime. + +exit 0 diff --git a/packaging/fnos/cmd/config_init b/packaging/fnos/cmd/config_init new file mode 100755 index 000000000..0d366f7a1 --- /dev/null +++ b/packaging/fnos/cmd/config_init @@ -0,0 +1,4 @@ +#!/bin/bash +# cmd/config_init - pre-config hook + +exit 0 diff --git a/packaging/fnos/cmd/install_callback b/packaging/fnos/cmd/install_callback new file mode 100755 index 000000000..4eac06acc --- /dev/null +++ b/packaging/fnos/cmd/install_callback @@ -0,0 +1,149 @@ +#!/bin/bash +# cmd/install_callback - post-install hook +# Prefers bundled uv binary, creates Python venv, syncs deps, verifies dist. +# Also validates the user-selected Node.js version is actually installed. + +APP_DIR="${TRIM_APPDEST}/langbot" + +# --- Resolve data directory --- +# LangBot loads config CWD-relative (data/config.yaml); cmd/main replaces +# APP_DIR/data with a symlink to this persistent dir on every start. +DATA_DIR="${TRIM_DATA_SHARE_PATHS%%:*}" +if [ -z "${DATA_DIR}" ]; then + DATA_DIR="${TRIM_PKGVAR}/data" +fi + +cd "${APP_DIR}" || { + echo "App directory missing after install" > "${TRIM_TEMP_LOGFILE}" + exit 1 +} + +# --- Ensure data directory exists --- +mkdir -p "${DATA_DIR}/plugins" "${DATA_DIR}/box" "${DATA_DIR}/logs" 2>/dev/null || true + +# --- Pre-seed config.yaml with the user-selected web port --- +# Data root points at the persistent share (LANGBOT_DATA_ROOT is exported by +# cmd/main at start), so this config survives app upgrades. +# LangBot copies templates/config.yaml there on first boot only if missing, +# so we seed it ourselves with the chosen port. +PORT="${wizard_port:-5300}" +case "${PORT}" in + ''|*[!0-9]*) PORT="5300" ;; +esac +TEMPLATE_FILE="${APP_DIR}/src/langbot/templates/config.yaml" + +_patch_config() { + local cfg_dir="$1" + local cfg_file="${cfg_dir}/config.yaml" + mkdir -p "${cfg_dir}" 2>/dev/null || true + if [ ! -f "${cfg_file}" ] && [ -f "${TEMPLATE_FILE}" ]; then + cp "${TEMPLATE_FILE}" "${cfg_file}" + fi + if [ -f "${cfg_file}" ]; then + sed -i -E "/^api:/,/^[a-z_]+:/ s/^([[:space:]]*port:).*/\1 ${PORT}/" "${cfg_file}" + sed -i "s#webhook_prefix: 'http://127\.0\.0\.1:[0-9]*'#webhook_prefix: 'http://127.0.0.1:${PORT}'#" "${cfg_file}" + fi +} + +# Seed the persistent dir AND the CWD-relative APP_DIR/data (cmd/main merges +# the latter into the persistent dir via symlink on first start, so the port +# survives regardless of which path ends up being read). +_patch_config "${DATA_DIR}" +if [ "${APP_DIR}/data" != "${DATA_DIR}" ]; then + _patch_config "${APP_DIR}/data" +fi + +# --- Fallback patch for desktop entry port --- +# fnOS natively substitutes ${wizard_port} in ui/config at install time. +# This only kicks in if the placeholder somehow survived (e.g. CLI install). +UI_CONFIG="${TRIM_APPDEST}/ui/config" +if [ -f "${UI_CONFIG}" ] && grep -q 'wizard_port\|{port}' "${UI_CONFIG}"; then + sed -i "s/\${wizard_port}/${PORT}/g; s/{port}/${PORT}/g" "${UI_CONFIG}" +fi + +# --- Validate user-selected Node.js version is installed --- +NODE_VERSION="${wizard_node_version:-22}" +if [ ! -d "/var/apps/nodejs_v${NODE_VERSION}" ]; then + echo "Node.js v${NODE_VERSION} 未安装：请先在应用中心安装 nodejs_v${NODE_VERSION}，再重新安装本应用。" > "${TRIM_TEMP_LOGFILE}" + exit 1 +fi + +# --- Python check --- +PYTHON_BIN="python3" +if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then + PYTHON_BIN="python" +fi +if ! command -v "${PYTHON_BIN}" >/dev/null 2>&1; then + echo "Python not found on this system" > "${TRIM_TEMP_LOGFILE}" + exit 1 +fi + +PY_VER=$("${PYTHON_BIN}" -c 'import sys; print(f"{sys.version_info.major}.{sys.version_info.minor}")' 2>/dev/null) +if [ -z "${PY_VER}" ]; then + echo "Python 3.11+ required but not found on this system" > "${TRIM_TEMP_LOGFILE}" + exit 1 +fi +PY_MAJOR=$(echo "${PY_VER}" | cut -d. -f1) +PY_MINOR=$(echo "${PY_VER}" | cut -d. -f2) +if [ "${PY_MAJOR}" -lt 3 ] || { [ "${PY_MAJOR}" -eq 3 ] && [ "${PY_MINOR}" -lt 11 ]; }; then + echo "Python 3.11+ required, found ${PY_VER}" > "${TRIM_TEMP_LOGFILE}" + exit 1 +fi + +# --- Resolve uv: bundled binary first, then online fallbacks --- +UV_BIN="" +ARCH=$(uname -m) +case "${ARCH}" in + x86_64) BUNDLED_UV="${TRIM_APPDEST}/bin/uv-x86_64" ;; + aarch64) BUNDLED_UV="${TRIM_APPDEST}/bin/uv-aarch64" ;; + *) BUNDLED_UV="" ;; +esac + +if [ -n "${BUNDLED_UV}" ] && [ -x "${BUNDLED_UV}" ]; then + mkdir -p "${TRIM_PKGVAR}/bin" + cp "${BUNDLED_UV}" "${TRIM_PKGVAR}/bin/uv" && chmod +x "${TRIM_PKGVAR}/bin/uv" + UV_BIN="${TRIM_PKGVAR}/bin/uv" +fi + +if [ -z "${UV_BIN}" ] && command -v uv >/dev/null 2>&1; then + UV_BIN="uv" +fi + +if [ -z "${UV_BIN}" ]; then + "${PYTHON_BIN}" -m pip install --user --no-cache-dir uv 2>/dev/null || \ + "${PYTHON_BIN}" -m pip install --no-cache-dir uv 2>/dev/null || \ + curl -LsSf https://astral.sh/uv/install.sh | sh 2>/dev/null || true + export PATH="${HOME}/.local/bin:${PATH}" + if command -v uv >/dev/null 2>&1; then + UV_BIN="uv" + elif [ -x "${HOME}/.local/bin/uv" ]; then + UV_BIN="${HOME}/.local/bin/uv" + fi +fi + +if [ -z "${UV_BIN}" ]; then + echo "无法获取 uv：内置二进制缺失且在线安装失败。请检查网络后重新安装。" > "${TRIM_TEMP_LOGFILE}" + exit 1 +fi + +# --- Create venv via uv --- +if [ ! -d ".venv" ]; then + "${UV_BIN}" venv .venv --python "${PYTHON_BIN}" || { + echo "Failed to create Python virtual environment via uv" > "${TRIM_TEMP_LOGFILE}" + exit 1 + } +fi + +# --- Sync dependencies --- +"${UV_BIN}" sync --extra seekdb || { + echo "Dependency sync failed. Check network connectivity." > "${TRIM_TEMP_LOGFILE}" + exit 1 +} + +# --- Verify frontend dist --- +if [ ! -d "web/dist" ] || [ -z "$(ls -A web/dist 2>/dev/null)" ]; then + echo "Frontend dist missing! Web UI will not be available." > "${TRIM_TEMP_LOGFILE}" + exit 1 +fi + +exit 0 diff --git a/packaging/fnos/cmd/install_init b/packaging/fnos/cmd/install_init new file mode 100755 index 000000000..2e940ee03 --- /dev/null +++ b/packaging/fnos/cmd/install_init @@ -0,0 +1,5 @@ +#!/bin/bash +# cmd/install_init - pre-install hook +# Nothing special to do before extraction. + +exit 0 diff --git a/packaging/fnos/cmd/main b/packaging/fnos/cmd/main new file mode 100755 index 000000000..a3181c383 --- /dev/null +++ b/packaging/fnos/cmd/main @@ -0,0 +1,194 @@ +#!/bin/bash +# cmd/main - LangBot lifecycle manager for fnOS +# Handles start / stop / status via standalone-runtime Python process. +# Node.js path is injected into PATH so Box sandbox npx MCP servers can run. + +PID_FILE="${TRIM_PKGVAR}/langbot.pid" +APP_DIR="${TRIM_APPDEST}/langbot" +LOG_FILE="${TRIM_PKGVAR}/langbot.log" + +# --- Locate fnOS Node.js bin path --- +# fnOS appname-based path: /var/apps/nodejs_vXX/target/bin +# This is a stable symlink regardless of which volume the app is on. +NODE_VERSION="${wizard_node_version:-22}" +NODE_BIN_DIR="/var/apps/nodejs_v${NODE_VERSION}/target/bin" + +if [ -d "${NODE_BIN_DIR}" ]; then + export PATH="${NODE_BIN_DIR}:${PATH}" +fi + +# --- Persistent data root --- +# LangBot loads data/config.yaml CWD-RELATIVE (see core/stages/load_config.py: +# load_yaml_config('data/config.yaml', ...)) and resolves its data root to +# /data in source-install mode — it does NOT honour LANGBOT_DATA_ROOT for +# config.yaml. So the real fix is the symlink below: APP_DIR/data -> DATA_DIR. +DATA_DIR="${TRIM_DATA_SHARE_PATHS%%:*}" +if [ -z "${DATA_DIR}" ]; then + DATA_DIR="${TRIM_PKGVAR}/data" +fi +export LANGBOT_DATA_ROOT="${DATA_DIR}" +mkdir -p "${DATA_DIR}" 2>/dev/null || true + +# --- Locate Python --- +PYTHON_BIN="python3" +! command -v "${PYTHON_BIN}" >/dev/null 2>&1 && PYTHON_BIN="python" + +# --- Locate uv --- +# install_callback puts the bundled uv binary at ${TRIM_PKGVAR}/bin/uv +UV_BIN="${TRIM_PKGVAR}/bin/uv" +if [ ! -x "${UV_BIN}" ]; then + UV_BIN="uv" +fi +if ! command -v "${UV_BIN}" >/dev/null 2>&1; then + UV_BIN="${HOME}/.local/bin/uv" +fi +if ! command -v "${UV_BIN}" >/dev/null 2>&1 && [ ! -x "${UV_BIN}" ]; then + UV_BIN="${HOME}/.cargo/bin/uv" +fi +if ! command -v "${UV_BIN}" >/dev/null 2>&1 && [ ! -x "${UV_BIN}" ]; then + UV_BIN="${APP_DIR}/.venv/bin/uv" +fi + +case $1 in + start) + if [ -f "${PID_FILE}" ]; then + PID=$(cat "${PID_FILE}" | tr -d '[:space:]') + if [ -n "${PID}" ] && kill -0 "${PID}" 2>/dev/null; then + exit 0 + fi + rm -f "${PID_FILE}" + fi + + if [ ! -d "${APP_DIR}" ]; then + echo "LangBot app directory missing: ${APP_DIR}" > "${TRIM_TEMP_LOGFILE}" + exit 1 + fi + + cd "${APP_DIR}" || { + echo "Cannot enter app directory: ${APP_DIR}" > "${TRIM_TEMP_LOGFILE}" + exit 1 + } + + if [ ! -d ".venv" ]; then + echo "Python virtual environment not found. Please reinstall LangBot." > "${TRIM_TEMP_LOGFILE}" + exit 1 + fi + + # --- Unify data location: APP_DIR/data -> symlink to persistent DATA_DIR --- + # LangBot reads config CWD-relative (data/config.yaml), so without this a + # fresh data/ with a default 5300 config gets recreated inside target/ on + # every install/upgrade. The symlink keeps everything on the persistent + # share; it is recreated here on each start (upgrades wipe target/). + APP_DATA="${APP_DIR}/data" + if [ -L "${APP_DATA}" ]; then + # already a symlink; re-point if the persistent dir changed + [ "$(readlink "${APP_DATA}")" != "${DATA_DIR}" ] && ln -sfn "${DATA_DIR}" "${APP_DATA}" + elif [ -d "${APP_DATA}" ]; then + # legacy real dir (created by LangBot before this fix): merge into the + # persistent dir without overwriting newer files already there + mkdir -p "${DATA_DIR}" + cp -an "${APP_DATA}/." "${DATA_DIR}/" 2>/dev/null || cp -a "${APP_DATA}/." "${DATA_DIR}/" + rm -rf "${APP_DATA}" + ln -s "${DATA_DIR}" "${APP_DATA}" + else + ln -s "${DATA_DIR}" "${APP_DATA}" + fi + + # Ensure LangBot's actual listen port always matches what fnOS shows in + # "应用设置 → 访问端口" (which is the single source of truth from the user's + # POV). Two sources, checked in priority order: + # 1. ${wizard_port} — only set during install/upgrade callbacks (not on + # normal `start`; kept for completeness). + # 2. target/ui/config — read the "port" field written by fnOS after + # ${wizard_port} substitution AND any later edit the user made via + # "应用设置 → 自定义 URL" pencil button. + # Without this: user picks 5303 in wizard, LangBot still listens on its + # default 5300, desktop shortcut hits 5303 → connection refused. + CONFIG_FILE="${DATA_DIR}/config.yaml" + _patch_port() { + local _port="$1" + case "${_port}" in + ''|*[!0-9]*) return ;; + esac + if [ ! -f "${CONFIG_FILE}" ]; then + local _tmpl="${APP_DIR}/src/langbot/templates/config.yaml" + [ -f "${_tmpl}" ] && cp "${_tmpl}" "${CONFIG_FILE}" + fi + if [ -f "${CONFIG_FILE}" ]; then + sed -i -E "/^api:/,/^[a-z_]+:/ s/^([[:space:]]*port:).*/\1 ${_port}/" "${CONFIG_FILE}" + sed -i "s#webhook_prefix: 'http://127\.0\.0\.1:[0-9]*'#webhook_prefix: 'http://127.0.0.1:${_port}'#" "${CONFIG_FILE}" + fi + } + if [ -n "${wizard_port:-}" ]; then + _patch_port "${wizard_port}" + fi + if [ -f "${TRIM_APPDEST}/ui/config" ]; then + _port_from_ui=$(python3 -c 'import json,sys +try: + d = json.load(open(sys.argv[1])) + for _name, _entry in (d.get(".url") or {}).items(): + p = _entry.get("port") + if isinstance(p, (int, float)): + print(int(p)) + elif isinstance(p, str) and p.isdigit(): + print(int(p)) + break +except Exception: + pass +' "${TRIM_APPDEST}/ui/config" 2>/dev/null) + if [ -n "${_port_from_ui}" ]; then + _patch_port "${_port_from_ui}" + fi + fi + + # Native deployment: no --standalone-runtime flag, LangBot spawns the + # plugin runtime as a stdio subprocess (same as official `uv run main.py`). + # (--standalone-runtime would require an external runtime at + # ws://langbot_plugin_runtime:5400, which only exists in Docker Compose.) + # --standalone-box omitted: Box sandbox defaults off, users enable via Web UI + nohup "${UV_BIN}" run --no-sync main.py \ + > "${LOG_FILE}" 2>&1 & + echo $! > "${PID_FILE}" + + sleep 3 + if [ -f "${PID_FILE}" ]; then + PID=$(cat "${PID_FILE}" | tr -d '[:space:]') + if [ -n "${PID}" ] && kill -0 "${PID}" 2>/dev/null; then + exit 0 + fi + fi + echo "LangBot failed to start. Check ${LOG_FILE}" > "${TRIM_TEMP_LOGFILE}" + exit 1 + ;; + + stop) + if [ -f "${PID_FILE}" ]; then + PID=$(cat "${PID_FILE}" | tr -d '[:space:]') + if [ -n "${PID}" ]; then + kill "${PID}" 2>/dev/null + for _ in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "${PID}" 2>/dev/null || break + sleep 1 + done + kill -9 "${PID}" 2>/dev/null + fi + rm -f "${PID_FILE}" + fi + exit 0 + ;; + + status) + if [ -f "${PID_FILE}" ]; then + PID=$(cat "${PID_FILE}" | tr -d '[:space:]') + if [ -n "${PID}" ] && kill -0 "${PID}" 2>/dev/null; then + exit 0 + fi + rm -f "${PID_FILE}" + fi + exit 3 + ;; + + *) + exit 1 + ;; +esac diff --git a/packaging/fnos/cmd/uninstall_callback b/packaging/fnos/cmd/uninstall_callback new file mode 100755 index 000000000..2baee90d9 --- /dev/null +++ b/packaging/fnos/cmd/uninstall_callback @@ -0,0 +1,21 @@ +#!/bin/bash +# cmd/uninstall_callback - post-uninstall hook +# The system preserves var/ and shares/ by default. Honor the user's +# wizard_keep_data choice: delete data only when explicitly requested. + +if [ "${wizard_keep_data:-yes}" = "no" ]; then + # 应用运行数据（pid、日志等） + if [ -n "${TRIM_PKGVAR}" ]; then + rm -rf "${TRIM_PKGVAR:?}"/langbot.pid \ + "${TRIM_PKGVAR:?}"/langbot.log \ + "${TRIM_PKGVAR:?}"/bin 2>/dev/null || true + fi + + # 共享数据目录（langbot/data） + DATA_DIR="${TRIM_DATA_SHARE_PATHS%%:*}" + if [ -n "${DATA_DIR}" ]; then + rm -rf "${DATA_DIR:?}" 2>/dev/null || true + fi +fi + +exit 0 diff --git a/packaging/fnos/cmd/uninstall_init b/packaging/fnos/cmd/uninstall_init new file mode 100755 index 000000000..800ced14c --- /dev/null +++ b/packaging/fnos/cmd/uninstall_init @@ -0,0 +1,20 @@ +#!/bin/bash +# cmd/uninstall_init - pre-uninstall hook +# Stop LangBot before files are removed. + +PID_FILE="${TRIM_PKGVAR}/langbot.pid" + +if [ -f "${PID_FILE}" ]; then + PID=$(cat "${PID_FILE}" | tr -d '[:space:]') + if [ -n "${PID}" ] && kill -0 "${PID}" 2>/dev/null; then + kill "${PID}" 2>/dev/null + for _ in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "${PID}" 2>/dev/null || break + sleep 1 + done + kill -9 "${PID}" 2>/dev/null + fi + rm -f "${PID_FILE}" +fi + +exit 0 diff --git a/packaging/fnos/cmd/upgrade_callback b/packaging/fnos/cmd/upgrade_callback new file mode 100755 index 000000000..9288c866c --- /dev/null +++ b/packaging/fnos/cmd/upgrade_callback @@ -0,0 +1,77 @@ +#!/bin/bash +# cmd/upgrade_callback - post-upgrade hook +# Re-sync dependencies after code replacement using uv. + +APP_DIR="${TRIM_APPDEST}/langbot" + +# Persistent data root (must match cmd/main) +DATA_DIR="${TRIM_DATA_SHARE_PATHS%%:*}" +[ -z "${DATA_DIR}" ] && DATA_DIR="${TRIM_PKGVAR}/data" + +# Apply port from upgrade wizard (config persists across upgrades; this +# only rewrites it when the user changed the value in the upgrade wizard) +CONFIG_FILE="${DATA_DIR}/config.yaml" +if [ -n "${wizard_port:-}" ] && [ -f "${CONFIG_FILE}" ]; then + case "${wizard_port}" in + ''|*[!0-9]*) ;; + *) + sed -i -E "/^api:/,/^[a-z_]+:/ s/^([[:space:]]*port:).*/\1 ${wizard_port}/" "${CONFIG_FILE}" + sed -i "s#webhook_prefix: 'http://127\.0\.0\.1:[0-9]*'#webhook_prefix: 'http://127.0.0.1:${wizard_port}'#" "${CONFIG_FILE}" + ;; + esac +fi + +cd "${APP_DIR}" || { + echo "App directory missing after upgrade" > "${TRIM_TEMP_LOGFILE}" + exit 1 +} + +# Find uv (bundled first, then PATH / ~/.local/bin / ~/.cargo/bin) +UV_BIN="${TRIM_PKGVAR}/bin/uv" +if [ ! -x "${UV_BIN}" ]; then + UV_BIN="uv" +fi +if ! command -v "${UV_BIN}" >/dev/null 2>&1; then + UV_BIN="${HOME}/.local/bin/uv" +fi +if ! command -v "${UV_BIN}" >/dev/null 2>&1 && [ ! -x "${UV_BIN}" ]; then + UV_BIN="${HOME}/.cargo/bin/uv" +fi + +PYTHON_BIN="python3" +! command -v "${PYTHON_BIN}" >/dev/null 2>&1 && PYTHON_BIN="python" + +# Re-sync deps +if [ -d ".venv" ]; then + "${UV_BIN}" sync --extra seekdb 2>/dev/null || { + echo "Dependency sync failed after upgrade" > "${TRIM_TEMP_LOGFILE}" + exit 1 + } +else + # Venv was lost, recreate via uv + if ! command -v "${UV_BIN}" >/dev/null 2>&1 && [ ! -x "${UV_BIN}" ]; then + "${PYTHON_BIN}" -m pip install --user --no-cache-dir uv 2>/dev/null || \ + "${PYTHON_BIN}" -m pip install --no-cache-dir uv 2>/dev/null || { + echo "Failed to install uv" > "${TRIM_TEMP_LOGFILE}" + exit 1 + } + export PATH="${HOME}/.local/bin:${PATH}" + UV_BIN="uv" + fi + "${UV_BIN}" venv .venv --python "${PYTHON_BIN}" || { + echo "Failed to recreate virtual environment" > "${TRIM_TEMP_LOGFILE}" + exit 1 + } + "${UV_BIN}" sync --extra seekdb || { + echo "Dependency sync failed" > "${TRIM_TEMP_LOGFILE}" + exit 1 + } +fi + +# Verify frontend dist still present +if [ ! -d "web/dist" ] || [ -z "$(ls -A web/dist 2>/dev/null)" ]; then + echo "Frontend dist missing after upgrade! Web UI will not be available." > "${TRIM_TEMP_LOGFILE}" + exit 1 +fi + +exit 0 diff --git a/packaging/fnos/cmd/upgrade_init b/packaging/fnos/cmd/upgrade_init new file mode 100755 index 000000000..fdac71831 --- /dev/null +++ b/packaging/fnos/cmd/upgrade_init @@ -0,0 +1,20 @@ +#!/bin/bash +# cmd/upgrade_init - pre-upgrade hook +# Stop the running LangBot process before files are replaced. + +PID_FILE="${TRIM_PKGVAR}/langbot.pid" + +if [ -f "${PID_FILE}" ]; then + PID=$(cat "${PID_FILE}" | tr -d '[:space:]') + if [ -n "${PID}" ] && kill -0 "${PID}" 2>/dev/null; then + kill "${PID}" 2>/dev/null + for _ in 1 2 3 4 5 6 7 8 9 10; do + kill -0 "${PID}" 2>/dev/null || break + sleep 1 + done + kill -9 "${PID}" 2>/dev/null + fi + rm -f "${PID_FILE}" +fi + +exit 0 diff --git a/packaging/fnos/config/privilege b/packaging/fnos/config/privilege new file mode 100644 index 000000000..e21db569b --- /dev/null +++ b/packaging/fnos/config/privilege @@ -0,0 +1,5 @@ +{ + "defaults": { + "run-as": "root" + } +} diff --git a/packaging/fnos/config/resource b/packaging/fnos/config/resource new file mode 100644 index 000000000..eef99d345 --- /dev/null +++ b/packaging/fnos/config/resource @@ -0,0 +1,9 @@ +{ + "data-share": { + "shares": [ + { + "name": "langbot/data" + } + ] + } +} diff --git a/packaging/fnos/manifest b/packaging/fnos/manifest new file mode 100644 index 000000000..98699b795 --- /dev/null +++ b/packaging/fnos/manifest @@ -0,0 +1,16 @@ +appname=langbot +version=4.10.10 +display_name=LangBot +desc=基于 LLM 的多平台智能对话机器人，支持 QQ、微信、飞书、钉钉、Telegram 等十余种即时通讯平台，内置 Web 管理界面和 AI Agent 能力。 +platform=all +source=thirdparty +maintainer=LangBot +maintainer_url=https://langbot.app +service_port=5300 +checkport=true +os_min_version=0.9.0 +desktop_uidir=ui +desktop_applaunchname=langbot.main +ctl_stop=true +install_dep_apps=nodejs_v22 +changelog=飞牛 fnOS 增强版首版：原生 Python 部署，依赖 Node.js v22 以启用 Box 沙箱 + npx MCP 能力 diff --git a/packaging/fnos/wizard/install b/packaging/fnos/wizard/install new file mode 100644 index 000000000..d9833facf --- /dev/null +++ b/packaging/fnos/wizard/install @@ -0,0 +1,47 @@ +[ + { + "stepTitle": "运行环境", + "items": [ + { + "type": "tips", + "helpText": "LangBot 依赖 Node.js v22 运行 Box 沙箱和 npx MCP。请先在应用中心安装 Node.js v22。" + }, + { + "type": "select", + "field": "wizard_node_version", + "label": "Node.js 版本", + "initValue": "22", + "options": [ + { "label": "Node.js v22 (推荐, LTS)", "value": "22" }, + { "label": "Node.js v24", "value": "24" }, + { "label": "Node.js v20", "value": "20" } + ] + } + ] + }, + { + "stepTitle": "访问配置", + "items": [ + { + "type": "text", + "field": "wizard_port", + "label": "Web 访问端口", + "initValue": "5300", + "rules": [ + { "required": true, "message": "请输入访问端口" }, + { "pattern": "^[0-9]+$", "message": "端口只能是数字" }, + { "min": 1, "max": 5, "message": "端口号长度不正确" } + ] + } + ] + }, + { + "stepTitle": "安装说明", + "items": [ + { + "type": "tips", + "helpText": "安装完成后，LangBot 首次启动约需 5-10 分钟完成依赖部署与初始化，部署完成后即可打开网页端使用。" + } + ] + } +] diff --git a/packaging/fnos/wizard/uninstall b/packaging/fnos/wizard/uninstall new file mode 100644 index 000000000..5b96987fb --- /dev/null +++ b/packaging/fnos/wizard/uninstall @@ -0,0 +1,17 @@ +[ + { + "stepTitle": "数据保留", + "items": [ + { + "type": "radio", + "field": "wizard_keep_data", + "label": "是否保留 LangBot 数据（插件、配置、日志）", + "initValue": "yes", + "options": [ + { "label": "保留数据（重新安装后可继续使用）", "value": "yes" }, + { "label": "彻底删除全部数据", "value": "no" } + ] + } + ] + } +] diff --git a/packaging/fnos/wizard/upgrade b/packaging/fnos/wizard/upgrade new file mode 100644 index 000000000..718264df7 --- /dev/null +++ b/packaging/fnos/wizard/upgrade @@ -0,0 +1,38 @@ +[ + { + "stepTitle": "运行环境", + "items": [ + { + "type": "tips", + "helpText": "LangBot 依赖 Node.js v22 运行 Box 沙箱和 npx MCP。如需更换版本，请先在应用中心安装对应版本。" + }, + { + "type": "select", + "field": "wizard_node_version", + "label": "Node.js 版本", + "initValue": "22", + "options": [ + { "label": "Node.js v22 (推荐, LTS)", "value": "22" }, + { "label": "Node.js v24", "value": "24" }, + { "label": "Node.js v20", "value": "20" } + ] + } + ] + }, + { + "stepTitle": "访问配置", + "items": [ + { + "type": "text", + "field": "wizard_port", + "label": "Web 访问端口", + "initValue": "5300", + "rules": [ + { "required": true, "message": "请输入访问端口" }, + { "pattern": "^[0-9]+$", "message": "端口只能是数字" }, + { "min": 1, "max": 5, "message": "端口号长度不正确" } + ] + } + ] + } +] From eb4563775dd37308376315b05c49090189c39dea Mon Sep 17 00:00:00 2001 From: Hyu Date: Tue, 15 Sep 2026 01:11:42 +0800 Subject: [PATCH 4/9] docs(readme): remove retired public demo from all languages (#2543) Co-authored-by: dadachann <185672915+dadachann@users.noreply.github.com> --- README.md | 11 ----------- README_CN.md | 10 ---------- README_ES.md | 10 ---------- README_FR.md | 10 ---------- README_JP.md | 10 ---------- README_KO.md | 10 ---------- README_RU.md | 10 ---------- README_TW.md | 10 ---------- README_VI.md | 10 ---------- 9 files changed, 91 deletions(-) diff --git a/README.md b/README.md index e0f2cae99..5de92d3ad 100644 --- a/README.md +++ b/README.md @@ -93,17 +93,6 @@ docker compose --profile all up -d --- -## Live Demo - -**Try it now:** https://demo.langbot.dev/ - -- Email: `demo@langbot.app` -- Password: `langbot123456` - -_Note: Public demo environment. Do not enter sensitive information._ - ---- - ## Supported Platforms | Platform | Status | Notes | diff --git a/README_CN.md b/README_CN.md index 49888f812..aad5c6efa 100644 --- a/README_CN.md +++ b/README_CN.md @@ -93,16 +93,6 @@ docker compose --profile all up -d --- -## 在线演示 - -**立即体验：** https://demo.langbot.dev/ -- 邮箱：`demo@langbot.app` -- 密码：`langbot123456` - -*注意：公开演示环境，请不要在其中填入任何敏感信息。* - ---- - ## 支持的平台 | 平台 | 状态 | 备注 | diff --git a/README_ES.md b/README_ES.md index 502047b9c..04ebc78a3 100644 --- a/README_ES.md +++ b/README_ES.md @@ -92,16 +92,6 @@ docker compose --profile all up -d --- -## Demo en Vivo - -**Pruébelo ahora:** https://demo.langbot.dev/ -- Correo electrónico: `demo@langbot.app` -- Contraseña: `langbot123456` - -*Nota: Entorno de demostración público. No ingrese información confidencial.* - ---- - ## Plataformas Soportadas | Plataforma | Estado | Notas | diff --git a/README_FR.md b/README_FR.md index 24aed3374..78d99c692 100644 --- a/README_FR.md +++ b/README_FR.md @@ -92,16 +92,6 @@ docker compose --profile all up -d --- -## Démo en Ligne - -**Essayez maintenant :** https://demo.langbot.dev/ -- Email : `demo@langbot.app` -- Mot de passe : `langbot123456` - -*Note : Environnement de démonstration public. Ne saisissez pas d'informations sensibles.* - ---- - ## Plateformes Supportées | Plateforme | Statut | Notes | diff --git a/README_JP.md b/README_JP.md index c8569de23..b876bd4ee 100644 --- a/README_JP.md +++ b/README_JP.md @@ -92,16 +92,6 @@ docker compose --profile all up -d --- -## ライブデモ - -**今すぐ試す:** https://demo.langbot.dev/ -- メール: `demo@langbot.app` -- パスワード: `langbot123456` - -*注意: 公開デモ環境です。機密情報を入力しないでください。* - ---- - ## 対応プラットフォーム | プラットフォーム | ステータス | 備考 | diff --git a/README_KO.md b/README_KO.md index 78e8fea55..b28d3ec2c 100644 --- a/README_KO.md +++ b/README_KO.md @@ -92,16 +92,6 @@ docker compose --profile all up -d --- -## 라이브 데모 - -**지금 체험:** https://demo.langbot.dev/ -- 이메일: `demo@langbot.app` -- 비밀번호: `langbot123456` - -*참고: 공개 데모 환경입니다. 민감한 정보를 입력하지 마세요.* - ---- - ## 지원 플랫폼 | 플랫폼 | 상태 | 비고 | diff --git a/README_RU.md b/README_RU.md index 33d03cfdf..f6c1f8bce 100644 --- a/README_RU.md +++ b/README_RU.md @@ -92,16 +92,6 @@ docker compose --profile all up -d --- -## Демо - -**Попробуйте прямо сейчас:** https://demo.langbot.dev/ -- Email: `demo@langbot.app` -- Пароль: `langbot123456` - -*Примечание: Публичная демо-среда. Не вводите конфиденциальную информацию.* - ---- - ## Поддерживаемые платформы | Платформа | Статус | Примечания | diff --git a/README_TW.md b/README_TW.md index 4a046d149..140515650 100644 --- a/README_TW.md +++ b/README_TW.md @@ -94,16 +94,6 @@ docker compose --profile all up -d --- -## 線上演示 - -**立即體驗：** https://demo.langbot.dev/ -- 信箱：`demo@langbot.app` -- 密碼：`langbot123456` - -*注意：公開演示環境，請不要在其中填入任何敏感資訊。* - ---- - ## 支援的平台 | 平台 | 狀態 | 備註 | diff --git a/README_VI.md b/README_VI.md index 50c64c280..356f577b3 100644 --- a/README_VI.md +++ b/README_VI.md @@ -92,16 +92,6 @@ docker compose --profile all up -d --- -## Demo trực tuyến - -**Thử ngay:** https://demo.langbot.dev/ -- Email: `demo@langbot.app` -- Mật khẩu: `langbot123456` - -*Lưu ý: Môi trường demo công khai. Không nhập thông tin nhạy cảm.* - ---- - ## Nền tảng được hỗ trợ | Nền tảng | Trạng thái | Ghi chú | From 38ff4766efedadb177d0ef34c0ec7f5d229e5971 Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 08:22:11 +0800 Subject: [PATCH 5/9] chore: update .gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 459a22c82..83b4faf7f 100644 --- a/.gitignore +++ b/.gitignore @@ -69,3 +69,5 @@ packaging/fnos/ICON_256.PNG packaging/fnos/app/ui/images/ packaging/fnos/app/desktop/images/ packaging/fnos/*.fpk + +r.ps1 From 1143d6a5ae1893f5eb29b9d52b6b4d69866300a8 Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 17:58:23 +0800 Subject: [PATCH 6/9] feat(storage): media content-addressable cache and monitoring base64 externalization - Add MediaCache using xxHash3-128 (with sha256 fallback) content-addressable storage - Externalize message chain image payloads before recording monitoring and discarded messages - Strip base64 payloads to null in SQLite monitoring_messages, dropping row size from megabytes to hundreds of bytes - Add GET /api/v1/files/media/ route with immutable HTTP cache headers to serve cached media - Integrate age-based retention (default 30 days) and configurable disk quota with MaintenanceService cleanup loop - Add defensive sanitizer in MonitoringService.record_message against oversized raw base64 payloads - Add comprehensive unit tests and end-to-end verification covering CAS deduplication, route serving, and LRU pruning --- pyproject.toml | 1 + .../pkg/api/http/controller/groups/files.py | 15 ++ .../pkg/api/http/service/maintenance.py | 31 +++ .../pkg/api/http/service/monitoring.py | 28 +++ src/langbot/pkg/pipeline/monitoring_helper.py | 10 +- src/langbot/pkg/platform/botmgr.py | 5 +- src/langbot/pkg/storage/media.py | 235 ++++++++++++++++++ src/langbot/pkg/storage/mgr.py | 3 + src/langbot/templates/config.yaml | 6 + tests/unit_tests/storage/test_media_cache.py | 215 ++++++++++++++++ uv.lock | 120 ++++----- 11 files changed, 607 insertions(+), 62 deletions(-) create mode 100644 src/langbot/pkg/storage/media.py create mode 100644 tests/unit_tests/storage/test_media_cache.py diff --git a/pyproject.toml b/pyproject.toml index 1dc5e3467..b8880a6b6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -58,6 +58,7 @@ dependencies = [ "python-docx>=1.1.0", "pandas>=2.2.2", "chardet>=5.2.0", + "xxhash>=3.5.0", "markdown>=3.6", "beautifulsoup4>=4.12.3", "ebooklib>=0.18", diff --git a/src/langbot/pkg/api/http/controller/groups/files.py b/src/langbot/pkg/api/http/controller/groups/files.py index 026f6c194..e38c181ae 100644 --- a/src/langbot/pkg/api/http/controller/groups/files.py +++ b/src/langbot/pkg/api/http/controller/groups/files.py @@ -47,6 +47,21 @@ class FilesRouterGroup(group.RouterGroup): return quart.Response(image_bytes, mimetype=mime_type) + @self.route( + '/media/', + methods=['GET'], + auth_type=group.AuthType.NONE, + ) + async def get_media_file(filename: str) -> quart.Response: + media = await self.ap.storage_mgr.media_cache.get_media(filename) + if media is None: + return quart.Response('Media not found or expired', status=404) + media_bytes, mime_type = media + headers = { + 'Cache-Control': 'public, max-age=2592000, immutable', + } + return quart.Response(media_bytes, mimetype=mime_type, headers=headers) + @self.route( '/images', methods=['POST'], diff --git a/src/langbot/pkg/api/http/service/maintenance.py b/src/langbot/pkg/api/http/service/maintenance.py index 0c61618c6..562c17d61 100644 --- a/src/langbot/pkg/api/http/service/maintenance.py +++ b/src/langbot/pkg/api/http/service/maintenance.py @@ -80,6 +80,25 @@ class MaintenanceService: DEFAULT_LOG_RETENTION_DAYS, 'storage.cleanup.log_retention_days', ) + media_cfg = self.ap.instance_config.data.get('storage', {}).get('media_cache', {}) + media_retention_days = self._positive_int( + media_cfg.get('retention_days'), + 30, + 'storage.media_cache.retention_days', + ) + media_max_size_mb = self._non_negative_int( + media_cfg.get('max_size_mb'), + 0, + 'storage.media_cache.max_size_mb', + ) + media_cleanup = ( + await self.ap.storage_mgr.media_cache.cleanup( + media_retention_days, + media_max_size_mb, + ) + if hasattr(self.ap.storage_mgr, 'media_cache') and await self._is_oss_singleton(context) + else {} + ) return { 'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days), @@ -89,6 +108,7 @@ class MaintenanceService: ) if await self._is_oss_singleton(context) else 0, + 'media_files': media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0), } async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]: @@ -466,6 +486,17 @@ class MaintenanceService: count += len(files) return count + def _non_negative_int(self, value: Any, default: int, name: str) -> int: + try: + parsed = int(value) + except (TypeError, ValueError): + self.ap.logger.warning(f'Invalid {name}: {value!r}, using {default}') + return default + if parsed < 0: + self.ap.logger.warning(f'{name} must be non-negative: {value!r}, using {default}') + return default + return parsed + def _positive_int(self, value: Any, default: int, name: str) -> int: try: parsed = int(value) diff --git a/src/langbot/pkg/api/http/service/monitoring.py b/src/langbot/pkg/api/http/service/monitoring.py index c90cec6d1..0bae983a6 100644 --- a/src/langbot/pkg/api/http/service/monitoring.py +++ b/src/langbot/pkg/api/http/service/monitoring.py @@ -1,5 +1,6 @@ from __future__ import annotations +import re import uuid import datetime import functools @@ -417,6 +418,32 @@ class MonitoringService: # ========== Recording Methods ========== + def _sanitize_message_content(self, content: str) -> str: + """Strip raw base64 data to protect database storage from unbounded bloating.""" + if not content or len(content) < 10000 or (';base64,' not in content and 'data:image/' not in content): + return content + try: + data = json.loads(content) + + def _strip_node(node): + if isinstance(node, list): + return [_strip_node(x) for x in node] + if isinstance(node, dict): + res = dict(node) + if res.get('type') == 'Image' and res.get('base64'): + res['base64'] = None + for k, v in list(res.items()): + if isinstance(v, (list, dict)): + res[k] = _strip_node(v) + return res + return node + + return json.dumps(_strip_node(data), ensure_ascii=False) + except Exception: + return re.sub( + r'data:image/[a-zA-Z0-9.+_-]+;base64,[\sA-Za-z0-9+/=]{1000,}', '[base64 image omitted]', content + ) + @_workspace_transaction async def record_message( self, @@ -439,6 +466,7 @@ class MonitoringService: """Record a message""" workspace_uuid = self._require_write_context(context) message_id = str(uuid.uuid4()) + message_content = self._sanitize_message_content(message_content) message_data = { 'id': message_id, 'workspace_uuid': workspace_uuid, diff --git a/src/langbot/pkg/pipeline/monitoring_helper.py b/src/langbot/pkg/pipeline/monitoring_helper.py index 1bab4bda4..15ae38f47 100644 --- a/src/langbot/pkg/pipeline/monitoring_helper.py +++ b/src/langbot/pkg/pipeline/monitoring_helper.py @@ -48,7 +48,10 @@ class MonitoringHelper: # Try to record message # Use JSON serialization to preserve message chain structure (including image URLs, etc.) if hasattr(query, 'message_chain') and hasattr(query.message_chain, 'model_dump'): - message_content = json.dumps(query.message_chain.model_dump(), ensure_ascii=False) + chain_dump = query.message_chain.model_dump() + if hasattr(ap, 'storage_mgr') and hasattr(ap.storage_mgr, 'media_cache'): + chain_dump = await ap.storage_mgr.media_cache.externalize_chain_dump(chain_dump) + message_content = json.dumps(chain_dump, ensure_ascii=False) else: message_content = str(query) @@ -168,7 +171,10 @@ class MonitoringHelper: if hasattr(last_resp, 'get_content_platform_message_chain'): chain = last_resp.get_content_platform_message_chain() if hasattr(chain, 'model_dump'): - message_content = json.dumps(chain.model_dump(), ensure_ascii=False) + chain_dump = chain.model_dump() + if hasattr(ap, 'storage_mgr') and hasattr(ap.storage_mgr, 'media_cache'): + chain_dump = await ap.storage_mgr.media_cache.externalize_chain_dump(chain_dump) + message_content = json.dumps(chain_dump, ensure_ascii=False) else: message_content = str(chain) else: diff --git a/src/langbot/pkg/platform/botmgr.py b/src/langbot/pkg/platform/botmgr.py index feaeea7a9..76a9ff03d 100644 --- a/src/langbot/pkg/platform/botmgr.py +++ b/src/langbot/pkg/platform/botmgr.py @@ -210,7 +210,10 @@ class RuntimeBot: """Record a discarded message in the monitoring system.""" try: if hasattr(message_chain, 'model_dump'): - message_content = json.dumps(message_chain.model_dump(), ensure_ascii=False) + chain_dump = message_chain.model_dump() + if hasattr(self.ap, 'storage_mgr') and hasattr(self.ap.storage_mgr, 'media_cache'): + chain_dump = await self.ap.storage_mgr.media_cache.externalize_chain_dump(chain_dump) + message_content = json.dumps(chain_dump, ensure_ascii=False) else: message_content = str(message_chain) diff --git a/src/langbot/pkg/storage/media.py b/src/langbot/pkg/storage/media.py new file mode 100644 index 000000000..74d882da1 --- /dev/null +++ b/src/langbot/pkg/storage/media.py @@ -0,0 +1,235 @@ +from __future__ import annotations + +import asyncio +import base64 +import copy +import datetime +import hashlib +import mimetypes +import os +import re +from pathlib import Path +from typing import TYPE_CHECKING, Any + +try: + import xxhash +except ImportError: + xxhash = None + +if TYPE_CHECKING: + from ...core import app + from . import mgr as storage_mgr + +DEFAULT_RETENTION_DAYS = 30 +DEFAULT_MAX_SIZE_MB = 0 +MEDIA_DIR = 'media_cache' +SAFE_MEDIA_FILENAME = re.compile(r'^[a-f0-9]{32,64}(\.[a-zA-Z0-9]{1,10})?$') + + +class MediaCache: + """Content-addressable storage cache for images and media attachments. + + Deduplicates media files using xxHash3-128 (with sha256 fallback), + offloads payloads from SQLite to StorageProvider, and implements LRU + and age-based retention cleanup. + """ + + def __init__(self, ap: app.Application, storage_mgr: storage_mgr.StorageMgr): + self.ap = ap + self.storage_mgr = storage_mgr + + @staticmethod + def hash_bytes(data: bytes) -> str: + """Compute content-addressable hash for binary data.""" + if xxhash is not None: + return xxhash.xxh3_128_hexdigest(data) + return hashlib.sha256(data).hexdigest()[:32] + + @staticmethod + def parse_data_url(data_url: str) -> tuple[bytes, str] | None: + """Parse a data URL or raw base64 string into bytes and mime type.""" + if not data_url or not isinstance(data_url, str): + return None + try: + if data_url.startswith('data:'): + split_index = data_url.find(';base64,') + if split_index != -1: + mime_type = data_url[5:split_index] + b64_data = data_url[split_index + 8 :] + return base64.b64decode(b64_data), mime_type + # Try raw base64 if sufficiently long + if len(data_url) > 20 and not data_url.startswith(('http://', 'https://', '/')): + return base64.b64decode(data_url), 'application/octet-stream' + except Exception: + return None + return None + + @staticmethod + def guess_extension(mime_type: str | None, default: str = '.jpg') -> str: + """Guess appropriate file extension from MIME type.""" + if not mime_type: + return default + mime_lower = mime_type.lower() + if 'png' in mime_lower: + return '.png' + if 'webp' in mime_lower: + return '.webp' + if 'gif' in mime_lower: + return '.gif' + if 'jpeg' in mime_lower or 'jpg' in mime_lower: + return '.jpg' + ext = mimetypes.guess_extension(mime_type) + if ext == '.jpe': + return '.jpg' + return ext or default + + async def save_media(self, data: bytes, mime_type: str | None = None) -> tuple[str, str, int]: + """Save media bytes into content-addressable storage cache. + + Returns: + Tuple of (hash_str, storage_key, byte_size) + """ + hash_str = self.hash_bytes(data) + ext = self.guess_extension(mime_type) + storage_key = f'{MEDIA_DIR}/{hash_str}{ext}' + provider = self.storage_mgr.storage_provider + + if not await provider.exists(storage_key): + await provider.save(storage_key, data) + else: + await self.touch(storage_key) + + return hash_str, storage_key, len(data) + + async def get_media(self, filename_or_key: str) -> tuple[bytes, str] | None: + """Retrieve media bytes and mime type by key or filename.""" + filename = os.path.basename(filename_or_key) + if not SAFE_MEDIA_FILENAME.match(filename): + return None + storage_key = f'{MEDIA_DIR}/{filename}' + provider = self.storage_mgr.storage_provider + + if not await provider.exists(storage_key): + return None + + data = await self.storage_mgr._load_object_bounded(storage_key) + mime_type = mimetypes.guess_type(filename)[0] or 'application/octet-stream' + await self.touch(storage_key) + return data, mime_type + + async def touch(self, storage_key: str) -> None: + """Update access/modified time of a media file for LRU tracking.""" + provider = getattr(self.storage_mgr, 'storage_provider', None) + if provider is not None and provider.__class__.__name__ == 'LocalStorageProvider': + full_path = os.path.join('data', 'storage', storage_key) + if os.path.exists(full_path): + now = datetime.datetime.now().timestamp() + try: + await asyncio.to_thread(os.utime, full_path, (now, now)) + except Exception: + pass + + async def cleanup( + self, + retention_days: int = DEFAULT_RETENTION_DAYS, + max_size_mb: int = DEFAULT_MAX_SIZE_MB, + ) -> dict[str, int]: + """Perform age-based and LRU size-based cleanup on media cache. + + Args: + retention_days: Retain media accessed within this many days (default 30). + max_size_mb: Maximum total size in MB (0 means unlimited). + + Returns: + Dictionary of cleanup metrics. + """ + provider = getattr(self.storage_mgr, 'storage_provider', None) + if provider is None or provider.__class__.__name__ != 'LocalStorageProvider': + return {'expired_deleted': 0, 'size_deleted': 0, 'bytes_freed': 0} + + target_dir = Path('data/storage') / MEDIA_DIR + if not target_dir.exists() or not target_dir.is_dir(): + return {'expired_deleted': 0, 'size_deleted': 0, 'bytes_freed': 0} + + now = datetime.datetime.now().timestamp() + cutoff = (now - retention_days * 86400) if retention_days > 0 else 0 + + expired_deleted = 0 + size_deleted = 0 + bytes_freed = 0 + remaining: list[tuple[Path, int, float]] = [] + + for entry in target_dir.iterdir(): + if not entry.is_file(): + continue + try: + stat = entry.stat() + except OSError: + continue + + if cutoff > 0 and stat.st_mtime < cutoff: + try: + entry.unlink(missing_ok=True) + expired_deleted += 1 + bytes_freed += stat.st_size + except OSError: + pass + else: + remaining.append((entry, stat.st_size, stat.st_mtime)) + + if max_size_mb > 0: + max_bytes = max_size_mb * 1024 * 1024 + total_bytes = sum(item[1] for item in remaining) + if total_bytes > max_bytes: + remaining.sort(key=lambda item: item[2]) + for path, size, _ in remaining: + if total_bytes <= max_bytes: + break + try: + path.unlink(missing_ok=True) + size_deleted += 1 + bytes_freed += size + total_bytes -= size + except OSError: + pass + + return { + 'expired_deleted': expired_deleted, + 'size_deleted': size_deleted, + 'bytes_freed': bytes_freed, + } + + async def externalize_chain_dump(self, chain_dump: Any) -> Any: + """Recursively extract raw base64 media into cache and replace with references.""" + if isinstance(chain_dump, list): + return [await self.externalize_chain_dump(item) for item in chain_dump] + if isinstance(chain_dump, dict): + node = copy.copy(chain_dump) + node_type = node.get('type') + if node_type == 'Image': + b64 = node.get('base64') + if b64 and isinstance(b64, str): + try: + parsed = self.parse_data_url(b64) + if parsed is not None: + raw_bytes, mime_type = parsed + hash_str, storage_key, size = await self.save_media(raw_bytes, mime_type) + filename = os.path.basename(storage_key) + current_url = node.get('url') or '' + if current_url and not current_url.startswith('data:'): + node['original_url'] = current_url + node['url'] = f'/api/v1/files/media/{filename}' + node['base64'] = None + node['hash'] = hash_str + node['storage_key'] = storage_key + node['size'] = size + node['mime_type'] = mime_type + except Exception as e: + if hasattr(self.ap, 'logger') and self.ap.logger: + self.ap.logger.warning(f'Failed to externalize image to media cache: {e}') + node['base64'] = None + for k, v in list(node.items()): + if isinstance(v, (list, dict)): + node[k] = await self.externalize_chain_dump(v) + return node + return chain_dump diff --git a/src/langbot/pkg/storage/mgr.py b/src/langbot/pkg/storage/mgr.py index c6c7c8a93..5a96eaa7d 100644 --- a/src/langbot/pkg/storage/mgr.py +++ b/src/langbot/pkg/storage/mgr.py @@ -34,6 +34,9 @@ class StorageMgr: def __init__(self, ap: app.Application): self.ap = ap + from . import media + + self.media_cache = media.MediaCache(ap, self) def _object_read_limit(self) -> int: config = getattr(getattr(self.ap, 'instance_config', None), 'data', {}) diff --git a/src/langbot/templates/config.yaml b/src/langbot/templates/config.yaml index 6b3c716fa..24efaba09 100644 --- a/src/langbot/templates/config.yaml +++ b/src/langbot/templates/config.yaml @@ -227,6 +227,12 @@ storage: # Bound every object materialized into Core memory. Built-in Local/S3 # providers enforce this while reading (hard cap: 64 MiB). max_object_read_bytes: 10485760 + # Media content cache (images & attachments externalized from monitoring and pipelines) + media_cache: + # Retention period in days for cached media (defaults to 30 days) + retention_days: 30 + # Maximum disk storage for cached media in MB (0 means unlimited, defaults to 0) + max_size_mb: 0 cleanup: # Enable periodic cleanup of local/S3 uploaded files and old log files enabled: true diff --git a/tests/unit_tests/storage/test_media_cache.py b/tests/unit_tests/storage/test_media_cache.py new file mode 100644 index 000000000..054ee725b --- /dev/null +++ b/tests/unit_tests/storage/test_media_cache.py @@ -0,0 +1,215 @@ +import base64 +import datetime +import os +import tempfile +from pathlib import Path +from unittest.mock import AsyncMock, Mock, patch +import pytest +from quart import Quart + +from langbot.pkg.storage.media import MediaCache, SAFE_MEDIA_FILENAME +from langbot.pkg.api.http.service.monitoring import MonitoringService +from langbot.pkg.api.http.controller.groups.files import FilesRouterGroup + + +class TestMediaCache: + def setup_method(self): + self.mock_app = Mock() + self.mock_app.logger = Mock() + self.mock_storage_mgr = Mock() + self.mock_provider = Mock() + self.mock_provider.__class__.__name__ = 'LocalStorageProvider' + self.mock_provider.exists = AsyncMock(return_value=False) + self.mock_provider.save = AsyncMock() + self.mock_provider.load = AsyncMock() + self.mock_storage_mgr.storage_provider = self.mock_provider + self.mock_storage_mgr._load_object_bounded = AsyncMock() + self.media_cache = MediaCache(self.mock_app, self.mock_storage_mgr) + + def test_safe_media_filename_regex(self): + assert SAFE_MEDIA_FILENAME.match('3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.png') + assert SAFE_MEDIA_FILENAME.match('3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.jpg') + assert SAFE_MEDIA_FILENAME.match('3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c') + assert not SAFE_MEDIA_FILENAME.match('../etc/passwd') + assert not SAFE_MEDIA_FILENAME.match('foo/bar.png') + assert not SAFE_MEDIA_FILENAME.match('test.exe') + assert not SAFE_MEDIA_FILENAME.match('3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3cXpng') + + def test_hash_bytes(self): + data1 = b'hello image content' + data2 = b'hello image content' + data3 = b'different content' + assert self.media_cache.hash_bytes(data1) == self.media_cache.hash_bytes(data2) + assert self.media_cache.hash_bytes(data1) != self.media_cache.hash_bytes(data3) + + def test_parse_data_url(self): + raw = b'png binary data here' + b64_str = base64.b64encode(raw).decode('ascii') + data_url = f'data:image/png;base64,{b64_str}' + + parsed = self.media_cache.parse_data_url(data_url) + assert parsed is not None + data, mime = parsed + assert data == raw + assert mime == 'image/png' + + @pytest.mark.asyncio + async def test_save_media_deduplication(self): + raw = b'fake png bytes' + hash_str = self.media_cache.hash_bytes(raw) + + # First save: provider.exists is False -> calls provider.save + h1, key1, size1 = await self.media_cache.save_media(raw, 'image/png') + assert h1 == hash_str + assert key1 == f'media_cache/{hash_str}.png' + assert size1 == len(raw) + self.mock_provider.save.assert_called_once_with(key1, raw) + + # Second save: provider.exists is True -> does not call provider.save again + self.mock_provider.exists.return_value = True + self.mock_provider.save.reset_mock() + h2, key2, size2 = await self.media_cache.save_media(raw, 'image/png') + assert h2 == h1 + assert key2 == key1 + self.mock_provider.save.assert_not_called() + + @pytest.mark.asyncio + async def test_get_media(self): + raw = b'stored bytes' + self.mock_provider.exists.return_value = True + self.mock_storage_mgr._load_object_bounded.return_value = raw + + valid_name = '3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.png' + res = await self.media_cache.get_media(valid_name) + assert res is not None + data, mime = res + assert data == raw + assert mime == 'image/png' + + # Rejects invalid names + assert await self.media_cache.get_media('../malicious.png') is None + + @pytest.mark.asyncio + async def test_externalize_chain_dump(self): + raw = b'tiny image' + b64 = f'data:image/png;base64,{base64.b64encode(raw).decode("ascii")}' + chain_dump = [ + {'type': 'Plain', 'text': 'hello'}, + {'type': 'Image', 'url': 'https://multimedia.nt.qq.com.cn/download?appid=1407', 'base64': b64}, + {'type': 'Quote', 'origin': [{'type': 'Image', 'url': '', 'base64': b64}]}, + ] + + result = await self.media_cache.externalize_chain_dump(chain_dump) + + # Root image + img = result[1] + assert img['base64'] is None + assert img['hash'] == self.media_cache.hash_bytes(raw) + assert img['storage_key'].startswith('media_cache/') + assert img['original_url'] == 'https://multimedia.nt.qq.com.cn/download?appid=1407' + assert img['url'] == f'/api/v1/files/media/{img["hash"]}.png' + assert img['size'] == len(raw) + + # Nested quote image + nested_img = result[2]['origin'][0] + assert nested_img['base64'] is None + assert nested_img['hash'] == self.media_cache.hash_bytes(raw) + assert nested_img['url'] == f'/api/v1/files/media/{nested_img["hash"]}.png' + + @pytest.mark.asyncio + async def test_externalize_chain_dump_error_resilience(self): + raw = b'broken image' + b64 = f'data:image/png;base64,{base64.b64encode(raw).decode("ascii")}' + chain_dump = [{'type': 'Image', 'base64': b64}] + + with patch.object(self.media_cache, 'save_media', side_effect=OSError('Disk full')): + result = await self.media_cache.externalize_chain_dump(chain_dump) + # Should not raise; base64 should be stripped as fallback + assert result[0]['base64'] is None + + @pytest.mark.asyncio + async def test_cleanup_retention_and_max_size(self): + with tempfile.TemporaryDirectory() as temp_dir: + base_path = Path(temp_dir) + cache_dir = base_path / 'data' / 'storage' / 'media_cache' + cache_dir.mkdir(parents=True) + + # Create 3 test files with different mtimes and sizes + f1 = cache_dir / 'old_expired.png' + f1.write_bytes(b'x' * 1000) + old_time = (datetime.datetime.now() - datetime.timedelta(days=35)).timestamp() + os.utime(f1, (old_time, old_time)) + + f2 = cache_dir / 'recent_large1.png' + f2.write_bytes(b'x' * 500) + t2 = (datetime.datetime.now() - datetime.timedelta(days=5)).timestamp() + os.utime(f2, (t2, t2)) + + f3 = cache_dir / 'recent_large2.png' + f3.write_bytes(b'x' * 500) + t3 = (datetime.datetime.now() - datetime.timedelta(days=1)).timestamp() + os.utime(f3, (t3, t3)) + + with patch('langbot.pkg.storage.media.Path') as mock_path: + mock_path.return_value = base_path / 'data' / 'storage' + # Run cleanup with 30-day retention and max_size_mb = 0 (unlimited) + stats = await self.media_cache.cleanup(retention_days=30, max_size_mb=0) + assert stats['expired_deleted'] == 1 + assert not f1.exists() + assert f2.exists() + assert f3.exists() + + # Run cleanup with max_size_mb limited to ~0.0006 MB (< 1000 bytes) + # Total is currently 1000 bytes (f2=500 + f3=500). Max size 600 bytes -> oldest f2 must be purged + stats2 = await self.media_cache.cleanup(retention_days=30, max_size_mb=0.0006) + assert stats2['size_deleted'] >= 1 + assert not f2.exists() + assert f3.exists() + + @pytest.mark.asyncio + async def test_files_media_endpoint(self): + quart_app = Quart(__name__) + mock_app = Mock() + mock_app.storage_mgr = self.mock_storage_mgr + + router = FilesRouterGroup(mock_app, quart_app) + await router.initialize() + + client = quart_app.test_client() + + # 1. 404 on not found + mock_cache = Mock() + mock_cache.get_media = AsyncMock(return_value=None) + self.mock_storage_mgr.media_cache = mock_cache + resp_404 = await client.get('/api/v1/files/media/3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.png') + assert resp_404.status_code == 404 + + # 2. 200 on found with cache headers + mock_cache.get_media.return_value = (b'fake image data', 'image/png') + resp_200 = await client.get('/api/v1/files/media/3f8a9b1c2d3e4f5a6b7c8d9e0f1a2b3c.png') + assert resp_200.status_code == 200 + assert await resp_200.get_data() == b'fake image data' + assert 'public' in resp_200.headers.get('Cache-Control', '') + assert 'image/png' in resp_200.headers.get('Content-Type', '') + + +class TestMonitoringServiceSanitization: + def test_sanitize_oversized_base64_payload(self): + svc = MonitoringService.__new__(MonitoringService) + small_content = '{"type": "Image", "base64": "data:image/png;base64,tiny"}' + # Should leave small contents untouched + assert svc._sanitize_message_content(small_content) == small_content + + # Large content with base64 data URL + huge_b64 = 'A' * 60000 + large_content = f'{{"type": "Image", "base64": "data:image/png;base64,{huge_b64}"}}' + sanitized = svc._sanitize_message_content(large_content) + assert huge_b64 not in sanitized + assert '"base64": null' in sanitized or '"base64":null' in sanitized + + # Non-JSON content with multi-line base64 + multiline_b64 = ('A' * 70 + '\r\n') * 300 + raw_corrupted = 'prefix data:image/png;base64,' + multiline_b64 + ' suffix' + sanitized_raw = svc._sanitize_message_content(raw_corrupted) + assert '[base64 image omitted]' in sanitized_raw + assert multiline_b64 not in sanitized_raw diff --git a/uv.lock b/uv.lock index 8d718eb73..107889d6b 100644 --- a/uv.lock +++ b/uv.lock @@ -1066,7 +1066,7 @@ name = "cuda-bindings" version = "13.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-pathfinder", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "cuda-pathfinder" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/51/6b/457ca12dad3ee9bfcc9a545cfd6b64b359ba49de40f776f6e028e678f262/cuda_bindings-13.3.1-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c5879712accf6e14bb01aa5e67440eb84998b8d104b509cc7a6dc0b8f656a474", size = 6053539, upload-time = "2026-05-29T23:11:43.19Z" }, @@ -1099,34 +1099,34 @@ wheels = [ [package.optional-dependencies] cudart = [ - { name = "nvidia-cuda-runtime", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime" }, ] cufft = [ - { name = "nvidia-cufft", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cufft" }, ] cufile = [ - { name = "nvidia-cufile", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cufile" }, ] cupti = [ - { name = "nvidia-cuda-cupti", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti" }, ] curand = [ - { name = "nvidia-curand", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-curand" }, ] cusolver = [ - { name = "nvidia-cusolver", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver" }, ] cusparse = [ - { name = "nvidia-cusparse", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse" }, ] nvjitlink = [ - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink" }, ] nvrtc = [ - { name = "nvidia-cuda-nvrtc", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc" }, ] nvtx = [ - { name = "nvidia-nvtx", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx" }, ] [[package]] @@ -2135,6 +2135,7 @@ dependencies = [ { name = "valkey-glide", marker = "sys_platform != 'win32'" }, { name = "webauthn" }, { name = "websockets" }, + { name = "xxhash" }, ] [package.optional-dependencies] @@ -2233,6 +2234,7 @@ requires-dist = [ { name = "valkey-glide", marker = "sys_platform != 'win32'", specifier = ">=2.4.1,<3.0.0" }, { name = "webauthn", specifier = ">=3.0.0" }, { name = "websockets", specifier = ">=15.0.1" }, + { name = "xxhash", specifier = ">=3.5.0" }, ] provides-extras = ["seekdb"] @@ -3299,7 +3301,7 @@ name = "nvidia-cublas" version = "13.1.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cuda-nvrtc", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cuda-nvrtc" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/a7/a1/0bd24ee8c8d03adac032fd2909426a00c88f8c57961b1277ded97f91119f/nvidia_cublas-13.1.1.3-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:b7a210458267ac818974c53038fbec2e969d5c99f305ab15c72522fa9f001dd5", size = 542848918, upload-time = "2026-04-08T18:46:22.985Z" }, @@ -3338,7 +3340,7 @@ name = "nvidia-cudnn-cu13" version = "9.20.0.48" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cublas" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/56/c5/83384d846b2fd17c44bd499b36c75a45ed4f095fbbb2252294e89cea5c5c/nvidia_cudnn_cu13-9.20.0.48-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:e31454ae00094b0c55319d9d15b6fa2fc50a9e1c0f5c8c80fb75258234e731e1", size = 444574296, upload-time = "2026-03-09T19:28:27.751Z" }, @@ -3350,7 +3352,7 @@ name = "nvidia-cufft" version = "12.0.0.61" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/8b/ae/f417a75c0259e85c1d2f83ca4e960289a5f814ed0cea74d18c353d3e989d/nvidia_cufft-12.0.0.61-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2708c852ef8cd89d1d2068bdbece0aa188813a0c934db3779b9b1faa8442e5f5", size = 214053554, upload-time = "2025-09-04T08:31:38.196Z" }, @@ -3380,9 +3382,9 @@ name = "nvidia-cusolver" version = "12.0.4.66" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "nvidia-cusparse", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-cublas" }, + { name = "nvidia-cusparse" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/c8/c3/b30c9e935fc01e3da443ec0116ed1b2a009bb867f5324d3f2d7e533e776b/nvidia_cusolver-12.0.4.66-py3-none-manylinux_2_27_aarch64.whl", hash = "sha256:02c2457eaa9e39de20f880f4bd8820e6a1cfb9f9a34f820eb12a155aa5bc92d2", size = 223467760, upload-time = "2025-09-04T08:33:04.222Z" }, @@ -3394,7 +3396,7 @@ name = "nvidia-cusparse" version = "12.6.3.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink", marker = "python_full_version >= '3.14' and sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "nvidia-nvjitlink" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/f8/94/5c26f33738ae35276672f12615a64bd008ed5be6d1ebcb23579285d960a9/nvidia_cusparse-12.6.3.3-py3-none-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:80bcc4662f23f1054ee334a15c72b8940402975e0eab63178fc7e670aa59472c", size = 162155568, upload-time = "2025-09-04T08:33:42.864Z" }, @@ -4487,7 +4489,7 @@ name = "pylibseekdb" version = "1.4.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pymysql", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "pymysql" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/ae/a8/7413d33218aff55a14ec9d20532b49243ffd0579e7a92244922c1885444e/pylibseekdb-1.4.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:5cb2efab9f1321cdb4b034d3a2bd92e41a402fc95e7dc9579c7473a426f96e24", size = 52173499, upload-time = "2026-08-27T13:05:09.347Z" }, @@ -5255,10 +5257,10 @@ name = "scikit-learn" version = "1.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "joblib", marker = "python_full_version >= '3.14'" }, - { name = "numpy", marker = "python_full_version >= '3.14'" }, - { name = "scipy", marker = "python_full_version >= '3.14'" }, - { name = "threadpoolctl", marker = "python_full_version >= '3.14'" }, + { name = "joblib" }, + { name = "numpy" }, + { name = "scipy" }, + { name = "threadpoolctl" }, ] sdist = { url = "https://files.pythonhosted.org/packages/0e/d4/40988bf3b8e34feec1d0e6a051446b1f66225f8529b9309becaeef62b6c4/scikit_learn-1.8.0.tar.gz", hash = "sha256:9bccbb3b40e3de10351f8f5068e105d0f4083b1a65fa07b6634fbc401a6287fd", size = 7335585, upload-time = "2025-12-10T07:08:53.618Z" } wheels = [ @@ -5305,7 +5307,7 @@ name = "scipy" version = "1.17.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "numpy", marker = "python_full_version >= '3.14'" }, + { name = "numpy" }, ] sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } wheels = [ @@ -5376,14 +5378,14 @@ name = "sentence-transformers" version = "5.2.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub", marker = "python_full_version >= '3.14'" }, - { name = "numpy", marker = "python_full_version >= '3.14'" }, - { name = "scikit-learn", marker = "python_full_version >= '3.14'" }, - { name = "scipy", marker = "python_full_version >= '3.14'" }, - { name = "torch", marker = "python_full_version >= '3.14'" }, - { name = "tqdm", marker = "python_full_version >= '3.14'" }, - { name = "transformers", marker = "python_full_version >= '3.14'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.14'" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "scikit-learn" }, + { name = "scipy" }, + { name = "torch" }, + { name = "tqdm" }, + { name = "transformers" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/5b/30/21664028fc0776eb1ca024879480bbbab36f02923a8ff9e4cae5a150fa35/sentence_transformers-5.2.3.tar.gz", hash = "sha256:3cd3044e1f3fe859b6a1b66336aac502eaae5d3dd7d5c8fc237f37fbf58137c7", size = 381623, upload-time = "2026-02-17T14:05:20.238Z" } wheels = [ @@ -5756,21 +5758,21 @@ name = "torch" version = "2.12.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "cuda-bindings", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "filelock", marker = "python_full_version >= '3.14'" }, - { name = "fsspec", marker = "python_full_version >= '3.14'" }, - { name = "jinja2", marker = "python_full_version >= '3.14'" }, - { name = "networkx", marker = "python_full_version >= '3.14'" }, - { name = "nvidia-cublas", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "nvidia-cusparselt-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "nvidia-nvshmem-cu13", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "setuptools", marker = "python_full_version >= '3.14'" }, - { name = "sympy", marker = "python_full_version >= '3.14'" }, - { name = "triton", marker = "python_full_version >= '3.14' and sys_platform == 'linux'" }, - { name = "typing-extensions", marker = "python_full_version >= '3.14'" }, + { name = "cuda-bindings", marker = "sys_platform == 'linux'" }, + { name = "cuda-toolkit", extra = ["cudart", "cufft", "cufile", "cupti", "curand", "cusolver", "cusparse", "nvjitlink", "nvrtc", "nvtx"], marker = "sys_platform == 'linux'" }, + { name = "filelock" }, + { name = "fsspec" }, + { name = "jinja2" }, + { name = "networkx" }, + { name = "nvidia-cublas", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparselt-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu13", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvshmem-cu13", marker = "sys_platform == 'linux'" }, + { name = "setuptools" }, + { name = "sympy" }, + { name = "triton", marker = "sys_platform == 'linux'" }, + { name = "typing-extensions" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/59/38/7028d3be540f1dcdf41660a2b01d0c51d2cb73915fe370d84e4d277a6d47/torch-2.12.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:ef81f503912effea2ce3d9b12a2e3a6ed488943e91271c90c7a829f60baf6aa2", size = 87975425, upload-time = "2026-06-17T21:08:34.094Z" }, @@ -5812,15 +5814,15 @@ name = "transformers" version = "5.3.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "huggingface-hub", marker = "python_full_version >= '3.14'" }, - { name = "numpy", marker = "python_full_version >= '3.14'" }, - { name = "packaging", marker = "python_full_version >= '3.14'" }, - { name = "pyyaml", marker = "python_full_version >= '3.14'" }, - { name = "regex", marker = "python_full_version >= '3.14'" }, - { name = "safetensors", marker = "python_full_version >= '3.14'" }, - { name = "tokenizers", marker = "python_full_version >= '3.14'" }, - { name = "tqdm", marker = "python_full_version >= '3.14'" }, - { name = "typer", marker = "python_full_version >= '3.14'" }, + { name = "huggingface-hub" }, + { name = "numpy" }, + { name = "packaging" }, + { name = "pyyaml" }, + { name = "regex" }, + { name = "safetensors" }, + { name = "tokenizers" }, + { name = "tqdm" }, + { name = "typer" }, ] sdist = { url = "https://files.pythonhosted.org/packages/fc/1a/70e830d53ecc96ce69cfa8de38f163712d2b43ac52fbd743f39f56025c31/transformers-5.3.0.tar.gz", hash = "sha256:009555b364029da9e2946d41f1c5de9f15e6b1df46b189b7293f33a161b9c557", size = 8830831, upload-time = "2026-03-04T17:41:46.119Z" } wheels = [ @@ -6081,9 +6083,9 @@ name = "valkey-glide" version = "2.4.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "anyio", marker = "sys_platform != 'win32'" }, - { name = "protobuf", marker = "sys_platform != 'win32'" }, - { name = "sniffio", marker = "sys_platform != 'win32'" }, + { name = "anyio" }, + { name = "protobuf" }, + { name = "sniffio" }, ] sdist = { url = "https://files.pythonhosted.org/packages/72/a2/582b34c6acc8dc857c537f6007459cba48dfa0dc404789a657e5c1a998c0/valkey_glide-2.4.1.tar.gz", hash = "sha256:f1155d84156d11b90488aa67e90102f0bf98a45314f5b99308ac9074c05f7241", size = 898030, upload-time = "2026-05-28T21:41:55.881Z" } wheels = [ From a32b3121d3e1525470b40cb62a53401f21777485 Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 18:14:29 +0800 Subject: [PATCH 7/9] build(boot): register xxhash in startup dependency check --- src/langbot/pkg/core/bootutils/deps.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/langbot/pkg/core/bootutils/deps.py b/src/langbot/pkg/core/bootutils/deps.py index 2cfd57e0c..b13d138b6 100644 --- a/src/langbot/pkg/core/bootutils/deps.py +++ b/src/langbot/pkg/core/bootutils/deps.py @@ -43,6 +43,7 @@ required_deps = { 'slack_sdk': 'slack_sdk', 'asyncpg': 'asyncpg', 'litellm': 'litellm', + 'xxhash': 'xxhash', } From c16cd433e0c53c89a6891df76cdce165d3fcffa4 Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 19:48:08 +0800 Subject: [PATCH 8/9] refactor(storage): enforce xxHash3-128 and remove sha256 fallback in MediaCache - Import xxhash directly instead of conditional try-except block - Remove unused hashlib import and sha256 fallback branch in hash_bytes - Update docstrings to reflect pure xxHash3-128 content-addressing --- src/langbot/pkg/storage/media.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/src/langbot/pkg/storage/media.py b/src/langbot/pkg/storage/media.py index 74d882da1..0029013ff 100644 --- a/src/langbot/pkg/storage/media.py +++ b/src/langbot/pkg/storage/media.py @@ -4,18 +4,13 @@ import asyncio import base64 import copy import datetime -import hashlib import mimetypes import os import re +import xxhash from pathlib import Path from typing import TYPE_CHECKING, Any -try: - import xxhash -except ImportError: - xxhash = None - if TYPE_CHECKING: from ...core import app from . import mgr as storage_mgr @@ -29,7 +24,7 @@ SAFE_MEDIA_FILENAME = re.compile(r'^[a-f0-9]{32,64}(\.[a-zA-Z0-9]{1,10})?$') class MediaCache: """Content-addressable storage cache for images and media attachments. - Deduplicates media files using xxHash3-128 (with sha256 fallback), + Deduplicates media files using xxHash3-128, offloads payloads from SQLite to StorageProvider, and implements LRU and age-based retention cleanup. """ @@ -41,9 +36,7 @@ class MediaCache: @staticmethod def hash_bytes(data: bytes) -> str: """Compute content-addressable hash for binary data.""" - if xxhash is not None: - return xxhash.xxh3_128_hexdigest(data) - return hashlib.sha256(data).hexdigest()[:32] + return xxhash.xxh3_128_hexdigest(data) @staticmethod def parse_data_url(data_url: str) -> tuple[bytes, str] | None: From 7e239f162942059a66a705a6af0720778262704b Mon Sep 17 00:00:00 2001 From: BiFangKNT <1320414964@qq.com> Date: Tue, 15 Sep 2026 20:30:34 +0800 Subject: [PATCH 9/9] fix(maintenance): guard media cache cleanup and annotate exception suppression - Safely resolve media_cache via getattr in cleanup_expired_files to prevent AttributeError when storage_mgr is unset in cloud/test fixtures - Only attach 'media_files' in cleanup return dictionary when media cache is active on singleton, preserving exact return contract for cloud maintenance tests - Add explanatory comments to exception suppression in MediaCache.touch and cleanup loops to address code-quality review findings --- src/langbot/pkg/api/http/service/maintenance.py | 14 +++++++++----- src/langbot/pkg/storage/media.py | 5 ++++- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/langbot/pkg/api/http/service/maintenance.py b/src/langbot/pkg/api/http/service/maintenance.py index 562c17d61..3ba4e77c5 100644 --- a/src/langbot/pkg/api/http/service/maintenance.py +++ b/src/langbot/pkg/api/http/service/maintenance.py @@ -91,25 +91,29 @@ class MaintenanceService: 0, 'storage.media_cache.max_size_mb', ) + media_cache = getattr(getattr(self.ap, 'storage_mgr', None), 'media_cache', None) + is_singleton = await self._is_oss_singleton(context) media_cleanup = ( - await self.ap.storage_mgr.media_cache.cleanup( + await media_cache.cleanup( media_retention_days, media_max_size_mb, ) - if hasattr(self.ap.storage_mgr, 'media_cache') and await self._is_oss_singleton(context) + if media_cache is not None and is_singleton else {} ) - return { + result = { 'uploaded_files': await self._cleanup_expired_uploaded_files(context, upload_retention_days), 'log_files': await asyncio.to_thread( self._cleanup_expired_log_files, log_retention_days, ) - if await self._is_oss_singleton(context) + if is_singleton else 0, - 'media_files': media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0), } + if media_cache is not None and is_singleton: + result['media_files'] = media_cleanup.get('expired_deleted', 0) + media_cleanup.get('size_deleted', 0) + return result async def get_storage_analysis(self, context: TenantContext) -> dict[str, Any]: require_workspace_uuid(context) diff --git a/src/langbot/pkg/storage/media.py b/src/langbot/pkg/storage/media.py index 0029013ff..8f5537fb3 100644 --- a/src/langbot/pkg/storage/media.py +++ b/src/langbot/pkg/storage/media.py @@ -120,6 +120,7 @@ class MediaCache: try: await asyncio.to_thread(os.utime, full_path, (now, now)) except Exception: + # Failures to update mtime are intentionally ignored because LRU touch is opportunistic. pass async def cleanup( @@ -166,6 +167,7 @@ class MediaCache: expired_deleted += 1 bytes_freed += stat.st_size except OSError: + # Best-effort cleanup; file may already be gone or temporarily inaccessible. pass else: remaining.append((entry, stat.st_size, stat.st_mtime)) @@ -184,7 +186,8 @@ class MediaCache: bytes_freed += size total_bytes -= size except OSError: - pass + # Best-effort cleanup; file may already be gone or temporarily inaccessible. + continue return { 'expired_deleted': expired_deleted,