From 43c0ba7e44cdb66d431f68bf29fedde6cb0e48c7 Mon Sep 17 00:00:00 2001 From: RockChinQ Date: Mon, 2 Nov 2026 23:51:22 +0800 Subject: [PATCH] feat(speech): implement TTS and STT model layer and pipeline stages (LANG-251) (#2839) * feat(speech): implement TTS and STT model layer and pipeline stages (LANG-251) Co-authored-by: multica-agent * fix(speech): fix invoke signatures and provider lookup in speech stages and model testing Co-authored-by: multica-agent * feat(speech): rename STT to ASR, improve TTS voice parameter handling and failure recovery Co-authored-by: multica-agent * fix(persistence, lint, web): fix migration revision id, legacy table check, ruff duplicates, and frontend types Co-authored-by: multica-agent * fix(speech): restore migration and CI contracts Co-authored-by: multica-agent * fix(pipeline,ui): fix select dropdown width gap, add speech to guided tour, and configure show_if for speech options Co-authored-by: multica-agent * feat(speech,openclaw): support silk audio transcoding, graceful fallback, duration estimation, and multi-lang deploy docs Co-authored-by: multica-agent * fix(audio): enhance ffmpeg executable detection via venv and imageio_ffmpeg Co-authored-by: multica-agent * fix(provider): improve LiteLLM error classification and avoid falling back to Unknown error Co-authored-by: multica-agent * fix(openclaw): serialize voice_item attributes and improve transcoding fallback logging Co-authored-by: multica-agent * fix(audio,openclaw): handle transient TMPDIR deletion and add size attribute to voice_item Co-authored-by: multica-agent * fix(openclaw): download and convert inbound voice messages and align voice sending protocol Co-authored-by: multica-agent * fix(models): redesign add model popover tabs layout and include ASR/TTS models in provider counts and scanning Co-authored-by: multica-agent * fix(audio): decrypt and transcode wecombot voice to mp3, align openclaw weixin voice delivery Co-authored-by: multica-agent * fix(wecombot): transcode outbound voice to silk and support voice_delivery mode Co-authored-by: multica-agent * fix(discord,web): map audio attachments to voice for ASR, prefer base64 voice in webui Co-authored-by: multica-agent * fix(telegram): deliver voice replies in streaming and non-streaming modes with quote origin Co-authored-by: multica-agent * feat(dingtalk): route voice messages through pipeline ASR and send TTS replies as voice Inbound DingTalk audio now arrives as a Voice component with its duration, so the pipeline speech recognition switch decides how it is transcribed. DingTalk's own recognition text is used only when the audio cannot be downloaded. Outbound Voice components are transcoded to ogg/opus, uploaded as voice media and sent as native sampleAudio messages in both the omni and legacy adapters. Audio over 60 seconds, or audio that cannot be transcoded or is rejected as a voice message, is sent as a file instead. Voice parts no longer leak into the reply text as "[Voice]". Also estimate ogg durations from the granule position and ignore non-audio MIME hints when extracting voice bytes, so ffmpeg can probe the input. Co-Authored-By: Claude Opus 5.5 Co-authored-by: multica-agent * fix(ci): include speech model counts in provider responses and fix lint/image build - Copy every count returned by get_provider_model_counts into the provider payload instead of indexing asr_count/tts_count, and update the test fakes to return the speech model counts. - Install libc6-dev in the runtime image so the pilk audio extra can compile. - Apply ruff and prettier formatting required by the lint jobs. Co-Authored-By: Claude Opus 5.5 Co-authored-by: multica-agent * test(e2e): expect the speech step in the pipeline setup guide Co-Authored-By: Claude Opus 5.5 Co-authored-by: multica-agent --------- Co-authored-by: multica-agent Co-authored-by: Claude Opus 5.5 --- Dockerfile | 4 +- docs/en/deploy/langbot/docker.mdx | 4 + docs/en/deploy/langbot/manual.mdx | 14 + docs/en/deploy/langbot/package.mdx | 12 + docs/zh/deploy/langbot/docker.mdx | 4 + docs/zh/deploy/langbot/manual.mdx | 14 + docs/zh/deploy/langbot/package.mdx | 12 + pyproject.toml | 3 + skills/skills/langbot-mcp-ops/SKILL.md | 2 +- src/langbot/libs/dingtalk_api/api.py | 90 ++- .../libs/dingtalk_api/dingtalkevent.py | 4 + .../libs/openclaw_weixin_api/client.py | 27 + src/langbot/libs/openclaw_weixin_api/types.py | 1 + src/langbot/libs/wecom_ai_bot_api/api.py | 53 +- src/langbot/libs/wecom_api/api.py | 9 + src/langbot/pkg/agent/runner/config_schema.py | 6 + .../http/controller/groups/provider/models.py | 226 ++++++++ .../controller/groups/provider/providers.py | 8 +- src/langbot/pkg/api/http/service/model.py | 518 ++++++++++++++++++ src/langbot/pkg/api/http/service/pipeline.py | 3 + .../api/http/service/pipeline_migration.py | 18 +- src/langbot/pkg/api/http/service/provider.py | 46 +- src/langbot/pkg/api/mcp/server.py | 10 + src/langbot/pkg/core/app.py | 5 + src/langbot/pkg/core/stages/build_app.py | 6 + src/langbot/pkg/core/stages/load_config.py | 1 + src/langbot/pkg/entity/persistence/model.py | 68 +++ src/langbot/pkg/i18n/locales/en_US.json | 5 +- src/langbot/pkg/i18n/locales/es_ES.json | 5 +- src/langbot/pkg/i18n/locales/ja_JP.json | 5 +- src/langbot/pkg/i18n/locales/ru_RU.json | 5 +- src/langbot/pkg/i18n/locales/th_TH.json | 5 +- src/langbot/pkg/i18n/locales/vi_VN.json | 5 +- src/langbot/pkg/i18n/locales/zh_Hans.json | 5 +- src/langbot/pkg/i18n/locales/zh_Hant.json | 5 +- .../alembic/versions/0043_speech_models.py | 183 +++++++ src/langbot/pkg/persistence/mgr.py | 2 + src/langbot/pkg/persistence/tenant_uow.py | 2 + src/langbot/pkg/pipeline/pipelinemgr.py | 3 + src/langbot/pkg/pipeline/speech/__init__.py | 1 + src/langbot/pkg/pipeline/speech/speech.py | 308 +++++++++++ .../adapters/aiocqhttp/message_converter.py | 13 +- .../pkg/platform/adapters/dingtalk/adapter.py | 14 +- .../adapters/dingtalk/message_converter.py | 9 +- .../adapters/discord/message_converter.py | 75 ++- .../adapters/lark/message_converter.py | 7 + .../pkg/platform/adapters/telegram/adapter.py | 24 +- .../pkg/platform/adapters/wecombot/adapter.py | 88 ++- .../platform/adapters/wecombot/manifest.yaml | 23 + .../adapters/wecombot/message_converter.py | 39 +- src/langbot/pkg/platform/sources/dingtalk.py | 26 +- src/langbot/pkg/platform/sources/discord.py | 101 +++- src/langbot/pkg/platform/sources/lark.py | 7 + .../pkg/platform/sources/openclaw_weixin.py | 175 +++++- .../pkg/platform/sources/openclaw_weixin.yaml | 23 + src/langbot/pkg/platform/sources/telegram.py | 112 +++- src/langbot/pkg/platform/sources/wecombot.py | 170 +++++- .../pkg/platform/sources/wecombot.yaml | 23 + src/langbot/pkg/provider/modelmgr/modelmgr.py | 214 ++++++++ .../pkg/provider/modelmgr/requester.py | 182 +++++- .../modelmgr/requesters/302aichatcmpl.yaml | 2 + .../modelmgr/requesters/chatcmpl.yaml | 2 + .../modelmgr/requesters/groqchatcmpl.yaml | 1 + .../modelmgr/requesters/litellmchat.py | 186 ++++++- .../modelmgr/requesters/litellmchat.yaml | 2 + .../modelmgr/requesters/newapichatcmpl.yaml | 2 + .../requesters/siliconflowchatcmpl.yaml | 2 + src/langbot/pkg/utils/audio.py | 358 ++++++++++++ .../templates/default-pipeline-config.json | 18 + .../templates/metadata/pipeline/speech.yaml | 209 +++++++ tests/e2e/test_startup.py | 2 +- tests/integration/api/test_providers.py | 7 +- .../persistence/test_migrations.py | 54 ++ .../api/service/test_pipeline_service.py | 6 +- .../api/service/test_provider_cascade.py | 20 +- .../api/service/test_provider_service.py | 19 +- .../api/test_provider_controller_secrets.py | 4 +- .../api/test_resource_secret_permissions.py | 2 +- .../unit_tests/pipeline/test_speech_stages.py | 406 ++++++++++++++ .../unit_tests/platform/test_dingtalk_api.py | 97 ++++ .../platform/test_dingtalk_eba_adapter.py | 44 ++ .../platform/test_discord_eba_adapter.py | 32 ++ .../platform/test_openclaw_weixin_client.py | 65 +++ .../platform/test_openclaw_weixin_tenancy.py | 41 ++ .../platform/test_telegram_adapter.py | 120 ++++ .../platform/test_telegram_eba_adapter.py | 18 + .../platform/test_wecombot_media.py | 23 + tests/unit_tests/provider/test_litellmchat.py | 28 + tests/unit_tests/test_audio_utils.py | 95 ++++ tests/unit_tests/test_speech_models_api.py | 279 ++++++++++ uv.lock | 18 +- .../bot-session/BotSessionMonitor.tsx | 2 +- .../dynamic-form/DynamicFormComponent.tsx | 2 + .../dynamic-form/DynamicFormItemComponent.tsx | 148 +++++ .../dynamic-form/DynamicFormSaveValues.ts | 2 + .../components/models-dialog/ModelsPanel.tsx | 81 ++- .../components/AddModelPopover.tsx | 70 ++- .../models-dialog/components/ModelItem.tsx | 6 +- .../models-dialog/components/ProviderCard.tsx | 134 ++++- .../home/components/models-dialog/types.ts | 8 +- .../components/debug-dialog/DebugDialog.tsx | 2 +- .../pipeline-form/PipelineFormComponent.tsx | 28 +- web/src/app/infra/entities/api/index.ts | 35 ++ web/src/app/infra/entities/form/dynamic.ts | 2 + web/src/app/infra/http/BackendClient.ts | 70 ++- web/src/components/ui/select.tsx | 6 +- web/src/i18n/locales/en-US.ts | 10 + web/src/i18n/locales/es-ES.ts | 12 +- web/src/i18n/locales/ja-JP.ts | 14 +- web/src/i18n/locales/ru-RU.ts | 12 +- web/src/i18n/locales/th-TH.ts | 12 +- web/src/i18n/locales/vi-VN.ts | 12 +- web/src/i18n/locales/zh-Hans.ts | 10 + web/src/i18n/locales/zh-Hant.ts | 10 + web/tests/e2e/contextual-guides.spec.ts | 5 +- .../e2e/processor-detail-workbench.spec.ts | 9 +- 116 files changed, 5640 insertions(+), 256 deletions(-) create mode 100644 src/langbot/pkg/persistence/alembic/versions/0043_speech_models.py create mode 100644 src/langbot/pkg/pipeline/speech/__init__.py create mode 100644 src/langbot/pkg/pipeline/speech/speech.py create mode 100644 src/langbot/pkg/utils/audio.py create mode 100644 src/langbot/templates/metadata/pipeline/speech.yaml create mode 100644 tests/unit_tests/pipeline/test_speech_stages.py create mode 100644 tests/unit_tests/test_audio_utils.py create mode 100644 tests/unit_tests/test_speech_models_api.py diff --git a/Dockerfile b/Dockerfile index 8c6310b53..ef68a93bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ COPY --from=node /app/web/dist ./web/dist COPY --from=nsjail-build /usr/local/bin/nsjail /usr/local/bin/nsjail RUN apt-get update \ - && apt-get install -y --no-install-recommends gcc ca-certificates curl git gnupg \ + && apt-get install -y --no-install-recommends gcc libc6-dev ca-certificates curl git gnupg ffmpeg \ # nsjail runtime libraries (the build toolchain stays in the nsjail-build # stage; only these shared libs are needed to execute the binary). && apt-get install -y --no-install-recommends libprotobuf32t64 libnl-route-3-200 \ @@ -73,7 +73,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends nodejs \ && rm -f /tmp/nodesource_setup.sh \ && python -m pip install --no-cache-dir uv \ - && uv sync --extra seekdb \ + && uv sync --extra seekdb --extra audio \ && apt-get purge -y --auto-remove curl git gnupg \ && rm -rf /var/lib/apt/lists/* \ && touch /.dockerenv diff --git a/docs/en/deploy/langbot/docker.mdx b/docs/en/deploy/langbot/docker.mdx index b13b8ba08..5e08559a9 100644 --- a/docs/en/deploy/langbot/docker.mdx +++ b/docs/en/deploy/langbot/docker.mdx @@ -44,6 +44,10 @@ docker compose --profile all up -d The first startup downloads the container images. On Windows / macOS, keep Docker Desktop running. + +The official Docker images come pre-equipped with minimal `ffmpeg` and `pilk` audio tools. Native voice transcoding (including SILK for WeChat) is supported out-of-the-box without extra host configuration. + + ## 4. Open the dashboard - **Local installation**: open `http://localhost:5300`. diff --git a/docs/en/deploy/langbot/manual.mdx b/docs/en/deploy/langbot/manual.mdx index 99c3d0f4f..cef3c2929 100644 --- a/docs/en/deploy/langbot/manual.mdx +++ b/docs/en/deploy/langbot/manual.mdx @@ -29,6 +29,20 @@ Embedded-mode platform support depends on the available `pylibseekdb` wheels; Apple Silicon currently requires macOS 15 or later. All other LangBot features can use the default `uv sync` installation. +### Optional Voice Transcoding Support (SILK / WeChat Native Voice) + +If you need native voice bubbles on platforms like WeChat (SILK encoding) or audio transcoding support, install system `ffmpeg` and the `audio` extra: + +- **System Dependency**: `ffmpeg` (macOS: `brew install ffmpeg`, Ubuntu/Debian: `sudo apt-get install -y ffmpeg`, Windows: `winget install Gyan.FFmpeg`) +- **Python Extra**: + ```bash + uv sync --extra audio + # or combine with seekdb + uv sync --extra seekdb --extra audio + ``` + +If transcoding dependencies are not installed, LangBot performs **graceful degradation** by delivering the audio as a playable audio file attachment rather than dropping the message. + 3. Run the main program once to generate configuration files ```bash diff --git a/docs/en/deploy/langbot/package.mdx b/docs/en/deploy/langbot/package.mdx index 6d8e063f3..0439a0fea 100644 --- a/docs/en/deploy/langbot/package.mdx +++ b/docs/en/deploy/langbot/package.mdx @@ -39,3 +39,15 @@ Silicon currently requires macOS 15 or later; follow macOS 14 ARM64 support plan. Because of the current `pyseekdb` package metadata, the extra also fails to resolve on macOS 14 ARM64 for server-only use. These platform limits do not affect LangBot when SeekDB is not installed. + +## Optional Voice Transcoding Support (SILK / WeChat Native Voice) + +If you need native voice bubbles on platforms like WeChat (SILK encoding) or audio transcoding support, make sure `ffmpeg` is installed on your system and launch LangBot with the `audio` extra: + +```bash +uvx --from 'langbot[audio]@latest' langbot +# or combine with seekdb +uvx --from 'langbot[seekdb,audio]@latest' langbot +``` + +If transcoding dependencies (`ffmpeg` / `pilk`) are not installed, LangBot automatically performs **graceful degradation** by delivering the audio as an audio file attachment. diff --git a/docs/zh/deploy/langbot/docker.mdx b/docs/zh/deploy/langbot/docker.mdx index 5d624ab65..c86673905 100644 --- a/docs/zh/deploy/langbot/docker.mdx +++ b/docs/zh/deploy/langbot/docker.mdx @@ -44,6 +44,10 @@ docker compose --profile all up -d 首次启动需要下载镜像。Windows / macOS 用户请保持 Docker Desktop 运行。 + +官方 Docker 镜像已默认预装 `ffmpeg` 与 `pilk` 音频工具链（最小化安装），开箱即用支持微信原生语音条（SILK）转码与语音处理，无需额外在宿主机安装音频环境。 + + ## 4. 打开管理页面 - **本机部署**：打开 `http://localhost:5300`。 diff --git a/docs/zh/deploy/langbot/manual.mdx b/docs/zh/deploy/langbot/manual.mdx index 0fd59e026..3593ab0b1 100644 --- a/docs/zh/deploy/langbot/manual.mdx +++ b/docs/zh/deploy/langbot/manual.mdx @@ -30,6 +30,20 @@ uv sync --extra seekdb SeekDB 嵌入式模式的平台支持取决于 `pylibseekdb` wheel；Apple Silicon 当前要求 macOS 15 或更高版本。其他功能仍可使用默认的 `uv sync` 安装。 +### 可选的语音转码支持（SILK / 微信原生语音） + +如果需要在微信等平台发送原生语音条（SILK 编码）或进行高级音频格式转码，需要系统安装 `ffmpeg` 并安装 `audio` 扩展依赖： + +- **系统依赖**：`ffmpeg`（macOS 可用 `brew install ffmpeg`，Ubuntu/Debian 可用 `sudo apt-get install -y ffmpeg`，Windows 可用 `winget install Gyan.FFmpeg`） +- **Python 扩展**： + ```bash + uv sync --extra audio + # 或同时启用 seekdb 与 audio + uv sync --extra seekdb --extra audio + ``` + +若未安装转码依赖，系统在需要转码为微信原生语音条时会自动进行**优雅降级**，将语音以音频文件的形式发送，确保用户仍能正常点击播放。 + 或者使用清华源 ```bash diff --git a/docs/zh/deploy/langbot/package.mdx b/docs/zh/deploy/langbot/package.mdx index 71fa0be6c..69a7f365d 100644 --- a/docs/zh/deploy/langbot/package.mdx +++ b/docs/zh/deploy/langbot/package.mdx @@ -37,3 +37,15 @@ SeekDB 嵌入式模式还要求当前平台存在兼容的 `pylibseekdb` wheel [oceanbase/seekdb#1324](https://github.com/oceanbase/seekdb/issues/1324)。受当前 `pyseekdb` 包元数据影响，macOS 14 ARM64 即使只连接远程 SeekDB，也无法解析该 extra。不使用 SeekDB 时，这些平台限制不影响 LangBot。 + +## 可选的语音转码支持（SILK / 微信原生语音） + +如果需要在微信等平台发送原生语音条（SILK 格式）或进行音频格式转换，需要系统已安装 `ffmpeg`，并使用 `audio` extra 启动： + +```bash +uvx --from 'langbot[audio]@latest' langbot +# 或同时启用 seekdb 与 audio +uvx --from 'langbot[seekdb,audio]@latest' langbot +``` + +若当前环境未安装 `ffmpeg` 或 `pilk`，LangBot 会自动**优雅降级**，将语音作为音频文件发送，保证用户端仍可正常接收播放。 diff --git a/pyproject.toml b/pyproject.toml index d1203092a..7e82defe7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,9 @@ classifiers = [ ] [project.optional-dependencies] +audio = [ + "pilk>=0.1.0", +] seekdb = [ "pyseekdb==1.4.0.post1", "pylibseekdb==1.4.0; sys_platform == 'linux' or (sys_platform == 'darwin' and platform_machine == 'arm64')", diff --git a/skills/skills/langbot-mcp-ops/SKILL.md b/skills/skills/langbot-mcp-ops/SKILL.md index a9d723de3..c8449375c 100644 --- a/skills/skills/langbot-mcp-ops/SKILL.md +++ b/skills/skills/langbot-mcp-ops/SKILL.md @@ -92,7 +92,7 @@ a pipeline retain these independent settings. | `list_processor_runs` / `get_processor_run_events` | Read one Agent or plugin processor run history and logs; paginate with `before_id` / `after_sequence`. | | `debug_agent` | Execute a synthetic Agent event (`processor_uuid`, `payload`); requires `runtime.operate`. Returns final text and up to 1000 execution events (thinking, text, tool arguments/results). Platform tools use Mock; other configured tools execute normally. Optional `payload.mock`: `errors`/`results` keyed by platform tool name, `unsupported_apis` lists unavailable platform APIs. | | `list_pipelines` / `get_pipeline` / `create_pipeline` / `update_pipeline` / `delete_pipeline` | Manage pipelines | -| `list_llm_models` / `get_llm_model` / `list_embedding_models` / `list_model_providers` | Inspect models & providers | +| `list_llm_models` / `get_llm_model` / `list_embedding_models` / `list_asr_models` / `list_tts_models` / `list_model_providers` | Inspect models & providers | | `list_knowledge_bases` / `get_knowledge_base` / `retrieve_knowledge_base` | RAG knowledge bases (incl. semantic search) | | `list_mcp_servers` | External MCP servers LangBot connects to (as a client) | | `list_skills` / `get_skill` | Installed skills | diff --git a/src/langbot/libs/dingtalk_api/api.py b/src/langbot/libs/dingtalk_api/api.py index 2eaff2447..700294579 100644 --- a/src/langbot/libs/dingtalk_api/api.py +++ b/src/langbot/libs/dingtalk_api/api.py @@ -34,6 +34,8 @@ DINGTALK_OPENAPI_BASE = 'https://api.dingtalk.com' # inbound event carried a forged destination. DINGTALK_WEBHOOK_DOMAINS = ('dingtalk.com',) _MAX_MEDIA_BYTES = 10 * 1024 * 1024 +# DingTalk voice bubbles play at most 60 seconds of audio. +_MAX_VOICE_DURATION_MS = 60 * 1000 _MAX_GATEWAY_MESSAGE_BYTES = 1024 * 1024 @@ -321,11 +323,77 @@ class DingTalkClient: return self._check_send_result(response) async def send_attachment(self, component, incoming_message): - from langbot_plugin.api.entities.builtin.platform.message import Image + from langbot_plugin.api.entities.builtin.platform.message import Image, Voice + + if isinstance(component, Voice): + return await self.send_voice(component, incoming_message) is_image = isinstance(component, Image) name = getattr(component, 'name', '') or ('image.png' if is_image else 'file') name = os.path.basename(name) + body = await self._read_component_bytes(component) + media_id = await self._upload_media(name, body, 'image' if is_image else 'file') + params = ( + {'photoURL': media_id} + if is_image + else {'mediaId': media_id, 'fileName': name, 'fileType': os.path.splitext(name)[1].lstrip('.')} + ) + return await self._send_robot_message('sampleImageMsg' if is_image else 'sampleFile', params, incoming_message) + + async def send_voice(self, component, incoming_message): + """Send a Voice component as a native DingTalk audio message. + + DingTalk plays ogg/opus and amr in its voice bubble, so other formats are + transcoded to ogg/opus. Audio longer than the voice limit, or audio that + cannot be transcoded or is rejected as a voice message, is delivered as + a regular file instead. + """ + from langbot.pkg.utils import audio as audio_utils + + body = await self._read_component_bytes(component) + fmt = audio_utils.detect_audio_format(body) + length = getattr(component, 'length', None) + duration = float(length) if length else audio_utils.estimate_audio_duration(body, fmt) + duration_ms = max(1000, int(round(duration * 1000))) + if fmt not in ('ogg', 'opus', 'amr'): + try: + body = await audio_utils.transcode_audio(body, 'ogg', fmt) + fmt = 'ogg' + except Exception as exc: + await self._log_warning(f'DingTalk voice transcode to ogg failed, sending as file: {exc}') + ext = 'amr' if fmt == 'amr' else ('ogg' if fmt in ('ogg', 'opus') else (fmt if fmt != 'unknown' else 'mp3')) + name = f'voice.{ext}' + if ext == 'ogg': + # Ogg carries an exact sample count, which beats bitrate-based estimates. + measured = audio_utils.estimate_audio_duration(body, fmt) + if measured > 0: + duration_ms = max(1000, int(round(measured * 1000))) + + if ext in ('ogg', 'amr') and duration_ms <= _MAX_VOICE_DURATION_MS: + try: + media_id = await self._upload_media(name, body, 'voice') + return await self._send_robot_message( + 'sampleAudio', + {'mediaId': media_id, 'duration': str(duration_ms)}, + incoming_message, + ) + except Exception as exc: + await self._log_warning(f'DingTalk voice message failed, sending as file: {exc}') + + media_id = await self._upload_media(name, body, 'file') + return await self._send_robot_message( + 'sampleFile', + {'mediaId': media_id, 'fileName': name, 'fileType': ext}, + incoming_message, + ) + + async def _log_warning(self, message: str): + if self.logger: + await self.logger.warning(message) + else: + _stdout_logger.warning(message) + + async def _read_component_bytes(self, component) -> bytes: encoded = getattr(component, 'base64', None) if encoded: encoded = encoded.split(',', 1)[-1] @@ -347,13 +415,16 @@ class DingTalkClient: raise ValueError('DingTalk attachment has no readable content') if len(body) > _MAX_MEDIA_BYTES: raise ValueError('DingTalk media exceeds the size limit') + return body + + async def _upload_media(self, name: str, body: bytes, media_type: str) -> str: token = await self.get_legacy_access_token() if not token: raise RuntimeError('DingTalk media authentication failed') async with self._http_client_context() as client: response = await client.post( 'https://oapi.dingtalk.com/media/upload', - params={'access_token': token, 'type': 'image' if is_image else 'file'}, + params={'access_token': token, 'type': media_type}, files={'media': (name, body)}, timeout=30.0, ) @@ -361,16 +432,14 @@ class DingTalkClient: media_id = uploaded.get('media_id') if not media_id: raise RuntimeError('DingTalk media upload returned no media ID') + return media_id + + async def _send_robot_message(self, msg_key: str, params: dict, incoming_message): if not await self.check_access_token(): await self.get_access_token() - params = ( - {'photoURL': media_id} - if is_image - else {'mediaId': media_id, 'fileName': name, 'fileType': os.path.splitext(name)[1].lstrip('.')} - ) payload = { 'robotCode': self.robot_code or self.key, - 'msgKey': 'sampleImageMsg' if is_image else 'sampleFile', + 'msgKey': msg_key, 'msgParam': json.dumps(params, ensure_ascii=False), } if str(incoming_message.conversation_type) == '2': @@ -609,6 +678,11 @@ class DingTalkClient: download_code = raw_content.get('downloadCode') if download_code: message_data['Audio'] = await self.get_audio_url(download_code) + try: + # DingTalk reports the voice duration in milliseconds. + message_data['AudioDuration'] = int(round(float(raw_content.get('duration') or 0) / 1000)) + except (TypeError, ValueError): + pass message_data['Type'] = 'audio' elif incoming_message.message_type == 'file': diff --git a/src/langbot/libs/dingtalk_api/dingtalkevent.py b/src/langbot/libs/dingtalk_api/dingtalkevent.py index be6842e97..2a994a367 100644 --- a/src/langbot/libs/dingtalk_api/dingtalkevent.py +++ b/src/langbot/libs/dingtalk_api/dingtalkevent.py @@ -35,6 +35,10 @@ class DingTalkEvent(dict): def audio(self): return self.get('Audio', '') + @property + def audio_duration(self) -> int: + return self.get('AudioDuration') or 0 + @property def file(self): return self.get('File', '') diff --git a/src/langbot/libs/openclaw_weixin_api/client.py b/src/langbot/libs/openclaw_weixin_api/client.py index fa89c0427..b7cfa9538 100644 --- a/src/langbot/libs/openclaw_weixin_api/client.py +++ b/src/langbot/libs/openclaw_weixin_api/client.py @@ -456,15 +456,29 @@ class OpenClawWeixinClient: to_user_id: str, voice_bytes: bytes, playtime: int = 0, + encode_type: Optional[int] = 6, context_token: str = '', + bits_per_sample: Optional[int] = None, + sample_rate: Optional[int] = None, + text: Optional[str] = None, ) -> None: """Upload a voice message to CDN and send it.""" + if encode_type == 6: + if bits_per_sample is None: + bits_per_sample = 16 + if sample_rate is None: + sample_rate = 24000 media = await self.upload_media(voice_bytes, to_user_id, media_type=4) item = MessageItem( type=MessageItem.VOICE, voice_item=VoiceItem( media=media, playtime=playtime, + encode_type=encode_type, + bits_per_sample=bits_per_sample, + sample_rate=sample_rate, + size=len(voice_bytes), + text=text, ), ) await self.send_message(to_user_id, [item], context_token) @@ -715,6 +729,9 @@ def _parse_message_item(data: dict) -> MessageItem: media=_parse_cdn_media(v.get('media')), encode_type=v.get('encode_type'), playtime=v.get('playtime'), + bits_per_sample=v.get('bits_per_sample'), + sample_rate=v.get('sample_rate'), + size=v.get('size'), text=v.get('text'), ) @@ -812,6 +829,16 @@ def _message_item_to_dict(item: MessageItem) -> dict: voice_d['media'] = _cdn_media_to_dict(item.voice_item.media) if item.voice_item.playtime is not None: voice_d['playtime'] = item.voice_item.playtime + if item.voice_item.encode_type is not None: + voice_d['encode_type'] = item.voice_item.encode_type + if item.voice_item.bits_per_sample is not None: + voice_d['bits_per_sample'] = item.voice_item.bits_per_sample + if item.voice_item.sample_rate is not None: + voice_d['sample_rate'] = item.voice_item.sample_rate + if item.voice_item.size is not None: + voice_d['size'] = item.voice_item.size + if item.voice_item.text is not None: + voice_d['text'] = item.voice_item.text d['voice_item'] = voice_d if item.file_item: diff --git a/src/langbot/libs/openclaw_weixin_api/types.py b/src/langbot/libs/openclaw_weixin_api/types.py index 93bb2e3c3..e7bdb750d 100644 --- a/src/langbot/libs/openclaw_weixin_api/types.py +++ b/src/langbot/libs/openclaw_weixin_api/types.py @@ -62,6 +62,7 @@ class VoiceItem: bits_per_sample: Optional[int] = None sample_rate: Optional[int] = None playtime: Optional[int] = None + size: Optional[int] = None text: Optional[str] = None _downloaded_bytes: Optional[bytes] = field(default=None, repr=False) diff --git a/src/langbot/libs/wecom_ai_bot_api/api.py b/src/langbot/libs/wecom_ai_bot_api/api.py index 207315d28..98c779556 100644 --- a/src/langbot/libs/wecom_ai_bot_api/api.py +++ b/src/langbot/libs/wecom_ai_bot_api/api.py @@ -415,6 +415,16 @@ def _bytes_to_data_uri(data: bytes) -> str: mime_type = 'application/pdf' elif data[:4] == b'PK\x03\x04': mime_type = 'application/zip' + elif data.startswith(b'#!AMR'): + mime_type = 'audio/amr' + elif data.startswith((b'#!SILK_V3', b'\x02#!SILK_V3')): + mime_type = 'audio/silk' + elif data.startswith(b'RIFF') and len(data) >= 12 and data[8:12] == b'WAVE': + mime_type = 'audio/wav' + elif data.startswith(b'ID3') or (len(data) >= 2 and data[0] == 0xFF and (data[1] & 0xE0) == 0xE0): + mime_type = 'audio/mpeg' + elif data.startswith(b'OggS'): + mime_type = 'audio/ogg' else: mime_type = 'application/octet-stream' @@ -508,6 +518,29 @@ async def parse_wecom_bot_message( return await asyncio.to_thread(_bytes_to_data_uri, data) return None + async def _safe_download_voice(url: str, per_msg_aeskey: str = '') -> Tuple[Optional[str], Optional[int]]: + """Download, decrypt, and transcode voice message to MP3 data URI for universal browser playback.""" + data, _filename = await _safe_download(url, per_msg_aeskey) + if not data: + return None, None + from langbot.pkg.utils import audio as audio_utils + + fmt = audio_utils.detect_audio_format(data) + duration = int(round(audio_utils.estimate_audio_duration(data, fmt))) if data else None + + # Transcode AMR / SILK to MP3 so web browsers and media players can decode directly + if fmt in ('amr', 'silk') and audio_utils.is_ffmpeg_available(): + try: + mp3_bytes = await audio_utils.transcode_audio(data, 'mp3', fmt) + b64 = base64.b64encode(mp3_bytes).decode('utf-8') + return f'data:audio/mpeg;base64,{b64}', duration + except Exception as exc: + await logger.warning(f'Failed to transcode {fmt} voice to MP3: {exc}') + + mime = 'audio/mpeg' if fmt == 'mp3' else ('audio/wav' if fmt == 'wav' else f'audio/{fmt}') + b64 = base64.b64encode(data).decode('utf-8') + return f'data:{mime};base64,{b64}', duration + if msg_type == 'text': message_data['content'] = msg_json.get('text', {}).get('content') elif msg_type == 'markdown': @@ -534,10 +567,12 @@ async def parse_wecom_bot_message( } if voice_info.get('content'): message_data['content'] = voice_info.get('content') - # if (message_data['voice'].get('filesize') or 0) <= max_inline_file_size: - # voice_base64 = await _safe_download_as_data_uri(download_url, per_msg_aeskey) - # if voice_base64: - # message_data['voice']['base64'] = voice_base64 + if (message_data['voice'].get('filesize') or 0) <= max_inline_file_size: + voice_base64, duration = await _safe_download_voice(download_url, per_msg_aeskey) + if voice_base64: + message_data['voice']['base64'] = voice_base64 + if duration: + message_data['voice']['length'] = duration elif msg_type == 'video': video_info = msg_json.get('video', {}) or {} download_url = video_info.get('url') @@ -638,9 +673,11 @@ async def parse_wecom_bot_message( if voice_info.get('content'): texts.append(voice_info.get('content')) if (voice_data.get('filesize') or 0) <= max_inline_file_size: - voice_base64 = await _safe_download_as_data_uri(download_url, item_aeskey) + voice_base64, duration = await _safe_download_voice(download_url, item_aeskey) if voice_base64: voice_data['base64'] = voice_base64 + if duration: + voice_data['length'] = duration voices.append(voice_data) elif item_type == 'video': video_info = item.get('video', {}) or {} @@ -740,6 +777,12 @@ async def parse_wecom_bot_message( } if voice_info.get('content'): quote_data['content'] = voice_info.get('content') + if (voice_data.get('filesize') or 0) <= max_inline_file_size: + voice_base64, duration = await _safe_download_voice(download_url, item_aeskey) + if voice_base64: + voice_data['base64'] = voice_base64 + if duration: + voice_data['length'] = duration # Same as private chat: append aeskey to url for plugin processing if download_url and item_aeskey: voice_data['url'] = download_url + f'?aeskey={item_aeskey}' diff --git a/src/langbot/libs/wecom_api/api.py b/src/langbot/libs/wecom_api/api.py index 72bd7ade7..75b59d254 100644 --- a/src/langbot/libs/wecom_api/api.py +++ b/src/langbot/libs/wecom_api/api.py @@ -562,6 +562,15 @@ class WecomClient: await self.logger.error('Voice对象出错') raise ValueError('voice对象出错') + from ...pkg.utils import audio as audio_utils + + if audio_utils.is_ffmpeg_available(): + try: + file_bytes = await audio_utils.transcode_audio(file_bytes, 'amr') + file_name = 'voice.amr' + except Exception: + pass + if len(file_bytes) > _MAX_MEDIA_BYTES: raise ValueError('WeCom media exceeds the size limit') boundary = '-------------------------acebdf13572468' diff --git a/src/langbot/pkg/agent/runner/config_schema.py b/src/langbot/pkg/agent/runner/config_schema.py index 1fad3193f..3cd533d79 100644 --- a/src/langbot/pkg/agent/runner/config_schema.py +++ b/src/langbot/pkg/agent/runner/config_schema.py @@ -222,6 +222,12 @@ def iter_config_model_refs( elif field_type == 'rerank-model-selector': if isinstance(value, str) and value not in NONE_SENTINELS: yield 'rerank', value + elif field_type == 'asr-model-selector': + if isinstance(value, str) and value not in NONE_SENTINELS: + yield 'asr', value + elif field_type == 'tts-model-selector': + if isinstance(value, str) and value not in NONE_SENTINELS: + yield 'tts', value def set_empty_llm_model_selection( diff --git a/src/langbot/pkg/api/http/controller/groups/provider/models.py b/src/langbot/pkg/api/http/controller/groups/provider/models.py index 678550b69..149f14b54 100644 --- a/src/langbot/pkg/api/http/controller/groups/provider/models.py +++ b/src/langbot/pkg/api/http/controller/groups/provider/models.py @@ -382,3 +382,229 @@ class RerankModelsRouterGroup(group.RouterGroup): except provider_errors.RequesterError as exc: return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc)) return self.success() + + +@group.group_class('models/asr', '/api/v1/provider/models/asr') +class ASRModelsRouterGroup(group.RouterGroup): + async def initialize(self) -> None: + @self.route( + '', + methods=['GET'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_VIEW, + ) + async def _(request_context: RequestContext) -> str: + provider_uuid = quart.request.args.get('provider_uuid') + include_secret, error = resolve_include_secret( + quart.request.args.get('include_secret'), + permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE), + ) + if error: + return self.http_status(400, -1, error) + if provider_uuid: + models = await self.ap.asr_models_service.get_asr_models_by_provider( + request_context, + provider_uuid, + include_secret=include_secret, + ) + else: + models = await self.ap.asr_models_service.get_asr_models( + request_context, + include_secret=include_secret, + ) + return self.success(data={'models': models}) + + @self.route( + '', + methods=['POST'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(request_context: RequestContext) -> str: + try: + model_uuid = await self.ap.asr_models_service.create_asr_model( + request_context, + await quart.request.json, + ) + except ValueError as exc: + return self.http_status(400, -1, str(exc)) + return self.success(data={'uuid': model_uuid}) + + @self.route( + '/', + methods=['GET'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_VIEW, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + include_secret, error = resolve_include_secret( + quart.request.args.get('include_secret'), + permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE), + ) + if error: + return self.http_status(400, -1, error) + model = await self.ap.asr_models_service.get_asr_model( + request_context, + model_uuid, + include_secret=include_secret, + ) + if model is None: + return self.http_status(404, -1, 'model not found') + return self.success(data={'model': model}) + + @self.route( + '/', + methods=['PUT'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + try: + await self.ap.asr_models_service.update_asr_model( + request_context, + model_uuid, + await quart.request.json, + ) + except ValueError as exc: + return self.http_status(400, -1, str(exc)) + return self.success() + + @self.route( + '/', + methods=['DELETE'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + await self.ap.asr_models_service.delete_asr_model(request_context, model_uuid) + return self.success() + + @self.route( + '//test', + methods=['POST'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + try: + await self.ap.asr_models_service.test_asr_model( + request_context, + model_uuid, + await quart.request.json, + ) + except provider_errors.RequesterError as exc: + return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc)) + return self.success() + + +@group.group_class('models/tts', '/api/v1/provider/models/tts') +class TTSModelsRouterGroup(group.RouterGroup): + async def initialize(self) -> None: + @self.route( + '', + methods=['GET'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_VIEW, + ) + async def _(request_context: RequestContext) -> str: + provider_uuid = quart.request.args.get('provider_uuid') + include_secret, error = resolve_include_secret( + quart.request.args.get('include_secret'), + permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE), + ) + if error: + return self.http_status(400, -1, error) + if provider_uuid: + models = await self.ap.tts_models_service.get_tts_models_by_provider( + request_context, + provider_uuid, + include_secret=include_secret, + ) + else: + models = await self.ap.tts_models_service.get_tts_models( + request_context, + include_secret=include_secret, + ) + return self.success(data={'models': models}) + + @self.route( + '', + methods=['POST'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(request_context: RequestContext) -> str: + try: + model_uuid = await self.ap.tts_models_service.create_tts_model( + request_context, + await quart.request.json, + ) + except ValueError as exc: + return self.http_status(400, -1, str(exc)) + return self.success(data={'uuid': model_uuid}) + + @self.route( + '/', + methods=['GET'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_VIEW, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + include_secret, error = resolve_include_secret( + quart.request.args.get('include_secret'), + permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE), + ) + if error: + return self.http_status(400, -1, error) + model = await self.ap.tts_models_service.get_tts_model( + request_context, + model_uuid, + include_secret=include_secret, + ) + if model is None: + return self.http_status(404, -1, 'model not found') + return self.success(data={'model': model}) + + @self.route( + '/', + methods=['PUT'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + try: + await self.ap.tts_models_service.update_tts_model( + request_context, + model_uuid, + await quart.request.json, + ) + except ValueError as exc: + return self.http_status(400, -1, str(exc)) + return self.success() + + @self.route( + '/', + methods=['DELETE'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + await self.ap.tts_models_service.delete_tts_model(request_context, model_uuid) + return self.success() + + @self.route( + '//test', + methods=['POST'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + try: + await self.ap.tts_models_service.test_tts_model( + request_context, + model_uuid, + await quart.request.json, + ) + except provider_errors.RequesterError as exc: + return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc)) + return self.success() diff --git a/src/langbot/pkg/api/http/controller/groups/provider/providers.py b/src/langbot/pkg/api/http/controller/groups/provider/providers.py index 0a7302b58..8e1847c56 100644 --- a/src/langbot/pkg/api/http/controller/groups/provider/providers.py +++ b/src/langbot/pkg/api/http/controller/groups/provider/providers.py @@ -103,9 +103,7 @@ class ModelProvidersRouterGroup(group.RouterGroup): ) for provider in providers: counts = await self.ap.provider_service.get_provider_model_counts(request_context, provider['uuid']) - provider['llm_count'] = counts['llm_count'] - provider['embedding_count'] = counts['embedding_count'] - provider['rerank_count'] = counts['rerank_count'] + provider.update(counts) return self.success(data={'providers': providers}) @self.route( @@ -143,9 +141,7 @@ class ModelProvidersRouterGroup(group.RouterGroup): if provider is None: return self.http_status(404, -1, 'provider not found') counts = await self.ap.provider_service.get_provider_model_counts(request_context, provider_uuid) - provider['llm_count'] = counts['llm_count'] - provider['embedding_count'] = counts['embedding_count'] - provider['rerank_count'] = counts['rerank_count'] + provider.update(counts) return self.success(data={'provider': provider}) @self.route( diff --git a/src/langbot/pkg/api/http/service/model.py b/src/langbot/pkg/api/http/service/model.py index ecd75f83b..44d82225c 100644 --- a/src/langbot/pkg/api/http/service/model.py +++ b/src/langbot/pkg/api/http/service/model.py @@ -1101,3 +1101,521 @@ class RerankModelsService: ], execution_context=runtime_rerank_model.execution_context, ) + + +def _generate_test_wav() -> bytes: + import io + import wave + + buf = io.BytesIO() + with wave.open(buf, 'wb') as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b'\x00' * 3200) + return buf.getvalue() + + +class ASRModelsService: + ap: app.Application + + def __init__(self, ap: app.Application) -> None: + self.ap = ap + + async def get_asr_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]: + """Get all ASR models with provider info""" + result = await self.ap.persistence_mgr.execute_async( + scope_statement(sqlalchemy.select(persistence_model.ASRModel), persistence_model.ASRModel, context) + ) + models = result.all() + + providers_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context + ) + ) + providers = {p.uuid: p for p in providers_result.all()} + + models_list = [] + for model in models: + model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ASRModel, model) + provider = providers.get(model.provider_uuid) + if provider: + provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider) + provider_dict = _parse_provider_api_keys(provider_dict) + model_dict['provider'] = provider_dict + if not include_secret: + model_dict = _redact_model_secrets(model_dict) + models_list.append(model_dict) + + return models_list + + async def get_asr_models_by_provider( + self, + context: TenantContext, + provider_uuid: str, + *, + include_secret: bool = False, + ) -> list[dict]: + """Get ASR models by provider UUID""" + await _require_workspace_provider(self.ap, context, provider_uuid) + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ASRModel).where( + persistence_model.ASRModel.provider_uuid == provider_uuid + ), + persistence_model.ASRModel, + context, + ) + ) + models = result.all() + serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.ASRModel, m) for m in models] + return serialized if include_secret else [_redact_model_secrets(model) for model in serialized] + + async def create_asr_model( + self, + context: TenantContext, + model_data: dict, + preserve_uuid: bool = False, + ) -> str: + """Create a new ASR model""" + model_data = model_data.copy() + if not preserve_uuid: + model_data['uuid'] = str(uuid.uuid4()) + model_data['workspace_uuid'] = require_workspace_uuid(context) + if 'extra_args' in model_data: + model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args']) + + if 'provider' in model_data: + provider_data = model_data.pop('provider') + if provider_data.get('uuid'): + model_data['provider_uuid'] = provider_data['uuid'] + else: + provider_uuid = await self.ap.provider_service.find_or_create_provider( + context, + requester=provider_data.get('requester', ''), + base_url=provider_data.get('base_url', ''), + api_keys=provider_data.get('api_keys', []), + ) + model_data['provider_uuid'] = provider_uuid + + await _require_workspace_provider(self.ap, context, model_data['provider_uuid']) + await _assert_cloud_managed_provider_mutable(self.ap, context, model_data['provider_uuid']) + await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'asr') + + await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_model.ASRModel).values(**model_data)) + + runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid']) + runtime_asr_model = await self.ap.model_mgr.load_asr_model_with_provider( + context, + persistence_model.ASRModel(**model_data), + runtime_provider, + ) + await self.ap.model_mgr.cache_asr_model(context, runtime_asr_model) + + return model_data['uuid'] + + async def get_asr_model( + self, + context: TenantContext, + model_uuid: str, + include_secret: bool = False, + ) -> dict | None: + """Get a single ASR model with provider info""" + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ASRModel).where(persistence_model.ASRModel.uuid == model_uuid), + persistence_model.ASRModel, + context, + ) + ) + model = result.first() + if model is None: + return None + + model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ASRModel, model) + + provider_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ModelProvider).where( + persistence_model.ModelProvider.uuid == model.provider_uuid + ), + persistence_model.ModelProvider, + context, + ) + ) + provider = provider_result.first() + if provider: + provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider) + provider_dict = _parse_provider_api_keys(provider_dict) + model_dict['provider'] = provider_dict + + if not include_secret: + model_dict = _redact_model_secrets(model_dict) + + return model_dict + + async def update_asr_model( + self, + context: TenantContext, + model_uuid: str, + model_data: dict, + ) -> None: + """Update an existing ASR model""" + existing_model = await self.get_asr_model(context, model_uuid, include_secret=True) + if existing_model is None: + raise WorkspaceNotFoundError('Model not found') + await _assert_cloud_managed_provider_mutable(self.ap, context, existing_model['provider_uuid']) + model_data = model_data.copy() + model_data.pop('uuid', None) + model_data.pop('workspace_uuid', None) + if 'extra_args' in model_data: + model_data['extra_args'] = restore_secret_placeholders( + model_data['extra_args'], + existing_model.get('extra_args', {}), + ) + + if 'provider' in model_data: + provider_data = model_data.pop('provider') + if provider_data.get('uuid'): + model_data['provider_uuid'] = provider_data['uuid'] + else: + provider_uuid = await self.ap.provider_service.find_or_create_provider( + context, + requester=provider_data.get('requester', ''), + base_url=provider_data.get('base_url', ''), + api_keys=provider_data.get('api_keys', []), + ) + model_data['provider_uuid'] = provider_uuid + + provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid']) + await _require_workspace_provider(self.ap, context, provider_uuid) + await _assert_cloud_managed_provider_mutable(self.ap, context, provider_uuid) + await _validate_provider_supports(self.ap, context, provider_uuid, 'asr') + + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.update(persistence_model.ASRModel) + .where(persistence_model.ASRModel.uuid == model_uuid) + .values(**model_data), + persistence_model.ASRModel, + context, + ) + ) + if getattr(result, 'rowcount', None) == 0: + raise WorkspaceNotFoundError('Model not found') + + await self.ap.model_mgr.remove_asr_model(context, model_uuid) + runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid) + runtime_asr_model = await self.ap.model_mgr.load_asr_model_with_provider( + context, + persistence_model.ASRModel( + **_runtime_model_data( + model_uuid, + { + key: value + for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items() + if key not in {'provider', 'created_at', 'updated_at'} + }, + ) + ), + runtime_provider, + ) + await self.ap.model_mgr.cache_asr_model(context, runtime_asr_model) + + async def delete_asr_model(self, context: TenantContext, model_uuid: str) -> None: + """Delete an ASR model""" + if _is_cloud_runtime(self.ap): + existing_model = await self.get_asr_model(context, model_uuid, include_secret=True) + if existing_model is None: + raise WorkspaceNotFoundError('Model not found') + await _assert_cloud_managed_provider_mutable(self.ap, context, existing_model['provider_uuid']) + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.delete(persistence_model.ASRModel).where(persistence_model.ASRModel.uuid == model_uuid), + persistence_model.ASRModel, + context, + ) + ) + if getattr(result, 'rowcount', None) == 0: + raise WorkspaceNotFoundError('Model not found') + await self.ap.model_mgr.remove_asr_model(context, model_uuid) + + async def test_asr_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None: + """Test an ASR model""" + require_workspace_uuid(context) + runtime_asr_model: model_requester.RuntimeASRModel | None = None + + if model_uuid != '_': + if await self.get_asr_model(context, model_uuid) is None: + raise WorkspaceNotFoundError('Model not found') + runtime_asr_model = await self.ap.model_mgr.get_asr_model_by_uuid(context, model_uuid) + else: + runtime_asr_model = await self.ap.model_mgr.init_temporary_runtime_asr_model( + context, + model_data, + ) + + await runtime_asr_model.provider.invoke_asr( + model=runtime_asr_model, + audio=_generate_test_wav(), + mime='audio/wav', + execution_context=runtime_asr_model.execution_context, + ) + + +class TTSModelsService: + ap: app.Application + + def __init__(self, ap: app.Application) -> None: + self.ap = ap + + async def get_tts_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]: + """Get all TTS models with provider info""" + result = await self.ap.persistence_mgr.execute_async( + scope_statement(sqlalchemy.select(persistence_model.TTSModel), persistence_model.TTSModel, context) + ) + models = result.all() + + providers_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context + ) + ) + providers = {p.uuid: p for p in providers_result.all()} + + models_list = [] + for model in models: + model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.TTSModel, model) + provider = providers.get(model.provider_uuid) + if provider: + provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider) + provider_dict = _parse_provider_api_keys(provider_dict) + model_dict['provider'] = provider_dict + if not include_secret: + model_dict = _redact_model_secrets(model_dict) + models_list.append(model_dict) + + return models_list + + async def get_tts_models_by_provider( + self, + context: TenantContext, + provider_uuid: str, + *, + include_secret: bool = False, + ) -> list[dict]: + """Get TTS models by provider UUID""" + await _require_workspace_provider(self.ap, context, provider_uuid) + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.TTSModel).where( + persistence_model.TTSModel.provider_uuid == provider_uuid + ), + persistence_model.TTSModel, + context, + ) + ) + models = result.all() + serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.TTSModel, m) for m in models] + return serialized if include_secret else [_redact_model_secrets(model) for model in serialized] + + async def create_tts_model( + self, + context: TenantContext, + model_data: dict, + preserve_uuid: bool = False, + ) -> str: + """Create a new TTS model""" + model_data = model_data.copy() + if not preserve_uuid: + model_data['uuid'] = str(uuid.uuid4()) + model_data['workspace_uuid'] = require_workspace_uuid(context) + if 'extra_args' in model_data: + model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args']) + + if 'provider' in model_data: + provider_data = model_data.pop('provider') + if provider_data.get('uuid'): + model_data['provider_uuid'] = provider_data['uuid'] + else: + provider_uuid = await self.ap.provider_service.find_or_create_provider( + context, + requester=provider_data.get('requester', ''), + base_url=provider_data.get('base_url', ''), + api_keys=provider_data.get('api_keys', []), + ) + model_data['provider_uuid'] = provider_uuid + + await _require_workspace_provider(self.ap, context, model_data['provider_uuid']) + await _assert_cloud_managed_provider_mutable(self.ap, context, model_data['provider_uuid']) + await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'tts') + + await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_model.TTSModel).values(**model_data)) + + runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid']) + runtime_tts_model = await self.ap.model_mgr.load_tts_model_with_provider( + context, + persistence_model.TTSModel(**model_data), + runtime_provider, + ) + await self.ap.model_mgr.cache_tts_model(context, runtime_tts_model) + + return model_data['uuid'] + + async def get_tts_model( + self, + context: TenantContext, + model_uuid: str, + include_secret: bool = False, + ) -> dict | None: + """Get a single TTS model with provider info""" + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.TTSModel).where(persistence_model.TTSModel.uuid == model_uuid), + persistence_model.TTSModel, + context, + ) + ) + model = result.first() + if model is None: + return None + + model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.TTSModel, model) + + provider_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ModelProvider).where( + persistence_model.ModelProvider.uuid == model.provider_uuid + ), + persistence_model.ModelProvider, + context, + ) + ) + provider = provider_result.first() + if provider: + provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider) + provider_dict = _parse_provider_api_keys(provider_dict) + model_dict['provider'] = provider_dict + + if not include_secret: + model_dict = _redact_model_secrets(model_dict) + + return model_dict + + async def update_tts_model( + self, + context: TenantContext, + model_uuid: str, + model_data: dict, + ) -> None: + """Update an existing TTS model""" + existing_model = await self.get_tts_model(context, model_uuid, include_secret=True) + if existing_model is None: + raise WorkspaceNotFoundError('Model not found') + await _assert_cloud_managed_provider_mutable(self.ap, context, existing_model['provider_uuid']) + model_data = model_data.copy() + model_data.pop('uuid', None) + model_data.pop('workspace_uuid', None) + if 'extra_args' in model_data: + model_data['extra_args'] = restore_secret_placeholders( + model_data['extra_args'], + existing_model.get('extra_args', {}), + ) + + if 'provider' in model_data: + provider_data = model_data.pop('provider') + if provider_data.get('uuid'): + model_data['provider_uuid'] = provider_data['uuid'] + else: + provider_uuid = await self.ap.provider_service.find_or_create_provider( + context, + requester=provider_data.get('requester', ''), + base_url=provider_data.get('base_url', ''), + api_keys=provider_data.get('api_keys', []), + ) + model_data['provider_uuid'] = provider_uuid + + provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid']) + await _require_workspace_provider(self.ap, context, provider_uuid) + await _assert_cloud_managed_provider_mutable(self.ap, context, provider_uuid) + await _validate_provider_supports(self.ap, context, provider_uuid, 'tts') + + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.update(persistence_model.TTSModel) + .where(persistence_model.TTSModel.uuid == model_uuid) + .values(**model_data), + persistence_model.TTSModel, + context, + ) + ) + if getattr(result, 'rowcount', None) == 0: + raise WorkspaceNotFoundError('Model not found') + + await self.ap.model_mgr.remove_tts_model(context, model_uuid) + runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid) + runtime_tts_model = await self.ap.model_mgr.load_tts_model_with_provider( + context, + persistence_model.TTSModel( + **_runtime_model_data( + model_uuid, + { + key: value + for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items() + if key not in {'provider', 'created_at', 'updated_at'} + }, + ) + ), + runtime_provider, + ) + await self.ap.model_mgr.cache_tts_model(context, runtime_tts_model) + + async def delete_tts_model(self, context: TenantContext, model_uuid: str) -> None: + """Delete a TTS model""" + if _is_cloud_runtime(self.ap): + existing_model = await self.get_tts_model(context, model_uuid, include_secret=True) + if existing_model is None: + raise WorkspaceNotFoundError('Model not found') + await _assert_cloud_managed_provider_mutable(self.ap, context, existing_model['provider_uuid']) + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.delete(persistence_model.TTSModel).where(persistence_model.TTSModel.uuid == model_uuid), + persistence_model.TTSModel, + context, + ) + ) + if getattr(result, 'rowcount', None) == 0: + raise WorkspaceNotFoundError('Model not found') + await self.ap.model_mgr.remove_tts_model(context, model_uuid) + + async def test_tts_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None: + """Test a TTS model""" + require_workspace_uuid(context) + runtime_tts_model: model_requester.RuntimeTTSModel | None = None + + if model_uuid != '_': + if await self.get_tts_model(context, model_uuid) is None: + raise WorkspaceNotFoundError('Model not found') + runtime_tts_model = await self.ap.model_mgr.get_tts_model_by_uuid(context, model_uuid) + else: + runtime_tts_model = await self.ap.model_mgr.init_temporary_runtime_tts_model( + context, + model_data, + ) + + model_extra = ( + getattr(runtime_tts_model, 'extra_args', None) + or getattr(runtime_tts_model.model_entity, 'extra_args', {}) + or {} + ) + voice = model_extra.get('voice') + extra_args: dict[str, typing.Any] = {'response_format': 'mp3'} + if voice: + extra_args['voice'] = voice + await runtime_tts_model.provider.invoke_tts( + model=runtime_tts_model, + text='Hello, this is a test.', + extra_args=extra_args, + execution_context=runtime_tts_model.execution_context, + ) diff --git a/src/langbot/pkg/api/http/service/pipeline.py b/src/langbot/pkg/api/http/service/pipeline.py index 934cf97f9..d362743aa 100644 --- a/src/langbot/pkg/api/http/service/pipeline.py +++ b/src/langbot/pkg/api/http/service/pipeline.py @@ -20,6 +20,7 @@ from .tenant import TenantContext, require_workspace_uuid, scope_statement default_stage_order = [ 'GroupRespondRuleCheckStage', # 群响应规则检查 'BanSessionCheckStage', # 封禁会话检查 + 'SpeechRecognitionStage', # 语音识别（ASR）阶段 'PreContentFilterStage', # 内容过滤前置阶段 'PreProcessor', # 预处理器 'RequireRateLimitOccupancy', # 请求速率限制占用 @@ -27,6 +28,7 @@ default_stage_order = [ 'ReleaseRateLimitOccupancy', # 释放速率限制占用 'PostContentFilterStage', # 内容过滤后置阶段 'ResponseWrapper', # 响应包装器 + 'TextToSpeechStage', # 文本转语音阶段 'LongTextProcessStage', # 长文本处理 'SendResponseBackStage', # 发送响应 ] @@ -105,6 +107,7 @@ class PipelineService: self.ap.pipeline_config_meta_trigger, self.ap.pipeline_config_meta_safety, ai_metadata, + self.ap.pipeline_config_meta_speech, self.ap.pipeline_config_meta_output, ] diff --git a/src/langbot/pkg/api/http/service/pipeline_migration.py b/src/langbot/pkg/api/http/service/pipeline_migration.py index b4ec9b502..c574d32f3 100644 --- a/src/langbot/pkg/api/http/service/pipeline_migration.py +++ b/src/langbot/pkg/api/http/service/pipeline_migration.py @@ -28,7 +28,7 @@ from ....agent.runner.config_resolver import RunnerConfigResolver from ....agent.runner import config_schema from ....agent.runner.resource_builder import AgentResourceBuilder from ....agent.runner.resource_policy import ResourcePolicyProjector -from ....entity.persistence.model import LLMModel, RerankModel, EmbeddingModel +from ....entity.persistence.model import LLMModel, RerankModel, EmbeddingModel, ASRModel, TTSModel from ....entity.persistence.rag import KnowledgeBase from ....entity.persistence.mcp import MCPServer from ....core.taskmgr import TaskContext @@ -681,6 +681,8 @@ class PipelineMigrationService: 'llm-model-selector', 'embedding-model-selector', 'rerank-model-selector', + 'asr-model-selector', + 'tts-model-selector', ) and isinstance(value, str) ) @@ -768,9 +770,19 @@ class PipelineMigrationService: permissions = getattr(descriptor, 'permissions', None) for model_type, model_uuid in config_schema.iter_config_model_refs(descriptor, runner_config): allowed = set(getattr(permissions, 'models', [])) - if not (allowed & ({'rerank'} if model_type == 'rerank' else {'invoke', 'stream'})): + needed = ( + {'rerank'} + if model_type == 'rerank' + else ({'asr'} if model_type == 'asr' else ({'tts'} if model_type == 'tts' else {'invoke', 'stream'})) + ) + if not (allowed & needed): raise MigrationError('runner_resource_unavailable') - await require_row(RerankModel if model_type == 'rerank' else LLMModel, model_uuid) + target_cls = ( + RerankModel + if model_type == 'rerank' + else (ASRModel if model_type == 'asr' else (TTSModel if model_type == 'tts' else LLMModel)) + ) + await require_row(target_cls, model_uuid) for field in config_schema.iter_schema_items(descriptor, {'embedding-model-selector'}): value = runner_config.get(field['name']) if value and value not in config_schema.NONE_SENTINELS: diff --git a/src/langbot/pkg/api/http/service/provider.py b/src/langbot/pkg/api/http/service/provider.py index b5909b404..c86dcfa27 100644 --- a/src/langbot/pkg/api/http/service/provider.py +++ b/src/langbot/pkg/api/http/service/provider.py @@ -203,6 +203,8 @@ class ModelProviderService: (persistence_model.LLMModel, 'LLM', 'remove_llm_model'), (persistence_model.EmbeddingModel, 'Embedding', 'remove_embedding_model'), (persistence_model.RerankModel, 'Rerank', 'remove_rerank_model'), + (persistence_model.ASRModel, 'ASR', 'remove_asr_model'), + (persistence_model.TTSModel, 'TTS', 'remove_tts_model'), ) deleted_models: list[tuple[str, list[str]]] = [] async with persistence.tenant_uow(workspace_uuid): @@ -335,7 +337,35 @@ class ModelProviderService: ) rerank_count = rerank_result.scalar() or 0 - return {'llm_count': llm_count, 'embedding_count': embedding_count, 'rerank_count': rerank_count} + asr_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(sqlalchemy.func.count()) + .select_from(persistence_model.ASRModel) + .where(persistence_model.ASRModel.provider_uuid == provider_uuid), + persistence_model.ASRModel, + workspace_uuid, + ) + ) + asr_count = asr_result.scalar() or 0 + + tts_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(sqlalchemy.func.count()) + .select_from(persistence_model.TTSModel) + .where(persistence_model.TTSModel.provider_uuid == provider_uuid), + persistence_model.TTSModel, + workspace_uuid, + ) + ) + tts_count = tts_result.scalar() or 0 + + return { + 'llm_count': llm_count, + 'embedding_count': embedding_count, + 'rerank_count': rerank_count, + 'asr_count': asr_count, + 'tts_count': tts_count, + } async def find_or_create_provider( self, @@ -439,9 +469,19 @@ class ModelProviderService: if rerank_service is not None else [] ) + asr_service = getattr(self.ap, 'asr_models_service', None) + asr_models = ( + await asr_service.get_asr_models_by_provider(context, provider_uuid) if asr_service is not None else [] + ) + tts_service = getattr(self.ap, 'tts_models_service', None) + tts_models = ( + await tts_service.get_tts_models_by_provider(context, provider_uuid) if tts_service is not None else [] + ) existing_llm_names = {model['name'] for model in llm_models} existing_embedding_names = {model['name'] for model in embedding_models} existing_rerank_names = {model['name'] for model in rerank_models} + existing_asr_names = {model['name'] for model in asr_models} + existing_tts_names = {model['name'] for model in tts_models} filtered_models = [] for model in scanned_models: @@ -470,6 +510,10 @@ class ModelProviderService: if scanned_type == 'embedding' else model_name in existing_rerank_names if scanned_type == 'rerank' + else model_name in existing_asr_names + if scanned_type == 'asr' + else model_name in existing_tts_names + if scanned_type == 'tts' else model_name in existing_llm_names ), } diff --git a/src/langbot/pkg/api/mcp/server.py b/src/langbot/pkg/api/mcp/server.py index 967a964ca..12c725eaa 100644 --- a/src/langbot/pkg/api/mcp/server.py +++ b/src/langbot/pkg/api/mcp/server.py @@ -583,6 +583,16 @@ class LangBotMCPServer: context = _authorized(Permission.RESOURCE_VIEW) return _dump(await ap.embedding_models_service.get_embedding_models(context, include_secret=False)) + @mcp.tool(description='List all configured ASR (speech recognition) models.') + async def list_asr_models() -> str: + context = _authorized(Permission.RESOURCE_VIEW) + return _dump(await ap.asr_models_service.get_asr_models(context, include_secret=False)) + + @mcp.tool(description='List all configured TTS (text-to-speech) models.') + async def list_tts_models() -> str: + context = _authorized(Permission.RESOURCE_VIEW) + return _dump(await ap.tts_models_service.get_tts_models(context, include_secret=False)) + @mcp.tool(description='List all model providers (OpenAI-compatible, Anthropic, etc.).') async def list_model_providers() -> str: context = _authorized(Permission.RESOURCE_VIEW) diff --git a/src/langbot/pkg/core/app.py b/src/langbot/pkg/core/app.py index 9f84ee10f..cc05321f9 100644 --- a/src/langbot/pkg/core/app.py +++ b/src/langbot/pkg/core/app.py @@ -118,6 +118,7 @@ class Application: pipeline_config_meta_safety: config_mgr.ConfigManager = None pipeline_config_meta_ai: config_mgr.ConfigManager = None pipeline_config_meta_output: config_mgr.ConfigManager = None + pipeline_config_meta_speech: config_mgr.ConfigManager = None # ========================= @@ -179,6 +180,10 @@ class Application: rerank_models_service: model_service.RerankModelsService = None + asr_models_service: model_service.ASRModelsService = None + + tts_models_service: model_service.TTSModelsService = None + provider_service: provider_service.ModelProviderService = None pipeline_service: pipeline_service.PipelineService = None diff --git a/src/langbot/pkg/core/stages/build_app.py b/src/langbot/pkg/core/stages/build_app.py index c42dd769e..94f073a6b 100644 --- a/src/langbot/pkg/core/stages/build_app.py +++ b/src/langbot/pkg/core/stages/build_app.py @@ -111,6 +111,12 @@ class BuildAppStage(stage.BootingStage): rerank_models_service_inst = model_service.RerankModelsService(ap) ap.rerank_models_service = rerank_models_service_inst + asr_models_service_inst = model_service.ASRModelsService(ap) + ap.asr_models_service = asr_models_service_inst + + tts_models_service_inst = model_service.TTSModelsService(ap) + ap.tts_models_service = tts_models_service_inst + provider_service_inst = provider_service.ModelProviderService(ap) ap.provider_service = provider_service_inst diff --git a/src/langbot/pkg/core/stages/load_config.py b/src/langbot/pkg/core/stages/load_config.py index b41e2accd..eac589a34 100644 --- a/src/langbot/pkg/core/stages/load_config.py +++ b/src/langbot/pkg/core/stages/load_config.py @@ -345,4 +345,5 @@ class LoadConfigStage(stage.BootingStage): ap.pipeline_config_meta_trigger = await load_resource_yaml_template_data('metadata/pipeline/trigger.yaml') ap.pipeline_config_meta_safety = await load_resource_yaml_template_data('metadata/pipeline/safety.yaml') ap.pipeline_config_meta_ai = await load_resource_yaml_template_data('metadata/pipeline/ai.yaml') + ap.pipeline_config_meta_speech = await load_resource_yaml_template_data('metadata/pipeline/speech.yaml') ap.pipeline_config_meta_output = await load_resource_yaml_template_data('metadata/pipeline/output.yaml') diff --git a/src/langbot/pkg/entity/persistence/model.py b/src/langbot/pkg/entity/persistence/model.py index 1e133475a..645b7f158 100644 --- a/src/langbot/pkg/entity/persistence/model.py +++ b/src/langbot/pkg/entity/persistence/model.py @@ -163,3 +163,71 @@ class RerankModel(Base): sqlalchemy.Index('ix_rerank_models_workspace_provider', 'workspace_uuid', 'provider_uuid'), sqlalchemy.Index('ix_rerank_models_workspace_name', 'workspace_uuid', 'name'), ) + + +class ASRModel(Base): + """ASR model""" + + __tablename__ = 'asr_models' + + uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True) + workspace_uuid = sqlalchemy.Column( + sqlalchemy.String(36), + sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'), + nullable=False, + ) + name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + provider_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={}) + prefered_ranking = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0) + created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now()) + updated_at = sqlalchemy.Column( + sqlalchemy.DateTime, + nullable=False, + server_default=sqlalchemy.func.now(), + onupdate=sqlalchemy.func.now(), + ) + + __table_args__ = ( + sqlalchemy.ForeignKeyConstraint( + ['workspace_uuid', 'provider_uuid'], + ['model_providers.workspace_uuid', 'model_providers.uuid'], + name='fk_asr_models_workspace_provider', + ), + sqlalchemy.Index('ix_asr_models_workspace_provider', 'workspace_uuid', 'provider_uuid'), + sqlalchemy.Index('ix_asr_models_workspace_name', 'workspace_uuid', 'name'), + ) + + +class TTSModel(Base): + """TTS model""" + + __tablename__ = 'tts_models' + + uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True) + workspace_uuid = sqlalchemy.Column( + sqlalchemy.String(36), + sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'), + nullable=False, + ) + name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + provider_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={}) + prefered_ranking = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0) + created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now()) + updated_at = sqlalchemy.Column( + sqlalchemy.DateTime, + nullable=False, + server_default=sqlalchemy.func.now(), + onupdate=sqlalchemy.func.now(), + ) + + __table_args__ = ( + sqlalchemy.ForeignKeyConstraint( + ['workspace_uuid', 'provider_uuid'], + ['model_providers.workspace_uuid', 'model_providers.uuid'], + name='fk_tts_models_workspace_provider', + ), + sqlalchemy.Index('ix_tts_models_workspace_provider', 'workspace_uuid', 'provider_uuid'), + sqlalchemy.Index('ix_tts_models_workspace_name', 'workspace_uuid', 'name'), + ) diff --git a/src/langbot/pkg/i18n/locales/en_US.json b/src/langbot/pkg/i18n/locales/en_US.json index dd07ff255..d5ae9f4b4 100644 --- a/src/langbot/pkg/i18n/locales/en_US.json +++ b/src/langbot/pkg/i18n/locales/en_US.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.", "Option {number}": "Option {number}", "Confirmation needed": "Confirmation needed", - "Session reset successfully": "Session reset successfully" + "Session reset successfully": "Session reset successfully", + "Voice transcription failed.": "Voice transcription failed.", + "Voice message exceeds maximum duration.": "Voice message exceeds maximum duration.", + "Voice synthesis failed.": "Voice synthesis failed." } diff --git a/src/langbot/pkg/i18n/locales/es_ES.json b/src/langbot/pkg/i18n/locales/es_ES.json index 116aa09a8..1aa2b69c2 100644 --- a/src/langbot/pkg/i18n/locales/es_ES.json +++ b/src/langbot/pkg/i18n/locales/es_ES.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "No se pudo iniciar el acceso por dispositivo. Activa el inicio de sesión con código de dispositivo en la configuración de ChatGPT.", "Option {number}": "Opción {number}", "Confirmation needed": "Se requiere confirmación", - "Session reset successfully": "Sesión restablecida" + "Session reset successfully": "Sesión restablecida", + "Voice transcription failed.": "Error en la transcripción de voz.", + "Voice message exceeds maximum duration.": "El mensaje de voz excede la duración máxima.", + "Voice synthesis failed.": "Error en la síntesis de voz." } diff --git a/src/langbot/pkg/i18n/locales/ja_JP.json b/src/langbot/pkg/i18n/locales/ja_JP.json index 292bb1b56..153eccb25 100644 --- a/src/langbot/pkg/i18n/locales/ja_JP.json +++ b/src/langbot/pkg/i18n/locales/ja_JP.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "ChatGPT のデバイスログインを開始できません。ChatGPT の設定でデバイスコードログインを有効にしてください。", "Option {number}": "選択肢 {number}", "Confirmation needed": "確認が必要です", - "Session reset successfully": "セッションをリセットしました" + "Session reset successfully": "セッションをリセットしました", + "Voice transcription failed.": "音声の文字起こしに失敗しました。", + "Voice message exceeds maximum duration.": "音声メッセージが最大時間を超えています。", + "Voice synthesis failed.": "音声合成に失敗しました。" } diff --git a/src/langbot/pkg/i18n/locales/ru_RU.json b/src/langbot/pkg/i18n/locales/ru_RU.json index ed9529018..b9a87de23 100644 --- a/src/langbot/pkg/i18n/locales/ru_RU.json +++ b/src/langbot/pkg/i18n/locales/ru_RU.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "Не удалось начать вход с устройства. Включите вход по коду устройства в настройках ChatGPT.", "Option {number}": "Вариант {number}", "Confirmation needed": "Требуется подтверждение", - "Session reset successfully": "Сессия сброшена" + "Session reset successfully": "Сессия сброшена", + "Voice transcription failed.": "Не удалось распознать голосовое сообщение.", + "Voice message exceeds maximum duration.": "Длительность голосового сообщения превышает максимум.", + "Voice synthesis failed.": "Сбой синтеза речи." } diff --git a/src/langbot/pkg/i18n/locales/th_TH.json b/src/langbot/pkg/i18n/locales/th_TH.json index ff40282cd..134809789 100644 --- a/src/langbot/pkg/i18n/locales/th_TH.json +++ b/src/langbot/pkg/i18n/locales/th_TH.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "ไม่สามารถเริ่มเข้าสู่ระบบด้วยอุปกรณ์ได้ โปรดเปิดการเข้าสู่ระบบด้วยรหัสอุปกรณ์ในการตั้งค่า ChatGPT", "Option {number}": "ตัวเลือก {number}", "Confirmation needed": "ต้องยืนยัน", - "Session reset successfully": "รีเซ็ตเซสชันแล้ว" + "Session reset successfully": "รีเซ็ตเซสชันแล้ว", + "Voice transcription failed.": "การแปลงเสียงเป็นข้อความล้มเหลว", + "Voice message exceeds maximum duration.": "ข้อความเสียงมีความยาวเกินกำหนดสูงสุด", + "Voice synthesis failed.": "การสังเคราะห์เสียงล้มเหลว" } diff --git a/src/langbot/pkg/i18n/locales/vi_VN.json b/src/langbot/pkg/i18n/locales/vi_VN.json index 39b3110a8..5ebe612e5 100644 --- a/src/langbot/pkg/i18n/locales/vi_VN.json +++ b/src/langbot/pkg/i18n/locales/vi_VN.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "Không thể bắt đầu đăng nhập thiết bị ChatGPT. Hãy bật đăng nhập bằng mã thiết bị trong cài đặt ChatGPT.", "Option {number}": "Lựa chọn {number}", "Confirmation needed": "Cần xác nhận", - "Session reset successfully": "Đã đặt lại phiên" + "Session reset successfully": "Đã đặt lại phiên", + "Voice transcription failed.": "Chuyển đổi giọng nói thành văn bản thất bại.", + "Voice message exceeds maximum duration.": "Tin nhắn thoại vượt quá thời lượng tối đa.", + "Voice synthesis failed.": "Tổng hợp giọng nói thất bại." } diff --git a/src/langbot/pkg/i18n/locales/zh_Hans.json b/src/langbot/pkg/i18n/locales/zh_Hans.json index bf507bd74..abd258dc2 100644 --- a/src/langbot/pkg/i18n/locales/zh_Hans.json +++ b/src/langbot/pkg/i18n/locales/zh_Hans.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "无法启动 ChatGPT 设备登录。请在 ChatGPT 设置中启用设备代码登录。", "Option {number}": "选项 {number}", "Confirmation needed": "需要确认", - "Session reset successfully": "会话已重置" + "Session reset successfully": "会话已重置", + "Voice transcription failed.": "语音转写失败。", + "Voice message exceeds maximum duration.": "语音消息超出最大时长。", + "Voice synthesis failed.": "语音合成失败。" } diff --git a/src/langbot/pkg/i18n/locales/zh_Hant.json b/src/langbot/pkg/i18n/locales/zh_Hant.json index 15f5e90c3..6cfe3a887 100644 --- a/src/langbot/pkg/i18n/locales/zh_Hant.json +++ b/src/langbot/pkg/i18n/locales/zh_Hant.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "無法啟動 ChatGPT 裝置登入。請在 ChatGPT 設定中啟用裝置代碼登入。", "Option {number}": "選項 {number}", "Confirmation needed": "需要確認", - "Session reset successfully": "對話已重設" + "Session reset successfully": "對話已重設", + "Voice transcription failed.": "語音轉寫失敗。", + "Voice message exceeds maximum duration.": "語音訊息超過最大時長。", + "Voice synthesis failed.": "語音合成失敗。" } diff --git a/src/langbot/pkg/persistence/alembic/versions/0043_speech_models.py b/src/langbot/pkg/persistence/alembic/versions/0043_speech_models.py new file mode 100644 index 000000000..c662adb86 --- /dev/null +++ b/src/langbot/pkg/persistence/alembic/versions/0043_speech_models.py @@ -0,0 +1,183 @@ +"""Add ASR and TTS models and speech pipeline stages. + +Revision ID: 0043_speech_models +Revises: 0042_sqlite_incremental_vacuum +""" + +import json +from alembic import op +import sqlalchemy as sa + +revision = '0043_speech_models' +down_revision = '0042_sqlite_incremental_vacuum' +branch_labels = None +depends_on = None + +DEFAULT_SPEECH_CONFIG = { + 'speech-recognition': { + 'enabled': False, + 'model': '', + 'max-duration': 60, + 'keep-original-voice': False, + 'on-failure': 'ignore', + }, + 'text-to-speech': { + 'enabled': False, + 'model': '', + 'trigger': 'voice-input-only', + 'output': 'voice-only', + 'max-text-length': 1000, + 'voice': '', + 'speed': 1.0, + }, +} + + +def upgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + existing_tables = set(inspector.get_table_names()) + model_parents_exist = {'workspaces', 'model_providers'} <= existing_tables + + if model_parents_exist and 'asr_models' not in existing_tables: + op.create_table( + 'asr_models', + sa.Column('uuid', sa.String(255), primary_key=True, unique=True), + sa.Column( + 'workspace_uuid', + sa.String(36), + sa.ForeignKey('workspaces.uuid', ondelete='CASCADE'), + nullable=False, + ), + sa.Column('name', sa.String(255), nullable=False), + sa.Column('provider_uuid', sa.String(255), nullable=False), + sa.Column('extra_args', sa.JSON, nullable=False, server_default='{}'), + sa.Column('prefered_ranking', sa.Integer, nullable=False, server_default='0'), + sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.ForeignKeyConstraint( + ['workspace_uuid', 'provider_uuid'], + ['model_providers.workspace_uuid', 'model_providers.uuid'], + name='fk_asr_models_workspace_provider', + ), + ) + op.create_index( + 'ix_asr_models_workspace_provider', + 'asr_models', + ['workspace_uuid', 'provider_uuid'], + ) + op.create_index( + 'ix_asr_models_workspace_name', + 'asr_models', + ['workspace_uuid', 'name'], + ) + + if model_parents_exist and 'tts_models' not in existing_tables: + op.create_table( + 'tts_models', + sa.Column('uuid', sa.String(255), primary_key=True, unique=True), + sa.Column( + 'workspace_uuid', + sa.String(36), + sa.ForeignKey('workspaces.uuid', ondelete='CASCADE'), + nullable=False, + ), + sa.Column('name', sa.String(255), nullable=False), + sa.Column('provider_uuid', sa.String(255), nullable=False), + sa.Column('extra_args', sa.JSON, nullable=False, server_default='{}'), + sa.Column('prefered_ranking', sa.Integer, nullable=False, server_default='0'), + sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.ForeignKeyConstraint( + ['workspace_uuid', 'provider_uuid'], + ['model_providers.workspace_uuid', 'model_providers.uuid'], + name='fk_tts_models_workspace_provider', + ), + ) + op.create_index( + 'ix_tts_models_workspace_provider', + 'tts_models', + ['workspace_uuid', 'provider_uuid'], + ) + op.create_index( + 'ix_tts_models_workspace_name', + 'tts_models', + ['workspace_uuid', 'name'], + ) + + if conn.dialect.name == 'postgresql': + for tbl in ('asr_models', 'tts_models'): + if not sa.inspect(conn).has_table(tbl): + continue + op.execute(f'ALTER TABLE {tbl} ENABLE ROW LEVEL SECURITY') + op.execute(f'ALTER TABLE {tbl} FORCE ROW LEVEL SECURITY') + op.execute(f'DROP POLICY IF EXISTS langbot_workspace_isolation ON {tbl}') + op.execute(f"""CREATE POLICY langbot_workspace_isolation ON {tbl} + USING (workspace_uuid::text = NULLIF(current_setting('langbot.workspace_uuid', true), '')) + WITH CHECK (workspace_uuid::text = NULLIF(current_setting('langbot.workspace_uuid', true), ''))""") + + # Migrate legacy_pipelines stages and config + if 'legacy_pipelines' in existing_tables: + pipeline_columns = {col['name'] for col in inspector.get_columns('legacy_pipelines')} + if {'stages', 'config', 'uuid'} <= pipeline_columns: + metadata = sa.MetaData() + pipelines_tbl = sa.Table('legacy_pipelines', metadata, autoload_with=conn) + rows = conn.execute( + sa.select(pipelines_tbl.c.uuid, pipelines_tbl.c.stages, pipelines_tbl.c.config) + ).fetchall() + + for row in rows: + p_uuid, stages_raw, config_raw = row[0], row[1], row[2] + stages = stages_raw if isinstance(stages_raw, list) else json.loads(stages_raw or '[]') + config = config_raw if isinstance(config_raw, dict) else json.loads(config_raw or '{}') + + stages_changed = False + config_changed = False + + if 'SpeechRecognitionStage' not in stages: + # Place after BanSessionCheckStage or GroupRespondRuleCheckStage, and before PreContentFilterStage + if 'BanSessionCheckStage' in stages: + idx = stages.index('BanSessionCheckStage') + 1 + stages.insert(idx, 'SpeechRecognitionStage') + elif 'GroupRespondRuleCheckStage' in stages: + idx = stages.index('GroupRespondRuleCheckStage') + 1 + stages.insert(idx, 'SpeechRecognitionStage') + elif 'PreContentFilterStage' in stages: + idx = stages.index('PreContentFilterStage') + stages.insert(idx, 'SpeechRecognitionStage') + else: + stages.append('SpeechRecognitionStage') + stages_changed = True + + if 'TextToSpeechStage' not in stages: + # Place after ResponseWrapper, before LongTextProcessStage + if 'ResponseWrapper' in stages: + idx = stages.index('ResponseWrapper') + 1 + stages.insert(idx, 'TextToSpeechStage') + elif 'LongTextProcessStage' in stages: + idx = stages.index('LongTextProcessStage') + stages.insert(idx, 'TextToSpeechStage') + else: + stages.append('TextToSpeechStage') + stages_changed = True + + if 'speech' not in config: + config['speech'] = DEFAULT_SPEECH_CONFIG.copy() + config_changed = True + + if stages_changed or config_changed: + conn.execute( + sa.update(pipelines_tbl) + .where(pipelines_tbl.c.uuid == p_uuid) + .values(stages=stages, config=config) + ) + + +def downgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + existing_tables = set(inspector.get_table_names()) + if 'tts_models' in existing_tables: + op.drop_table('tts_models') + if 'asr_models' in existing_tables: + op.drop_table('asr_models') diff --git a/src/langbot/pkg/persistence/mgr.py b/src/langbot/pkg/persistence/mgr.py index 445af890a..50a24981d 100644 --- a/src/langbot/pkg/persistence/mgr.py +++ b/src/langbot/pkg/persistence/mgr.py @@ -70,6 +70,8 @@ _ALEMBIC_TENANT_TABLES = { 'llm_models', 'embedding_models', 'rerank_models', + 'asr_models', + 'tts_models', 'legacy_pipelines', 'pipeline_run_records', 'pipeline_migration_snapshots', diff --git a/src/langbot/pkg/persistence/tenant_uow.py b/src/langbot/pkg/persistence/tenant_uow.py index 51974d66a..4821fa597 100644 --- a/src/langbot/pkg/persistence/tenant_uow.py +++ b/src/langbot/pkg/persistence/tenant_uow.py @@ -59,6 +59,8 @@ TENANT_TABLE_COLUMNS: dict[str, str] = { 'llm_models': 'workspace_uuid', 'embedding_models': 'workspace_uuid', 'rerank_models': 'workspace_uuid', + 'asr_models': 'workspace_uuid', + 'tts_models': 'workspace_uuid', 'legacy_pipelines': 'workspace_uuid', 'pipeline_run_records': 'workspace_uuid', 'pipeline_migration_snapshots': 'workspace_uuid', diff --git a/src/langbot/pkg/pipeline/pipelinemgr.py b/src/langbot/pkg/pipeline/pipelinemgr.py index 194f61478..baf31619b 100644 --- a/src/langbot/pkg/pipeline/pipelinemgr.py +++ b/src/langbot/pkg/pipeline/pipelinemgr.py @@ -39,6 +39,7 @@ from . import ( wrapper, preproc, ratelimit, + speech, ) importutil.import_modules_in_pkgs( @@ -52,6 +53,7 @@ importutil.import_modules_in_pkgs( wrapper, preproc, ratelimit, + speech, ] ) @@ -915,6 +917,7 @@ class PipelineManager: getattr(self.ap, 'pipeline_config_meta_trigger', {'name': 'trigger', 'stages': []}), getattr(self.ap, 'pipeline_config_meta_safety', {'name': 'safety', 'stages': []}), getattr(self.ap, 'pipeline_config_meta_ai', {'name': 'ai', 'stages': []}), + getattr(self.ap, 'pipeline_config_meta_speech', {'name': 'speech', 'stages': []}), getattr(self.ap, 'pipeline_config_meta_output', {'name': 'output', 'stages': []}), ) diff --git a/src/langbot/pkg/pipeline/speech/__init__.py b/src/langbot/pkg/pipeline/speech/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/src/langbot/pkg/pipeline/speech/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/src/langbot/pkg/pipeline/speech/speech.py b/src/langbot/pkg/pipeline/speech/speech.py new file mode 100644 index 000000000..6ad99aa23 --- /dev/null +++ b/src/langbot/pkg/pipeline/speech/speech.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import base64 +import logging +import re +import typing + +import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query +import langbot_plugin.api.entities.builtin.platform.message as platform_message + +from ...i18n import pipeline_notice +from ...utils import audio as audio_utils +from .. import entities, stage +from ..pool import get_query_execution_context + +logger = logging.getLogger(__name__) + + +def strip_markdown(text: str) -> str: + """Strip markdown formatting from text for TTS input.""" + # Remove code blocks + text = re.sub(r'```[\s\S]*?```', '', text) + # Remove inline code + text = re.sub(r'`([^`]+)`', r'\1', text) + # Remove images: ![alt](url) -> '' + text = re.sub(r'!\[([^\]]*)\]\([^\)]+\)', '', text) + # Replace links: [text](url) -> text + text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text) + # Remove headers (# Heading) + text = re.sub(r'^\s*#{1,6}\s*', '', text, flags=re.MULTILINE) + # Remove bold/italic (**text**, *text*, __text__, _text_) + text = re.sub(r'(\*\*|__)(.*?)\1', r'\2', text) + text = re.sub(r'(\*|_)(.*?)\1', r'\2', text) + # Remove strikethrough (~~text~~) + text = re.sub(r'~~(.*?)~~', r'\1', text) + # Remove blockquotes + text = re.sub(r'^\s*>\s*', '', text, flags=re.MULTILINE) + # Remove horizontal rules + text = re.sub(r'^\s*[-*_]{3,}\s*$', '', text, flags=re.MULTILINE) + # Normalize whitespace + text = re.sub(r'\n{3,}', '\n\n', text).strip() + return text + + +@stage.stage_class('SpeechRecognitionStage') +class SpeechRecognitionStage(stage.PipelineStage): + """Speech Recognition (ASR) Stage + + Transcribes incoming voice messages into plain text and updates the message chain. + """ + + async def initialize(self, pipeline_config: dict): + pass + + async def process( + self, + query: pipeline_query.Query, + stage_inst_name: str, + ) -> entities.StageProcessResult: + config = query.pipeline_config.get('speech', {}).get('speech-recognition', {}) + if not config.get('enable', False): + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + model_uuid = config.get('model') + if not model_uuid: + self.ap.logger.debug('SpeechRecognitionStage enabled but no model configured; skipping.') + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + voice_components = [comp for comp in query.message_chain if isinstance(comp, platform_message.Voice)] + if not voice_components: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + query.variables['_input_was_voice'] = True + primary_voice = voice_components[0] + + max_duration = config.get('max-duration', 60) + if primary_voice.length and primary_voice.length > max_duration: + self.ap.logger.info( + f'Voice message duration ({primary_voice.length}s) exceeds max-duration ({max_duration}s).' + ) + if config.get('on-failure', 'notify') == 'notify': + notice = pipeline_notice( + query, + 'voice-duration-exceeded-hint', + 'Voice message exceeds maximum duration.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.INTERRUPT, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + try: + raw_bytes, source_format = await audio_utils.extract_voice_bytes(primary_voice) + audio_bytes = raw_bytes + target_format = source_format if source_format in ('wav', 'mp3', 'ogg', 'm4a', 'flac') else 'wav' + if source_format != target_format and audio_utils.is_ffmpeg_available(): + audio_bytes = await audio_utils.transcode_audio(raw_bytes, target_format, source_format) + except Exception as exc: + self.ap.logger.error(f'Failed to extract or transcode voice bytes: {exc}') + if config.get('on-failure', 'notify') == 'notify': + notice = pipeline_notice( + query, + 'voice-transcribe-failed-hint', + 'Voice transcription failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.INTERRUPT, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + execution_context = get_query_execution_context(query) + asr_model = await self.ap.model_mgr.get_asr_model_by_uuid(execution_context, model_uuid) + if asr_model is None: + self.ap.logger.error(f'ASR model {model_uuid} not found.') + if config.get('on-failure', 'notify') == 'notify': + notice = pipeline_notice( + query, + 'voice-transcribe-failed-hint', + 'Voice transcription failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.INTERRUPT, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + try: + transcript = await asr_model.provider.invoke_asr( + model=asr_model, + audio=audio_bytes, + mime=f'audio/{target_format}', + execution_context=asr_model.execution_context, + ) + transcript = (transcript or '').strip() + except Exception as exc: + self.ap.logger.error(f'ASR model invocation failed: {exc}') + if config.get('on-failure', 'notify') == 'notify': + notice = pipeline_notice( + query, + 'voice-transcribe-failed-hint', + 'Voice transcription failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.INTERRUPT, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + query.variables['voice_transcript'] = transcript + + keep_original = config.get('keep-original-voice', False) + new_chain = [] + for comp in query.message_chain: + if isinstance(comp, platform_message.Voice): + if keep_original: + new_chain.append(comp) + new_chain.append(platform_message.Plain(text=transcript)) + else: + new_chain.append(comp) + query.message_chain = platform_message.MessageChain(new_chain) + + quote = getattr(query, 'quote', None) + if quote and getattr(quote, 'origin', None): + new_origin = [] + for comp in quote.origin: + if isinstance(comp, platform_message.Voice): + if keep_original: + new_origin.append(comp) + new_origin.append(platform_message.Plain(text=transcript)) + else: + new_origin.append(comp) + quote.origin = platform_message.MessageChain(new_origin) + + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + +@stage.stage_class('TextToSpeechStage') +class TextToSpeechStage(stage.PipelineStage): + """Text to Speech Stage + + Synthesizes outgoing response text into voice and attaches/replaces with Voice component. + """ + + async def initialize(self, pipeline_config: dict): + pass + + async def process( + self, + query: pipeline_query.Query, + stage_inst_name: str, + ) -> entities.StageProcessResult: + config = query.pipeline_config.get('speech', {}).get('text-to-speech', {}) + if not config.get('enable', False): + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + model_uuid = config.get('model') + if not model_uuid: + self.ap.logger.debug('TextToSpeechStage enabled but no model configured; skipping.') + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + trigger = config.get('trigger', 'voice-input-only') + if trigger == 'never': + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + if trigger == 'voice-input-only' and not query.variables.get('_input_was_voice', False): + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + # Skip command messages, tool call announcements, or streaming chunks that are not final + if query.resp_messages: + last_resp = query.resp_messages[-1] + if getattr(last_resp, 'role', None) == 'command': + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + if getattr(last_resp, 'tool_calls', None): + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + if hasattr(last_resp, 'is_final') and not last_resp.is_final: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + if not query.resp_message_chain: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + plain_texts = [comp.text for comp in query.resp_message_chain[-1] if isinstance(comp, platform_message.Plain)] + full_text = ''.join(plain_texts).strip() + if not full_text: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + clean_text = strip_markdown(full_text) + if not clean_text: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + max_text_length = config.get('max-text-length', 500) + if len(clean_text) > max_text_length: + self.ap.logger.info( + f'Response text length ({len(clean_text)}) exceeds max-text-length ({max_text_length}); falling back to text.' + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + execution_context = get_query_execution_context(query) + tts_model = await self.ap.model_mgr.get_tts_model_by_uuid(execution_context, model_uuid) + if tts_model is None: + self.ap.logger.warning(f'TTS model {model_uuid} not found; falling back to text.') + if config.get('on-failure', 'fallback') == 'notify': + notice = pipeline_notice( + query, + 'voice-synthesis-failed-hint', + 'Voice synthesis failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.CONTINUE, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + model_extra = getattr(tts_model, 'extra_args', None) + if model_extra is None and hasattr(tts_model, 'model_entity') and tts_model.model_entity is not None: + model_extra = getattr(tts_model.model_entity, 'extra_args', {}) + model_extra = model_extra or {} + voice_param = config.get('voice') or model_extra.get('voice') + extra_args: dict[str, typing.Any] = {'response_format': 'mp3'} + if voice_param: + extra_args['voice'] = voice_param + + try: + audio_bytes, mime = await tts_model.provider.invoke_tts( + model=tts_model, + text=clean_text, + extra_args=extra_args, + execution_context=tts_model.execution_context, + ) + except Exception as exc: + self.ap.logger.error(f'TTS invocation failed: {exc}; falling back to text.') + if config.get('on-failure', 'fallback') == 'notify': + notice = pipeline_notice( + query, + 'voice-synthesis-failed-hint', + 'Voice synthesis failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.CONTINUE, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + b64_audio = base64.b64encode(audio_bytes).decode('ascii') + audio_duration = audio_utils.estimate_audio_duration( + audio_bytes, mime.split('/')[-1] if '/' in mime else 'unknown' + ) + voice_comp = platform_message.Voice( + base64=f'data:{mime};base64,{b64_audio}', + length=int(round(audio_duration)) if audio_duration > 0 else None, + ) + + output_mode = config.get('output', 'voice-only') + if output_mode == 'voice-only': + non_plain_components = [ + comp for comp in query.resp_message_chain[-1] if not isinstance(comp, platform_message.Plain) + ] + query.resp_message_chain[-1] = platform_message.MessageChain([voice_comp] + non_plain_components) + else: + query.resp_message_chain[-1].append(voice_comp) + + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) diff --git a/src/langbot/pkg/platform/adapters/aiocqhttp/message_converter.py b/src/langbot/pkg/platform/adapters/aiocqhttp/message_converter.py index a8e2fe2a2..106a3b397 100644 --- a/src/langbot/pkg/platform/adapters/aiocqhttp/message_converter.py +++ b/src/langbot/pkg/platform/adapters/aiocqhttp/message_converter.py @@ -99,7 +99,18 @@ class AiocqhttpMessageConverter(abstract_platform_adapter.AbstractMessageConvert if file_arg: target.append(aiocqhttp.MessageSegment.image(file_arg)) elif isinstance(component, platform_message.Voice): - file_arg = AiocqhttpMessageConverter._file_arg(component) + file_arg = None + from ....utils import audio as audio_utils + + if audio_utils.is_silk_available() and audio_utils.is_ffmpeg_available(): + try: + voice_bytes, src_fmt = await audio_utils.extract_voice_bytes(component) + silk_bytes = await audio_utils.transcode_audio(voice_bytes, 'silk', src_fmt) + file_arg = f'base64://{base64.b64encode(silk_bytes).decode("ascii")}' + except Exception: + pass + if not file_arg: + file_arg = AiocqhttpMessageConverter._file_arg(component) if file_arg: target.append(aiocqhttp.MessageSegment.record(file_arg)) elif isinstance(component, platform_message.File): diff --git a/src/langbot/pkg/platform/adapters/dingtalk/adapter.py b/src/langbot/pkg/platform/adapters/dingtalk/adapter.py index 60a14087d..6dd0f55df 100644 --- a/src/langbot/pkg/platform/adapters/dingtalk/adapter.py +++ b/src/langbot/pkg/platform/adapters/dingtalk/adapter.py @@ -167,7 +167,11 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo incoming_message = message_source.source_platform_object.incoming_message markdown_enabled = self.config.get('markdown_card', False) text_message = platform_message.MessageChain( - [part for part in message if not isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if not isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) content, at = await DingTalkMessageConverter.yiri2target(text_message, markdown_enabled) raw = await self.bot.send_message(content, incoming_message, at) @@ -180,7 +184,7 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo async def _send_attachments(self, message, incoming_message): failures = [] for part in message: - if isinstance(part, (platform_message.Image, platform_message.File)): + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)): try: await self.bot.send_attachment(part, incoming_message) except Exception as exc: @@ -207,7 +211,11 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo markdown_enabled = self.config.get('markdown_card', False) text_message = platform_message.MessageChain( - [part for part in message if not isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if not isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) content, _ = await DingTalkMessageConverter.yiri2target(text_message, markdown_enabled) card_instance, card_instance_id = self.card_instance_id_dict[message_id] diff --git a/src/langbot/pkg/platform/adapters/dingtalk/message_converter.py b/src/langbot/pkg/platform/adapters/dingtalk/message_converter.py index 253eed8d1..c9090d13a 100644 --- a/src/langbot/pkg/platform/adapters/dingtalk/message_converter.py +++ b/src/langbot/pkg/platform/adapters/dingtalk/message_converter.py @@ -125,10 +125,11 @@ class DingTalkMessageConverter(abstract_platform_adapter.AbstractMessageConverte if event.file: components.append(platform_message.File(url=event.file, name=event.name or 'file')) if event.audio: - if event.content and event.type == 'audio': - components.append(platform_message.Plain(text=event.content)) - else: - components.append(platform_message.Voice(base64=event.audio)) + # Keep the audio so the pipeline's speech recognition setting decides how it is transcribed. + components.append(platform_message.Voice(base64=event.audio, length=event.audio_duration or None)) + elif event.type == 'audio' and event.content: + # The audio could not be downloaded; fall back to DingTalk's own transcription. + components.append(platform_message.Plain(text=event.content)) quote = DingTalkMessageConverter._quote_component(event) if quote: diff --git a/src/langbot/pkg/platform/adapters/discord/message_converter.py b/src/langbot/pkg/platform/adapters/discord/message_converter.py index 82bb5aa6c..f8faedfa5 100644 --- a/src/langbot/pkg/platform/adapters/discord/message_converter.py +++ b/src/langbot/pkg/platform/adapters/discord/message_converter.py @@ -66,7 +66,35 @@ class DiscordMessageConverter: elements.extend(DiscordMessageConverter._text_components(message.content)) for attachment in message.attachments: - if DiscordMessageConverter._is_image_attachment(attachment): + if DiscordMessageConverter._is_audio_attachment(attachment): + audio_data = b'' + mime_type = getattr(attachment, 'content_type', None) or 'audio/ogg' + try: + audio_data, downloaded_mime = await DiscordMessageConverter._download(attachment.url) + if downloaded_mime: + mime_type = downloaded_mime + except Exception: + pass + from ....utils import audio as audio_utils + + detected_fmt = audio_utils.detect_audio_format(audio_data) if audio_data else 'unknown' + duration = getattr(attachment, 'duration_secs', None) + if duration is not None: + duration = int(round(duration)) + elif audio_data: + duration = int(round(audio_utils.estimate_audio_duration(audio_data, detected_fmt))) + + import base64 as _b64 + + b64_val = _b64.b64encode(audio_data).decode('utf-8') if audio_data else '' + elements.append( + platform_message.Voice( + url=attachment.url, + base64=f'data:{mime_type};base64,{b64_val}' if b64_val else '', + length=duration, + ) + ) + elif DiscordMessageConverter._is_image_attachment(attachment): elements.append(platform_message.Image(url=attachment.url)) else: elements.append( @@ -134,22 +162,22 @@ class DiscordMessageConverter: @staticmethod async def _load_voice(element: platform_message.Voice) -> tuple[bytes | None, str]: - filename = f'{uuid.uuid4()}.mp3' - if element.base64: - header, _, payload = element.base64.partition(',') - data = payload or header - for ext in ('wav', 'mp3', 'ogg', 'm4a', 'aac', 'flac', 'opus', 'webm'): - if ext in header: - filename = f'{uuid.uuid4()}.{ext}' - break - return await asyncio.to_thread(_decode_discord_base64_limited, data), filename - if element.url: - data, _ = await DiscordMessageConverter._download(element.url) - return data, filename - return None, filename + from ....utils import audio as audio_utils + + raw_bytes = await DiscordMessageConverter._load_file(element) + fmt = audio_utils.detect_audio_format(raw_bytes) if raw_bytes else 'unknown' + if fmt in ('silk', 'amr') and audio_utils.is_ffmpeg_available() and raw_bytes: + try: + raw_bytes = await audio_utils.transcode_audio(raw_bytes, 'mp3', fmt) + fmt = 'mp3' + except Exception: + pass + ext = fmt if fmt != 'unknown' else 'mp3' + filename = getattr(element, 'name', '') or f'{uuid.uuid4()}.{ext}' + return raw_bytes, filename @staticmethod - async def _load_file(element: platform_message.File) -> bytes | None: + async def _load_file(element: platform_message.File | platform_message.Voice) -> bytes | None: if element.base64: return await asyncio.to_thread(_decode_discord_base64_limited, element.base64) if element.url: @@ -169,7 +197,20 @@ class DiscordMessageConverter: @staticmethod def _is_image_attachment(attachment: discord.Attachment) -> bool: - content_type = attachment.content_type or '' - return content_type.startswith('image/') or attachment.filename.lower().endswith( + content_type = getattr(attachment, 'content_type', '') or '' + filename = getattr(attachment, 'filename', '') or '' + return content_type.startswith('image/') or filename.lower().endswith( ('.png', '.jpg', '.jpeg', '.gif', '.webp') ) + + @staticmethod + def _is_audio_attachment(attachment: discord.Attachment) -> bool: + content_type = getattr(attachment, 'content_type', '') or '' + filename = (getattr(attachment, 'filename', '') or '').lower() + flags = getattr(attachment, 'flags', None) + is_voice = getattr(flags, 'voice', False) if flags else False + return ( + is_voice + or content_type.startswith('audio/') + or filename.endswith(('.mp3', '.ogg', '.oga', '.opus', '.wav', '.m4a', '.aac', '.flac', '.weba')) + ) diff --git a/src/langbot/pkg/platform/adapters/lark/message_converter.py b/src/langbot/pkg/platform/adapters/lark/message_converter.py index 40db36bde..797ac1af3 100644 --- a/src/langbot/pkg/platform/adapters/lark/message_converter.py +++ b/src/langbot/pkg/platform/adapters/lark/message_converter.py @@ -132,6 +132,13 @@ class LarkMessageConverter(abstract_platform_adapter.AbstractMessageConverter): elif isinstance(msg, platform_message.Voice): data = await LarkMessageConverter._get_component_bytes(msg) if data: + from ....utils import audio as audio_utils + + if audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'opus') + except Exception: + pass duration = int(msg.length * 1000) if msg.length else None file_key = await LarkMessageConverter.upload_file_to_lark( data, api_client, file_type='opus', file_name='voice.opus', duration=duration diff --git a/src/langbot/pkg/platform/adapters/telegram/adapter.py b/src/langbot/pkg/platform/adapters/telegram/adapter.py index 5d6519267..4f1664726 100644 --- a/src/langbot/pkg/platform/adapters/telegram/adapter.py +++ b/src/langbot/pkg/platform/adapters/telegram/adapter.py @@ -239,6 +239,14 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo filename = component.get('filename', 'file') args['document'] = telegram.InputFile(doc, filename=filename) sent_messages.append(await self.bot.send_document(**args)) + elif component_type == 'voice': + voice = component.get('voice') + if voice is None: + continue + args['voice'] = telegram.InputFile(voice, filename='voice.ogg') + if component.get('duration'): + args['duration'] = component['duration'] + sent_messages.append(await self.bot.send_voice(**args)) return platform_events.MessageResult( message_id=sent_messages[-1].message_id if sent_messages else None, @@ -290,6 +298,14 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo filename = component.get('filename', 'file') args['document'] = telegram.InputFile(doc, filename=filename) sent_messages.append(await self.bot.send_document(**args)) + elif component_type == 'voice': + voice = component.get('voice') + if voice is None: + continue + args['voice'] = telegram.InputFile(voice, filename='voice.ogg') + if component.get('duration'): + args['duration'] = component['duration'] + sent_messages.append(await self.bot.send_voice(**args)) return platform_events.MessageResult( message_id=sent_messages[-1].message_id if sent_messages else None, @@ -382,7 +398,11 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo # through the regular reply path, including attachment-only completions. if is_final and not bot_message.tool_calls: attachments = platform_message.MessageChain( - [part for part in message if isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) if attachments: await self.reply_message(message_source, attachments, quote_origin=quote_origin) @@ -425,6 +445,8 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo if is_final and bot_message.tool_calls is None: # Finalise: send the real message, discard the draft args = self._build_message_args(chat_id, content, message_thread_id) + if quote_origin and effective_message: + args['reply_to_message_id'] = effective_message.id try: await self.bot.send_message(**args) except telegram.error.BadRequest as exc: diff --git a/src/langbot/pkg/platform/adapters/wecombot/adapter.py b/src/langbot/pkg/platform/adapters/wecombot/adapter.py index 90ee727e7..09f7c8e16 100644 --- a/src/langbot/pkg/platform/adapters/wecombot/adapter.py +++ b/src/langbot/pkg/platform/adapters/wecombot/adapter.py @@ -137,26 +137,88 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo _iter_media_components = staticmethod(LegacyWecomBotAdapter._iter_media_components) @staticmethod - async def _send_media(bot, req_id: str, item: dict) -> bool: + async def _send_media(bot, req_id: str, item: dict, config: dict | None = None) -> bool: """Require upload and delivery acknowledgements instead of dropping attachments.""" kind = item['type'] payload = item.get('base64') or '' - if payload.startswith('data:'): - payload = payload.split(',', 1)[-1] - try: - data = base64.b64decode(payload, validate=True) - except ValueError as exc: - raise ValueError(f'Invalid {kind} attachment data') from exc + data = b'' + if payload: + if payload.startswith('data:'): + payload = payload.split(',', 1)[-1] + try: + data = base64.b64decode(payload, validate=False) + except Exception as exc: + raise ValueError(f'Invalid {kind} attachment data') from exc + elif item.get('path'): + with open(item['path'], 'rb') as f: + data = f.read() + if not data: raise ValueError(f'{kind} attachment has no data to upload') - result = await bot.upload_media(data, item.get('name') or f'attachment.{kind}', media_type=kind) - media_id = result.get('media_id') if isinstance(result, dict) else None + + media_id = None + delivery_kind = kind + filename = item.get('name') + voice_mode = (config or {}).get('voice_delivery', 'voice') + + if kind == 'voice': + from langbot.pkg.utils import audio as audio_utils + + fmt = audio_utils.detect_audio_format(data) + duration = audio_utils.estimate_audio_duration(data, fmt) if data else 0 + + if voice_mode == 'file' or (duration and duration > 60): + delivery_kind = 'file' + if fmt in ('silk', 'amr') and audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'mp3', fmt) + fmt = 'mp3' + except Exception: + pass + filename = filename or f'voice.{fmt if fmt != "unknown" else "mp3"}' + else: + # WeCom native voice message ONLY supports SILK or AMR format. + # If audio is MP3/WAV/etc. (e.g. from TTS), transcode it to SILK + # so the WeCom client can play the native voice bubble. + if fmt != 'silk' and fmt != 'amr': + if audio_utils.is_silk_available() and audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'silk', fmt) + fmt = 'silk' + except Exception: + pass + + if fmt == 'silk': + delivery_kind = 'voice' + filename = filename or 'voice.silk' + elif fmt == 'amr': + delivery_kind = 'voice' + filename = filename or 'voice.amr' + elif fmt in ('mp3', 'wav', 'ogg', 'flac'): + delivery_kind = 'file' + filename = filename or f'voice.{fmt}' + else: + delivery_kind = 'voice' + filename = filename or 'voice.silk' + else: + filename = filename or f'attachment.{kind}' + + result = await bot.upload_media(data, filename, media_type=delivery_kind) + if isinstance(result, dict): + media_id = result.get('media_id') + if not media_id and delivery_kind == 'voice': + # Fall back to uploading as audio file if voice-type upload is not accepted + delivery_kind = 'file' + fallback_filename = filename.rsplit('.', 1)[0] + '.mp3' if filename else 'voice.mp3' + result = await bot.upload_media(data, fallback_filename, media_type='file') + if isinstance(result, dict): + media_id = result.get('media_id') if not media_id: raise RuntimeError(f'WeCom {kind} upload failed: no media ID returned') - ack = await getattr(bot, f'reply_{kind}')(req_id, media_id) + ack = await getattr(bot, f'reply_{delivery_kind}')(req_id, media_id) if ack is None or ack.get('errcode', 0) != 0: code = ack.get('errcode') if isinstance(ack, dict) else 'no acknowledgement' - raise RuntimeError(f'WeCom {kind} delivery failed: {code}') + raise RuntimeError(f'WeCom {delivery_kind} delivery failed: {code}') return True async def send_message( @@ -225,7 +287,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo if not finished: raw = await self.bot.reply_text(event.get('req_id'), content) for item in self._iter_media_components(items): - await self._send_media(self.bot, event.get('req_id'), item) + await self._send_media(self.bot, event.get('req_id'), item, self.config) else: raw = await self.bot.set_message(event.message_id, content) return platform_events.MessageResult(message_id=event.message_id, raw={'result': raw}) @@ -253,7 +315,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo if is_final: if not self.config.get('enable-webhook', False) and event.get('req_id'): for item in self._iter_media_components(items): - await self._send_media(self.bot, event.get('req_id'), item) + await self._send_media(self.bot, event.get('req_id'), item, self.config) elif not success: await self.bot.set_message(event.message_id, content) return {'stream': success} diff --git a/src/langbot/pkg/platform/adapters/wecombot/manifest.yaml b/src/langbot/pkg/platform/adapters/wecombot/manifest.yaml index d5b7c1eeb..5b3f18c5f 100644 --- a/src/langbot/pkg/platform/adapters/wecombot/manifest.yaml +++ b/src/langbot/pkg/platform/adapters/wecombot/manifest.yaml @@ -170,6 +170,29 @@ spec: type: boolean required: false default: true + - name: voice_delivery + label: + en_US: Voice Delivery Mode + zh_Hans: 语音发送模式 + zh_Hant: 語音傳送模式 + description: + en_US: Choose how voice responses are delivered. 'voice' sends native voice messages (requires SILK transcoding). 'file' sends audio files. + zh_Hans: 选择语音回复的发送方式。'voice' 为企微原生语音条（默认自动转码为 SILK），'file' 为音频文件卡片。 + zh_Hant: 選擇語音回覆的傳送方式。'voice' 為企微原生語音條（預設自動轉碼為 SILK），'file' 為音訊檔案卡片。 + type: select + options: + - label: + en_US: Native Voice (Recommended) + zh_Hans: 原生语音条（推荐） + zh_Hant: 原生語音條（推薦） + value: voice + - label: + en_US: Audio File + zh_Hans: 音频文件 + zh_Hant: 音訊檔案 + value: file + required: false + default: voice supported_events: - message.received diff --git a/src/langbot/pkg/platform/adapters/wecombot/message_converter.py b/src/langbot/pkg/platform/adapters/wecombot/message_converter.py index 95ce52202..9c13e9c5e 100644 --- a/src/langbot/pkg/platform/adapters/wecombot/message_converter.py +++ b/src/langbot/pkg/platform/adapters/wecombot/message_converter.py @@ -112,13 +112,46 @@ class WecomBotMessageConverter(abstract_platform_adapter.AbstractMessageConverte voice_payload = voice_info.get('base64') or voice_info.get('url') if not voice_payload: return + length = voice_info.get('length') + import base64 + from langbot.pkg.utils import audio as audio_utils + if voice_info.get('base64') and not voice_payload.startswith('data:'): - voice_payload = f'data:audio/mpeg;base64,{voice_info.get("base64")}' + try: + raw_bytes = base64.b64decode(voice_info.get('base64'), validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + if fmt in ('amr', 'silk') and audio_utils.is_ffmpeg_available(): + try: + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + except Exception: + voice_payload = f'data:audio/{fmt};base64,{voice_info.get("base64")}' + else: + mime = 'audio/mpeg' if fmt == 'mp3' else ('audio/wav' if fmt == 'wav' else f'audio/{fmt}') + voice_payload = f'data:{mime};base64,{voice_info.get("base64")}' + except Exception: + voice_payload = f'data:audio/mpeg;base64,{voice_info.get("base64")}' + elif voice_payload.startswith('data:'): + header, _, b64_body = voice_payload.partition(',') + if ('amr' in header or 'silk' in header or 'octet-stream' in header) and audio_utils.is_ffmpeg_available(): + try: + raw_bytes = base64.b64decode(b64_body, validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if fmt in ('amr', 'silk'): + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + except Exception: + pass + try: if voice_payload.startswith('data:'): - components.append(platform_message.Voice(base64=voice_payload)) + components.append(platform_message.Voice(base64=voice_payload, length=length)) else: - components.append(platform_message.Voice(url=voice_payload)) + components.append(platform_message.Voice(url=voice_payload, length=length)) except Exception: components.append(platform_message.Unknown(text='[voice message unsupported]')) diff --git a/src/langbot/pkg/platform/sources/dingtalk.py b/src/langbot/pkg/platform/sources/dingtalk.py index a8fd2349d..7e3efcbcd 100644 --- a/src/langbot/pkg/platform/sources/dingtalk.py +++ b/src/langbot/pkg/platform/sources/dingtalk.py @@ -96,11 +96,11 @@ class DingTalkMessageConverter(abstract_platform_adapter.AbstractMessageConverte if event.file: yiri_msg_list.append(platform_message.File(url=event.file, name=event.name)) if event.audio: - # 优先使用钉钉自带的语音转写文字（recognition字段） - if event.content and event.type == 'audio': - yiri_msg_list.append(platform_message.Plain(text=event.content)) - else: - yiri_msg_list.append(platform_message.Voice(base64=event.audio)) + # Keep the audio so the pipeline's speech recognition setting decides how it is transcribed. + yiri_msg_list.append(platform_message.Voice(base64=event.audio, length=event.audio_duration or None)) + elif event.type == 'audio' and event.content: + # The audio could not be downloaded; fall back to DingTalk's own transcription. + yiri_msg_list.append(platform_message.Plain(text=event.content)) # Handle quoted/replied message - extract content as top-level components # so that plugins like FileReader can process them the same way as direct messages @@ -614,12 +614,16 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): markdown_enabled = self.config.get('markdown_card', False) text_message = platform_message.MessageChain( - [part for part in message if not isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if not isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) content, at = await DingTalkMessageConverter.yiri2target(text_message, markdown_enabled) await self.bot.send_message(content, incoming_message, at) for part in message: - if isinstance(part, (platform_message.Image, platform_message.File)): + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)): await self.bot.send_attachment(part, incoming_message) async def reply_message_chunk( @@ -648,7 +652,11 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if (msg_seq - 1) % 8 == 0 or is_final: markdown_enabled = self.config.get('markdown_card', False) text_message = platform_message.MessageChain( - [part for part in message if not isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if not isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) content, at = await DingTalkMessageConverter.yiri2target(text_message, markdown_enabled) if not content and bot_message.content: @@ -721,7 +729,7 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): self.card_instance_id_dict.pop(message_id, None) event = await DingTalkEventConverter.yiri2target(message_source) for part in message: - if isinstance(part, (platform_message.Image, platform_message.File)): + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)): await self.bot.send_attachment(part, event.incoming_message) async def send_message(self, target_type: str, target_id: str, message: platform_message.MessageChain): diff --git a/src/langbot/pkg/platform/sources/discord.py b/src/langbot/pkg/platform/sources/discord.py index 1d9a29234..610d51482 100644 --- a/src/langbot/pkg/platform/sources/discord.py +++ b/src/langbot/pkg/platform/sources/discord.py @@ -659,33 +659,15 @@ class DiscordMessageConverter(abstract_platform_adapter.AbstractMessageConverter text_string += ele.text elif isinstance(ele, platform_message.Voice): file_bytes = None - filename = f'{uuid.uuid4()}.mp3' + filename = getattr(ele, 'name', '') or '' if ele.base64: - if ele.base64.startswith('data:'): - data_header = ele.base64.split(',')[0] - if 'wav' in data_header: - filename = f'{uuid.uuid4()}.wav' - elif 'mp3' in data_header: - filename = f'{uuid.uuid4()}.mp3' - elif 'ogg' in data_header: - filename = f'{uuid.uuid4()}.ogg' - elif 'm4a' in data_header: - filename = f'{uuid.uuid4()}.m4a' - elif 'aac' in data_header: - filename = f'{uuid.uuid4()}.aac' - elif 'flac' in data_header: - filename = f'{uuid.uuid4()}.flac' - elif 'alac' in data_header: - filename = f'{uuid.uuid4()}.alac' - elif 'opus' in data_header: - filename = f'{uuid.uuid4()}.opus' - elif 'webm' in data_header: - filename = f'{uuid.uuid4()}.webm' - file_bytes = await asyncio.to_thread( _decode_discord_base64_limited, ele.base64, ) + elif getattr(ele, 'path', None): + with open(ele.path, 'rb') as f: + file_bytes = await asyncio.to_thread(f.read) elif ele.url: session = httpclient.get_session() async with session.get(ele.url) as response: @@ -694,6 +676,19 @@ class DiscordMessageConverter(abstract_platform_adapter.AbstractMessageConverter max_bytes=_MAX_DISCORD_MEDIA_BYTES, ) if file_bytes: + from langbot.pkg.utils import audio as audio_utils + + fmt = audio_utils.detect_audio_format(file_bytes) + # If format is silk or amr, Discord client cannot decode it; transcode to mp3 + if fmt in ('silk', 'amr') and audio_utils.is_ffmpeg_available(): + try: + file_bytes = await audio_utils.transcode_audio(file_bytes, 'mp3', fmt) + fmt = 'mp3' + except Exception: + pass + if not filename: + ext = fmt if fmt != 'unknown' else 'mp3' + filename = f'{uuid.uuid4()}.{ext}' files.append(discord.File(fp=io.BytesIO(file_bytes), filename=filename)) elif isinstance(ele, platform_message.File): file_bytes = None @@ -767,16 +762,68 @@ class DiscordMessageConverter(abstract_platform_adapter.AbstractMessageConverter for attachment in message.attachments: session = httpclient.get_session(trust_env=True) async with session.get(attachment.url) as response: - image_data = await httpclient.read_limited( + media_data = await httpclient.read_limited( response, max_bytes=_MAX_DISCORD_MEDIA_BYTES, ) - image_base64 = (await asyncio.to_thread(base64.b64encode, image_data)).decode('utf-8') - image_format = response.headers['Content-Type'] - element_list.append( - platform_message.Image(url=attachment.url, base64=f'data:{image_format};base64,{image_base64}') + media_base64 = (await asyncio.to_thread(base64.b64encode, media_data)).decode('utf-8') + content_type = response.headers.get('Content-Type') or getattr(attachment, 'content_type', None) or '' + filename = (getattr(attachment, 'filename', '') or '').lower() + flags = getattr(attachment, 'flags', None) + is_voice_flag = getattr(flags, 'voice', False) if flags else False + + from langbot.pkg.utils import audio as audio_utils + + detected_fmt = audio_utils.detect_audio_format(media_data) + is_audio = ( + is_voice_flag + or content_type.startswith('audio/') + or any( + filename.endswith(ext) + for ext in ('.mp3', '.ogg', '.oga', '.opus', '.wav', '.m4a', '.aac', '.flac', '.weba') + ) + or detected_fmt in ('wav', 'mp3', 'ogg', 'opus', 'flac', 'silk', 'amr') ) + if is_audio: + mime = ( + content_type + if content_type.startswith('audio/') + else ( + 'audio/ogg' + if detected_fmt in ('ogg', 'opus') + else f'audio/{detected_fmt if detected_fmt != "unknown" else "mpeg"}' + ) + ) + duration = getattr(attachment, 'duration_secs', None) + if duration is not None: + duration = int(round(duration)) + elif media_data: + duration = int(round(audio_utils.estimate_audio_duration(media_data, detected_fmt))) + element_list.append( + platform_message.Voice( + url=attachment.url, + base64=f'data:{mime};base64,{media_base64}', + length=duration, + ) + ) + elif content_type.startswith('image/') or any( + filename.endswith(ext) for ext in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg') + ): + mime = content_type if content_type.startswith('image/') else 'image/png' + element_list.append( + platform_message.Image(url=attachment.url, base64=f'data:{mime};base64,{media_base64}') + ) + else: + mime = content_type or 'application/octet-stream' + element_list.append( + platform_message.File( + url=attachment.url, + base64=f'data:{mime};base64,{media_base64}', + name=getattr(attachment, 'filename', 'file'), + ) + ) + return platform_message.MessageChain(element_list) diff --git a/src/langbot/pkg/platform/sources/lark.py b/src/langbot/pkg/platform/sources/lark.py index 4dc600c8f..b38fb9331 100644 --- a/src/langbot/pkg/platform/sources/lark.py +++ b/src/langbot/pkg/platform/sources/lark.py @@ -600,6 +600,13 @@ class LarkMessageConverter(abstract_platform_adapter.AbstractMessageConverter): elif isinstance(msg, platform_message.Voice): data = await LarkMessageConverter._get_media_bytes(msg) if data: + from ...utils import audio as audio_utils + + if audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'opus') + except Exception: + pass duration = int(msg.length * 1000) if msg.length else None file_key = await LarkMessageConverter.upload_file_to_lark( data, api_client, file_type='opus', file_name='voice.opus', duration=duration diff --git a/src/langbot/pkg/platform/sources/openclaw_weixin.py b/src/langbot/pkg/platform/sources/openclaw_weixin.py index b4c9dc270..3b4751eef 100644 --- a/src/langbot/pkg/platform/sources/openclaw_weixin.py +++ b/src/langbot/pkg/platform/sources/openclaw_weixin.py @@ -12,6 +12,7 @@ import asyncio import base64 import contextlib import os +import logging import traceback import typing @@ -39,6 +40,7 @@ import langbot_plugin.api.entities.builtin.platform.message as platform_message from langbot.pkg.api.http.context import ExecutionContext _MAX_OPENCLAW_COMPONENT_BYTES = 10 * 1024 * 1024 +logger = logging.getLogger('langbot.openclaw-weixin') class OpenClawWeixinMessageConverter(abstract_platform_adapter.AbstractMessageConverter): @@ -130,31 +132,32 @@ class OpenClawWeixinMessageConverter(abstract_platform_adapter.AbstractMessageCo components.append(platform_message.Unknown(text='[Image]')) elif item.type == MessageItem.VOICE and item.voice_item: - # Voice with speech-to-text: use the transcribed text - if item.voice_item.text: + if hasattr(item.voice_item, '_downloaded_bytes') and item.voice_item._downloaded_bytes: + b64 = ( + await asyncio.to_thread( + base64.b64encode, + item.voice_item._downloaded_bytes, + ) + ).decode('utf-8') + playtime = item.voice_item.playtime or 0 + length_sec = int(round(playtime / 1000.0)) if playtime > 0 else 0 + if length_sec == 0: + from langbot.pkg.utils import audio as audio_utils + + dur = audio_utils.estimate_audio_duration(item.voice_item._downloaded_bytes, 'silk') + length_sec = int(round(dur)) if dur > 0 else 0 + + components.append( + platform_message.Voice( + base64=f'data:audio/silk;base64,{b64}', + length=length_sec, + ) + ) + elif item.voice_item.text: components.append(platform_message.Plain(text=item.voice_item.text)) else: components.append(platform_message.Unknown(text='[Voice]')) - # TODO: enable after full testing - # elif item.type == MessageItem.VOICE and item.voice_item: - # if item.voice_item.text: - # components.append(platform_message.Plain(text=item.voice_item.text)) - # elif hasattr(item.voice_item, '_downloaded_bytes') and item.voice_item._downloaded_bytes: - # b64 = base64.b64encode(item.voice_item._downloaded_bytes).decode('utf-8') - # components.append( - # platform_message.Voice( - # base64=b64, - # length=item.voice_item.playtime or 0, - # ) - # ) - # else: - # components.append( - # platform_message.Voice( - # length=item.voice_item.playtime or 0, - # ) - # ) - elif item.type == MessageItem.FILE and item.file_item: components.append(platform_message.Unknown(text=f'[File: {item.file_item.file_name or ""}]')) @@ -389,7 +392,123 @@ class OpenClawWeixinAdapter(abstract_platform_adapter.AbstractMessagePlatformAda elif isinstance(component, platform_message.Voice): voice_bytes = await self._get_component_bytes(component) if voice_bytes: - await self.client.send_voice(target_id, voice_bytes, component.length or 0, context_token) + from langbot.pkg.utils import audio as audio_utils + + audio_fmt = audio_utils.detect_audio_format(voice_bytes) + delivery_mode = self.config.get('voice_delivery', 'file') + + if delivery_mode == 'voice': + duration_sec = component.length or audio_utils.estimate_audio_duration( + voice_bytes, audio_fmt + ) + playtime_ms = int(duration_sec * 1000) if duration_sec > 0 else 0 + + if audio_fmt == 'silk': + if playtime_ms <= 0: + silk_dur = audio_utils.estimate_audio_duration(voice_bytes, 'silk') + playtime_ms = int(silk_dur * 1000) if silk_dur > 0 else 1000 + + await self.client.send_voice( + target_id, + voice_bytes, + playtime=playtime_ms, + encode_type=6, + sample_rate=24000, + bits_per_sample=16, + context_token=context_token, + ) + elif audio_utils.is_silk_available() and audio_utils.is_ffmpeg_available(): + silk_bytes = None + try: + silk_bytes = await audio_utils.transcode_audio(voice_bytes, 'silk', audio_fmt) + except Exception as exc: + logger.warning( + f'Failed to transcode voice to SILK: {exc}; falling back to audio file.' + ) + await self.logger.warning( + f'Failed to transcode voice to SILK: {exc}; gracefully falling back to audio file.' + ) + + if silk_bytes: + try: + if playtime_ms <= 0: + silk_dur = audio_utils.estimate_audio_duration(silk_bytes, 'silk') + playtime_ms = int(silk_dur * 1000) if silk_dur > 0 else 1000 + + await self.client.send_voice( + target_id, + silk_bytes, + playtime=playtime_ms, + encode_type=6, + sample_rate=24000, + bits_per_sample=16, + context_token=context_token, + ) + logger.info( + f'Successfully sent SILK voice message to {target_id} (playtime={playtime_ms}ms)' + ) + except Exception as exc: + logger.warning( + f'Failed to send WeChat voice: {exc}; falling back to audio file.' + ) + await self.logger.warning( + f'Failed to send WeChat voice: {exc}; gracefully falling back to audio file.' + ) + # Deliver as playable MP3 file + out_bytes = voice_bytes + if audio_fmt != 'mp3' and audio_utils.is_ffmpeg_available(): + try: + out_bytes = await audio_utils.transcode_audio( + voice_bytes, 'mp3', audio_fmt + ) + except Exception: + pass + await self.client.send_file( + target_id, + out_bytes, + 'voice.mp3', + context_token, + ) + else: + out_bytes = voice_bytes + if audio_fmt != 'mp3' and audio_utils.is_ffmpeg_available(): + try: + out_bytes = await audio_utils.transcode_audio(voice_bytes, 'mp3', audio_fmt) + except Exception: + pass + await self.client.send_file( + target_id, + out_bytes, + 'voice.mp3', + context_token, + ) + else: + out_bytes = voice_bytes + if audio_fmt != 'mp3' and audio_utils.is_ffmpeg_available(): + try: + out_bytes = await audio_utils.transcode_audio(voice_bytes, 'mp3', audio_fmt) + except Exception: + pass + await self.client.send_file( + target_id, + out_bytes, + 'voice.mp3', + context_token, + ) + else: + # Standard OpenClaw behavior: deliver as playable MP3 audio file card + out_bytes = voice_bytes + if audio_fmt != 'mp3' and audio_utils.is_ffmpeg_available(): + try: + out_bytes = await audio_utils.transcode_audio(voice_bytes, 'mp3', audio_fmt) + except Exception: + pass + await self.client.send_file( + target_id, + out_bytes, + 'voice.mp3', + context_token, + ) elif isinstance(component, platform_message.Forward): for node in component.node_list: @@ -426,6 +545,8 @@ class OpenClawWeixinAdapter(abstract_platform_adapter.AbstractMessagePlatformAda path_val = getattr(component, 'path', None) if b64_val: + if b64_val.startswith('data:'): + _, _, b64_val = b64_val.partition(',') max_encoded_chars = 4 * ((_MAX_OPENCLAW_COMPONENT_BYTES + 2) // 3) + 4 if len(b64_val) > max_encoded_chars: raise ValueError('OpenClaw media exceeds the size limit') @@ -618,17 +739,17 @@ class OpenClawWeixinAdapter(abstract_platform_adapter.AbstractMessagePlatformAda img_bytes = await self.client.download_media(item.image_item.media) item.image_item._downloaded_bytes = img_bytes + elif item.type == MessageItem.VOICE and item.voice_item and item.voice_item.media: + if item.voice_item.media.encrypt_query_param and item.voice_item.media.aes_key: + voice_bytes = await self.client.download_media(item.voice_item.media) + item.voice_item._downloaded_bytes = voice_bytes + # TODO: enable after full testing # elif item.type == MessageItem.FILE and item.file_item and item.file_item.media: # if item.file_item.media.encrypt_query_param and item.file_item.media.aes_key: # file_bytes = await self.client.download_media(item.file_item.media) # item.file_item._downloaded_bytes = file_bytes # - # elif item.type == MessageItem.VOICE and item.voice_item and item.voice_item.media: - # if item.voice_item.media.encrypt_query_param and item.voice_item.media.aes_key: - # voice_bytes = await self.client.download_media(item.voice_item.media) - # item.voice_item._downloaded_bytes = voice_bytes - # # elif item.type == MessageItem.VIDEO and item.video_item and item.video_item.media: # if item.video_item.media.encrypt_query_param and item.video_item.media.aes_key: # video_bytes = await self.client.download_media(item.video_item.media) diff --git a/src/langbot/pkg/platform/sources/openclaw_weixin.yaml b/src/langbot/pkg/platform/sources/openclaw_weixin.yaml index 963b6da7a..5335cb925 100644 --- a/src/langbot/pkg/platform/sources/openclaw_weixin.yaml +++ b/src/langbot/pkg/platform/sources/openclaw_weixin.yaml @@ -82,6 +82,29 @@ spec: type: integer required: false default: 35 + - name: voice_delivery + label: + en_US: Voice Delivery Format + zh_Hans: 语音发送方式 + zh_Hant: 語音發送方式 + description: + en_US: Choose whether to send audio as a playable audio file card (recommended, as WeChat personal bots often silently drop native voice bubbles) or attempt native SILK voice bubble. + zh_Hans: 选择以可播放音频文件卡片发送（推荐，微信官方个人助手常静默丢弃机器人的原生语音条），或尝试原生 SILK 语音条。 + zh_Hant: 選擇以可播放音訊檔案卡片發送（推薦，微信官方個人助手常靜默丟棄機器人的原生語音條），或嘗試原生 SILK 語音條。 + type: select + options: + - label: + en_US: Playable Audio File (Recommended) + zh_Hans: 可播放音频文件（推荐，稳定投递） + zh_Hant: 可播放音訊檔案（推薦，穩定投遞） + value: file + - label: + en_US: Native Voice Bubble (Experimental) + zh_Hans: 原生语音条（实验性，可能被微信平台静默拦截） + zh_Hant: 原生語音條（實驗性，可能被微信平台靜默攔截） + value: voice + required: false + default: file execution: python: path: ./openclaw_weixin.py diff --git a/src/langbot/pkg/platform/sources/telegram.py b/src/langbot/pkg/platform/sources/telegram.py index 92aa91705..794e71299 100644 --- a/src/langbot/pkg/platform/sources/telegram.py +++ b/src/langbot/pkg/platform/sources/telegram.py @@ -116,6 +116,24 @@ class TelegramMessageConverter(abstract_platform_adapter.AbstractMessageConverte photo_bytes, _mime_type = await component.get_bytes() components.append({'type': 'photo', 'photo': photo_bytes}) + elif isinstance(component, platform_message.Voice): + from ...utils import audio as audio_utils + + voice_bytes, src_fmt = await audio_utils.extract_voice_bytes(component) + if audio_utils.is_ffmpeg_available(): + try: + voice_bytes = await audio_utils.transcode_audio(voice_bytes, 'ogg', src_fmt) + except Exception: + pass + duration = component.length + if duration is not None: + duration = int(round(duration)) + elif voice_bytes: + try: + duration = int(round(audio_utils.estimate_audio_duration(voice_bytes, 'ogg'))) + except Exception: + duration = None + components.append({'type': 'voice', 'voice': voice_bytes, 'duration': duration}) elif isinstance(component, platform_message.File): file_bytes = None @@ -542,6 +560,14 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): filename = component.get('filename', 'file') args['document'] = telegram.InputFile(doc, filename=filename) await self.bot.send_document(**args) + elif component_type == 'voice': + voice = component.get('voice') + if voice is None: + continue + args['voice'] = telegram.InputFile(voice, filename='voice.ogg') + if component.get('duration'): + args['duration'] = component['duration'] + await self.bot.send_voice(**args) async def reply_message( self, @@ -552,28 +578,53 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): assert isinstance(message_source.source_platform_object, Update) components = await TelegramMessageConverter.yiri2target(message, self.bot) + update = message_source.source_platform_object + chat_id = update.effective_chat.id + effective_message = getattr(update, 'effective_message', None) or getattr(update, 'message', None) + message_thread_id = getattr(effective_message, 'message_thread_id', None) if effective_message else None + reply_to_message_id = effective_message.id if (quote_origin and effective_message) else None + + sent_messages = [] for component in components: - if component['type'] == 'text': - if self.config['markdown_card'] is True: - content = telegramify_markdown.markdownify( - content=component['text'], - ) - else: - content = component['text'] - args = { - 'chat_id': message_source.source_platform_object.effective_chat.id, - 'text': content, - } - if self.config['markdown_card'] is True: + component_type = component.get('type') + args = {'chat_id': chat_id} + + if message_thread_id: + args['message_thread_id'] = message_thread_id + + if reply_to_message_id: + args['reply_to_message_id'] = reply_to_message_id + + if component_type == 'text': + content = component.get('text', '') + if self.config.get('markdown_card', False): + content = telegramify_markdown.markdownify(content=content) args['parse_mode'] = 'MarkdownV2' + args['text'] = content + sent_messages.append(await self.bot.send_message(**args)) + elif component_type == 'photo': + photo = component.get('photo') + if photo is None: + continue + args['photo'] = telegram.InputFile(photo) + sent_messages.append(await self.bot.send_photo(**args)) + elif component_type == 'document': + doc = component.get('document') + if doc is None: + continue + filename = component.get('filename', 'file') + args['document'] = telegram.InputFile(doc, filename=filename) + sent_messages.append(await self.bot.send_document(**args)) + elif component_type == 'voice': + voice = component.get('voice') + if voice is None: + continue + args['voice'] = telegram.InputFile(voice, filename='voice.ogg') + if component.get('duration'): + args['duration'] = component['duration'] + sent_messages.append(await self.bot.send_voice(**args)) - if message_source.source_platform_object.message.message_thread_id: - args['message_thread_id'] = message_source.source_platform_object.message.message_thread_id - - if quote_origin: - args['reply_to_message_id'] = message_source.source_platform_object.message.id - - await self.bot.send_message(**args) + return sent_messages def _process_markdown(self, text: str) -> str: if self.config.get('markdown_card', False): @@ -589,7 +640,7 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): return args async def _delete_group_stream_message(self, chat_mode: str, chat_id: int, stream_id: int | None): - if chat_mode != 'group' or stream_id is None: + if chat_mode == 'private' or stream_id is None: return try: await self.bot.delete_message(chat_id=chat_id, message_id=stream_id) @@ -639,14 +690,29 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if message_id not in self.msg_stream_id: return + # Media cannot be edited into a text stream. Deliver terminal attachments + # through the regular reply path, including attachment-only completions. + if is_final and not getattr(bot_message, 'tool_calls', None): + attachments = platform_message.MessageChain( + [ + part + for part in message + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] + ) + if attachments: + await self.reply_message(message_source, attachments, quote_origin=quote_origin) + stream_state = self.msg_stream_id[message_id] chat_mode, stream_id = stream_state[:2] has_visible_content = len(stream_state) > 2 and stream_state[2] components = await TelegramMessageConverter.yiri2target(message, self.bot) if not components or components[0]['type'] != 'text': - if is_final and bot_message.tool_calls is None: - self.msg_stream_id.pop(message_id) + if is_final and getattr(bot_message, 'tool_calls', None) is None: + if not has_visible_content: + await self._delete_group_stream_message(chat_mode, chat_id, stream_id) + self.msg_stream_id.pop(message_id, None) return content = components[0]['text'] @@ -685,6 +751,8 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if is_final and bot_message.tool_calls is None: # Finalise: send the real message, discard the draft args = self._build_message_args(chat_id, content, message_thread_id) + if quote_origin and effective_message: + args['reply_to_message_id'] = effective_message.id try: await self.bot.send_message(**args) except telegram.error.BadRequest as exc: diff --git a/src/langbot/pkg/platform/sources/wecombot.py b/src/langbot/pkg/platform/sources/wecombot.py index 92d58da7d..18459a002 100644 --- a/src/langbot/pkg/platform/sources/wecombot.py +++ b/src/langbot/pkg/platform/sources/wecombot.py @@ -98,10 +98,47 @@ class WecomBotMessageConverter(abstract_platform_adapter.AbstractMessageConverte if voice_info: voice_payload = voice_info.get('base64') or voice_info.get('url') if voice_payload: + length = voice_info.get('length') + from langbot.pkg.utils import audio as audio_utils + if voice_info.get('base64') and not voice_payload.startswith('data:'): - voice_payload = f'data:audio/mpeg;base64,{voice_info.get("base64")}' + try: + raw_bytes = base64.b64decode(voice_info.get('base64'), validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + if fmt in ('amr', 'silk') and audio_utils.is_ffmpeg_available(): + try: + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + except Exception: + voice_payload = f'data:audio/{fmt};base64,{voice_info.get("base64")}' + else: + mime = 'audio/mpeg' if fmt == 'mp3' else ('audio/wav' if fmt == 'wav' else f'audio/{fmt}') + voice_payload = f'data:{mime};base64,{voice_info.get("base64")}' + except Exception: + voice_payload = f'data:audio/mpeg;base64,{voice_info.get("base64")}' + elif voice_payload.startswith('data:'): + header, _, b64_body = voice_payload.partition(',') + if ( + 'amr' in header or 'silk' in header or 'octet-stream' in header + ) and audio_utils.is_ffmpeg_available(): + try: + raw_bytes = base64.b64decode(b64_body, validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if fmt in ('amr', 'silk'): + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + except Exception: + pass + try: - yiri_msg_list.append(platform_message.Voice(base64=voice_payload)) + if voice_payload.startswith('data:'): + yiri_msg_list.append(platform_message.Voice(base64=voice_payload, length=length)) + else: + yiri_msg_list.append(platform_message.Voice(url=voice_payload, length=length)) except Exception: try: voice_kwargs = {'url': voice_payload} @@ -194,10 +231,53 @@ class WecomBotMessageConverter(abstract_platform_adapter.AbstractMessageConverte if quote_voice: voice_payload = quote_voice.get('base64') or quote_voice.get('url') if voice_payload: + length = quote_voice.get('length') + from langbot.pkg.utils import audio as audio_utils + if quote_voice.get('base64') and not voice_payload.startswith('data:'): - voice_payload = f'data:audio/mpeg;base64,{quote_voice.get("base64")}' + try: + raw_bytes = base64.b64decode(quote_voice.get('base64'), validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + if fmt in ('amr', 'silk') and audio_utils.is_ffmpeg_available(): + try: + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = ( + f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + ) + except Exception: + voice_payload = f'data:audio/{fmt};base64,{quote_voice.get("base64")}' + else: + mime = ( + 'audio/mpeg' if fmt == 'mp3' else ('audio/wav' if fmt == 'wav' else f'audio/{fmt}') + ) + voice_payload = f'data:{mime};base64,{quote_voice.get("base64")}' + except Exception: + voice_payload = f'data:audio/mpeg;base64,{quote_voice.get("base64")}' + elif voice_payload.startswith('data:'): + header, _, b64_body = voice_payload.partition(',') + if ( + 'amr' in header or 'silk' in header or 'octet-stream' in header + ) and audio_utils.is_ffmpeg_available(): + try: + raw_bytes = base64.b64decode(b64_body, validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if fmt in ('amr', 'silk'): + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = ( + f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + ) + except Exception: + pass + try: - yiri_msg_list.append(platform_message.Voice(base64=voice_payload)) + if voice_payload.startswith('data:'): + yiri_msg_list.append(platform_message.Voice(base64=voice_payload, length=length)) + else: + yiri_msg_list.append(platform_message.Voice(url=voice_payload, length=length)) except Exception: try: voice_kwargs = {'url': voice_payload} @@ -394,6 +474,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): bot, req_id: str, item: dict, + config: dict | None = None, ) -> bool: """Upload *item* to the WeCom AI Bot CDN and send it as a media reply. @@ -407,30 +488,87 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if upload is None: return False b64_text = item.get('base64') or '' - if not b64_text: - return False - if b64_text.startswith('data:') and ',' in b64_text: - b64_text = b64_text.split(',', 1)[1] - try: - data = base64.b64decode(b64_text, validate=False) - except Exception: - return False + data = b'' + if b64_text: + if b64_text.startswith('data:') and ',' in b64_text: + b64_text = b64_text.split(',', 1)[1] + try: + data = base64.b64decode(b64_text, validate=False) + except Exception: + return False + elif item.get('path'): + with open(item['path'], 'rb') as f: + data = f.read() if not data: return False + + delivery_kind = kind + filename = item.get('name') + voice_mode = (config or {}).get('voice_delivery', 'voice') + + if kind == 'voice': + from langbot.pkg.utils import audio as audio_utils + + fmt = audio_utils.detect_audio_format(data) + duration = audio_utils.estimate_audio_duration(data, fmt) if data else 0 + + if voice_mode == 'file' or (duration and duration > 60): + delivery_kind = 'file' + if fmt in ('silk', 'amr') and audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'mp3', fmt) + fmt = 'mp3' + except Exception: + pass + filename = filename or f'voice.{fmt if fmt != "unknown" else "mp3"}' + else: + if fmt != 'silk' and fmt != 'amr': + if audio_utils.is_silk_available() and audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'silk', fmt) + fmt = 'silk' + except Exception: + pass + + if fmt == 'silk': + delivery_kind = 'voice' + filename = filename or 'voice.silk' + elif fmt == 'amr': + delivery_kind = 'voice' + filename = filename or 'voice.amr' + elif fmt in ('mp3', 'wav', 'ogg', 'flac'): + delivery_kind = 'file' + filename = filename or f'voice.{fmt}' + else: + delivery_kind = 'voice' + filename = filename or 'voice.silk' + else: + filename = filename or f'attachment.{kind}' + try: - upload_result = await upload(data, item.get('name') or f'attachment.{kind}', media_type=kind) + upload_result = await upload(data, filename, media_type=delivery_kind) except Exception: return False media_id = getattr(upload_result, 'media_id', None) or ( isinstance(upload_result, dict) and upload_result.get('media_id') ) + if not media_id and delivery_kind == 'voice': + delivery_kind = 'file' + fallback_filename = filename.rsplit('.', 1)[0] + '.mp3' if filename else 'voice.mp3' + try: + upload_result = await upload(data, fallback_filename, media_type='file') + media_id = getattr(upload_result, 'media_id', None) or ( + isinstance(upload_result, dict) and upload_result.get('media_id') + ) + except Exception: + return False if not media_id: return False reply_fn = { 'image': getattr(bot, 'reply_image', None), 'file': getattr(bot, 'reply_file', None), 'voice': getattr(bot, 'reply_voice', None), - }.get(kind) + }.get(delivery_kind) if reply_fn is None: return False try: @@ -480,7 +618,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): else: await self.bot.set_message(event.message_id, text) for item in self._iter_media_components(items): - await self._send_media(self.bot, req_id, item) + await self._send_media(self.bot, req_id, item, config=self.config) else: await self.bot.set_message(event.message_id, text) @@ -555,7 +693,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): event = message_source.source_platform_object req_id = event.get('req_id', '') for item in self._iter_media_components(items): - await self._send_media(self.bot, req_id, item) + await self._send_media(self.bot, req_id, item, config=self.config) return {'stream': success} else: success = await self.bot.push_stream_chunk(msg_id, text, is_final=is_final) diff --git a/src/langbot/pkg/platform/sources/wecombot.yaml b/src/langbot/pkg/platform/sources/wecombot.yaml index 51961d357..d4bf3366b 100644 --- a/src/langbot/pkg/platform/sources/wecombot.yaml +++ b/src/langbot/pkg/platform/sources/wecombot.yaml @@ -148,6 +148,29 @@ spec: type: boolean required: false default: true + - name: voice_delivery + label: + en_US: Voice Delivery Mode + zh_Hans: 语音发送模式 + zh_Hant: 語音傳送模式 + description: + en_US: Choose how voice responses are delivered. 'voice' sends native voice messages (requires SILK transcoding). 'file' sends audio files. + zh_Hans: 选择语音回复的发送方式。'voice' 为企微原生语音条（默认自动转码为 SILK），'file' 为音频文件卡片。 + zh_Hant: 選擇語音回覆的傳送方式。'voice' 為企微原生語音條（預設自動轉碼為 SILK），'file' 為音訊檔案卡片。 + type: select + options: + - label: + en_US: Native Voice (Recommended) + zh_Hans: 原生语音条（推荐） + zh_Hant: 原生語音條（推薦） + value: voice + - label: + en_US: Audio File + zh_Hans: 音频文件 + zh_Hant: 音訊檔案 + value: file + required: false + default: voice execution: python: path: ./wecombot.py diff --git a/src/langbot/pkg/provider/modelmgr/modelmgr.py b/src/langbot/pkg/provider/modelmgr/modelmgr.py index e13464a9c..e0713f732 100644 --- a/src/langbot/pkg/provider/modelmgr/modelmgr.py +++ b/src/langbot/pkg/provider/modelmgr/modelmgr.py @@ -40,6 +40,8 @@ class ModelManager: llm_model_dict: dict[_CacheKey, requester.RuntimeLLMModel] embedding_model_dict: dict[_CacheKey, requester.RuntimeEmbeddingModel] rerank_model_dict: dict[_CacheKey, requester.RuntimeRerankModel] + asr_model_dict: dict[_CacheKey, requester.RuntimeASRModel] + tts_model_dict: dict[_CacheKey, requester.RuntimeTTSModel] requester_components: list[engine.Component] requester_dict: dict[str, type[requester.ProviderAPIRequester]] @@ -50,6 +52,8 @@ class ModelManager: self.llm_model_dict = {} self.embedding_model_dict = {} self.rerank_model_dict = {} + self.asr_model_dict = {} + self.tts_model_dict = {} self.requester_components = [] self.requester_dict = {} self._scope_generations: dict[tuple[str, str], int] = {} @@ -57,6 +61,8 @@ class ModelManager: self._llm_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} self._embedding_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} self._rerank_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} + self._asr_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} + self._tts_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} def _cache_index(self, cache: dict) -> dict[tuple[str, str], set[_CacheKey]]: if cache is self.provider_dict: @@ -67,6 +73,10 @@ class ModelManager: return self._embedding_keys_by_scope if cache is self.rerank_model_dict: return self._rerank_keys_by_scope + if cache is self.asr_model_dict: + return self._asr_keys_by_scope + if cache is self.tts_model_dict: + return self._tts_keys_by_scope raise ValueError('Unknown model runtime cache') def _cache_set(self, cache: dict, key: _CacheKey, value: object) -> None: @@ -89,6 +99,8 @@ class ModelManager: self._llm_keys_by_scope, self._embedding_keys_by_scope, self._rerank_keys_by_scope, + self._asr_keys_by_scope, + self._tts_keys_by_scope, ) ): self._scope_generations.pop(scope, None) @@ -113,6 +125,8 @@ class ModelManager: (self.llm_model_dict, self._llm_keys_by_scope), (self.embedding_model_dict, self._embedding_keys_by_scope), (self.rerank_model_dict, self._rerank_keys_by_scope), + (self.asr_model_dict, self._asr_keys_by_scope), + (self.tts_model_dict, self._tts_keys_by_scope), ): for key in index.pop(scope, ()): removed = cache.pop(key, None) @@ -156,6 +170,8 @@ class ModelManager: self._llm_keys_by_scope, self._embedding_keys_by_scope, self._rerank_keys_by_scope, + self._asr_keys_by_scope, + self._tts_keys_by_scope, ) ): self._scope_generations.pop(scope, None) @@ -168,11 +184,15 @@ class ModelManager: self.llm_model_dict = {} self.embedding_model_dict = {} self.rerank_model_dict = {} + self.asr_model_dict = {} + self.tts_model_dict = {} self._scope_generations = {} self._provider_keys_by_scope = {} self._llm_keys_by_scope = {} self._embedding_keys_by_scope = {} self._rerank_keys_by_scope = {} + self._asr_keys_by_scope = {} + self._tts_keys_by_scope = {} await self._close_runtime_providers(providers) @staticmethod @@ -336,11 +356,15 @@ class ModelManager: self.llm_model_dict = {} self.embedding_model_dict = {} self.rerank_model_dict = {} + self.asr_model_dict = {} + self.tts_model_dict = {} self._scope_generations = {} self._provider_keys_by_scope = {} self._llm_keys_by_scope = {} self._embedding_keys_by_scope = {} self._rerank_keys_by_scope = {} + self._asr_keys_by_scope = {} + self._tts_keys_by_scope = {} list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None) tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None) @@ -412,6 +436,18 @@ class ModelManager: self._build_rerank_model, context_for, ) + await self._load_model_kind( + persistence_model.ASRModel, + self.asr_model_dict, + self._build_asr_model, + context_for, + ) + await self._load_model_kind( + persistence_model.TTSModel, + self.tts_model_dict, + self._build_tts_model, + context_for, + ) async def _load_model_kind(self, entity_type, cache: dict, builder, context_for) -> None: result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(entity_type)) @@ -480,6 +516,18 @@ class ModelManager: self.rerank_model_dict, self._build_rerank_model, ) + await self._load_workspace_model_kind( + context, + persistence_model.ASRModel, + self.asr_model_dict, + self._build_asr_model, + ) + await self._load_workspace_model_kind( + context, + persistence_model.TTSModel, + self.tts_model_dict, + self._build_tts_model, + ) async def _load_workspace_model_kind(self, context, entity_type, cache: dict, builder) -> None: result = await self.ap.persistence_mgr.execute_async( @@ -716,6 +764,68 @@ class ModelManager: ) return self._build_rerank_model(execution_context, model_entity, runtime_provider) + async def init_temporary_runtime_asr_model( + self, + context: TenantContext, + model_info: dict, + ) -> requester.RuntimeASRModel: + execution_context = await self.resolve_execution_context(context) + provider_info = {**model_info.get('provider', {}), 'workspace_uuid': execution_context.workspace_uuid} + provider_uuid = model_info.get('provider_uuid') or provider_info.get('uuid') + provider_entity = persistence_model.ModelProvider(**provider_info) + if provider_uuid and not provider_entity.requester: + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(persistence_model.ModelProvider).where( + persistence_model.ModelProvider.workspace_uuid == execution_context.workspace_uuid, + persistence_model.ModelProvider.uuid == provider_uuid, + ) + ) + saved_provider = result.first() + if saved_provider is not None: + provider_entity = self._coerce_provider(saved_provider, execution_context) + elif model_info.get('provider_uuid'): + raise WorkspaceNotFoundError('Provider not found') + runtime_provider = await self._build_provider(execution_context, provider_entity) + model_entity = persistence_model.ASRModel( + workspace_uuid=execution_context.workspace_uuid, + uuid=model_info.get('uuid', ''), + name=model_info.get('name', ''), + provider_uuid=runtime_provider.provider_entity.uuid, + extra_args=model_info.get('extra_args', {}), + ) + return self._build_asr_model(execution_context, model_entity, runtime_provider) + + async def init_temporary_runtime_tts_model( + self, + context: TenantContext, + model_info: dict, + ) -> requester.RuntimeTTSModel: + execution_context = await self.resolve_execution_context(context) + provider_info = {**model_info.get('provider', {}), 'workspace_uuid': execution_context.workspace_uuid} + provider_uuid = model_info.get('provider_uuid') or provider_info.get('uuid') + provider_entity = persistence_model.ModelProvider(**provider_info) + if provider_uuid and not provider_entity.requester: + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(persistence_model.ModelProvider).where( + persistence_model.ModelProvider.workspace_uuid == execution_context.workspace_uuid, + persistence_model.ModelProvider.uuid == provider_uuid, + ) + ) + saved_provider = result.first() + if saved_provider is not None: + provider_entity = self._coerce_provider(saved_provider, execution_context) + elif model_info.get('provider_uuid'): + raise WorkspaceNotFoundError('Provider not found') + runtime_provider = await self._build_provider(execution_context, provider_entity) + model_entity = persistence_model.TTSModel( + workspace_uuid=execution_context.workspace_uuid, + uuid=model_info.get('uuid', ''), + name=model_info.get('name', ''), + provider_uuid=runtime_provider.provider_entity.uuid, + extra_args=model_info.get('extra_args', {}), + ) + return self._build_tts_model(execution_context, model_entity, runtime_provider) + @staticmethod def _coerce_provider( provider_info: persistence_model.ModelProvider | sqlalchemy.Row | dict, @@ -906,6 +1016,34 @@ class ModelManager: provider=provider, ) + def _build_asr_model( + self, + context: ExecutionContext, + model_info: persistence_model.ASRModel | sqlalchemy.Row, + provider: requester.RuntimeProvider, + ) -> requester.RuntimeASRModel: + model_entity = self._coerce_model(model_info, persistence_model.ASRModel) + self._validate_model_provider(context, model_entity, provider) + return requester.RuntimeASRModel( + execution_context=context, + model_entity=model_entity, + provider=provider, + ) + + def _build_tts_model( + self, + context: ExecutionContext, + model_info: persistence_model.TTSModel | sqlalchemy.Row, + provider: requester.RuntimeProvider, + ) -> requester.RuntimeTTSModel: + model_entity = self._coerce_model(model_info, persistence_model.TTSModel) + self._validate_model_provider(context, model_entity, provider) + return requester.RuntimeTTSModel( + execution_context=context, + model_entity=model_entity, + provider=provider, + ) + async def load_llm_model_with_provider( self, context: TenantContext, @@ -933,6 +1071,24 @@ class ModelManager: execution_context = await self.resolve_execution_context(context) return self._build_rerank_model(execution_context, model_info, provider) + async def load_asr_model_with_provider( + self, + context: TenantContext, + model_info: persistence_model.ASRModel | sqlalchemy.Row, + provider: requester.RuntimeProvider, + ) -> requester.RuntimeASRModel: + execution_context = await self.resolve_execution_context(context) + return self._build_asr_model(execution_context, model_info, provider) + + async def load_tts_model_with_provider( + self, + context: TenantContext, + model_info: persistence_model.TTSModel | sqlalchemy.Row, + provider: requester.RuntimeProvider, + ) -> requester.RuntimeTTSModel: + execution_context = await self.resolve_execution_context(context) + return self._build_tts_model(execution_context, model_info, provider) + async def cache_llm_model(self, context: TenantContext, model: requester.RuntimeLLMModel) -> None: execution_context = await self.resolve_execution_context(context) self._ensure_same_scope(execution_context, model.execution_context, resource='LLM model') @@ -967,6 +1123,26 @@ class ModelManager: model, ) + async def cache_asr_model(self, context: TenantContext, model: requester.RuntimeASRModel) -> None: + execution_context = await self.resolve_execution_context(context) + self._ensure_same_scope(execution_context, model.execution_context, resource='ASR model') + self._observe_execution_context(execution_context) + self._cache_set( + self.asr_model_dict, + self._cache_key(execution_context, model.model_entity.uuid), + model, + ) + + async def cache_tts_model(self, context: TenantContext, model: requester.RuntimeTTSModel) -> None: + execution_context = await self.resolve_execution_context(context) + self._ensure_same_scope(execution_context, model.execution_context, resource='TTS model') + self._observe_execution_context(execution_context) + self._cache_set( + self.tts_model_dict, + self._cache_key(execution_context, model.model_entity.uuid), + model, + ) + async def get_model_by_uuid(self, context: TenantContext, model_uuid: str) -> requester.RuntimeLLMModel: execution_context = await self.resolve_execution_context(context) model = self.llm_model_dict.get(self._cache_key(execution_context, model_uuid)) @@ -999,6 +1175,30 @@ class ModelManager: self._ensure_same_scope(execution_context, model.execution_context, resource='Rerank model') return model + async def get_asr_model_by_uuid( + self, + context: TenantContext, + model_uuid: str, + ) -> requester.RuntimeASRModel: + execution_context = await self.resolve_execution_context(context) + model = self.asr_model_dict.get(self._cache_key(execution_context, model_uuid)) + if model is None: + raise ValueError(f'ASR model {model_uuid} not found') + self._ensure_same_scope(execution_context, model.execution_context, resource='ASR model') + return model + + async def get_tts_model_by_uuid( + self, + context: TenantContext, + model_uuid: str, + ) -> requester.RuntimeTTSModel: + execution_context = await self.resolve_execution_context(context) + model = self.tts_model_dict.get(self._cache_key(execution_context, model_uuid)) + if model is None: + raise ValueError(f'TTS model {model_uuid} not found') + self._ensure_same_scope(execution_context, model.execution_context, resource='TTS model') + return model + async def remove_llm_model(self, context: TenantContext, model_uuid: str) -> None: execution_context = await self.resolve_execution_context(context) self._cache_pop( @@ -1020,6 +1220,20 @@ class ModelManager: self._cache_key(execution_context, model_uuid), ) + async def remove_asr_model(self, context: TenantContext, model_uuid: str) -> None: + execution_context = await self.resolve_execution_context(context) + self._cache_pop( + self.asr_model_dict, + self._cache_key(execution_context, model_uuid), + ) + + async def remove_tts_model(self, context: TenantContext, model_uuid: str) -> None: + execution_context = await self.resolve_execution_context(context) + self._cache_pop( + self.tts_model_dict, + self._cache_key(execution_context, model_uuid), + ) + def get_available_requesters_info(self, model_type: str) -> list[dict]: if model_type: return [ diff --git a/src/langbot/pkg/provider/modelmgr/requester.py b/src/langbot/pkg/provider/modelmgr/requester.py index 41dd62a8c..f03a1f93e 100644 --- a/src/langbot/pkg/provider/modelmgr/requester.py +++ b/src/langbot/pkg/provider/modelmgr/requester.py @@ -75,7 +75,7 @@ class RuntimeProvider: def _validate_invocation( self, - model: RuntimeLLMModel | RuntimeEmbeddingModel | RuntimeRerankModel, + model: RuntimeLLMModel | RuntimeEmbeddingModel | RuntimeRerankModel | RuntimeASRModel | RuntimeTTSModel, execution_context: ExecutionContext, ) -> None: _ensure_same_execution_scope(self.execution_context, execution_context, resource='Provider invocation') @@ -380,6 +380,74 @@ class RuntimeProvider: except Exception as monitor_err: self.requester.ap.logger.error(f'[Monitoring] Failed to record rerank call: {monitor_err}') + async def invoke_asr( + self, + model: RuntimeASRModel, + audio: bytes, + mime: str, + extra_args: dict[str, typing.Any] = {}, + *, + execution_context: ExecutionContext, + ) -> str: + """Bridge method for invoking ASR with monitoring""" + self._validate_invocation(model, execution_context) + start_time = time.time() + status = 'success' + + try: + result = await self.requester.invoke_asr( + model=model, + audio=audio, + mime=mime, + extra_args=extra_args, + ) + return result + except Exception: + status = 'error' + raise + finally: + duration_ms = int((time.time() - start_time) * 1000) + try: + self.requester.ap.logger.debug( + f'[ASR] model={model.model_entity.name} bytes={len(audio)} mime={mime} ' + f'duration={duration_ms}ms status={status}' + ) + except Exception as monitor_err: + self.requester.ap.logger.error(f'[Monitoring] Failed to record ASR call: {monitor_err}') + + async def invoke_tts( + self, + model: RuntimeTTSModel, + text: str, + extra_args: dict[str, typing.Any] = {}, + *, + execution_context: ExecutionContext, + ) -> tuple[bytes, str]: + """Bridge method for invoking TTS with monitoring""" + self._validate_invocation(model, execution_context) + start_time = time.time() + status = 'success' + + try: + result = await self.requester.invoke_tts( + model=model, + text=text, + extra_args=extra_args, + ) + return result + except Exception: + status = 'error' + raise + finally: + duration_ms = int((time.time() - start_time) * 1000) + try: + self.requester.ap.logger.debug( + f'[TTS] model={model.model_entity.name} text_len={len(text)} ' + f'duration={duration_ms}ms status={status}' + ) + except Exception as monitor_err: + self.requester.ap.logger.error(f'[Monitoring] Failed to record TTS call: {monitor_err}') + class RuntimeLLMModel: """运行时模型""" @@ -461,6 +529,80 @@ class RuntimeRerankModel: self.provider = provider +class RuntimeASRModel: + """运行时 ASR 模型""" + + model_entity: persistence_model.ASRModel + """模型数据""" + + provider: RuntimeProvider + """提供商实例""" + + def __init__( + self, + execution_context: ExecutionContext, + model_entity: persistence_model.ASRModel, + provider: RuntimeProvider, + ): + _ensure_same_execution_scope(provider.execution_context, execution_context, resource='ASR model') + if model_entity.workspace_uuid != execution_context.workspace_uuid: + raise WorkspaceInvariantError('ASR model belongs to another Workspace') + if model_entity.provider_uuid != provider.provider_entity.uuid: + raise WorkspaceInvariantError('ASR model references another provider') + self.execution_context = execution_context + self.model_entity = model_entity + self.provider = provider + + @property + def uuid(self) -> str: + return self.model_entity.uuid + + @property + def name(self) -> str: + return self.model_entity.name + + @property + def extra_args(self) -> dict[str, typing.Any]: + return self.model_entity.extra_args or {} + + +class RuntimeTTSModel: + """运行时 TTS 模型""" + + model_entity: persistence_model.TTSModel + """模型数据""" + + provider: RuntimeProvider + """提供商实例""" + + def __init__( + self, + execution_context: ExecutionContext, + model_entity: persistence_model.TTSModel, + provider: RuntimeProvider, + ): + _ensure_same_execution_scope(provider.execution_context, execution_context, resource='TTS model') + if model_entity.workspace_uuid != execution_context.workspace_uuid: + raise WorkspaceInvariantError('TTS model belongs to another Workspace') + if model_entity.provider_uuid != provider.provider_entity.uuid: + raise WorkspaceInvariantError('TTS model references another provider') + self.execution_context = execution_context + self.model_entity = model_entity + self.provider = provider + + @property + def uuid(self) -> str: + return self.model_entity.uuid + + @property + def name(self) -> str: + return self.model_entity.name + + @property + def extra_args(self) -> dict[str, typing.Any]: + return self.model_entity.extra_args or {} + + class ProviderAPIRequester(metaclass=abc.ABCMeta): """Provider API请求器""" @@ -606,3 +748,41 @@ class ProviderAPIRequester(metaclass=abc.ABCMeta): typing.List[dict]: [{"index": int, "relevance_score": float}, ...] """ raise NotImplementedError('This requester does not support rerank') + + async def invoke_asr( + self, + model: RuntimeASRModel, + audio: bytes, + mime: str, + extra_args: dict[str, typing.Any] = {}, + ) -> str: + """调用 ASR API + + Args: + model (RuntimeASRModel): 使用的模型信息 + audio (bytes): 音频字节数据 + mime (str): 音频 mime 类型 + extra_args (dict[str, typing.Any], optional): 额外参数. Defaults to {}. + + Returns: + str: 转写文本 + """ + raise NotImplementedError('This requester does not support asr') + + async def invoke_tts( + self, + model: RuntimeTTSModel, + text: str, + extra_args: dict[str, typing.Any] = {}, + ) -> tuple[bytes, str]: + """调用 TTS API + + Args: + model (RuntimeTTSModel): 使用的模型信息 + text (str): 待合成文本 + extra_args (dict[str, typing.Any], optional): 额外参数. Defaults to {}. + + Returns: + tuple[bytes, str]: (音频字节数据, mime 类型) + """ + raise NotImplementedError('This requester does not support tts') diff --git a/src/langbot/pkg/provider/modelmgr/requesters/302aichatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/302aichatcmpl.yaml index 3cfec1982..6cdeaac18 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/302aichatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/302aichatcmpl.yaml @@ -28,6 +28,8 @@ spec: - llm - text-embedding - rerank + - asr + - tts provider_category: maas execution: python: diff --git a/src/langbot/pkg/provider/modelmgr/requesters/chatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/chatcmpl.yaml index 526721b0e..72a1118dd 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/chatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/chatcmpl.yaml @@ -27,6 +27,8 @@ spec: support_type: - llm - text-embedding + - asr + - tts provider_category: manufacturer execution: python: diff --git a/src/langbot/pkg/provider/modelmgr/requesters/groqchatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/groqchatcmpl.yaml index d51367473..85997daec 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/groqchatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/groqchatcmpl.yaml @@ -26,4 +26,5 @@ spec: alias: "groq Groq 高速 llama mixtral 推理加速 lpu" support_type: - llm + - asr provider_category: manufacturer diff --git a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py index 16eb776c4..2becbd5bf 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py +++ b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py @@ -19,7 +19,7 @@ from urllib.parse import urlsplit from ..request_policy import current_call, RequestPolicy, error_metadata import litellm -from litellm import acompletion, aembedding, arerank +from litellm import acompletion, aembedding, arerank, atranscription, aspeech from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.secret_managers.main import get_secret, get_secret_bool @@ -182,6 +182,17 @@ class LiteLLMRequester(requester.ProviderAPIRequester): _EMBEDDING_MODEL_HINTS = ('embedding', 'embed', 'bge-', 'e5-', 'm3e', 'gte-', 'text-embedding') _RERANK_MODEL_HINTS = ('rerank', 're-rank', 're_rank') + _ASR_MODEL_HINTS = ( + 'whisper', + 'sensevoice', + 'asr', + 'paraformer', + 'conformer', + 'speech-to-text', + 'transcription', + 'stt', + ) + _TTS_MODEL_HINTS = ('tts', 'text-to-speech', 'speech-synthesis', 'cosyvoice', 'voice-clone', 'sambert', 'chattts') _QWEN_DEDICATED_THINKING_MODELS = frozenset( { 'qwen3.7-max-preview', @@ -786,12 +797,27 @@ class LiteLLMRequester(requester.ProviderAPIRequester): def _reasoning_level(self, model: requester.RuntimeLLMModel) -> str: return reasoning.normalize_reasoning_config(self._reasoning_config_value(model))['level'] - def _infer_model_type(self, model_id: str) -> str: + def _infer_model_type(self, model_id: str, model_payload: dict[str, typing.Any] | None = None) -> str: + if model_payload: + explicit_type = (model_payload.get('type') or model_payload.get('task') or '').lower() + if explicit_type in ('asr', 'audio_transcription', 'speech-to-text', 'stt'): + return 'asr' + if explicit_type in ('tts', 'text_to_speech', 'text-to-speech', 'speech-synthesis'): + return 'tts' + if explicit_type in ('embedding', 'text-embedding'): + return 'embedding' + if explicit_type in ('rerank', 'reranking'): + return 'rerank' + normalized_id = (model_id or '').lower() if any(kw in normalized_id for kw in self._RERANK_MODEL_HINTS): return 'rerank' if any(kw in normalized_id for kw in self._EMBEDDING_MODEL_HINTS): return 'embedding' + if any(kw in normalized_id for kw in self._ASR_MODEL_HINTS): + return 'asr' + if any(kw in normalized_id for kw in self._TTS_MODEL_HINTS): + return 'tts' return 'llm' def _enrich_scanned_model( @@ -799,7 +825,7 @@ class LiteLLMRequester(requester.ProviderAPIRequester): model_id: str, model_payload: dict[str, typing.Any] | None = None, ) -> dict[str, typing.Any]: - model_type = self._infer_model_type(model_id) + model_type = self._infer_model_type(model_id, model_payload) scanned_model: dict[str, typing.Any] = { 'id': model_id, 'name': model_id, @@ -1231,11 +1257,15 @@ class LiteLLMRequester(requester.ProviderAPIRequester): mappings = ( (litellm.ContextWindowExceededError, 'context_length', t('Context length exceeded'), False), (litellm.AuthenticationError, 'authentication', t('Invalid API key'), False), + (getattr(litellm, 'PermissionDeniedError', ()), 'permission_denied', t('Permission denied'), False), (litellm.NotFoundError, 'not_found', t('Invalid model or endpoint'), False), (litellm.RateLimitError, 'rate_limit', t('Rate limit exceeded or insufficient balance'), True), (litellm.Timeout, 'timeout', t('Request timed out'), True), (litellm.APIConnectionError, 'connection', t('Connection failed'), True), (litellm.BadRequestError, 'bad_request', t('Invalid request parameters'), False), + (getattr(litellm, 'InternalServerError', ()), 'internal_error', t('Internal server error'), True), + (getattr(litellm, 'BadGatewayError', ()), 'bad_gateway', t('Internal server error'), True), + (getattr(litellm, 'ServiceUnavailableError', ()), 'service_unavailable', t('Internal server error'), True), ) for exception_type, code, message, retryable in mappings: if isinstance(e, exception_type): @@ -1247,9 +1277,38 @@ class LiteLLMRequester(requester.ProviderAPIRequester): code, retryable = 'quota_exceeded', False raise errors.RequesterError(message, code=code, retryable=retryable, **metadata) from e status = metadata['status_code'] + code = 'upstream_error' + if status in (500, 502, 503): + message = t('Internal server error') + code = 'internal_error' + elif status == 504: + message = t('Request timed out') + code = 'timeout' + elif status == 401: + message = t('Invalid API key') + code = 'authentication' + elif status == 403: + message = t('Permission denied') + code = 'permission_denied' + elif status == 404: + message = t('Invalid model or endpoint') + code = 'not_found' + elif status == 429: + message = t('Rate limit exceeded or insufficient balance') + code = 'rate_limit' + elif status == 400: + message = t('Invalid request parameters') + code = 'bad_request' + elif status is not None: + message = f'{t("API error")} (HTTP {status})' + elif isinstance(e, (litellm.APIError, getattr(litellm, 'OpenAIError', ()))): + message = t('API error') + else: + message = t('Unknown error') + raise errors.RequesterError( - t('API error') if isinstance(e, litellm.APIError) else t('Unknown error'), - code='upstream_error', + message, + code=code, retryable=status in (408, 409, 429, 500, 502, 503, 504), **metadata, ) from e @@ -1881,6 +1940,123 @@ class LiteLLMRequester(requester.ProviderAPIRequester): return results + async def invoke_asr( + self, + model: requester.RuntimeASRModel, + audio: bytes, + mime: str, + extra_args: dict[str, typing.Any] = {}, + ) -> str: + """Invoke speech recognition (ASR) and return transcript.""" + model_name = self._build_litellm_model_name(model.model_entity.name) + api_key = model.provider.token_mgr.get_token() + + ext = 'wav' + if 'mp3' in mime or 'mpeg' in mime: + ext = 'mp3' + elif 'ogg' in mime: + ext = 'ogg' + elif 'flac' in mime: + ext = 'flac' + elif 'm4a' in mime or 'mp4' in mime: + ext = 'm4a' + elif 'wav' in mime: + ext = 'wav' + filename = f'audio.{ext}' + + args: dict[str, typing.Any] = { + 'model': model_name, + 'file': (filename, audio, mime), + 'api_key': api_key, + 'response_format': 'json', + } + self._build_common_args(args, include_retry_params=False) + + provider = self._get_custom_llm_provider() + if provider: + args['custom_llm_provider'] = provider + + if model.model_entity.extra_args: + args.update(model.model_entity.extra_args) + args.update(extra_args) + + try: + response = await atranscription(**args) + if hasattr(response, 'text'): + return response.text or '' + if isinstance(response, dict): + return response.get('text', '') + return str(response) + except errors.RequesterError: + raise + except Exception as e: + self._handle_litellm_error(e) + + async def invoke_tts( + self, + model: requester.RuntimeTTSModel, + text: str, + extra_args: dict[str, typing.Any] = {}, + ) -> tuple[bytes, str]: + """Invoke text-to-speech and return (audio_bytes, mime_type).""" + model_name = self._build_litellm_model_name(model.model_entity.name) + api_key = model.provider.token_mgr.get_token() + + combined_extra = {**(model.model_entity.extra_args or {}), **extra_args} + voice = combined_extra.pop('voice', None) + response_format = combined_extra.pop('response_format', None) or combined_extra.pop('format', None) or 'mp3' + + args: dict[str, typing.Any] = { + 'model': model_name, + 'input': text, + 'response_format': response_format, + 'api_key': api_key, + } + if voice: + args['voice'] = voice + self._build_common_args(args, include_retry_params=False) + + provider = self._get_custom_llm_provider() + if provider: + args['custom_llm_provider'] = provider + + args.update(combined_extra) + + try: + response = await aspeech(**args) + content = b'' + if hasattr(response, 'content'): + content = response.content + elif hasattr(response, 'read'): + res = response.read() + if asyncio.iscoroutine(res): + content = await res + else: + content = res + elif isinstance(response, bytes): + content = response + elif hasattr(response, 'response') and hasattr(response.response, 'content'): + content = response.response.content + + # Determine mime type + mime = 'audio/mpeg' + if response_format == 'wav': + mime = 'audio/wav' + elif response_format in ('ogg', 'opus'): + mime = 'audio/ogg' + elif response_format == 'flac': + mime = 'audio/flac' + elif response_format == 'aac': + mime = 'audio/aac' + elif hasattr(response, 'response') and hasattr(response.response, 'headers'): + mime = response.response.headers.get('content-type', mime) + + return content, mime + except errors.RequesterError: + raise + except Exception as e: + self._handle_litellm_error(e) + async def scan_models(self, api_key: str | None = None) -> dict[str, typing.Any]: """Scan models supported by the provider.""" import httpx diff --git a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.yaml b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.yaml index 1d5452d50..79b8c3a72 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.yaml @@ -58,6 +58,8 @@ spec: - llm - text-embedding - rerank + - asr + - tts provider_category: unified execution: python: diff --git a/src/langbot/pkg/provider/modelmgr/requesters/newapichatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/newapichatcmpl.yaml index 694af440e..3fa339720 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/newapichatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/newapichatcmpl.yaml @@ -28,6 +28,8 @@ spec: - llm - text-embedding - rerank + - asr + - tts provider_category: maas execution: python: diff --git a/src/langbot/pkg/provider/modelmgr/requesters/siliconflowchatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/siliconflowchatcmpl.yaml index b4e5d736e..d0ac01dfe 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/siliconflowchatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/siliconflowchatcmpl.yaml @@ -28,6 +28,8 @@ spec: - llm - text-embedding - rerank + - asr + - tts provider_category: maas execution: python: diff --git a/src/langbot/pkg/utils/audio.py b/src/langbot/pkg/utils/audio.py new file mode 100644 index 000000000..b56fa7c97 --- /dev/null +++ b/src/langbot/pkg/utils/audio.py @@ -0,0 +1,358 @@ +from __future__ import annotations + +import asyncio +import base64 +import importlib.util +import os +import shutil +import subprocess +import sys +import tempfile +import typing + +if typing.TYPE_CHECKING: + import langbot_plugin.api.entities.builtin.platform.message as platform_message + + +class AudioDependencyError(RuntimeError): + """Raised when an audio transcoding tool or optional dependency is missing or fails.""" + + pass + + +def _safe_tempdir() -> str | None: + """Return a valid temporary directory path even if TMPDIR was pointed to a deleted directory.""" + current = tempfile.gettempdir() + if not os.path.isdir(current): + tempfile.tempdir = None + for candidate in ('/tmp', '/var/tmp'): + if os.path.isdir(candidate): + tempfile.tempdir = candidate + return candidate + current = tempfile.gettempdir() + return current if os.path.isdir(current) else None + + +def get_ffmpeg_cmd() -> str: + """Get the path to the ffmpeg executable.""" + cmd = shutil.which('ffmpeg') + if cmd: + return cmd + venv_ffmpeg = os.path.join(os.path.dirname(sys.executable), 'ffmpeg') + if os.path.isfile(venv_ffmpeg) and os.access(venv_ffmpeg, os.X_OK): + return venv_ffmpeg + try: + import imageio_ffmpeg + + return imageio_ffmpeg.get_ffmpeg_exe() + except (ImportError, Exception): + return 'ffmpeg' + + +def is_ffmpeg_available() -> bool: + """Check if the ffmpeg command line tool is installed and available.""" + if shutil.which('ffmpeg') is not None: + return True + venv_ffmpeg = os.path.join(os.path.dirname(sys.executable), 'ffmpeg') + if os.path.isfile(venv_ffmpeg) and os.access(venv_ffmpeg, os.X_OK): + return True + try: + import imageio_ffmpeg + + return bool(imageio_ffmpeg.get_ffmpeg_exe()) + except (ImportError, Exception): + return False + + +def is_silk_available() -> bool: + """Check if the pilk library is installed.""" + return importlib.util.find_spec('pilk') is not None + + +def detect_audio_format(data: bytes) -> str: + """Detect audio format from magic bytes.""" + if not data or len(data) < 4: + return 'unknown' + + if data.startswith(b'#!AMR'): + return 'amr' + + if data.startswith(b'\x02#!SILK_V3') or data.startswith(b'#!SILK_V3'): + return 'silk' + + if data.startswith(b'RIFF') and len(data) >= 12 and data[8:12] == b'WAVE': + return 'wav' + + if data.startswith(b'OggS'): + if b'OpusHead' in data[:100]: + return 'opus' + return 'ogg' + + if data.startswith(b'ID3'): + return 'mp3' + + if len(data) >= 2 and data[0] == 0xFF and (data[1] & 0xE0) == 0xE0: + return 'mp3' + + if data.startswith(b'fLaC'): + return 'flac' + + if len(data) >= 8 and data[4:8] == b'ftyp': + return 'm4a' + + return 'unknown' + + +def transcode_audio_sync( + audio_bytes: bytes, + target_format: str, + source_format: str | None = None, +) -> bytes: + """Synchronously transcode audio bytes to target_format.""" + target_format = target_format.lower().lstrip('.') + if target_format == 'opus': + target_format = 'ogg' + + if not source_format or source_format == 'unknown': + source_format = detect_audio_format(audio_bytes) + + if source_format == 'opus': + source_format = 'ogg' + + if source_format == target_format and source_format != 'unknown': + return audio_bytes + + # Handle SILK decoding + if source_format == 'silk': + if not is_silk_available(): + raise AudioDependencyError( + "Transcoding SILK audio requires 'pilk'. Install it via 'pip install pilk' or 'pip install langbot[audio]'." + ) + if not is_ffmpeg_available(): + raise AudioDependencyError("Transcoding requires 'ffmpeg' which is not installed on the system.") + + import pilk + + with tempfile.TemporaryDirectory(dir=_safe_tempdir()) as tmpdir: + silk_path = os.path.join(tmpdir, 'input.silk') + pcm_path = os.path.join(tmpdir, 'decoded.pcm') + with open(silk_path, 'wb') as f: + f.write(audio_bytes) + + try: + pilk.decode(silk_path, pcm_path) + except Exception as exc: + raise AudioDependencyError(f'Failed to decode SILK audio: {exc}') from exc + + # Convert PCM to target format via ffmpeg + cmd = [get_ffmpeg_cmd(), '-y', '-f', 's16le', '-ar', '24000', '-ac', '1', '-i', pcm_path] + if target_format == 'mp3': + cmd.extend(['-f', 'mp3']) + elif target_format == 'ogg': + cmd.extend(['-c:a', 'libopus', '-f', 'ogg']) + elif target_format == 'wav': + cmd.extend(['-f', 'wav']) + elif target_format == 'amr': + cmd.extend(['-ar', '8000', '-ac', '1', '-c:a', 'libopencore_amrnb', '-f', 'amr']) + else: + cmd.extend(['-f', target_format]) + cmd.append('pipe:1') + + proc = subprocess.run(cmd, capture_output=True) + if proc.returncode != 0: + raise AudioDependencyError( + f'ffmpeg conversion failed from silk(pcm) to {target_format}: {proc.stderr.decode("utf-8", errors="replace")}' + ) + return proc.stdout + + # Handle SILK encoding + if target_format == 'silk': + if not is_silk_available(): + raise AudioDependencyError( + "Transcoding to SILK requires 'pilk'. Install it via 'pip install pilk' or 'pip install langbot[audio]'." + ) + if not is_ffmpeg_available(): + raise AudioDependencyError("Transcoding requires 'ffmpeg' which is not installed on the system.") + + import pilk + + # Convert source audio to PCM s16le, 24000Hz, mono + cmd = [get_ffmpeg_cmd(), '-y'] + if source_format and source_format != 'unknown': + cmd.extend(['-f', source_format]) + cmd.extend(['-i', 'pipe:0', '-f', 's16le', '-ar', '24000', '-ac', '1', 'pipe:1']) + + proc = subprocess.run(cmd, input=audio_bytes, capture_output=True) + if proc.returncode != 0: + raise AudioDependencyError( + f'ffmpeg conversion failed from {source_format} to pcm: {proc.stderr.decode("utf-8", errors="replace")}' + ) + pcm_bytes = proc.stdout + + with tempfile.TemporaryDirectory(dir=_safe_tempdir()) as tmpdir: + pcm_path = os.path.join(tmpdir, 'input.pcm') + silk_path = os.path.join(tmpdir, 'output.silk') + with open(pcm_path, 'wb') as f: + f.write(pcm_bytes) + + try: + pilk.encode(pcm_path, silk_path, pcm_rate=24000, tencent=True) + except Exception as exc: + raise AudioDependencyError(f'Failed to encode SILK audio: {exc}') from exc + + with open(silk_path, 'rb') as f: + return f.read() + + # Standard ffmpeg transcoding + if not is_ffmpeg_available(): + raise AudioDependencyError("Transcoding requires 'ffmpeg' which is not installed on the system.") + + cmd = [get_ffmpeg_cmd(), '-y'] + if source_format and source_format != 'unknown': + cmd.extend(['-f', source_format]) + cmd.extend(['-i', 'pipe:0']) + + if target_format == 'mp3': + cmd.extend(['-f', 'mp3']) + elif target_format == 'ogg': + cmd.extend(['-c:a', 'libopus', '-f', 'ogg']) + elif target_format == 'wav': + cmd.extend(['-f', 'wav']) + elif target_format == 'amr': + cmd.extend(['-ar', '8000', '-ac', '1', '-c:a', 'libopencore_amrnb', '-f', 'amr']) + else: + cmd.extend(['-f', target_format]) + cmd.append('pipe:1') + + proc = subprocess.run(cmd, input=audio_bytes, capture_output=True) + if proc.returncode != 0: + raise AudioDependencyError( + f'ffmpeg conversion failed from {source_format} to {target_format}: {proc.stderr.decode("utf-8", errors="replace")}' + ) + return proc.stdout + + +async def transcode_audio( + audio_bytes: bytes, + target_format: str, + source_format: str | None = None, +) -> bytes: + """Asynchronously transcode audio bytes to target_format.""" + return await asyncio.to_thread(transcode_audio_sync, audio_bytes, target_format, source_format) + + +async def extract_voice_bytes(voice: platform_message.Voice) -> tuple[bytes, str]: + """Extract raw bytes and audio format from a platform_message.Voice component.""" + if voice.base64: + b64_str = voice.base64 + format_hint = 'unknown' + if b64_str.startswith('data:'): + header, _, b64_str = b64_str.partition(',') + if ';' in header: + mime_part = header[5:].split(';')[0] + if '/' in mime_part: + format_hint = mime_part.split('/')[-1] + raw_bytes = await asyncio.to_thread(base64.b64decode, b64_str) + detected = detect_audio_format(raw_bytes) + if detected != 'unknown': + return raw_bytes, detected + # Only trust audio subtypes ffmpeg can parse; generic hints such as + # "octet-stream" would make ffmpeg reject the input format. + format_hint = {'mpeg': 'mp3', 'x-wav': 'wav', 'wave': 'wav', 'x-m4a': 'm4a', 'mp4': 'm4a'}.get( + format_hint, format_hint + ) + if format_hint not in ('wav', 'mp3', 'ogg', 'opus', 'amr', 'silk', 'flac', 'm4a', 'aac', 'webm'): + format_hint = 'unknown' + return raw_bytes, format_hint + + if voice.path: + with open(voice.path, 'rb') as f: + raw_bytes = await asyncio.to_thread(f.read) + detected = detect_audio_format(raw_bytes) + return raw_bytes, detected + + if voice.url: + from ...http import client as httpclient + + session = httpclient.get_session() + async with session.get(voice.url) as response: + raw_bytes = await response.read() + detected = detect_audio_format(raw_bytes) + return raw_bytes, detected + + raise ValueError('Voice component contains no base64, path, or url') + + +def estimate_audio_duration(data: bytes, format_hint: str = 'unknown') -> float: + """Estimate or calculate audio duration in seconds from raw bytes.""" + if not data: + return 0.0 + + fmt = detect_audio_format(data) + if fmt == 'unknown': + fmt = format_hint.lower().lstrip('.') + + if fmt == 'wav' and len(data) >= 44: + try: + byte_rate = int.from_bytes(data[28:32], byteorder='little') + if byte_rate > 0: + pos = 12 + while pos + 8 <= len(data): + chunk_id = data[pos : pos + 4] + chunk_size = int.from_bytes(data[pos + 4 : pos + 8], byteorder='little') + if chunk_id == b'data': + return max(0.1, round(chunk_size / byte_rate, 2)) + pos += 8 + chunk_size + return max(0.1, round((len(data) - 44) / byte_rate, 2)) + except Exception: + pass + + if fmt == 'mp3': + offset = 0 + if data.startswith(b'ID3') and len(data) >= 10: + size = ((data[6] & 0x7F) << 21) | ((data[7] & 0x7F) << 14) | ((data[8] & 0x7F) << 7) | (data[9] & 0x7F) + offset = 10 + size + audio_data = data[offset:] + if audio_data: + for i in range(min(4096, len(audio_data) - 4)): + if audio_data[i] == 0xFF and (audio_data[i + 1] & 0xE0) == 0xE0: + bitrate_idx = (audio_data[i + 2] >> 4) & 15 + bitrates = [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0] + if 0 < bitrate_idx < len(bitrates) and bitrates[bitrate_idx] > 0: + kbps = bitrates[bitrate_idx] + return max(0.1, round(len(audio_data) * 8 / (kbps * 1000), 2)) + return max(0.1, round(len(audio_data) * 8 / 128000, 2)) + + if fmt == 'silk': + header_len = 10 if data.startswith(b'\x02#!SILK_V3') else (9 if data.startswith(b'#!SILK_V3') else 0) + pos = header_len + frames = 0 + while pos + 2 <= len(data): + size = int.from_bytes(data[pos : pos + 2], byteorder='little') + if size <= 0 or pos + 2 + size > len(data): + break + frames += 1 + pos += 2 + size + if frames > 0: + return max(0.1, round(frames * 0.02, 2)) + + if fmt == 'amr': + body_len = max(0, len(data) - 6) + return max(0.1, round((body_len / 32) * 0.02, 2)) + + if fmt in ('ogg', 'opus'): + # The granule position of the last Ogg page is the total sample count. + last_page = data.rfind(b'OggS') + if last_page >= 0 and last_page + 14 <= len(data): + granule = int.from_bytes(data[last_page + 6 : last_page + 14], byteorder='little') + sample_rate = 0 + if fmt == 'opus': + sample_rate = 48000 + else: + vorbis = data.find(b'\x01vorbis') + if vorbis >= 0 and vorbis + 16 <= len(data): + sample_rate = int.from_bytes(data[vorbis + 12 : vorbis + 16], byteorder='little') + if sample_rate > 0 and 0 < granule < 2**62: + return max(0.1, round(granule / sample_rate, 2)) + + return 0.0 diff --git a/src/langbot/templates/default-pipeline-config.json b/src/langbot/templates/default-pipeline-config.json index b692c3e0e..83fe401f6 100644 --- a/src/langbot/templates/default-pipeline-config.json +++ b/src/langbot/templates/default-pipeline-config.json @@ -43,6 +43,24 @@ }, "runner_config": {} }, + "speech": { + "speech-recognition": { + "enable": false, + "model": "", + "max-duration": 60, + "keep-original-voice": false, + "on-failure": "notify" + }, + "text-to-speech": { + "enable": false, + "model": "", + "trigger": "voice-input-only", + "output": "voice-only", + "max-text-length": 500, + "voice": "", + "on-failure": "fallback" + } + }, "output": { "long-text-processing": { "threshold": 1000, diff --git a/src/langbot/templates/metadata/pipeline/speech.yaml b/src/langbot/templates/metadata/pipeline/speech.yaml new file mode 100644 index 000000000..38d484bd4 --- /dev/null +++ b/src/langbot/templates/metadata/pipeline/speech.yaml @@ -0,0 +1,209 @@ +name: speech +label: + en_US: Speech + zh_Hans: 语音 +stages: + - name: speech-recognition + label: + en_US: Speech Recognition (ASR) + zh_Hans: 语音识别 (ASR) + description: + en_US: Transcribe incoming voice messages to text + zh_Hans: 将收到的语音消息识别为文本 + config: + - name: enable + label: + en_US: Enable + zh_Hans: 启用 + type: boolean + required: true + default: false + - name: model + label: + en_US: Model + zh_Hans: 模型 + type: asr-model-selector + required: false + default: "" + show_if: + field: enable + operator: eq + value: true + - name: max-duration + label: + en_US: Maximum Duration (seconds) + zh_Hans: 最大语音时长（秒） + description: + en_US: Voice messages longer than this limit will not be processed + zh_Hans: 超过此时长的语音消息将不会被转写 + type: integer + required: true + default: 60 + show_if: + field: enable + operator: eq + value: true + - name: keep-original-voice + label: + en_US: Keep Original Voice + zh_Hans: 保留原始语音 + description: + en_US: Keep the original Voice component in the message chain alongside transcribed text + zh_Hans: 在消息链中保留原始语音组件（与转写后的文本共存） + type: boolean + required: true + default: false + show_if: + field: enable + operator: eq + value: true + - name: on-failure + label: + en_US: On Failure + zh_Hans: 失败处理 + description: + en_US: Action to take when transcription fails + zh_Hans: 语音识别失败时的处理策略 + type: select + required: true + default: notify + options: + - name: notify + label: + en_US: Notify User + zh_Hans: 提示用户 + - name: ignore + label: + en_US: Ignore + zh_Hans: 忽略 + show_if: + field: enable + operator: eq + value: true + - name: text-to-speech + label: + en_US: Text to Speech (TTS) + zh_Hans: 文本转语音 (TTS) + description: + en_US: "Synthesize outgoing response text to voice. Note: Native WeChat voice messages require SILK transcoding (ffmpeg & pilk); if not installed, the system gracefully degrades to audio file attachments or text." + zh_Hans: "将回复的文本内容转换为语音。注：微信原生语音气泡依赖 SILK 转码环境（ffmpeg 与 pilk）；若未安装，系统将自动优雅降级为音频文件或文本。" + config: + - name: enable + label: + en_US: Enable + zh_Hans: 启用 + type: boolean + required: true + default: false + - name: model + label: + en_US: Model + zh_Hans: 模型 + type: tts-model-selector + required: false + default: "" + show_if: + field: enable + operator: eq + value: true + - name: trigger + label: + en_US: Trigger Mode + zh_Hans: 触发模式 + description: + en_US: When to convert response text to voice + zh_Hans: 何时将回复文本转换为语音 + type: select + required: true + default: voice-input-only + options: + - name: voice-input-only + label: + en_US: Voice Input Only + zh_Hans: 仅语音输入时 + - name: always + label: + en_US: Always + zh_Hans: 总是转换为语音 + - name: never + label: + en_US: Never + zh_Hans: 从不转换为语音 + show_if: + field: enable + operator: eq + value: true + - name: output + label: + en_US: Output Mode + zh_Hans: 输出模式 + description: + en_US: "How to output synthesized voice. If platform-native voice transcoding (e.g. SILK for WeChat) is unavailable, output gracefully degrades to an audio file." + zh_Hans: "合成语音的输出形式。若平台原生语音转码环境（如微信 SILK）未安装，将自动优雅降级为音频文件。" + type: select + required: true + default: voice-only + options: + - name: voice-only + label: + en_US: Voice Only + zh_Hans: 仅语音 + - name: voice-and-text + label: + en_US: Voice and Text + zh_Hans: 语音与文本并存 + show_if: + field: enable + operator: eq + value: true + - name: max-text-length + label: + en_US: Maximum Text Length + zh_Hans: 最大文本长度 + description: + en_US: Text longer than this limit will fall back to text output + zh_Hans: 超过此时长的文本将降级为纯文本输出 + type: integer + required: true + default: 500 + show_if: + field: enable + operator: eq + value: true + - name: voice + label: + en_US: Voice + zh_Hans: 发音人 / 音色 + description: + en_US: Optional voice parameter to pass to the TTS model + zh_Hans: 传给 TTS 模型的发音人音色参数（可选） + type: string + required: false + default: "" + show_if: + field: enable + operator: eq + value: true + - name: on-failure + label: + en_US: On Failure + zh_Hans: 失败处理 + description: + en_US: Action to take when synthesis fails + zh_Hans: 语音合成失败时的处理策略 + type: select + required: true + default: fallback + options: + - name: fallback + label: + en_US: Fallback to Text + zh_Hans: 降级为纯文本 + - name: notify + label: + en_US: Notify User and Fallback + zh_Hans: 提示用户并降级 + show_if: + field: enable + operator: eq + value: true diff --git a/tests/e2e/test_startup.py b/tests/e2e/test_startup.py index 939effc6b..5e63a6b48 100644 --- a/tests/e2e/test_startup.py +++ b/tests/e2e/test_startup.py @@ -243,7 +243,7 @@ class TestMinimalStartupNoLLM: payload = response.json() assert payload['code'] == 0 configs = payload['data']['configs'] - assert [config['name'] for config in configs] == ['trigger', 'safety', 'ai', 'output'] + assert [config['name'] for config in configs] == ['trigger', 'safety', 'ai', 'speech', 'output'] ai = configs[2] runner = next(stage for stage in ai['stages'] if stage['name'] == 'runner') assert any(item['name'] == 'id' for item in runner['config']) diff --git a/tests/integration/api/test_providers.py b/tests/integration/api/test_providers.py index cc01466b4..b7627bd27 100644 --- a/tests/integration/api/test_providers.py +++ b/tests/integration/api/test_providers.py @@ -102,7 +102,7 @@ def fake_provider_app(): app.provider_service.update_provider = AsyncMock(return_value={}) app.provider_service.delete_provider = AsyncMock() app.provider_service.get_provider_model_counts = AsyncMock( - return_value={'llm_count': 2, 'embedding_count': 1, 'rerank_count': 0} + return_value={'llm_count': 2, 'embedding_count': 1, 'rerank_count': 0, 'asr_count': 1, 'tts_count': 0} ) # LLM model service @@ -271,10 +271,13 @@ class TestProviderEndpoints: assert called.await_args.args[0].workspace_uuid == 'workspace-test' assert called.await_args.args[1:] == ('test-provider-uuid',) assert called.await_args.kwargs == {} - assert {key: provider_data[key] for key in ('llm_count', 'embedding_count', 'rerank_count')} == { + count_keys = ('llm_count', 'embedding_count', 'rerank_count', 'asr_count', 'tts_count') + assert {key: provider_data[key] for key in count_keys} == { 'llm_count': 2, 'embedding_count': 1, 'rerank_count': 0, + 'asr_count': 1, + 'tts_count': 0, } diff --git a/tests/integration/persistence/test_migrations.py b/tests/integration/persistence/test_migrations.py index cbe7a8cba..63081abc1 100644 --- a/tests/integration/persistence/test_migrations.py +++ b/tests/integration/persistence/test_migrations.py @@ -122,6 +122,60 @@ class TestSQLiteMigrationBaseline: class TestSQLiteMigrationUpgrade: """Tests for upgrade to head workflow.""" + @pytest.mark.asyncio + @pytest.mark.parametrize('with_model_parents', [False, True]) + async def test_speech_upgrade_preserves_pipeline_and_model_constraints(self, sqlite_engine, with_model_parents): + metadata = sqlalchemy.MetaData() + if with_model_parents: + sqlalchemy.Table('workspaces', metadata, sqlalchemy.Column('uuid', sqlalchemy.String(36), primary_key=True)) + sqlalchemy.Table( + 'model_providers', + metadata, + sqlalchemy.Column('uuid', sqlalchemy.String(255), primary_key=True), + sqlalchemy.Column('workspace_uuid', sqlalchemy.String(36), nullable=False), + sqlalchemy.UniqueConstraint('workspace_uuid', 'uuid'), + ) + pipelines = sqlalchemy.Table( + 'legacy_pipelines', + metadata, + sqlalchemy.Column('uuid', sqlalchemy.String(255), primary_key=True), + sqlalchemy.Column('stages', sqlalchemy.JSON), + sqlalchemy.Column('config', sqlalchemy.JSON), + ) + stages = ['GroupRespondRuleCheckStage', 'PreContentFilterStage', 'ResponseWrapper', 'LongTextProcessStage'] + async with sqlite_engine.begin() as conn: + await conn.run_sync(metadata.create_all) + await conn.execute( + pipelines.insert().values(uuid='pipeline', stages=stages, config={'ai': {'model': 'llm'}}) + ) + await run_alembic_stamp(sqlite_engine, '0042_sqlite_incremental_vacuum') + await run_alembic_upgrade(sqlite_engine, '0043_speech_models') + async with sqlite_engine.connect() as conn: + pipeline = (await conn.execute(pipelines.select())).one() + assert pipeline.stages == [ + 'GroupRespondRuleCheckStage', + 'SpeechRecognitionStage', + 'PreContentFilterStage', + 'ResponseWrapper', + 'TextToSpeechStage', + 'LongTextProcessStage', + ] + assert pipeline.config['ai'] == {'model': 'llm'} + assert pipeline.config['speech']['speech-recognition']['enabled'] is False + assert pipeline.config['speech']['text-to-speech']['enabled'] is False + for table_name in ('asr_models', 'tts_models'): + assert ( + await conn.run_sync(lambda sync: sqlalchemy.inspect(sync).has_table(table_name)) + is with_model_parents + ) + if with_model_parents: + foreign_keys = await conn.run_sync( + lambda sync: sqlalchemy.inspect(sync).get_foreign_keys(table_name) + ) + provider_key = next(key for key in foreign_keys if key['referred_table'] == 'model_providers') + assert provider_key['constrained_columns'] == ['workspace_uuid', 'provider_uuid'] + assert provider_key['referred_columns'] == ['workspace_uuid', 'uuid'] + @pytest.mark.asyncio async def test_execution_filter_index_round_trip_preserves_history(self, sqlite_engine): from langbot.pkg.entity.persistence.monitoring import MonitoringMessage diff --git a/tests/unit_tests/api/service/test_pipeline_service.py b/tests/unit_tests/api/service/test_pipeline_service.py index 6f324c9b8..30f2507d2 100644 --- a/tests/unit_tests/api/service/test_pipeline_service.py +++ b/tests/unit_tests/api/service/test_pipeline_service.py @@ -76,6 +76,7 @@ class TestPipelineServiceGetPipelineMetadata: ap.pipeline_config_meta_trigger = {'trigger': {}} ap.pipeline_config_meta_safety = {'safety': {}} ap.pipeline_config_meta_ai = {'ai': {}} + ap.pipeline_config_meta_speech = {'speech': {}} ap.pipeline_config_meta_output = {'output': {}} service = PipelineService(ap) @@ -86,11 +87,12 @@ class TestPipelineServiceGetPipelineMetadata: ) # Verify - assert len(result) == 4 + assert len(result) == 5 assert 'trigger' in result[0] assert 'safety' in result[1] assert 'ai' in result[2] - assert 'output' in result[3] + assert 'speech' in result[3] + assert 'output' in result[4] class TestPipelineServiceGetPipelines: diff --git a/tests/unit_tests/api/service/test_provider_cascade.py b/tests/unit_tests/api/service/test_provider_cascade.py index 971dafc3b..75f7091cc 100644 --- a/tests/unit_tests/api/service/test_provider_cascade.py +++ b/tests/unit_tests/api/service/test_provider_cascade.py @@ -13,7 +13,15 @@ from sqlalchemy.ext.asyncio import create_async_engine from langbot.pkg.api.http.context import ExecutionContext from langbot.pkg.api.http.controller.groups.provider.providers import ModelProvidersRouterGroup from langbot.pkg.api.http.service.provider import ModelProviderService -from langbot.pkg.entity.persistence.model import CodexCredential, EmbeddingModel, LLMModel, ModelProvider, RerankModel +from langbot.pkg.entity.persistence.model import ( + ASRModel, + TTSModel, + CodexCredential, + EmbeddingModel, + LLMModel, + ModelProvider, + RerankModel, +) from langbot.pkg.entity.persistence.user import User from langbot.pkg.entity.persistence.metadata import WorkspaceMetadata from langbot.pkg.entity.persistence.workspace import Workspace @@ -22,7 +30,7 @@ from langbot.pkg.provider.modelmgr.modelmgr import ModelManager from langbot.pkg.workspace.errors import WorkspaceNotFoundError pytestmark = pytest.mark.asyncio -MODEL_TYPES = (LLMModel, EmbeddingModel, RerankModel) +MODEL_TYPES = (LLMModel, EmbeddingModel, RerankModel, ASRModel, TTSModel) TABLES = (*MODEL_TYPES, CodexCredential, ModelProvider) @@ -100,6 +108,8 @@ async def deletion(tmp_path): manager.llm_model_dict, manager.embedding_model_dict, manager.rerank_model_dict, + manager.asr_model_dict, + manager.tts_model_dict, ), ): for index in range(2): @@ -121,6 +131,8 @@ async def deletion(tmp_path): manager.llm_model_dict, manager.embedding_model_dict, manager.rerank_model_dict, + manager.asr_model_dict, + manager.tts_model_dict, ) ] try: @@ -148,6 +160,8 @@ def assert_caches_unchanged(deletion): deletion.manager.llm_model_dict, deletion.manager.embedding_model_dict, deletion.manager.rerank_model_dict, + deletion.manager.asr_model_dict, + deletion.manager.tts_model_dict, ) ] @@ -167,6 +181,8 @@ async def test_cascade_deletes_all_model_types_and_credentials_after_commit(dele deletion.manager.llm_model_dict, deletion.manager.embedding_model_dict, deletion.manager.rerank_model_dict, + deletion.manager.asr_model_dict, + deletion.manager.tts_model_dict, ): assert all(not key[-1].startswith('target') for key in cache) assert any(key[1] == 'b' for key in cache) diff --git a/tests/unit_tests/api/service/test_provider_service.py b/tests/unit_tests/api/service/test_provider_service.py index 62612892d..391cd34b8 100644 --- a/tests/unit_tests/api/service/test_provider_service.py +++ b/tests/unit_tests/api/service/test_provider_service.py @@ -21,7 +21,14 @@ from unittest.mock import AsyncMock, Mock from types import SimpleNamespace from langbot.pkg.api.http.service.provider import ModelProviderService -from langbot.pkg.entity.persistence.model import ModelProvider, LLMModel, EmbeddingModel, RerankModel +from langbot.pkg.entity.persistence.model import ( + ModelProvider, + LLMModel, + EmbeddingModel, + RerankModel, + ASRModel, + TTSModel, +) from langbot.pkg.workspace.errors import WorkspaceNotFoundError @@ -412,12 +419,12 @@ class TestModelProviderServiceUpdateProvider: class TestModelProviderServiceDeleteProvider: """Fast guard coverage; real transaction/cache behavior is in test_provider_cascade.""" - @pytest.mark.parametrize('label', ['LLM', 'Embedding', 'Rerank', None]) + @pytest.mark.parametrize('label', ['LLM', 'Embedding', 'Rerank', 'ASR', 'TTS', None]) async def test_delete_provider_requires_no_references(self, label): provider_result = Mock() provider_result.first.return_value = SimpleNamespace(requester='openai') results = [provider_result] - for model_label in ('LLM', 'Embedding', 'Rerank'): + for model_label in ('LLM', 'Embedding', 'Rerank', 'ASR', 'TTS'): result = Mock() result.scalars.return_value = ['model'] if label == model_label else [] results.append(result) @@ -462,6 +469,8 @@ class TestModelProviderServiceGetProviderModelCounts: LLMModel.__tablename__: (LLMModel, llm_result), EmbeddingModel.__tablename__: (EmbeddingModel, embedding_result), RerankModel.__tablename__: (RerankModel, rerank_result), + ASRModel.__tablename__: (ASRModel, Mock(scalar=Mock(return_value=4))), + TTSModel.__tablename__: (TTSModel, Mock(scalar=Mock(return_value=5))), } async def mock_execute(query): @@ -485,6 +494,8 @@ class TestModelProviderServiceGetProviderModelCounts: assert result['llm_count'] == 3 assert result['embedding_count'] == 2 assert result['rerank_count'] == 1 + assert result['asr_count'] == 4 + assert result['tts_count'] == 5 async def test_get_model_counts_zero_counts(self): """Returns zero counts when no models.""" @@ -507,6 +518,8 @@ class TestModelProviderServiceGetProviderModelCounts: assert result['llm_count'] == 0 assert result['embedding_count'] == 0 assert result['rerank_count'] == 0 + assert result['asr_count'] == 0 + assert result['tts_count'] == 0 class TestModelProviderServiceFindOrCreateProvider: diff --git a/tests/unit_tests/api/test_provider_controller_secrets.py b/tests/unit_tests/api/test_provider_controller_secrets.py index 73d9fe990..c156bfddc 100644 --- a/tests/unit_tests/api/test_provider_controller_secrets.py +++ b/tests/unit_tests/api/test_provider_controller_secrets.py @@ -62,7 +62,7 @@ async def _create_client(role: str): application.provider_service = SimpleNamespace( get_providers=AsyncMock(side_effect=get_providers), get_provider=AsyncMock(side_effect=get_provider), - get_provider_model_counts=AsyncMock(return_value={'llm_count': 1, 'embedding_count': 1, 'rerank_count': 1}), + get_provider_model_counts=AsyncMock(return_value={'llm_count': 1, 'embedding_count': 1, 'rerank_count': 1, 'asr_count': 1, 'tts_count': 1}), ) def model_service(list_name: str, get_name: str): @@ -135,7 +135,7 @@ async def test_default_preserves_secrets_and_explicit_false_redacts_high_permiss default_value = default_data['data'].get('providers', default_data['data'].get('models'))[0] false_value = false_data['data'].get('providers', false_data['data'].get('models'))[0] expected = ( - {**RAW_PROVIDER, 'llm_count': 1, 'embedding_count': 1, 'rerank_count': 1} + {**RAW_PROVIDER, 'llm_count': 1, 'embedding_count': 1, 'rerank_count': 1, 'asr_count': 1, 'tts_count': 1} if endpoint.endswith('/providers') else RAW_MODEL ) diff --git a/tests/unit_tests/api/test_resource_secret_permissions.py b/tests/unit_tests/api/test_resource_secret_permissions.py index 9e4e31d98..ffd604f30 100644 --- a/tests/unit_tests/api/test_resource_secret_permissions.py +++ b/tests/unit_tests/api/test_resource_secret_permissions.py @@ -94,7 +94,7 @@ async def _create_client(role: str): application.provider_service = SimpleNamespace( get_providers=AsyncMock(side_effect=get_providers), - get_provider_model_counts=AsyncMock(return_value={'llm_count': 0, 'embedding_count': 0, 'rerank_count': 0}), + get_provider_model_counts=AsyncMock(return_value={'llm_count': 0, 'embedding_count': 0, 'rerank_count': 0, 'asr_count': 0, 'tts_count': 0}), ) async def get_knowledge_bases(_context, *, include_secret=False): diff --git a/tests/unit_tests/pipeline/test_speech_stages.py b/tests/unit_tests/pipeline/test_speech_stages.py new file mode 100644 index 000000000..4812d057d --- /dev/null +++ b/tests/unit_tests/pipeline/test_speech_stages.py @@ -0,0 +1,406 @@ +from __future__ import annotations + +import base64 +import io +from unittest.mock import AsyncMock, Mock +import wave + +import langbot_plugin.api.entities.builtin.platform.message as platform_message +import pytest + +from langbot.pkg.pipeline.entities import ResultType +from langbot.pkg.pipeline.speech import speech as speech_module + + +def _create_wav_bytes() -> bytes: + buf = io.BytesIO() + with wave.open(buf, 'wb') as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b'\x00' * 3200) + return buf.getvalue() + + +@pytest.mark.asyncio +async def test_asr_disabled_continues(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': False, + 'model': 'asr-model-uuid', + } + } + } + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + assert result.result_type == ResultType.CONTINUE + + +@pytest.mark.asyncio +async def test_asr_no_voice_continues(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + } + } + } + sample_query.message_chain = platform_message.MessageChain([platform_message.Plain(text='hello')]) + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + assert result.result_type == ResultType.CONTINUE + + +@pytest.mark.asyncio +async def test_asr_max_duration_exceeded_notify_stops(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'max-duration': 30, + 'on-failure': 'notify', + } + } + } + sample_query.message_chain = platform_message.MessageChain( + [platform_message.Voice(base64='dummy', length=45)] + ) + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + assert result.result_type == ResultType.INTERRUPT + assert result.user_notice == 'Voice message exceeds maximum duration.' + + +@pytest.mark.asyncio +async def test_asr_max_duration_exceeded_ignore_continues(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'max-duration': 30, + 'on-failure': 'ignore', + } + } + } + sample_query.message_chain = platform_message.MessageChain( + [platform_message.Voice(base64='dummy', length=45)] + ) + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + assert result.result_type == ResultType.CONTINUE + + +@pytest.mark.asyncio +async def test_asr_success_replaces_voice(mock_app, sample_query): + wav_bytes = _create_wav_bytes() + b64_str = base64.b64encode(wav_bytes).decode('ascii') + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'keep-original-voice': False, + 'max-duration': 60, + } + } + } + sample_query.message_chain = platform_message.MessageChain([platform_message.Voice(base64=b64_str)]) + + mock_asr_model = Mock() + mock_asr_model.execution_context = Mock() + mock_asr_model.provider = Mock() + mock_asr_model.provider.invoke_asr = AsyncMock(return_value='Transcribed voice message') + mock_app.model_mgr.get_asr_model_by_uuid = AsyncMock(return_value=mock_asr_model) + + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + + assert result.result_type == ResultType.CONTINUE + assert sample_query.variables['_input_was_voice'] is True + assert sample_query.variables['voice_transcript'] == 'Transcribed voice message' + assert len(sample_query.message_chain) == 1 + assert isinstance(sample_query.message_chain[0], platform_message.Plain) + assert sample_query.message_chain[0].text == 'Transcribed voice message' + + +@pytest.mark.asyncio +async def test_asr_success_keep_original_voice(mock_app, sample_query): + wav_bytes = _create_wav_bytes() + b64_str = base64.b64encode(wav_bytes).decode('ascii') + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'keep-original-voice': True, + 'max-duration': 60, + } + } + } + sample_query.message_chain = platform_message.MessageChain([platform_message.Voice(base64=b64_str)]) + + mock_asr_model = Mock() + mock_asr_model.execution_context = Mock() + mock_asr_model.provider = Mock() + mock_asr_model.provider.invoke_asr = AsyncMock(return_value='Transcribed voice message') + mock_app.model_mgr.get_asr_model_by_uuid = AsyncMock(return_value=mock_asr_model) + + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.message_chain) == 2 + assert isinstance(sample_query.message_chain[0], platform_message.Voice) + assert isinstance(sample_query.message_chain[1], platform_message.Plain) + assert sample_query.message_chain[1].text == 'Transcribed voice message' + + +@pytest.mark.asyncio +async def test_asr_failure_notify_stops(mock_app, sample_query): + wav_bytes = _create_wav_bytes() + b64_str = base64.b64encode(wav_bytes).decode('ascii') + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'on-failure': 'notify', + } + } + } + sample_query.message_chain = platform_message.MessageChain([platform_message.Voice(base64=b64_str)]) + + mock_asr_model = Mock() + mock_asr_model.execution_context = Mock() + mock_asr_model.provider = Mock() + mock_asr_model.provider.invoke_asr = AsyncMock(side_effect=RuntimeError('Transcription API error')) + mock_app.model_mgr.get_asr_model_by_uuid = AsyncMock(return_value=mock_asr_model) + + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + + assert result.result_type == ResultType.INTERRUPT + assert result.user_notice == 'Voice transcription failed.' + + +@pytest.mark.asyncio +async def test_tts_disabled_continues(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': False, + 'model': 'tts-model-uuid', + } + } + } + sample_query.resp_message_chain = [platform_message.MessageChain([platform_message.Plain(text='Hello')])] + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + assert result.result_type == ResultType.CONTINUE + + +@pytest.mark.asyncio +async def test_tts_voice_input_only_skips_when_not_voice(mock_app, sample_query): + sample_query.variables['_input_was_voice'] = False + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'voice-input-only', + } + } + } + sample_query.resp_message_chain = [platform_message.MessageChain([platform_message.Plain(text='Hello')])] + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.resp_message_chain[-1]) == 1 + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + + +@pytest.mark.asyncio +async def test_tts_voice_omitted_when_not_configured(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'always', + 'output': 'voice-only', + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='Hello without voice')]) + ] + + mock_tts_model = Mock() + mock_tts_model.execution_context = Mock() + mock_tts_model.extra_args = {} + mock_tts_model.model_entity = Mock(extra_args={}) + mock_tts_model.provider = Mock() + mock_tts_model.provider.invoke_tts = AsyncMock(return_value=(b'fake-mp3', 'audio/mp3')) + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + extra_args = mock_tts_model.provider.invoke_tts.call_args[1]['extra_args'] + assert 'voice' not in extra_args + + +@pytest.mark.asyncio +async def test_tts_failure_on_failure_fallback_and_notify(mock_app, sample_query): + # Test on-failure=fallback + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'always', + 'on-failure': 'fallback', + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='Fallback test')]) + ] + + mock_tts_model = Mock() + mock_tts_model.execution_context = Mock() + mock_tts_model.extra_args = {} + mock_tts_model.model_entity = Mock(extra_args={}) + mock_tts_model.provider = Mock() + mock_tts_model.provider.invoke_tts = AsyncMock(side_effect=RuntimeError('TTS provider error')) + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + assert not result.user_notice + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + + # Test on-failure=notify + sample_query.pipeline_config['speech']['text-to-speech']['on-failure'] = 'notify' + result = await stage.process(sample_query, 'TextToSpeechStage') + assert result.result_type == ResultType.CONTINUE + assert result.user_notice == 'Voice synthesis failed.' + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + + +@pytest.mark.asyncio +async def test_tts_voice_only_success(mock_app, sample_query): + sample_query.variables['_input_was_voice'] = True + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'voice-input-only', + 'output': 'voice-only', + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='**Hello** [world](https://example.com)!')]) + ] + + mock_tts_model = Mock() + mock_tts_model.execution_context = Mock() + mock_tts_model.extra_args = {'voice': 'alloy'} + mock_tts_model.provider = Mock() + mock_tts_model.provider.invoke_tts = AsyncMock(return_value=(b'fake-mp3-audio-content', 'audio/mp3')) + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.resp_message_chain[-1]) == 1 + voice_comp = sample_query.resp_message_chain[-1][0] + assert isinstance(voice_comp, platform_message.Voice) + assert voice_comp.base64.startswith('data:audio/mp3;base64,') + mock_tts_model.provider.invoke_tts.assert_called_once() + assert mock_tts_model.provider.invoke_tts.call_args[1]['text'] == 'Hello world!' + + +@pytest.mark.asyncio +async def test_tts_voice_and_text_success(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'always', + 'output': 'voice-and-text', + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='Response with both voice and text')]) + ] + + mock_tts_model = Mock() + mock_tts_model.execution_context = Mock() + mock_tts_model.extra_args = {} + mock_tts_model.provider = Mock() + mock_tts_model.provider.invoke_tts = AsyncMock(return_value=(b'fake-audio', 'audio/mp3')) + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.resp_message_chain[-1]) == 2 + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + assert isinstance(sample_query.resp_message_chain[-1][1], platform_message.Voice) + + +@pytest.mark.asyncio +async def test_tts_max_text_length_fallback(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'always', + 'max-text-length': 10, + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='This is a long response text exceeding 10 chars')]) + ] + + mock_tts_model = Mock() + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.resp_message_chain[-1]) == 1 + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + + diff --git a/tests/unit_tests/platform/test_dingtalk_api.py b/tests/unit_tests/platform/test_dingtalk_api.py index 91803baac..12cecff0d 100644 --- a/tests/unit_tests/platform/test_dingtalk_api.py +++ b/tests/unit_tests/platform/test_dingtalk_api.py @@ -1,9 +1,12 @@ """Tests for DingTalk API payload helpers.""" +import base64 import json from contextlib import asynccontextmanager from unittest.mock import AsyncMock +import pytest + from langbot.libs.dingtalk_api.api import DingTalkClient, _stringify_card_param_map from langbot.pkg.utils import httpclient @@ -82,3 +85,97 @@ async def test_create_card_embeds_layout_config_as_template_parameter(monkeypatc } } assert original_params == {'content': 'hello'} + + +def _voice_client(): + from langbot.libs.dingtalk_api.api import DingTalkClient + + client = DingTalkClient.__new__(DingTalkClient) + client.logger = None + client._upload_media = AsyncMock(return_value='media-1') + client._send_robot_message = AsyncMock(return_value={'ok': True}) + return client + + +@pytest.mark.asyncio +async def test_send_voice_transcodes_to_ogg_and_sends_sample_audio(monkeypatch): + from langbot.pkg.utils import audio as audio_utils + from langbot_plugin.api.entities.builtin.platform.message import Voice + + transcode = AsyncMock(return_value=b'OggS' + b'\x00' * 32) + monkeypatch.setattr(audio_utils, 'transcode_audio', transcode) + client = _voice_client() + voice = Voice(base64='data:audio/mpeg;base64,' + base64.b64encode(b'ID3' + b'\x00' * 32).decode(), length=5) + + await client.send_voice(voice, 'incoming') + + transcode.assert_awaited_once() + assert transcode.await_args.args[1] == 'ogg' + client._upload_media.assert_awaited_once() + name, _body, media_type = client._upload_media.await_args.args + assert (name, media_type) == ('voice.ogg', 'voice') + msg_key, params, incoming = client._send_robot_message.await_args.args + assert msg_key == 'sampleAudio' + assert params == {'mediaId': 'media-1', 'duration': '5000'} + assert incoming == 'incoming' + + +@pytest.mark.asyncio +async def test_send_voice_over_limit_is_sent_as_file(monkeypatch): + from langbot.pkg.utils import audio as audio_utils + from langbot_plugin.api.entities.builtin.platform.message import Voice + + monkeypatch.setattr(audio_utils, 'transcode_audio', AsyncMock()) + client = _voice_client() + voice = Voice(base64='data:audio/ogg;base64,' + base64.b64encode(b'OggS' + b'\x00' * 32).decode(), length=90) + + await client.send_voice(voice, 'incoming') + + audio_utils.transcode_audio.assert_not_awaited() + assert client._upload_media.await_args.args[2] == 'file' + msg_key, params, _ = client._send_robot_message.await_args.args + assert msg_key == 'sampleFile' + assert params == {'mediaId': 'media-1', 'fileName': 'voice.ogg', 'fileType': 'ogg'} + + +@pytest.mark.asyncio +async def test_send_voice_falls_back_to_file_when_transcode_fails(monkeypatch): + from langbot.pkg.utils import audio as audio_utils + from langbot_plugin.api.entities.builtin.platform.message import Voice + + monkeypatch.setattr(audio_utils, 'transcode_audio', AsyncMock(side_effect=RuntimeError('no ffmpeg'))) + client = _voice_client() + voice = Voice(base64='data:audio/mpeg;base64,' + base64.b64encode(b'ID3' + b'\x00' * 32).decode(), length=3) + + await client.send_voice(voice, 'incoming') + + assert client._upload_media.await_args.args[2] == 'file' + msg_key, params, _ = client._send_robot_message.await_args.args + assert msg_key == 'sampleFile' + assert params['fileName'] == 'voice.mp3' + + +def test_estimate_audio_duration_reads_ogg_opus_granule_position(): + from langbot.pkg.utils import audio as audio_utils + + def page(granule: int, payload: bytes) -> bytes: + return b'OggS\x00\x00' + granule.to_bytes(8, 'little') + b'\x00' * 12 + payload + + data = page(0, b'OpusHead' + b'\x00' * 11) + page(0, b'OpusTags') + page(48000 * 3, b'\x00' * 8) + + assert audio_utils.detect_audio_format(data) == 'opus' + assert audio_utils.estimate_audio_duration(data) == 3.0 + + +@pytest.mark.asyncio +async def test_extract_voice_bytes_ignores_non_audio_mime_hint(): + from langbot.pkg.utils import audio as audio_utils + from langbot_plugin.api.entities.builtin.platform.message import Voice + + voice = Voice(base64='data:application/octet-stream;base64,' + base64.b64encode(b'unrecognized').decode()) + _, fmt = await audio_utils.extract_voice_bytes(voice) + assert fmt == 'unknown' + + voice = Voice(base64='data:audio/mpeg;base64,' + base64.b64encode(b'unrecognized').decode()) + _, fmt = await audio_utils.extract_voice_bytes(voice) + assert fmt == 'mp3' diff --git a/tests/unit_tests/platform/test_dingtalk_eba_adapter.py b/tests/unit_tests/platform/test_dingtalk_eba_adapter.py index 74d5a1c04..015aa2fe7 100644 --- a/tests/unit_tests/platform/test_dingtalk_eba_adapter.py +++ b/tests/unit_tests/platform/test_dingtalk_eba_adapter.py @@ -559,3 +559,47 @@ async def test_dingtalk_subscription_retry_does_not_repeat_successful_members(): assert (await handler.process(message))[0] == 200 assert (await handler.process(message))[0] == 200 assert [call.args[0].member.id for call in adapter._dispatch_eba_event.await_args_list] == ['a', 'b', 'b'] + + +@pytest.mark.asyncio +async def test_dingtalk_inbound_voice_keeps_audio_for_pipeline_speech_recognition(): + event = dingtalk_event( + conversation='OneToOneMessage', + msg_type='audio', + content='DingTalk recognition text', + audio='data:audio/ogg;base64,AAAA', + ) + event['AudioDuration'] = 4 + chain = await DingTalkMessageConverter.target2yiri(event, 'LangBot') + + voices = [part for part in chain if isinstance(part, platform_message.Voice)] + assert len(voices) == 1 + assert voices[0].base64 == 'data:audio/ogg;base64,AAAA' + assert voices[0].length == 4 + assert not any(isinstance(part, platform_message.Plain) for part in chain) + + +@pytest.mark.asyncio +async def test_dingtalk_inbound_voice_falls_back_to_recognition_without_audio(): + event = dingtalk_event(conversation='OneToOneMessage', msg_type='audio', content='DingTalk recognition text') + chain = await DingTalkMessageConverter.target2yiri(event, 'LangBot') + + assert not any(isinstance(part, platform_message.Voice) for part in chain) + plains = [part for part in chain if isinstance(part, platform_message.Plain)] + assert [part.text for part in plains] == ['DingTalk recognition text'] + + +@pytest.mark.asyncio +async def test_dingtalk_reply_sends_voice_as_attachment_not_text(): + adapter = make_adapter() + adapter.bot.send_attachment = AsyncMock() + voice = platform_message.Voice(base64='data:audio/mpeg;base64,AAAA', length=3) + message = platform_message.MessageChain([platform_message.Plain(text='hello'), voice]) + + source_event = await DingTalkEventConverter.target2yiri(dingtalk_event(), 'LangBot') + await adapter.reply_message(source_event, message) + + content = adapter.bot.send_message.await_args.args[0] + assert content == 'hello' + adapter.bot.send_attachment.assert_awaited_once() + assert adapter.bot.send_attachment.await_args.args[0] is voice diff --git a/tests/unit_tests/platform/test_discord_eba_adapter.py b/tests/unit_tests/platform/test_discord_eba_adapter.py index 0e776bf1a..db86c1d21 100644 --- a/tests/unit_tests/platform/test_discord_eba_adapter.py +++ b/tests/unit_tests/platform/test_discord_eba_adapter.py @@ -181,6 +181,38 @@ async def test_discord_message_converter_maps_mentions_and_text_to_target(): assert files == [] +@pytest.mark.asyncio +async def test_discord_message_converter_maps_audio_attachments_to_voice(monkeypatch): + wav_header = b'RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80>\x00\x00\x00}\x00\x00\x02\x00\x10\x00data\x00\x00\x00\x00' + attachment = SimpleNamespace( + id=999, + filename='voice-message.ogg', + url='https://cdn.discordapp.com/attachments/1/2/voice-message.ogg', + size=len(wav_header), + content_type='audio/ogg', + flags=SimpleNamespace(voice=True), + duration_secs=4.5, + ) + message = SimpleNamespace( + id=123, + content='Voice test', + created_at=datetime.datetime.now(), + attachments=[attachment], + ) + monkeypatch.setattr( + DiscordMessageConverter, + '_download', + AsyncMock(return_value=(wav_header, 'audio/ogg')), + ) + + chain = await DiscordMessageConverter.target2yiri(message) + voices = [c for c in chain if isinstance(c, platform_message.Voice)] + assert len(voices) == 1 + assert voices[0].url == attachment.url + assert voices[0].length == 4 + assert voices[0].base64.startswith('data:audio/ogg;base64,') + + def test_discord_message_converter_splits_discord_mentions(): components = DiscordMessageConverter._text_components('hi <@123> and @everyone') diff --git a/tests/unit_tests/platform/test_openclaw_weixin_client.py b/tests/unit_tests/platform/test_openclaw_weixin_client.py index 11f0779ab..7762dec2a 100644 --- a/tests/unit_tests/platform/test_openclaw_weixin_client.py +++ b/tests/unit_tests/platform/test_openclaw_weixin_client.py @@ -37,3 +37,68 @@ async def test_upload_media_rejects_oversized_input_before_network_access(): 'recipient', 3, ) + + +def test_voice_item_serialization_includes_encode_type_and_playtime(): + from langbot.libs.openclaw_weixin_api.client import _message_item_to_dict + from langbot.libs.openclaw_weixin_api.types import CDNMedia, MessageItem, VoiceItem + + media = CDNMedia(encrypt_query_param='param', aes_key='key', encrypt_type=1) + item = MessageItem( + type=MessageItem.VOICE, + voice_item=VoiceItem( + media=media, + playtime=4500, + encode_type=6, + bits_per_sample=16, + sample_rate=24000, + size=1024, + ), + ) + d = _message_item_to_dict(item) + + assert d['type'] == MessageItem.VOICE + assert 'voice_item' in d + assert d['voice_item']['playtime'] == 4500 + assert d['voice_item']['encode_type'] == 6 + assert d['voice_item']['bits_per_sample'] == 16 + assert d['voice_item']['sample_rate'] == 24000 + assert d['voice_item']['size'] == 1024 + assert d['voice_item']['media']['encrypt_query_param'] == 'param' + + +@pytest.mark.asyncio +async def test_inbound_voice_conversion_to_voice_component(): + import base64 + import langbot_plugin.api.entities.builtin.platform.message as platform_message + from langbot.libs.openclaw_weixin_api.types import CDNMedia, MessageItem, VoiceItem, WeixinMessage + from langbot.pkg.platform.sources.openclaw_weixin import OpenClawWeixinMessageConverter + + fake_silk_data = b'\x02#!SILK_V3' + b'\x05\x00abcde' * 50 + item = MessageItem( + type=MessageItem.VOICE, + voice_item=VoiceItem( + media=CDNMedia(encrypt_query_param='param', aes_key='key'), + playtime=1000, + _downloaded_bytes=fake_silk_data, + ), + ) + msg = WeixinMessage( + message_id=1, + from_user_id='user1', + to_user_id='bot1', + item_list=[item], + ) + + converter = OpenClawWeixinMessageConverter() + chain = await converter.target2yiri(msg) + + assert len(chain) == 1 + voice = chain[0] + assert isinstance(voice, platform_message.Voice) + assert voice.length == 1 + assert voice.base64.startswith('data:audio/silk;base64,') + b64_payload = voice.base64.split(',', 1)[1] + assert base64.b64decode(b64_payload) == fake_silk_data + + diff --git a/tests/unit_tests/platform/test_openclaw_weixin_tenancy.py b/tests/unit_tests/platform/test_openclaw_weixin_tenancy.py index 8a2f53e9b..cbc236b15 100644 --- a/tests/unit_tests/platform/test_openclaw_weixin_tenancy.py +++ b/tests/unit_tests/platform/test_openclaw_weixin_tenancy.py @@ -1,3 +1,4 @@ +import base64 from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -26,6 +27,8 @@ def make_adapter(*, execution_context: ExecutionContext | None): ap=app, execution_context=execution_context, warning=AsyncMock(), + info=AsyncMock(), + error=AsyncMock(), ) adapter = OpenClawWeixinAdapter.model_construct( config={'token': 'refreshed-token'}, @@ -93,3 +96,41 @@ async def test_component_base64_decode_is_bounded(monkeypatch): with pytest.raises(ValueError, match='exceeds'): await OpenClawWeixinAdapter._get_component_bytes(component) + + +@pytest.mark.asyncio +async def test_component_base64_strips_data_uri_prefix(): + raw_payload = b'hello world audio payload' + encoded = base64.b64encode(raw_payload).decode('ascii') + component = platform_message.Voice(base64=f'data:audio/mpeg;base64,{encoded}') + + result = await OpenClawWeixinAdapter._get_component_bytes(component) + assert result == raw_payload + + +@pytest.mark.asyncio +async def test_voice_send_gracefully_degrades_to_file_when_silk_unavailable(monkeypatch): + adapter, app, logger = make_adapter(execution_context=None) + adapter.client = AsyncMock() + + # Emulate SILK or ffmpeg unavailable + monkeypatch.setattr('langbot.pkg.utils.audio.is_silk_available', lambda: False) + monkeypatch.setattr('langbot.pkg.utils.audio.is_ffmpeg_available', lambda: False) + + raw_mp3 = b'ID3\x04\x00\x00\x00\x00\x00\x00\xff\xfb\x90\x00fake-mp3-audio' + encoded = base64.b64encode(raw_mp3).decode('ascii') + voice_comp = platform_message.Voice(base64=f'data:audio/mpeg;base64,{encoded}', length=3) + + msg = platform_message.MessageChain([voice_comp]) + await adapter.send_message('friend', 'target-wxid', msg) + + # Should NOT call send_voice (which WeChat mobile would silently drop for MP3) + adapter.client.send_voice.assert_not_awaited() + # Should call send_file as graceful degradation + adapter.client.send_file.assert_awaited_once_with( + 'target-wxid', + raw_mp3, + 'voice.mp3', + '', + ) + diff --git a/tests/unit_tests/platform/test_telegram_adapter.py b/tests/unit_tests/platform/test_telegram_adapter.py index cf661d638..daeef1799 100644 --- a/tests/unit_tests/platform/test_telegram_adapter.py +++ b/tests/unit_tests/platform/test_telegram_adapter.py @@ -367,3 +367,123 @@ async def test_telegram_text_field_does_not_show_action_buttons(): assert args['text'].startswith('[人工介入]') assert 'us_input (paragraph)' in args['text'] assert adapter._form_action_titles == {} + + +@pytest.mark.asyncio +async def test_telegram_source_reply_message_sends_text_and_voice(): + import base64 as _b64 + import telegram as _tg + + bot = MagicMock() + bot.send_message = AsyncMock() + bot.send_voice = AsyncMock() + adapter = TelegramAdapter.model_construct(bot=bot, config={}, msg_stream_id={}, seq=1, listeners={}) + + update = _tg.Update.de_json( + { + 'update_id': 1000, + 'message': { + 'message_id': 99, + 'date': 1700000000, + 'chat': {'id': 123, 'type': 'private'}, + 'from': {'id': 1, 'is_bot': False, 'first_name': 'User'}, + 'text': 'hi', + }, + }, + None, + ) + event = platform_events.FriendMessage( + sender=platform_entities.Friend(id='user-1', nickname='', remark=''), + message_chain=platform_message.MessageChain([]), + source_platform_object=update, + ) + + # Coexist mode: text and voice + chain = platform_message.MessageChain( + [ + platform_message.Plain(text='Hello'), + platform_message.Voice( + base64='data:audio/ogg;base64,' + _b64.b64encode(b'OggS-voice').decode('utf-8'), + length=6, + ), + ] + ) + await adapter.reply_message(event, chain, quote_origin=True) + bot.send_message.assert_awaited_once() + bot.send_voice.assert_awaited_once() + assert bot.send_message.await_args.kwargs['reply_to_message_id'] == 99 + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 99 + assert bot.send_voice.await_args.kwargs['duration'] == 6 + assert bot.send_voice.await_args.kwargs['voice'].filename == 'voice.ogg' + + # Voice only mode + bot.send_message.reset_mock() + bot.send_voice.reset_mock() + voice_only_chain = platform_message.MessageChain( + [ + platform_message.Voice( + base64='data:audio/ogg;base64,' + _b64.b64encode(b'OggS-voice-only').decode('utf-8'), + length=4, + ), + ] + ) + await adapter.reply_message(event, voice_only_chain, quote_origin=True) + bot.send_message.assert_not_awaited() + bot.send_voice.assert_awaited_once() + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 99 + assert bot.send_voice.await_args.kwargs['duration'] == 4 + + +@pytest.mark.asyncio +async def test_telegram_source_reply_message_chunk_delivers_voice(): + import base64 as _b64 + import telegram as _tg + + bot = MagicMock() + bot.send_message = AsyncMock() + bot.send_voice = AsyncMock() + bot.send_message_draft = AsyncMock() + adapter = TelegramAdapter.model_construct(bot=bot, config={}, msg_stream_id={}, seq=1, listeners={}) + adapter.msg_stream_id['stream_1'] = ('private', 101, True) + + update = _tg.Update.de_json( + { + 'update_id': 1000, + 'message': { + 'message_id': 99, + 'date': 1700000000, + 'chat': {'id': 123, 'type': 'private'}, + 'from': {'id': 1, 'is_bot': False, 'first_name': 'User'}, + 'text': 'hi', + }, + }, + None, + ) + event = platform_events.FriendMessage( + sender=platform_entities.Friend(id='user-1', nickname='', remark=''), + message_chain=platform_message.MessageChain([]), + source_platform_object=update, + ) + + chunk = SimpleNamespace(resp_message_id='stream_1', msg_sequence=1, tool_calls=None) + chain = platform_message.MessageChain( + [ + platform_message.Plain(text='Streaming text'), + platform_message.Voice( + base64='data:audio/ogg;base64,' + _b64.b64encode(b'OggS-voice').decode('utf-8'), + length=5, + ), + ] + ) + + # Not final chunk: no voice sent yet + await adapter.reply_message_chunk(event, chunk, chain, is_final=False) + bot.send_voice.assert_not_awaited() + + # Final chunk: voice sent and message finalized + await adapter.reply_message_chunk(event, chunk, chain, quote_origin=True, is_final=True) + bot.send_voice.assert_awaited_once() + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 99 + assert bot.send_voice.await_args.kwargs['duration'] == 5 + bot.send_message.assert_awaited_once() + assert 'stream_1' not in adapter.msg_stream_id diff --git a/tests/unit_tests/platform/test_telegram_eba_adapter.py b/tests/unit_tests/platform/test_telegram_eba_adapter.py index 3c2a0a15f..f3808679a 100644 --- a/tests/unit_tests/platform/test_telegram_eba_adapter.py +++ b/tests/unit_tests/platform/test_telegram_eba_adapter.py @@ -531,6 +531,7 @@ async def test_telegram_reply_message_sends_text_image_and_file_components(): send_message=AsyncMock(), send_photo=AsyncMock(), send_document=AsyncMock(), + send_voice=AsyncMock(), ) object.__setattr__(adapter, 'bot', bot) update = make_update({'message': base_message_payload(message_id=88)}) @@ -552,6 +553,10 @@ async def test_telegram_reply_message_sends_text_image_and_file_components(): size=4, base64='data:text/plain;base64,' + base64.b64encode(b'test').decode('utf-8'), ), + platform_message.Voice( + base64='data:audio/ogg;base64,' + base64.b64encode(b'OggS-voice').decode('utf-8'), + length=5, + ), ] ), quote_origin=True, @@ -560,11 +565,15 @@ async def test_telegram_reply_message_sends_text_image_and_file_components(): bot.send_message.assert_awaited_once() bot.send_photo.assert_awaited_once() bot.send_document.assert_awaited_once() + bot.send_voice.assert_awaited_once() assert bot.send_message.await_args.kwargs['reply_to_message_id'] == 88 assert bot.send_photo.await_args.kwargs['reply_to_message_id'] == 88 assert bot.send_photo.await_args.kwargs['photo'].input_file_content == b'image-bytes' assert bot.send_document.await_args.kwargs['document'].filename == 'test.txt' assert bot.send_document.await_args.kwargs['document'].input_file_content == b'test' + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 88 + assert bot.send_voice.await_args.kwargs['duration'] == 5 + assert bot.send_voice.await_args.kwargs['voice'].filename == 'voice.ogg' @pytest.mark.asyncio @@ -683,6 +692,7 @@ async def test_stream_completion_delivers_attachments_once(with_text): edit_message_text=AsyncMock(), send_photo=AsyncMock(return_value=SimpleNamespace(message_id=901)), send_document=AsyncMock(return_value=SimpleNamespace(message_id=902)), + send_voice=AsyncMock(return_value=SimpleNamespace(message_id=903)), ) object.__setattr__(adapter, 'bot', bot) adapter.msg_stream_id['stream'] = ('message', 999, True) @@ -696,6 +706,10 @@ async def test_stream_completion_delivers_attachments_once(with_text): platform_message.File( name='result.txt', base64='data:text/plain;base64,' + base64.b64encode(b'result').decode() ), + platform_message.Voice( + base64='data:audio/ogg;base64,' + base64.b64encode(b'OggS-voice').decode(), + length=3, + ), ] ) chunk = SimpleNamespace(resp_message_id='stream', msg_sequence=2, tool_calls=None) @@ -703,12 +717,16 @@ async def test_stream_completion_delivers_attachments_once(with_text): await adapter.reply_message_chunk(source, chunk, chain, is_final=False) bot.send_photo.assert_not_awaited() bot.send_document.assert_not_awaited() + bot.send_voice.assert_not_awaited() await adapter.reply_message_chunk(source, chunk, chain, quote_origin=True, is_final=True) await adapter.reply_message_chunk(source, chunk, chain, quote_origin=True, is_final=True) bot.send_photo.assert_awaited_once() bot.send_document.assert_awaited_once() + bot.send_voice.assert_awaited_once() assert bot.send_document.await_args.kwargs['document'].filename == 'result.txt' assert bot.send_photo.await_args.kwargs['reply_to_message_id'] == 88 + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 88 + assert bot.send_voice.await_args.kwargs['duration'] == 3 assert 'stream' not in adapter.msg_stream_id diff --git a/tests/unit_tests/platform/test_wecombot_media.py b/tests/unit_tests/platform/test_wecombot_media.py index 8bbff53e1..4bd3f9429 100644 --- a/tests/unit_tests/platform/test_wecombot_media.py +++ b/tests/unit_tests/platform/test_wecombot_media.py @@ -51,6 +51,12 @@ class Bot: async def reply_image(self, req_id, media_id): self.calls.append(('reply_image', req_id, media_id)) + async def reply_voice(self, req_id, media_id): + self.calls.append(('reply_voice', req_id, media_id)) + + async def reply_file(self, req_id, media_id): + self.calls.append(('reply_file', req_id, media_id)) + async def send_message(self, target_id, content): self.calls.append(('send_message', target_id, content)) @@ -106,6 +112,23 @@ async def test_reply_message_uploads_and_replies_image_media(): ] +@pytest.mark.asyncio +async def test_reply_message_uploads_and_replies_voice_media(): + bot = Bot() + adapter = make_adapter(bot) + silk_data = b'\x02#!SILK_V3test' + silk_b64 = base64.b64encode(silk_data).decode('utf-8') + chain = platform_message.MessageChain([platform_message.Voice(base64=f'data:audio/silk;base64,{silk_b64}')]) + + items = await WecomBotMessageConverter.yiri2target(chain) + await adapter._send_media(bot, 'req-1', items[0]) + + assert bot.calls == [ + ('upload_media', 'voice', 'voice.silk', silk_data), + ('reply_voice', 'req-1', 'media-1'), + ] + + @pytest.mark.asyncio async def test_send_message_sends_text_and_skips_proactive_image(): bot = Bot() diff --git a/tests/unit_tests/provider/test_litellmchat.py b/tests/unit_tests/provider/test_litellmchat.py index b5dff4b34..5f88aed19 100644 --- a/tests/unit_tests/provider/test_litellmchat.py +++ b/tests/unit_tests/provider/test_litellmchat.py @@ -12,6 +12,7 @@ import pytest from unittest.mock import Mock, AsyncMock, patch import litellm +import httpx from langbot.pkg.provider.modelmgr.requesters import litellmchat from langbot.pkg.provider.modelmgr import errors @@ -813,6 +814,33 @@ class TestHandleLiteLLMError: assert 'Context length exceeded' in str(exc_info.value) + def test_internal_server_error(self): + """Test InternalServerError translation""" + requester = litellmchat.LiteLLMRequester(ap=Mock(), config={}) + + error = litellm.InternalServerError(message='server error', model='gpt-4o', llm_provider='openai') + + with pytest.raises(errors.RequesterError) as exc_info: + requester._handle_litellm_error(error) + + assert 'Internal server error' in str(exc_info.value) + assert exc_info.value.retryable + + def test_permission_denied_error(self): + """Test PermissionDeniedError translation""" + requester = litellmchat.LiteLLMRequester(ap=Mock(), config={}) + + response = httpx.Response(403, request=httpx.Request('POST', 'https://example.com')) + error = litellm.PermissionDeniedError( + message='forbidden', model='gpt-4o', llm_provider='openai', response=response + ) + + with pytest.raises(errors.RequesterError) as exc_info: + requester._handle_litellm_error(error) + + assert 'Permission denied' in str(exc_info.value) + assert not exc_info.value.retryable + def test_unknown_error(self): """Test unknown error translation""" requester = litellmchat.LiteLLMRequester(ap=Mock(), config={}) diff --git a/tests/unit_tests/test_audio_utils.py b/tests/unit_tests/test_audio_utils.py new file mode 100644 index 000000000..9bab3d4f8 --- /dev/null +++ b/tests/unit_tests/test_audio_utils.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import io +import unittest +from unittest.mock import patch +import wave + +import langbot_plugin.api.entities.builtin.platform.message as platform_message + +from langbot.pkg.utils import audio as audio_utils + + +def _create_sample_wav() -> bytes: + buf = io.BytesIO() + with wave.open(buf, 'wb') as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b'\x00' * 3200) + return buf.getvalue() + + +class TestAudioUtils(unittest.IsolatedAsyncioTestCase): + def test_detect_audio_format(self): + sample_wav = _create_sample_wav() + self.assertEqual(audio_utils.detect_audio_format(sample_wav), 'wav') + self.assertEqual(audio_utils.detect_audio_format(b'#!AMR\n12345'), 'amr') + self.assertEqual(audio_utils.detect_audio_format(b'#!SILK_V3\n'), 'silk') + self.assertEqual(audio_utils.detect_audio_format(b'\x02#!SILK_V3\n'), 'silk') + self.assertEqual(audio_utils.detect_audio_format(b'OggS\x00\x02...OpusHead...'), 'opus') + self.assertEqual(audio_utils.detect_audio_format(b'OggS\x00\x02...'), 'ogg') + self.assertEqual(audio_utils.detect_audio_format(b'ID3\x03\x00\x00'), 'mp3') + self.assertEqual(audio_utils.detect_audio_format(b'\xff\xfb\x90\x64'), 'mp3') + self.assertEqual(audio_utils.detect_audio_format(b'fLaC\x00\x00'), 'flac') + self.assertEqual(audio_utils.detect_audio_format(b''), 'unknown') + self.assertEqual(audio_utils.detect_audio_format(b'12'), 'unknown') + + def test_is_ffmpeg_and_silk_available(self): + with patch.object(audio_utils.shutil, 'which', return_value='/usr/bin/ffmpeg'): + self.assertTrue(audio_utils.is_ffmpeg_available()) + with ( + patch.object(audio_utils.shutil, 'which', return_value=None), + patch.object(audio_utils.os.path, 'isfile', return_value=False), + patch.dict('sys.modules', {'imageio_ffmpeg': None}), + ): + self.assertFalse(audio_utils.is_ffmpeg_available()) + with patch.object(audio_utils.importlib.util, 'find_spec', return_value=object()): + self.assertTrue(audio_utils.is_silk_available()) + with patch.object(audio_utils.importlib.util, 'find_spec', return_value=None): + self.assertFalse(audio_utils.is_silk_available()) + + @unittest.skipUnless(audio_utils.is_ffmpeg_available(), 'ffmpeg is not installed') + async def test_transcode_audio_wav_to_mp3_and_ogg(self): + sample_wav = _create_sample_wav() + mp3_bytes = await audio_utils.transcode_audio(sample_wav, 'mp3', 'wav') + self.assertTrue(len(mp3_bytes) > 0) + self.assertEqual(audio_utils.detect_audio_format(mp3_bytes), 'mp3') + + ogg_bytes = await audio_utils.transcode_audio(sample_wav, 'ogg', 'wav') + self.assertTrue(len(ogg_bytes) > 0) + fmt = audio_utils.detect_audio_format(ogg_bytes) + self.assertIn(fmt, ('ogg', 'opus')) + + async def test_transcode_same_format_returns_input(self): + sample_wav = _create_sample_wav() + result = await audio_utils.transcode_audio(sample_wav, 'wav', 'wav') + self.assertEqual(result, sample_wav) + + def test_transcode_missing_ffmpeg_raises_error(self): + sample_wav = _create_sample_wav() + with patch.object(audio_utils, 'is_ffmpeg_available', return_value=False): + with self.assertRaises(audio_utils.AudioDependencyError): + audio_utils.transcode_audio_sync(sample_wav, 'mp3', 'wav') + + def test_transcode_missing_silk_raises_error(self): + sample_wav = _create_sample_wav() + with patch.object(audio_utils, 'is_silk_available', return_value=False): + with self.assertRaises(audio_utils.AudioDependencyError): + audio_utils.transcode_audio_sync(sample_wav, 'silk', 'wav') + + async def test_extract_voice_bytes_from_base64(self): + import base64 + + sample_wav = _create_sample_wav() + b64_raw = base64.b64encode(sample_wav).decode('ascii') + voice_raw = platform_message.Voice(base64=b64_raw) + data, fmt = await audio_utils.extract_voice_bytes(voice_raw) + self.assertEqual(data, sample_wav) + self.assertEqual(fmt, 'wav') + + data_uri = f'data:audio/wav;base64,{b64_raw}' + voice_uri = platform_message.Voice(base64=data_uri) + data2, fmt2 = await audio_utils.extract_voice_bytes(voice_uri) + self.assertEqual(data2, sample_wav) + self.assertEqual(fmt2, 'wav') diff --git a/tests/unit_tests/test_speech_models_api.py b/tests/unit_tests/test_speech_models_api.py new file mode 100644 index 000000000..dda0d8496 --- /dev/null +++ b/tests/unit_tests/test_speech_models_api.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from langbot.pkg.api.http.context import ExecutionContext +from langbot.pkg.entity.persistence.model import ModelProvider, ASRModel, TTSModel +from langbot.pkg.provider.modelmgr.modelmgr import ModelManager +from langbot.pkg.provider.modelmgr.requester import RuntimeProvider, RuntimeASRModel, RuntimeTTSModel +from langbot.pkg.provider.modelmgr.requesters.litellmchat import LiteLLMRequester + + +class TestSpeechModels(unittest.IsolatedAsyncioTestCase): + async def test_model_manager_asr_and_tts_cache_and_get(self): + ap = Mock() + ap.logger = Mock() + ap.workspace_service = Mock() + ap.workspace_service.get_execution_binding = AsyncMock( + return_value=Mock( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + ) + mgr = ModelManager(ap) + + context = ExecutionContext( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + + provider_entity = ModelProvider( + uuid='prov-1', + name='Test Provider', + requester='openai', + base_url='https://api.openai.com/v1', + api_keys=['sk-test'], + workspace_uuid='ws-1', + ) + runtime_provider = RuntimeProvider( + execution_context=context, + provider_entity=provider_entity, + token_mgr=Mock(), + requester=Mock(), + ) + + asr_entity = ASRModel( + uuid='asr-1', + name='whisper-1', + provider_uuid='prov-1', + workspace_uuid='ws-1', + ) + tts_entity = TTSModel( + uuid='tts-1', + name='tts-1', + provider_uuid='prov-1', + workspace_uuid='ws-1', + ) + + runtime_asr = await mgr.load_asr_model_with_provider(context, asr_entity, runtime_provider) + await mgr.cache_asr_model(context, runtime_asr) + + runtime_tts = await mgr.load_tts_model_with_provider(context, tts_entity, runtime_provider) + await mgr.cache_tts_model(context, runtime_tts) + + retrieved_asr = await mgr.get_asr_model_by_uuid(context, 'asr-1') + self.assertIsNotNone(retrieved_asr) + self.assertEqual(retrieved_asr.model_entity.uuid, 'asr-1') + self.assertEqual(retrieved_asr.model_entity.name, 'whisper-1') + + retrieved_tts = await mgr.get_tts_model_by_uuid(context, 'tts-1') + self.assertIsNotNone(retrieved_tts) + self.assertEqual(retrieved_tts.model_entity.uuid, 'tts-1') + self.assertEqual(retrieved_tts.model_entity.name, 'tts-1') + + await mgr.remove_asr_model(context, 'asr-1') + with self.assertRaises(ValueError): + await mgr.get_asr_model_by_uuid(context, 'asr-1') + + await mgr.remove_tts_model(context, 'tts-1') + with self.assertRaises(ValueError): + await mgr.get_tts_model_by_uuid(context, 'tts-1') + + async def test_litellm_requester_invoke_asr(self): + ap = Mock() + requester = LiteLLMRequester(ap, {'base_url': 'https://api.openai.com/v1'}) + + context = ExecutionContext( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + + provider_entity = ModelProvider( + uuid='prov-1', + name='OpenAI', + requester='openai', + base_url='https://api.openai.com/v1', + api_keys=['sk-test-key'], + workspace_uuid='ws-1', + ) + runtime_provider = RuntimeProvider( + execution_context=context, + provider_entity=provider_entity, + token_mgr=Mock(get_token=Mock(return_value='sk-test-key')), + requester=requester, + ) + asr_entity = ASRModel( + uuid='asr-1', + name='whisper-1', + provider_uuid='prov-1', + workspace_uuid='ws-1', + ) + runtime_asr = RuntimeASRModel( + execution_context=context, + model_entity=asr_entity, + provider=runtime_provider, + ) + + mock_resp = Mock(text='Hello world transcription') + + with patch('langbot.pkg.provider.modelmgr.requesters.litellmchat.atranscription', new_callable=AsyncMock) as mock_atranscription: + mock_atranscription.return_value = mock_resp + + result = await requester.invoke_asr( + model=runtime_asr, + audio=b'fake-wav-bytes', + mime='audio/wav', + ) + + self.assertEqual(result, 'Hello world transcription') + mock_atranscription.assert_called_once() + call_kwargs = mock_atranscription.call_args[1] + self.assertEqual(call_kwargs['model'], 'whisper-1') + self.assertEqual(call_kwargs['api_key'], 'sk-test-key') + self.assertEqual(call_kwargs['api_base'], 'https://api.openai.com/v1') + self.assertEqual(call_kwargs['file'], ('audio.wav', b'fake-wav-bytes', 'audio/wav')) + + async def test_litellm_requester_invoke_tts(self): + ap = Mock() + requester = LiteLLMRequester(ap, {}) + + context = ExecutionContext( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + + provider_entity = ModelProvider( + uuid='prov-1', + name='OpenAI', + requester='openai', + base_url='https://api.openai.com/v1', + api_keys=['sk-test-key'], + workspace_uuid='ws-1', + ) + runtime_provider = RuntimeProvider( + execution_context=context, + provider_entity=provider_entity, + token_mgr=Mock(get_token=Mock(return_value='sk-test-key')), + requester=requester, + ) + tts_entity = TTSModel( + uuid='tts-1', + name='tts-1', + provider_uuid='prov-1', + workspace_uuid='ws-1', + extra_args={'voice': 'alloy'}, + ) + runtime_tts = RuntimeTTSModel( + execution_context=context, + model_entity=tts_entity, + provider=runtime_provider, + ) + + mock_resp = Mock(content=b'fake-mp3-bytes', spec=['content']) + + with patch('langbot.pkg.provider.modelmgr.requesters.litellmchat.aspeech', new_callable=AsyncMock) as mock_aspeech: + mock_aspeech.return_value = mock_resp + + content, mime = await requester.invoke_tts( + model=runtime_tts, + text='Hello, world!', + extra_args={'voice': 'alloy', 'response_format': 'mp3'}, + ) + + self.assertEqual(content, b'fake-mp3-bytes') + self.assertEqual(mime, 'audio/mpeg') + mock_aspeech.assert_called_once() + call_kwargs = mock_aspeech.call_args[1] + self.assertEqual(call_kwargs['model'], 'tts-1') + self.assertEqual(call_kwargs['input'], 'Hello, world!') + self.assertEqual(call_kwargs['voice'], 'alloy') + self.assertEqual(call_kwargs['response_format'], 'mp3') + self.assertEqual(call_kwargs['api_key'], 'sk-test-key') + + # Test without voice configured + tts_entity_no_voice = TTSModel( + uuid='tts-2', + name='CosyVoice2', + provider_uuid='prov-1', + workspace_uuid='ws-1', + extra_args={}, + ) + runtime_tts_no_voice = RuntimeTTSModel( + execution_context=context, + model_entity=tts_entity_no_voice, + provider=runtime_provider, + ) + with patch('langbot.pkg.provider.modelmgr.requesters.litellmchat.aspeech', new_callable=AsyncMock) as mock_aspeech_no_voice: + mock_aspeech_no_voice.return_value = mock_resp + await requester.invoke_tts( + model=runtime_tts_no_voice, + text='Hello without voice', + extra_args={}, + ) + mock_aspeech_no_voice.assert_called_once() + call_kwargs_no_voice = mock_aspeech_no_voice.call_args[1] + self.assertNotIn('voice', call_kwargs_no_voice) + + async def test_init_temporary_runtime_tts_model_by_provider_uuid(self): + ap = Mock() + ap.logger = Mock() + ap.workspace_service = Mock() + ap.workspace_service.get_execution_binding = AsyncMock( + return_value=Mock( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + ) + ap.persistence_mgr = Mock() + saved_prov = ModelProvider( + uuid='prov-1', + name='Test Provider', + requester='new-api-chat-completions', + base_url='https://api.openai.com/v1', + api_keys=['sk-test'], + workspace_uuid='ws-1', + ) + ap.persistence_mgr.execute_async = AsyncMock(return_value=Mock(first=Mock(return_value=saved_prov))) + + mgr = ModelManager(ap) + context = ExecutionContext( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + + with patch.object(mgr, '_build_provider', new_callable=AsyncMock) as mock_build_prov: + mock_runtime_prov = Mock(provider_entity=saved_prov, execution_context=context) + mock_build_prov.return_value = mock_runtime_prov + + model = await mgr.init_temporary_runtime_tts_model( + context, + { + 'name': 'gpt-4o-mini-tts', + 'provider_uuid': 'prov-1', + 'extra_args': {'voice': 'alloy'}, + }, + ) + + self.assertEqual(model.name, 'gpt-4o-mini-tts') + self.assertEqual(model.uuid, '') + self.assertEqual(model.extra_args, {'voice': 'alloy'}) + mock_build_prov.assert_called_once() + + def test_litellm_requester_infer_model_type_speech(self): + req = LiteLLMRequester('openai', {'base_url': 'https://api.openai.com/v1'}) + self.assertEqual(req._infer_model_type('whisper-1'), 'asr') + self.assertEqual(req._infer_model_type('FunAudioLLM/SenseVoiceSmall'), 'asr') + self.assertEqual(req._infer_model_type('tts-1'), 'tts') + self.assertEqual(req._infer_model_type('tts-1-hd'), 'tts') + self.assertEqual(req._infer_model_type('FunAudioLLM/CosyVoice2-0.5B'), 'tts') + self.assertEqual(req._infer_model_type('text-embedding-3-small'), 'embedding') + self.assertEqual(req._infer_model_type('bge-reranker-v2-m3'), 'rerank') + self.assertEqual(req._infer_model_type('gpt-4o'), 'llm') + diff --git a/uv.lock b/uv.lock index 410790882..6ba1a6d82 100644 --- a/uv.lock +++ b/uv.lock @@ -2199,6 +2199,9 @@ dependencies = [ ] [package.optional-dependencies] +audio = [ + { name = "pilk" }, +] seekdb = [ { name = "pylibseekdb", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or sys_platform == 'linux'" }, { name = "pyseekdb" }, @@ -2259,6 +2262,7 @@ requires-dist = [ { name = "openai", specifier = ">1.0.0" }, { name = "pandas", specifier = ">=2.2.2" }, { name = "pgvector", specifier = ">=0.4.1" }, + { name = "pilk", marker = "extra == 'audio'", specifier = ">=0.1.0" }, { name = "pillow", specifier = ">=13.0.0" }, { name = "pip", specifier = ">=26.2.0" }, { name = "pre-commit", specifier = ">=4.2.0" }, @@ -2297,7 +2301,7 @@ requires-dist = [ { name = "websockets", specifier = ">=15.0.1" }, { name = "xxhash", specifier = ">=3.5.0" }, ] -provides-extras = ["seekdb"] +provides-extras = ["audio", "seekdb"] [package.metadata.requires-dev] dev = [ @@ -3958,6 +3962,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/26/6cee8a1ce8c43625ec561aff19df07f9776b7525d9002c86bceb3e0ac970/pgvector-0.4.2-py3-none-any.whl", hash = "sha256:549d45f7a18593783d5eec609ea1684a724ba8405c4cb182a0b2b08aeff04e08", size = 27441, upload-time = "2025-12-05T01:07:16.536Z" }, ] +[[package]] +name = "pilk" +version = "0.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/bb/938dd697b6bc2d851ffec4ffe82ed20078e58bd0ef049e84f4d038d6f991/pilk-0.2.4.tar.gz", hash = "sha256:d4a1bcf93dc6ef5e95e0cfd728ed4ef4d49f9c0476d70816fecbe456cc762e7f", size = 226451, upload-time = "2023-05-26T03:32:34.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/06/80cac61bc7f791bcaae552ba90fb868f7af7503ef1c74e423cc20fca1a53/pilk-0.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:a6692096607e8d77d348aeec00df633788c85b527aa83cbd803ddf508936db7f", size = 127024, upload-time = "2023-05-26T03:32:24.524Z" }, +] + [[package]] name = "pillow" version = "13.0.0" diff --git a/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx b/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx index b6edb70dd..e6c59b4ee 100644 --- a/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx +++ b/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx @@ -575,7 +575,7 @@ const BotSessionMonitor = forwardRef< case 'Voice': { const voice = component as Voice; - const voiceUrl = voice.url || (voice.base64 ? voice.base64 : ''); + const voiceUrl = voice.base64 || voice.url || ''; if (!voiceUrl) { return ( ([]); const [embeddingModels, setEmbeddingModels] = useState([]); const [rerankModels, setRerankModels] = useState([]); + const [asrModels, setAsrModels] = useState([]); + const [ttsModels, setTtsModels] = useState([]); const [knowledgeBases, setKnowledgeBases] = useState([]); const [bots, setBots] = useState([]); const [tools, setTools] = useState([]); @@ -167,6 +171,8 @@ export default function DynamicFormItemComponent({ DynamicFormItemType.LLM_MODEL_SELECTOR, DynamicFormItemType.EMBEDDING_MODEL_SELECTOR, DynamicFormItemType.RERANK_MODEL_SELECTOR, + DynamicFormItemType.ASR_MODEL_SELECTOR, + DynamicFormItemType.TTS_MODEL_SELECTOR, DynamicFormItemType.MODEL_FALLBACK_SELECTOR, ].includes(config.type); const { @@ -244,12 +250,36 @@ export default function DynamicFormItemComponent({ }); }; + const fetchAsrModels = () => { + httpClient + .getProviderASRModels() + .then((resp) => { + setAsrModels(resp.models.filter(hasUsableUuid)); + }) + .catch((err) => { + toast.error('Failed to load ASR models: ' + err.msg); + }); + }; + + const fetchTtsModels = () => { + httpClient + .getProviderTTSModels() + .then((resp) => { + setTtsModels(resp.models.filter(hasUsableUuid)); + }) + .catch((err) => { + toast.error('Failed to load TTS models: ' + err.msg); + }); + }; + const handleModelsDialogChange = (open: boolean) => { setModelsDialogOpen(open); if (!open) { fetchLlmModels(); fetchEmbeddingModels(); fetchRerankModels(); + fetchAsrModels(); + fetchTtsModels(); } }; @@ -326,6 +356,18 @@ export default function DynamicFormItemComponent({ } }, [config.type]); + useEffect(() => { + if (config.type === DynamicFormItemType.ASR_MODEL_SELECTOR) { + fetchAsrModels(); + } + }, [config.type]); + + useEffect(() => { + if (config.type === DynamicFormItemType.TTS_MODEL_SELECTOR) { + fetchTtsModels(); + } + }, [config.type]); + useEffect(() => { if (config.type === DynamicFormItemType.MODEL_FALLBACK_SELECTOR) { fetchLlmModels(); @@ -1060,6 +1102,112 @@ export default function DynamicFormItemComponent({ ); + case DynamicFormItemType.ASR_MODEL_SELECTOR: + const groupedAsrModels = asrModels.reduce( + (acc, model) => { + const providerName = model.provider?.name || 'Unknown'; + if (!acc[providerName]) acc[providerName] = []; + acc[providerName].push(model); + return acc; + }, + {} as Record, + ); + for (const [providerName, models] of Object.entries(groupedAsrModels)) { + if ( + models[0]?.provider?.requester === LANGBOT_MODELS_PROVIDER_REQUESTER + ) { + groupedAsrModels[providerName] = sortModelsByCatalog( + models, + langbotModelMetadata, + ); + } + } + + return ( +
+ +
+ ); + + case DynamicFormItemType.TTS_MODEL_SELECTOR: + const groupedTtsModels = ttsModels.reduce( + (acc, model) => { + const providerName = model.provider?.name || 'Unknown'; + if (!acc[providerName]) acc[providerName] = []; + acc[providerName].push(model); + return acc; + }, + {} as Record, + ); + for (const [providerName, models] of Object.entries(groupedTtsModels)) { + if ( + models[0]?.provider?.requester === LANGBOT_MODELS_PROVIDER_REQUESTER + ) { + groupedTtsModels[providerName] = sortModelsByCatalog( + models, + langbotModelMetadata, + ); + } + } + + return ( +
+ +
+ ); + case DynamicFormItemType.MODEL_FALLBACK_SELECTOR: { // Separate space models from regular models const fbSpaceModels = sortModelsByCatalog( diff --git a/web/src/app/home/components/dynamic-form/DynamicFormSaveValues.ts b/web/src/app/home/components/dynamic-form/DynamicFormSaveValues.ts index 2ddc942d6..f229796d2 100644 --- a/web/src/app/home/components/dynamic-form/DynamicFormSaveValues.ts +++ b/web/src/app/home/components/dynamic-form/DynamicFormSaveValues.ts @@ -21,6 +21,8 @@ const STRING_FIELD_TYPES = new Set([ 'llm-model-selector', 'embedding-model-selector', 'rerank-model-selector', + 'asr-model-selector', + 'tts-model-selector', 'knowledge-base-selector', 'bot-selector', ]); diff --git a/web/src/app/home/components/models-dialog/ModelsPanel.tsx b/web/src/app/home/components/models-dialog/ModelsPanel.tsx index 789082b0b..091b3ea44 100644 --- a/web/src/app/home/components/models-dialog/ModelsPanel.tsx +++ b/web/src/app/home/components/models-dialog/ModelsPanel.tsx @@ -213,17 +213,22 @@ export default function ModelsPanel({ setLoadingProviders((prev) => new Set(prev).add(providerUuid)); } try { - const [llmResp, embeddingResp, rerankResp] = await Promise.all([ - httpClient.getProviderLLMModels(providerUuid), - httpClient.getProviderEmbeddingModels(providerUuid), - httpClient.getProviderRerankModels(providerUuid), - ]); + const [llmResp, embeddingResp, rerankResp, asrResp, ttsResp] = + await Promise.all([ + httpClient.getProviderLLMModels(providerUuid), + httpClient.getProviderEmbeddingModels(providerUuid), + httpClient.getProviderRerankModels(providerUuid), + httpClient.getProviderASRModels(providerUuid), + httpClient.getProviderTTSModels(providerUuid), + ]); setProviderModels((prev) => ({ ...prev, [providerUuid]: { llm: llmResp.models, embedding: embeddingResp.models, rerank: rerankResp.models, + asr: asrResp.models, + tts: ttsResp.models, }, })); } catch (err) { @@ -320,12 +325,24 @@ export default function ModelsPanel({ provider_uuid: providerUuid, extra_args: extraArgsObj, } as never); - } else { + } else if (modelType === 'rerank') { await httpClient.createProviderRerankModel({ name, provider_uuid: providerUuid, extra_args: extraArgsObj, } as never); + } else if (modelType === 'asr') { + await httpClient.createProviderASRModel({ + name, + provider_uuid: providerUuid, + extra_args: extraArgsObj, + } as never); + } else { + await httpClient.createProviderTTSModel({ + name, + provider_uuid: providerUuid, + extra_args: extraArgsObj, + } as never); } setAddModelPopoverOpen(null); loadProviderModels(providerUuid, true); @@ -379,12 +396,24 @@ export default function ModelsPanel({ provider_uuid: providerUuid, extra_args: {}, } as never); - } else { + } else if (effectiveType === 'rerank') { await httpClient.createProviderRerankModel({ name: item.model.name, provider_uuid: providerUuid, extra_args: {}, } as never); + } else if (effectiveType === 'asr') { + await httpClient.createProviderASRModel({ + name: item.model.name, + provider_uuid: providerUuid, + extra_args: {}, + } as never); + } else { + await httpClient.createProviderTTSModel({ + name: item.model.name, + provider_uuid: providerUuid, + extra_args: {}, + } as never); } } setAddModelPopoverOpen(null); @@ -436,12 +465,24 @@ export default function ModelsPanel({ provider_uuid: providerUuid, extra_args: extraArgsObj, } as never); - } else { + } else if (modelType === 'rerank') { await httpClient.updateProviderRerankModel(modelId, { name, provider_uuid: providerUuid, extra_args: extraArgsObj, } as never); + } else if (modelType === 'asr') { + await httpClient.updateProviderASRModel(modelId, { + name, + provider_uuid: providerUuid, + extra_args: extraArgsObj, + } as never); + } else { + await httpClient.updateProviderTTSModel(modelId, { + name, + provider_uuid: providerUuid, + extra_args: extraArgsObj, + } as never); } setEditModelPopoverOpen(null); loadProviderModels(providerUuid, true); @@ -463,8 +504,12 @@ export default function ModelsPanel({ await httpClient.deleteProviderLLMModel(modelId); } else if (modelType === 'embedding') { await httpClient.deleteProviderEmbeddingModel(modelId); - } else { + } else if (modelType === 'rerank') { await httpClient.deleteProviderRerankModel(modelId); + } else if (modelType === 'asr') { + await httpClient.deleteProviderASRModel(modelId); + } else { + await httpClient.deleteProviderTTSModel(modelId); } toast.success(t('models.deleteSuccess')); loadProviderModels(providerUuid, true); @@ -515,7 +560,7 @@ export default function ModelsPanel({ provider: providerData, extra_args: extraArgsObj, } as never); - } else { + } else if (modelType === 'rerank') { await httpClient.testRerankModel('_', { uuid: '', name, @@ -523,6 +568,22 @@ export default function ModelsPanel({ provider: providerData, extra_args: extraArgsObj, } as never); + } else if (modelType === 'asr') { + await httpClient.testASRModel('_', { + uuid: '', + name, + provider_uuid: providerUuid, + provider: providerData, + extra_args: extraArgsObj, + } as never); + } else { + await httpClient.testTTSModel('_', { + uuid: '', + name, + provider_uuid: providerUuid, + provider: providerData, + extra_args: extraArgsObj, + } as never); } const duration = Date.now() - startTime; setTestResult({ success: true, duration }); diff --git a/web/src/app/home/components/models-dialog/components/AddModelPopover.tsx b/web/src/app/home/components/models-dialog/components/AddModelPopover.tsx index bbf71c6a2..ffc97b5c9 100644 --- a/web/src/app/home/components/models-dialog/components/AddModelPopover.tsx +++ b/web/src/app/home/components/models-dialog/components/AddModelPopover.tsx @@ -4,6 +4,8 @@ import { MessageSquareText, Cpu, ArrowUpDown, + Mic, + Volume2, Eye, Wrench, BrainCircuit, @@ -93,8 +95,10 @@ export default function AddModelPopover({ llm: 'llm', embedding: 'text-embedding', rerank: 'rerank', + asr: 'asr', + tts: 'tts', }; - const allTabs: ModelType[] = ['llm', 'embedding', 'rerank']; + const allTabs: ModelType[] = ['llm', 'embedding', 'rerank', 'asr', 'tts']; // When supportTypes is undefined (unknown requester), show all tabs for // backward compatibility. Otherwise only show supported tabs. const visibleTabs: ModelType[] = supportTypes @@ -284,7 +288,7 @@ export default function AddModelPopover({ )} {!(trigger && initialMode === 'scan') && visibleTabs.length > 1 && ( {visibleTabs.includes('llm') && ( - - - {t('models.chat')} + + + {t('models.chat')} )} {visibleTabs.includes('embedding') && ( - - - {t('models.embedding')} + + + {t('models.embedding')} )} {visibleTabs.includes('rerank') && ( - - - {t('models.rerank')} + + + {t('models.rerank')} + + )} + {visibleTabs.includes('asr') && ( + + + + {t('models.asrTab', { defaultValue: 'ASR' })} + + + )} + {visibleTabs.includes('tts') && ( + + + + {t('models.ttsTab', { defaultValue: 'TTS' })} + )} @@ -531,7 +571,11 @@ export default function AddModelPopover({ ? t('models.chat') : model.type === 'embedding' ? t('models.embedding') - : t('models.rerank')} + : model.type === 'rerank' + ? t('models.rerank') + : model.type === 'asr' + ? t('models.asr') + : t('models.tts')} diff --git a/web/src/app/home/components/models-dialog/components/ModelItem.tsx b/web/src/app/home/components/models-dialog/components/ModelItem.tsx index aa2f47041..4a6881b64 100644 --- a/web/src/app/home/components/models-dialog/components/ModelItem.tsx +++ b/web/src/app/home/components/models-dialog/components/ModelItem.tsx @@ -211,7 +211,11 @@ export default function ModelItem({ ? t('models.chat') : modelType === 'embedding' ? t('models.embedding') - : t('models.rerank')} + : modelType === 'rerank' + ? t('models.rerank') + : modelType === 'asr' + ? t('models.asr') + : t('models.tts')} {modelType === 'llm' && (model as LLMModel).abilities?.includes('vision') && ( diff --git a/web/src/app/home/components/models-dialog/components/ProviderCard.tsx b/web/src/app/home/components/models-dialog/components/ProviderCard.tsx index 2ba418c56..07457b08a 100644 --- a/web/src/app/home/components/models-dialog/components/ProviderCard.tsx +++ b/web/src/app/home/components/models-dialog/components/ProviderCard.tsx @@ -153,15 +153,23 @@ export default function ProviderCard({ useState(false); const [addModelMode, setAddModelMode] = useState<'manual' | 'scan'>('manual'); - const canDelete = - !isLangBotModels && - (provider.llm_count || 0) === 0 && - (provider.embedding_count || 0) === 0 && - (provider.rerank_count || 0) === 0; + const modelsCountFromLoaded = models + ? (models.llm?.length || 0) + + (models.embedding?.length || 0) + + (models.rerank?.length || 0) + + (models.asr?.length || 0) + + (models.tts?.length || 0) + : undefined; const totalModels = - (provider.llm_count || 0) + - (provider.embedding_count || 0) + - (provider.rerank_count || 0); + modelsCountFromLoaded !== undefined + ? modelsCountFromLoaded + : (provider.llm_count || 0) + + (provider.embedding_count || 0) + + (provider.rerank_count || 0) + + (provider.asr_count || 0) + + (provider.tts_count || 0); + + const canDelete = !isLangBotModels && totalModels === 0; return ( @@ -601,9 +609,117 @@ export default function ProviderCard({ onResetTestResult={onResetTestResult} /> ))} + {(models.asr || []).map((model) => ( + onDeleteModel(model.uuid, 'asr')} + onUpdateModel={( + name, + abilities, + extraArgs, + reasoningConfig, + ) => + onUpdateModel( + model.uuid, + 'asr', + name, + abilities, + extraArgs, + reasoningConfig, + ) + } + onTestModel={( + name, + abilities, + extraArgs, + reasoningConfig, + ) => + onTestModel( + name, + 'asr', + abilities, + extraArgs, + reasoningConfig, + ) + } + isSubmitting={isSubmitting} + isTesting={isTesting} + testResult={testResult} + onResetTestResult={onResetTestResult} + /> + ))} + {(models.tts || []).map((model) => ( + onDeleteModel(model.uuid, 'tts')} + onUpdateModel={( + name, + abilities, + extraArgs, + reasoningConfig, + ) => + onUpdateModel( + model.uuid, + 'tts', + name, + abilities, + extraArgs, + reasoningConfig, + ) + } + onTestModel={( + name, + abilities, + extraArgs, + reasoningConfig, + ) => + onTestModel( + name, + 'tts', + abilities, + extraArgs, + reasoningConfig, + ) + } + isSubmitting={isSubmitting} + isTesting={isTesting} + testResult={testResult} + onResetTestResult={onResetTestResult} + /> + ))} {models.llm.length === 0 && models.embedding.length === 0 && - models.rerank.length === 0 && ( + models.rerank.length === 0 && + (!models.asr || models.asr.length === 0) && + (!models.tts || models.tts.length === 0) && (

{t('models.noModels')}

diff --git a/web/src/app/home/components/models-dialog/types.ts b/web/src/app/home/components/models-dialog/types.ts index 6b4eaf614..48754afbd 100644 --- a/web/src/app/home/components/models-dialog/types.ts +++ b/web/src/app/home/components/models-dialog/types.ts @@ -2,6 +2,8 @@ import { LLMModel, EmbeddingModel, RerankModel, + ASRModel, + TTSModel, ModelProvider, ProviderScanDebugInfo, ScannedProviderModel, @@ -15,7 +17,7 @@ export type ExtraArg = { value: string; }; -export type ModelType = 'llm' | 'embedding' | 'rerank'; +export type ModelType = 'llm' | 'embedding' | 'rerank' | 'asr' | 'tts'; export const DEFAULT_REASONING_CONFIG: ReasoningConfig = { level: 'provider_default', @@ -25,6 +27,8 @@ export interface ProviderModels { llm: LLMModel[]; embedding: EmbeddingModel[]; rerank: RerankModel[]; + asr: ASRModel[]; + tts: TTSModel[]; } export type TestResult = @@ -41,7 +45,7 @@ export type ScanModelsResult = { }; export interface ModelItemProps { - model: LLMModel | EmbeddingModel; + model: LLMModel | EmbeddingModel | RerankModel | ASRModel | TTSModel; modelType: ModelType; providerUuid: string; isLangBotModels: boolean; diff --git a/web/src/app/home/pipelines/components/debug-dialog/DebugDialog.tsx b/web/src/app/home/pipelines/components/debug-dialog/DebugDialog.tsx index 4b8c7be09..1b7ef0557 100644 --- a/web/src/app/home/pipelines/components/debug-dialog/DebugDialog.tsx +++ b/web/src/app/home/pipelines/components/debug-dialog/DebugDialog.tsx @@ -646,7 +646,7 @@ export default function DebugDialog({ case 'Voice': { const voice = component as Voice; - const voiceUrl = voice.url || (voice.base64 ? voice.base64 : ''); + const voiceUrl = voice.base64 || voice.url || ''; if (!voiceUrl) { return [{t('pipelines.debugDialog.voice')}]; diff --git a/web/src/app/home/pipelines/components/pipeline-form/PipelineFormComponent.tsx b/web/src/app/home/pipelines/components/pipeline-form/PipelineFormComponent.tsx index a401e41f7..b5cd5bc44 100644 --- a/web/src/app/home/pipelines/components/pipeline-form/PipelineFormComponent.tsx +++ b/web/src/app/home/pipelines/components/pipeline-form/PipelineFormComponent.tsx @@ -60,6 +60,7 @@ import { Info, Brain, Zap, + Mic, Shield, FileOutput, Puzzle, @@ -170,6 +171,7 @@ const PipelineFormComponent = forwardRef< }), ai: z.record(z.string(), z.any()), trigger: z.record(z.string(), z.any()), + speech: z.record(z.string(), z.any()).optional(), safety: z.record(z.string(), z.any()), output: z.record(z.string(), z.any()), }) @@ -181,6 +183,7 @@ const PipelineFormComponent = forwardRef< }), ai: z.record(z.string(), z.any()).optional(), trigger: z.record(z.string(), z.any()).optional(), + speech: z.record(z.string(), z.any()).optional(), safety: z.record(z.string(), z.any()).optional(), output: z.record(z.string(), z.any()).optional(), }); @@ -191,6 +194,7 @@ const PipelineFormComponent = forwardRef< basic: Info, ai: Brain, trigger: Zap, + speech: Mic, safety: Shield, output: FileOutput, extensions: Puzzle, @@ -213,6 +217,11 @@ const PipelineFormComponent = forwardRef< name: 'trigger', icon: SECTION_ICONS.trigger, }, + { + label: t('pipelines.speechProcessing'), + name: 'speech', + icon: SECTION_ICONS.speech, + }, { label: t('pipelines.safetyControls'), name: 'safety', @@ -243,6 +252,7 @@ const PipelineFormComponent = forwardRef< const guideSteps = useMemo(() => { const sections = [ 'trigger', + 'speech', 'ai', 'output', 'safety', @@ -286,7 +296,7 @@ const PipelineFormComponent = forwardRef< }, ]; }, [t, debugGuideEnabled, monitoringGuideEnabled]); - const primarySectionNames = ['trigger', 'ai', 'output']; + const primarySectionNames = ['trigger', 'speech', 'ai', 'output']; const primarySections = primarySectionNames .map((name) => formLabelList.find((section) => section.name === name)) .filter((section): section is SectionItem => Boolean(section)); @@ -302,6 +312,8 @@ const PipelineFormComponent = forwardRef< useState(); const [triggerConfigTabSchema, setTriggerConfigTabSchema] = useState(); + const [speechConfigTabSchema, setSpeechConfigTabSchema] = + useState(); const [safetyConfigTabSchema, setSafetyConfigTabSchema] = useState(); const [outputConfigTabSchema, setOutputConfigTabSchema] = @@ -371,6 +383,8 @@ const PipelineFormComponent = forwardRef< setAIConfigTabSchema(config); } else if (config.name === 'trigger') { setTriggerConfigTabSchema(config); + } else if (config.name === 'speech') { + setSpeechConfigTabSchema(config); } else if (config.name === 'safety') { setSafetyConfigTabSchema(config); } else if (config.name === 'output') { @@ -404,6 +418,7 @@ const PipelineFormComponent = forwardRef< }, ai: resp.pipeline.config.ai, trigger: resp.pipeline.config.trigger, + speech: resp.pipeline.config.speech, safety: resp.pipeline.config.safety, output: resp.pipeline.config.output, }; @@ -528,6 +543,7 @@ const PipelineFormComponent = forwardRef< const realConfig = { ai: values.ai, trigger: values.trigger, + speech: values.speech, safety: values.safety, output: values.output, }; @@ -902,7 +918,7 @@ const PipelineFormComponent = forwardRef<