diff --git a/Dockerfile b/Dockerfile index 8c6310b53..ef68a93bf 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ COPY --from=node /app/web/dist ./web/dist COPY --from=nsjail-build /usr/local/bin/nsjail /usr/local/bin/nsjail RUN apt-get update \ - && apt-get install -y --no-install-recommends gcc ca-certificates curl git gnupg \ + && apt-get install -y --no-install-recommends gcc libc6-dev ca-certificates curl git gnupg ffmpeg \ # nsjail runtime libraries (the build toolchain stays in the nsjail-build # stage; only these shared libs are needed to execute the binary). && apt-get install -y --no-install-recommends libprotobuf32t64 libnl-route-3-200 \ @@ -73,7 +73,7 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends nodejs \ && rm -f /tmp/nodesource_setup.sh \ && python -m pip install --no-cache-dir uv \ - && uv sync --extra seekdb \ + && uv sync --extra seekdb --extra audio \ && apt-get purge -y --auto-remove curl git gnupg \ && rm -rf /var/lib/apt/lists/* \ && touch /.dockerenv diff --git a/docs/en/deploy/langbot/docker.mdx b/docs/en/deploy/langbot/docker.mdx index b13b8ba08..5e08559a9 100644 --- a/docs/en/deploy/langbot/docker.mdx +++ b/docs/en/deploy/langbot/docker.mdx @@ -44,6 +44,10 @@ docker compose --profile all up -d The first startup downloads the container images. On Windows / macOS, keep Docker Desktop running. + +The official Docker images come pre-equipped with minimal `ffmpeg` and `pilk` audio tools. Native voice transcoding (including SILK for WeChat) is supported out-of-the-box without extra host configuration. + + ## 4. Open the dashboard - **Local installation**: open `http://localhost:5300`. diff --git a/docs/en/deploy/langbot/manual.mdx b/docs/en/deploy/langbot/manual.mdx index 99c3d0f4f..cef3c2929 100644 --- a/docs/en/deploy/langbot/manual.mdx +++ b/docs/en/deploy/langbot/manual.mdx @@ -29,6 +29,20 @@ Embedded-mode platform support depends on the available `pylibseekdb` wheels; Apple Silicon currently requires macOS 15 or later. All other LangBot features can use the default `uv sync` installation. +### Optional Voice Transcoding Support (SILK / WeChat Native Voice) + +If you need native voice bubbles on platforms like WeChat (SILK encoding) or audio transcoding support, install system `ffmpeg` and the `audio` extra: + +- **System Dependency**: `ffmpeg` (macOS: `brew install ffmpeg`, Ubuntu/Debian: `sudo apt-get install -y ffmpeg`, Windows: `winget install Gyan.FFmpeg`) +- **Python Extra**: + ```bash + uv sync --extra audio + # or combine with seekdb + uv sync --extra seekdb --extra audio + ``` + +If transcoding dependencies are not installed, LangBot performs **graceful degradation** by delivering the audio as a playable audio file attachment rather than dropping the message. + 3. Run the main program once to generate configuration files ```bash diff --git a/docs/en/deploy/langbot/package.mdx b/docs/en/deploy/langbot/package.mdx index 6d8e063f3..0439a0fea 100644 --- a/docs/en/deploy/langbot/package.mdx +++ b/docs/en/deploy/langbot/package.mdx @@ -39,3 +39,15 @@ Silicon currently requires macOS 15 or later; follow macOS 14 ARM64 support plan. Because of the current `pyseekdb` package metadata, the extra also fails to resolve on macOS 14 ARM64 for server-only use. These platform limits do not affect LangBot when SeekDB is not installed. + +## Optional Voice Transcoding Support (SILK / WeChat Native Voice) + +If you need native voice bubbles on platforms like WeChat (SILK encoding) or audio transcoding support, make sure `ffmpeg` is installed on your system and launch LangBot with the `audio` extra: + +```bash +uvx --from 'langbot[audio]@latest' langbot +# or combine with seekdb +uvx --from 'langbot[seekdb,audio]@latest' langbot +``` + +If transcoding dependencies (`ffmpeg` / `pilk`) are not installed, LangBot automatically performs **graceful degradation** by delivering the audio as an audio file attachment. diff --git a/docs/zh/deploy/langbot/docker.mdx b/docs/zh/deploy/langbot/docker.mdx index 5d624ab65..c86673905 100644 --- a/docs/zh/deploy/langbot/docker.mdx +++ b/docs/zh/deploy/langbot/docker.mdx @@ -44,6 +44,10 @@ docker compose --profile all up -d 首次启动需要下载镜像。Windows / macOS 用户请保持 Docker Desktop 运行。 + +官方 Docker 镜像已默认预装 `ffmpeg` 与 `pilk` 音频工具链(最小化安装),开箱即用支持微信原生语音条(SILK)转码与语音处理,无需额外在宿主机安装音频环境。 + + ## 4. 打开管理页面 - **本机部署**:打开 `http://localhost:5300`。 diff --git a/docs/zh/deploy/langbot/manual.mdx b/docs/zh/deploy/langbot/manual.mdx index 0fd59e026..3593ab0b1 100644 --- a/docs/zh/deploy/langbot/manual.mdx +++ b/docs/zh/deploy/langbot/manual.mdx @@ -30,6 +30,20 @@ uv sync --extra seekdb SeekDB 嵌入式模式的平台支持取决于 `pylibseekdb` wheel;Apple Silicon 当前要求 macOS 15 或更高版本。其他功能仍可使用默认的 `uv sync` 安装。 +### 可选的语音转码支持(SILK / 微信原生语音) + +如果需要在微信等平台发送原生语音条(SILK 编码)或进行高级音频格式转码,需要系统安装 `ffmpeg` 并安装 `audio` 扩展依赖: + +- **系统依赖**:`ffmpeg`(macOS 可用 `brew install ffmpeg`,Ubuntu/Debian 可用 `sudo apt-get install -y ffmpeg`,Windows 可用 `winget install Gyan.FFmpeg`) +- **Python 扩展**: + ```bash + uv sync --extra audio + # 或同时启用 seekdb 与 audio + uv sync --extra seekdb --extra audio + ``` + +若未安装转码依赖,系统在需要转码为微信原生语音条时会自动进行**优雅降级**,将语音以音频文件的形式发送,确保用户仍能正常点击播放。 + 或者使用清华源 ```bash diff --git a/docs/zh/deploy/langbot/package.mdx b/docs/zh/deploy/langbot/package.mdx index 71fa0be6c..69a7f365d 100644 --- a/docs/zh/deploy/langbot/package.mdx +++ b/docs/zh/deploy/langbot/package.mdx @@ -37,3 +37,15 @@ SeekDB 嵌入式模式还要求当前平台存在兼容的 `pylibseekdb` wheel [oceanbase/seekdb#1324](https://github.com/oceanbase/seekdb/issues/1324)。受当前 `pyseekdb` 包元数据影响,macOS 14 ARM64 即使只连接远程 SeekDB,也无法解析该 extra。不使用 SeekDB 时,这些平台限制不影响 LangBot。 + +## 可选的语音转码支持(SILK / 微信原生语音) + +如果需要在微信等平台发送原生语音条(SILK 格式)或进行音频格式转换,需要系统已安装 `ffmpeg`,并使用 `audio` extra 启动: + +```bash +uvx --from 'langbot[audio]@latest' langbot +# 或同时启用 seekdb 与 audio +uvx --from 'langbot[seekdb,audio]@latest' langbot +``` + +若当前环境未安装 `ffmpeg` 或 `pilk`,LangBot 会自动**优雅降级**,将语音作为音频文件发送,保证用户端仍可正常接收播放。 diff --git a/pyproject.toml b/pyproject.toml index d1203092a..7e82defe7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -111,6 +111,9 @@ classifiers = [ ] [project.optional-dependencies] +audio = [ + "pilk>=0.1.0", +] seekdb = [ "pyseekdb==1.4.0.post1", "pylibseekdb==1.4.0; sys_platform == 'linux' or (sys_platform == 'darwin' and platform_machine == 'arm64')", diff --git a/skills/skills/langbot-mcp-ops/SKILL.md b/skills/skills/langbot-mcp-ops/SKILL.md index a9d723de3..c8449375c 100644 --- a/skills/skills/langbot-mcp-ops/SKILL.md +++ b/skills/skills/langbot-mcp-ops/SKILL.md @@ -92,7 +92,7 @@ a pipeline retain these independent settings. | `list_processor_runs` / `get_processor_run_events` | Read one Agent or plugin processor run history and logs; paginate with `before_id` / `after_sequence`. | | `debug_agent` | Execute a synthetic Agent event (`processor_uuid`, `payload`); requires `runtime.operate`. Returns final text and up to 1000 execution events (thinking, text, tool arguments/results). Platform tools use Mock; other configured tools execute normally. Optional `payload.mock`: `errors`/`results` keyed by platform tool name, `unsupported_apis` lists unavailable platform APIs. | | `list_pipelines` / `get_pipeline` / `create_pipeline` / `update_pipeline` / `delete_pipeline` | Manage pipelines | -| `list_llm_models` / `get_llm_model` / `list_embedding_models` / `list_model_providers` | Inspect models & providers | +| `list_llm_models` / `get_llm_model` / `list_embedding_models` / `list_asr_models` / `list_tts_models` / `list_model_providers` | Inspect models & providers | | `list_knowledge_bases` / `get_knowledge_base` / `retrieve_knowledge_base` | RAG knowledge bases (incl. semantic search) | | `list_mcp_servers` | External MCP servers LangBot connects to (as a client) | | `list_skills` / `get_skill` | Installed skills | diff --git a/src/langbot/libs/dingtalk_api/api.py b/src/langbot/libs/dingtalk_api/api.py index 2eaff2447..700294579 100644 --- a/src/langbot/libs/dingtalk_api/api.py +++ b/src/langbot/libs/dingtalk_api/api.py @@ -34,6 +34,8 @@ DINGTALK_OPENAPI_BASE = 'https://api.dingtalk.com' # inbound event carried a forged destination. DINGTALK_WEBHOOK_DOMAINS = ('dingtalk.com',) _MAX_MEDIA_BYTES = 10 * 1024 * 1024 +# DingTalk voice bubbles play at most 60 seconds of audio. +_MAX_VOICE_DURATION_MS = 60 * 1000 _MAX_GATEWAY_MESSAGE_BYTES = 1024 * 1024 @@ -321,11 +323,77 @@ class DingTalkClient: return self._check_send_result(response) async def send_attachment(self, component, incoming_message): - from langbot_plugin.api.entities.builtin.platform.message import Image + from langbot_plugin.api.entities.builtin.platform.message import Image, Voice + + if isinstance(component, Voice): + return await self.send_voice(component, incoming_message) is_image = isinstance(component, Image) name = getattr(component, 'name', '') or ('image.png' if is_image else 'file') name = os.path.basename(name) + body = await self._read_component_bytes(component) + media_id = await self._upload_media(name, body, 'image' if is_image else 'file') + params = ( + {'photoURL': media_id} + if is_image + else {'mediaId': media_id, 'fileName': name, 'fileType': os.path.splitext(name)[1].lstrip('.')} + ) + return await self._send_robot_message('sampleImageMsg' if is_image else 'sampleFile', params, incoming_message) + + async def send_voice(self, component, incoming_message): + """Send a Voice component as a native DingTalk audio message. + + DingTalk plays ogg/opus and amr in its voice bubble, so other formats are + transcoded to ogg/opus. Audio longer than the voice limit, or audio that + cannot be transcoded or is rejected as a voice message, is delivered as + a regular file instead. + """ + from langbot.pkg.utils import audio as audio_utils + + body = await self._read_component_bytes(component) + fmt = audio_utils.detect_audio_format(body) + length = getattr(component, 'length', None) + duration = float(length) if length else audio_utils.estimate_audio_duration(body, fmt) + duration_ms = max(1000, int(round(duration * 1000))) + if fmt not in ('ogg', 'opus', 'amr'): + try: + body = await audio_utils.transcode_audio(body, 'ogg', fmt) + fmt = 'ogg' + except Exception as exc: + await self._log_warning(f'DingTalk voice transcode to ogg failed, sending as file: {exc}') + ext = 'amr' if fmt == 'amr' else ('ogg' if fmt in ('ogg', 'opus') else (fmt if fmt != 'unknown' else 'mp3')) + name = f'voice.{ext}' + if ext == 'ogg': + # Ogg carries an exact sample count, which beats bitrate-based estimates. + measured = audio_utils.estimate_audio_duration(body, fmt) + if measured > 0: + duration_ms = max(1000, int(round(measured * 1000))) + + if ext in ('ogg', 'amr') and duration_ms <= _MAX_VOICE_DURATION_MS: + try: + media_id = await self._upload_media(name, body, 'voice') + return await self._send_robot_message( + 'sampleAudio', + {'mediaId': media_id, 'duration': str(duration_ms)}, + incoming_message, + ) + except Exception as exc: + await self._log_warning(f'DingTalk voice message failed, sending as file: {exc}') + + media_id = await self._upload_media(name, body, 'file') + return await self._send_robot_message( + 'sampleFile', + {'mediaId': media_id, 'fileName': name, 'fileType': ext}, + incoming_message, + ) + + async def _log_warning(self, message: str): + if self.logger: + await self.logger.warning(message) + else: + _stdout_logger.warning(message) + + async def _read_component_bytes(self, component) -> bytes: encoded = getattr(component, 'base64', None) if encoded: encoded = encoded.split(',', 1)[-1] @@ -347,13 +415,16 @@ class DingTalkClient: raise ValueError('DingTalk attachment has no readable content') if len(body) > _MAX_MEDIA_BYTES: raise ValueError('DingTalk media exceeds the size limit') + return body + + async def _upload_media(self, name: str, body: bytes, media_type: str) -> str: token = await self.get_legacy_access_token() if not token: raise RuntimeError('DingTalk media authentication failed') async with self._http_client_context() as client: response = await client.post( 'https://oapi.dingtalk.com/media/upload', - params={'access_token': token, 'type': 'image' if is_image else 'file'}, + params={'access_token': token, 'type': media_type}, files={'media': (name, body)}, timeout=30.0, ) @@ -361,16 +432,14 @@ class DingTalkClient: media_id = uploaded.get('media_id') if not media_id: raise RuntimeError('DingTalk media upload returned no media ID') + return media_id + + async def _send_robot_message(self, msg_key: str, params: dict, incoming_message): if not await self.check_access_token(): await self.get_access_token() - params = ( - {'photoURL': media_id} - if is_image - else {'mediaId': media_id, 'fileName': name, 'fileType': os.path.splitext(name)[1].lstrip('.')} - ) payload = { 'robotCode': self.robot_code or self.key, - 'msgKey': 'sampleImageMsg' if is_image else 'sampleFile', + 'msgKey': msg_key, 'msgParam': json.dumps(params, ensure_ascii=False), } if str(incoming_message.conversation_type) == '2': @@ -609,6 +678,11 @@ class DingTalkClient: download_code = raw_content.get('downloadCode') if download_code: message_data['Audio'] = await self.get_audio_url(download_code) + try: + # DingTalk reports the voice duration in milliseconds. + message_data['AudioDuration'] = int(round(float(raw_content.get('duration') or 0) / 1000)) + except (TypeError, ValueError): + pass message_data['Type'] = 'audio' elif incoming_message.message_type == 'file': diff --git a/src/langbot/libs/dingtalk_api/dingtalkevent.py b/src/langbot/libs/dingtalk_api/dingtalkevent.py index be6842e97..2a994a367 100644 --- a/src/langbot/libs/dingtalk_api/dingtalkevent.py +++ b/src/langbot/libs/dingtalk_api/dingtalkevent.py @@ -35,6 +35,10 @@ class DingTalkEvent(dict): def audio(self): return self.get('Audio', '') + @property + def audio_duration(self) -> int: + return self.get('AudioDuration') or 0 + @property def file(self): return self.get('File', '') diff --git a/src/langbot/libs/openclaw_weixin_api/client.py b/src/langbot/libs/openclaw_weixin_api/client.py index fa89c0427..b7cfa9538 100644 --- a/src/langbot/libs/openclaw_weixin_api/client.py +++ b/src/langbot/libs/openclaw_weixin_api/client.py @@ -456,15 +456,29 @@ class OpenClawWeixinClient: to_user_id: str, voice_bytes: bytes, playtime: int = 0, + encode_type: Optional[int] = 6, context_token: str = '', + bits_per_sample: Optional[int] = None, + sample_rate: Optional[int] = None, + text: Optional[str] = None, ) -> None: """Upload a voice message to CDN and send it.""" + if encode_type == 6: + if bits_per_sample is None: + bits_per_sample = 16 + if sample_rate is None: + sample_rate = 24000 media = await self.upload_media(voice_bytes, to_user_id, media_type=4) item = MessageItem( type=MessageItem.VOICE, voice_item=VoiceItem( media=media, playtime=playtime, + encode_type=encode_type, + bits_per_sample=bits_per_sample, + sample_rate=sample_rate, + size=len(voice_bytes), + text=text, ), ) await self.send_message(to_user_id, [item], context_token) @@ -715,6 +729,9 @@ def _parse_message_item(data: dict) -> MessageItem: media=_parse_cdn_media(v.get('media')), encode_type=v.get('encode_type'), playtime=v.get('playtime'), + bits_per_sample=v.get('bits_per_sample'), + sample_rate=v.get('sample_rate'), + size=v.get('size'), text=v.get('text'), ) @@ -812,6 +829,16 @@ def _message_item_to_dict(item: MessageItem) -> dict: voice_d['media'] = _cdn_media_to_dict(item.voice_item.media) if item.voice_item.playtime is not None: voice_d['playtime'] = item.voice_item.playtime + if item.voice_item.encode_type is not None: + voice_d['encode_type'] = item.voice_item.encode_type + if item.voice_item.bits_per_sample is not None: + voice_d['bits_per_sample'] = item.voice_item.bits_per_sample + if item.voice_item.sample_rate is not None: + voice_d['sample_rate'] = item.voice_item.sample_rate + if item.voice_item.size is not None: + voice_d['size'] = item.voice_item.size + if item.voice_item.text is not None: + voice_d['text'] = item.voice_item.text d['voice_item'] = voice_d if item.file_item: diff --git a/src/langbot/libs/openclaw_weixin_api/types.py b/src/langbot/libs/openclaw_weixin_api/types.py index 93bb2e3c3..e7bdb750d 100644 --- a/src/langbot/libs/openclaw_weixin_api/types.py +++ b/src/langbot/libs/openclaw_weixin_api/types.py @@ -62,6 +62,7 @@ class VoiceItem: bits_per_sample: Optional[int] = None sample_rate: Optional[int] = None playtime: Optional[int] = None + size: Optional[int] = None text: Optional[str] = None _downloaded_bytes: Optional[bytes] = field(default=None, repr=False) diff --git a/src/langbot/libs/wecom_ai_bot_api/api.py b/src/langbot/libs/wecom_ai_bot_api/api.py index 207315d28..98c779556 100644 --- a/src/langbot/libs/wecom_ai_bot_api/api.py +++ b/src/langbot/libs/wecom_ai_bot_api/api.py @@ -415,6 +415,16 @@ def _bytes_to_data_uri(data: bytes) -> str: mime_type = 'application/pdf' elif data[:4] == b'PK\x03\x04': mime_type = 'application/zip' + elif data.startswith(b'#!AMR'): + mime_type = 'audio/amr' + elif data.startswith((b'#!SILK_V3', b'\x02#!SILK_V3')): + mime_type = 'audio/silk' + elif data.startswith(b'RIFF') and len(data) >= 12 and data[8:12] == b'WAVE': + mime_type = 'audio/wav' + elif data.startswith(b'ID3') or (len(data) >= 2 and data[0] == 0xFF and (data[1] & 0xE0) == 0xE0): + mime_type = 'audio/mpeg' + elif data.startswith(b'OggS'): + mime_type = 'audio/ogg' else: mime_type = 'application/octet-stream' @@ -508,6 +518,29 @@ async def parse_wecom_bot_message( return await asyncio.to_thread(_bytes_to_data_uri, data) return None + async def _safe_download_voice(url: str, per_msg_aeskey: str = '') -> Tuple[Optional[str], Optional[int]]: + """Download, decrypt, and transcode voice message to MP3 data URI for universal browser playback.""" + data, _filename = await _safe_download(url, per_msg_aeskey) + if not data: + return None, None + from langbot.pkg.utils import audio as audio_utils + + fmt = audio_utils.detect_audio_format(data) + duration = int(round(audio_utils.estimate_audio_duration(data, fmt))) if data else None + + # Transcode AMR / SILK to MP3 so web browsers and media players can decode directly + if fmt in ('amr', 'silk') and audio_utils.is_ffmpeg_available(): + try: + mp3_bytes = await audio_utils.transcode_audio(data, 'mp3', fmt) + b64 = base64.b64encode(mp3_bytes).decode('utf-8') + return f'data:audio/mpeg;base64,{b64}', duration + except Exception as exc: + await logger.warning(f'Failed to transcode {fmt} voice to MP3: {exc}') + + mime = 'audio/mpeg' if fmt == 'mp3' else ('audio/wav' if fmt == 'wav' else f'audio/{fmt}') + b64 = base64.b64encode(data).decode('utf-8') + return f'data:{mime};base64,{b64}', duration + if msg_type == 'text': message_data['content'] = msg_json.get('text', {}).get('content') elif msg_type == 'markdown': @@ -534,10 +567,12 @@ async def parse_wecom_bot_message( } if voice_info.get('content'): message_data['content'] = voice_info.get('content') - # if (message_data['voice'].get('filesize') or 0) <= max_inline_file_size: - # voice_base64 = await _safe_download_as_data_uri(download_url, per_msg_aeskey) - # if voice_base64: - # message_data['voice']['base64'] = voice_base64 + if (message_data['voice'].get('filesize') or 0) <= max_inline_file_size: + voice_base64, duration = await _safe_download_voice(download_url, per_msg_aeskey) + if voice_base64: + message_data['voice']['base64'] = voice_base64 + if duration: + message_data['voice']['length'] = duration elif msg_type == 'video': video_info = msg_json.get('video', {}) or {} download_url = video_info.get('url') @@ -638,9 +673,11 @@ async def parse_wecom_bot_message( if voice_info.get('content'): texts.append(voice_info.get('content')) if (voice_data.get('filesize') or 0) <= max_inline_file_size: - voice_base64 = await _safe_download_as_data_uri(download_url, item_aeskey) + voice_base64, duration = await _safe_download_voice(download_url, item_aeskey) if voice_base64: voice_data['base64'] = voice_base64 + if duration: + voice_data['length'] = duration voices.append(voice_data) elif item_type == 'video': video_info = item.get('video', {}) or {} @@ -740,6 +777,12 @@ async def parse_wecom_bot_message( } if voice_info.get('content'): quote_data['content'] = voice_info.get('content') + if (voice_data.get('filesize') or 0) <= max_inline_file_size: + voice_base64, duration = await _safe_download_voice(download_url, item_aeskey) + if voice_base64: + voice_data['base64'] = voice_base64 + if duration: + voice_data['length'] = duration # Same as private chat: append aeskey to url for plugin processing if download_url and item_aeskey: voice_data['url'] = download_url + f'?aeskey={item_aeskey}' diff --git a/src/langbot/libs/wecom_api/api.py b/src/langbot/libs/wecom_api/api.py index 72bd7ade7..75b59d254 100644 --- a/src/langbot/libs/wecom_api/api.py +++ b/src/langbot/libs/wecom_api/api.py @@ -562,6 +562,15 @@ class WecomClient: await self.logger.error('Voice对象出错') raise ValueError('voice对象出错') + from ...pkg.utils import audio as audio_utils + + if audio_utils.is_ffmpeg_available(): + try: + file_bytes = await audio_utils.transcode_audio(file_bytes, 'amr') + file_name = 'voice.amr' + except Exception: + pass + if len(file_bytes) > _MAX_MEDIA_BYTES: raise ValueError('WeCom media exceeds the size limit') boundary = '-------------------------acebdf13572468' diff --git a/src/langbot/pkg/agent/runner/config_schema.py b/src/langbot/pkg/agent/runner/config_schema.py index 1fad3193f..3cd533d79 100644 --- a/src/langbot/pkg/agent/runner/config_schema.py +++ b/src/langbot/pkg/agent/runner/config_schema.py @@ -222,6 +222,12 @@ def iter_config_model_refs( elif field_type == 'rerank-model-selector': if isinstance(value, str) and value not in NONE_SENTINELS: yield 'rerank', value + elif field_type == 'asr-model-selector': + if isinstance(value, str) and value not in NONE_SENTINELS: + yield 'asr', value + elif field_type == 'tts-model-selector': + if isinstance(value, str) and value not in NONE_SENTINELS: + yield 'tts', value def set_empty_llm_model_selection( diff --git a/src/langbot/pkg/api/http/controller/groups/provider/models.py b/src/langbot/pkg/api/http/controller/groups/provider/models.py index 678550b69..149f14b54 100644 --- a/src/langbot/pkg/api/http/controller/groups/provider/models.py +++ b/src/langbot/pkg/api/http/controller/groups/provider/models.py @@ -382,3 +382,229 @@ class RerankModelsRouterGroup(group.RouterGroup): except provider_errors.RequesterError as exc: return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc)) return self.success() + + +@group.group_class('models/asr', '/api/v1/provider/models/asr') +class ASRModelsRouterGroup(group.RouterGroup): + async def initialize(self) -> None: + @self.route( + '', + methods=['GET'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_VIEW, + ) + async def _(request_context: RequestContext) -> str: + provider_uuid = quart.request.args.get('provider_uuid') + include_secret, error = resolve_include_secret( + quart.request.args.get('include_secret'), + permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE), + ) + if error: + return self.http_status(400, -1, error) + if provider_uuid: + models = await self.ap.asr_models_service.get_asr_models_by_provider( + request_context, + provider_uuid, + include_secret=include_secret, + ) + else: + models = await self.ap.asr_models_service.get_asr_models( + request_context, + include_secret=include_secret, + ) + return self.success(data={'models': models}) + + @self.route( + '', + methods=['POST'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(request_context: RequestContext) -> str: + try: + model_uuid = await self.ap.asr_models_service.create_asr_model( + request_context, + await quart.request.json, + ) + except ValueError as exc: + return self.http_status(400, -1, str(exc)) + return self.success(data={'uuid': model_uuid}) + + @self.route( + '/', + methods=['GET'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_VIEW, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + include_secret, error = resolve_include_secret( + quart.request.args.get('include_secret'), + permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE), + ) + if error: + return self.http_status(400, -1, error) + model = await self.ap.asr_models_service.get_asr_model( + request_context, + model_uuid, + include_secret=include_secret, + ) + if model is None: + return self.http_status(404, -1, 'model not found') + return self.success(data={'model': model}) + + @self.route( + '/', + methods=['PUT'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + try: + await self.ap.asr_models_service.update_asr_model( + request_context, + model_uuid, + await quart.request.json, + ) + except ValueError as exc: + return self.http_status(400, -1, str(exc)) + return self.success() + + @self.route( + '/', + methods=['DELETE'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + await self.ap.asr_models_service.delete_asr_model(request_context, model_uuid) + return self.success() + + @self.route( + '//test', + methods=['POST'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + try: + await self.ap.asr_models_service.test_asr_model( + request_context, + model_uuid, + await quart.request.json, + ) + except provider_errors.RequesterError as exc: + return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc)) + return self.success() + + +@group.group_class('models/tts', '/api/v1/provider/models/tts') +class TTSModelsRouterGroup(group.RouterGroup): + async def initialize(self) -> None: + @self.route( + '', + methods=['GET'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_VIEW, + ) + async def _(request_context: RequestContext) -> str: + provider_uuid = quart.request.args.get('provider_uuid') + include_secret, error = resolve_include_secret( + quart.request.args.get('include_secret'), + permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE), + ) + if error: + return self.http_status(400, -1, error) + if provider_uuid: + models = await self.ap.tts_models_service.get_tts_models_by_provider( + request_context, + provider_uuid, + include_secret=include_secret, + ) + else: + models = await self.ap.tts_models_service.get_tts_models( + request_context, + include_secret=include_secret, + ) + return self.success(data={'models': models}) + + @self.route( + '', + methods=['POST'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(request_context: RequestContext) -> str: + try: + model_uuid = await self.ap.tts_models_service.create_tts_model( + request_context, + await quart.request.json, + ) + except ValueError as exc: + return self.http_status(400, -1, str(exc)) + return self.success(data={'uuid': model_uuid}) + + @self.route( + '/', + methods=['GET'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_VIEW, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + include_secret, error = resolve_include_secret( + quart.request.args.get('include_secret'), + permitted=has_permission(request_context, Permission.PROVIDER_SECRET_MANAGE), + ) + if error: + return self.http_status(400, -1, error) + model = await self.ap.tts_models_service.get_tts_model( + request_context, + model_uuid, + include_secret=include_secret, + ) + if model is None: + return self.http_status(404, -1, 'model not found') + return self.success(data={'model': model}) + + @self.route( + '/', + methods=['PUT'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + try: + await self.ap.tts_models_service.update_tts_model( + request_context, + model_uuid, + await quart.request.json, + ) + except ValueError as exc: + return self.http_status(400, -1, str(exc)) + return self.success() + + @self.route( + '/', + methods=['DELETE'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.RESOURCE_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + await self.ap.tts_models_service.delete_tts_model(request_context, model_uuid) + return self.success() + + @self.route( + '//test', + methods=['POST'], + auth_type=group.AuthType.USER_TOKEN_OR_API_KEY, + permission=Permission.PROVIDER_SECRET_MANAGE, + ) + async def _(model_uuid: str, request_context: RequestContext) -> str: + try: + await self.ap.tts_models_service.test_tts_model( + request_context, + model_uuid, + await quart.request.json, + ) + except provider_errors.RequesterError as exc: + return self.http_status(400, MODEL_TEST_ERROR_CODE, str(exc)) + return self.success() diff --git a/src/langbot/pkg/api/http/controller/groups/provider/providers.py b/src/langbot/pkg/api/http/controller/groups/provider/providers.py index 0a7302b58..8e1847c56 100644 --- a/src/langbot/pkg/api/http/controller/groups/provider/providers.py +++ b/src/langbot/pkg/api/http/controller/groups/provider/providers.py @@ -103,9 +103,7 @@ class ModelProvidersRouterGroup(group.RouterGroup): ) for provider in providers: counts = await self.ap.provider_service.get_provider_model_counts(request_context, provider['uuid']) - provider['llm_count'] = counts['llm_count'] - provider['embedding_count'] = counts['embedding_count'] - provider['rerank_count'] = counts['rerank_count'] + provider.update(counts) return self.success(data={'providers': providers}) @self.route( @@ -143,9 +141,7 @@ class ModelProvidersRouterGroup(group.RouterGroup): if provider is None: return self.http_status(404, -1, 'provider not found') counts = await self.ap.provider_service.get_provider_model_counts(request_context, provider_uuid) - provider['llm_count'] = counts['llm_count'] - provider['embedding_count'] = counts['embedding_count'] - provider['rerank_count'] = counts['rerank_count'] + provider.update(counts) return self.success(data={'provider': provider}) @self.route( diff --git a/src/langbot/pkg/api/http/service/model.py b/src/langbot/pkg/api/http/service/model.py index ecd75f83b..44d82225c 100644 --- a/src/langbot/pkg/api/http/service/model.py +++ b/src/langbot/pkg/api/http/service/model.py @@ -1101,3 +1101,521 @@ class RerankModelsService: ], execution_context=runtime_rerank_model.execution_context, ) + + +def _generate_test_wav() -> bytes: + import io + import wave + + buf = io.BytesIO() + with wave.open(buf, 'wb') as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b'\x00' * 3200) + return buf.getvalue() + + +class ASRModelsService: + ap: app.Application + + def __init__(self, ap: app.Application) -> None: + self.ap = ap + + async def get_asr_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]: + """Get all ASR models with provider info""" + result = await self.ap.persistence_mgr.execute_async( + scope_statement(sqlalchemy.select(persistence_model.ASRModel), persistence_model.ASRModel, context) + ) + models = result.all() + + providers_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context + ) + ) + providers = {p.uuid: p for p in providers_result.all()} + + models_list = [] + for model in models: + model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ASRModel, model) + provider = providers.get(model.provider_uuid) + if provider: + provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider) + provider_dict = _parse_provider_api_keys(provider_dict) + model_dict['provider'] = provider_dict + if not include_secret: + model_dict = _redact_model_secrets(model_dict) + models_list.append(model_dict) + + return models_list + + async def get_asr_models_by_provider( + self, + context: TenantContext, + provider_uuid: str, + *, + include_secret: bool = False, + ) -> list[dict]: + """Get ASR models by provider UUID""" + await _require_workspace_provider(self.ap, context, provider_uuid) + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ASRModel).where( + persistence_model.ASRModel.provider_uuid == provider_uuid + ), + persistence_model.ASRModel, + context, + ) + ) + models = result.all() + serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.ASRModel, m) for m in models] + return serialized if include_secret else [_redact_model_secrets(model) for model in serialized] + + async def create_asr_model( + self, + context: TenantContext, + model_data: dict, + preserve_uuid: bool = False, + ) -> str: + """Create a new ASR model""" + model_data = model_data.copy() + if not preserve_uuid: + model_data['uuid'] = str(uuid.uuid4()) + model_data['workspace_uuid'] = require_workspace_uuid(context) + if 'extra_args' in model_data: + model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args']) + + if 'provider' in model_data: + provider_data = model_data.pop('provider') + if provider_data.get('uuid'): + model_data['provider_uuid'] = provider_data['uuid'] + else: + provider_uuid = await self.ap.provider_service.find_or_create_provider( + context, + requester=provider_data.get('requester', ''), + base_url=provider_data.get('base_url', ''), + api_keys=provider_data.get('api_keys', []), + ) + model_data['provider_uuid'] = provider_uuid + + await _require_workspace_provider(self.ap, context, model_data['provider_uuid']) + await _assert_cloud_managed_provider_mutable(self.ap, context, model_data['provider_uuid']) + await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'asr') + + await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_model.ASRModel).values(**model_data)) + + runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid']) + runtime_asr_model = await self.ap.model_mgr.load_asr_model_with_provider( + context, + persistence_model.ASRModel(**model_data), + runtime_provider, + ) + await self.ap.model_mgr.cache_asr_model(context, runtime_asr_model) + + return model_data['uuid'] + + async def get_asr_model( + self, + context: TenantContext, + model_uuid: str, + include_secret: bool = False, + ) -> dict | None: + """Get a single ASR model with provider info""" + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ASRModel).where(persistence_model.ASRModel.uuid == model_uuid), + persistence_model.ASRModel, + context, + ) + ) + model = result.first() + if model is None: + return None + + model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ASRModel, model) + + provider_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ModelProvider).where( + persistence_model.ModelProvider.uuid == model.provider_uuid + ), + persistence_model.ModelProvider, + context, + ) + ) + provider = provider_result.first() + if provider: + provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider) + provider_dict = _parse_provider_api_keys(provider_dict) + model_dict['provider'] = provider_dict + + if not include_secret: + model_dict = _redact_model_secrets(model_dict) + + return model_dict + + async def update_asr_model( + self, + context: TenantContext, + model_uuid: str, + model_data: dict, + ) -> None: + """Update an existing ASR model""" + existing_model = await self.get_asr_model(context, model_uuid, include_secret=True) + if existing_model is None: + raise WorkspaceNotFoundError('Model not found') + await _assert_cloud_managed_provider_mutable(self.ap, context, existing_model['provider_uuid']) + model_data = model_data.copy() + model_data.pop('uuid', None) + model_data.pop('workspace_uuid', None) + if 'extra_args' in model_data: + model_data['extra_args'] = restore_secret_placeholders( + model_data['extra_args'], + existing_model.get('extra_args', {}), + ) + + if 'provider' in model_data: + provider_data = model_data.pop('provider') + if provider_data.get('uuid'): + model_data['provider_uuid'] = provider_data['uuid'] + else: + provider_uuid = await self.ap.provider_service.find_or_create_provider( + context, + requester=provider_data.get('requester', ''), + base_url=provider_data.get('base_url', ''), + api_keys=provider_data.get('api_keys', []), + ) + model_data['provider_uuid'] = provider_uuid + + provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid']) + await _require_workspace_provider(self.ap, context, provider_uuid) + await _assert_cloud_managed_provider_mutable(self.ap, context, provider_uuid) + await _validate_provider_supports(self.ap, context, provider_uuid, 'asr') + + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.update(persistence_model.ASRModel) + .where(persistence_model.ASRModel.uuid == model_uuid) + .values(**model_data), + persistence_model.ASRModel, + context, + ) + ) + if getattr(result, 'rowcount', None) == 0: + raise WorkspaceNotFoundError('Model not found') + + await self.ap.model_mgr.remove_asr_model(context, model_uuid) + runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid) + runtime_asr_model = await self.ap.model_mgr.load_asr_model_with_provider( + context, + persistence_model.ASRModel( + **_runtime_model_data( + model_uuid, + { + key: value + for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items() + if key not in {'provider', 'created_at', 'updated_at'} + }, + ) + ), + runtime_provider, + ) + await self.ap.model_mgr.cache_asr_model(context, runtime_asr_model) + + async def delete_asr_model(self, context: TenantContext, model_uuid: str) -> None: + """Delete an ASR model""" + if _is_cloud_runtime(self.ap): + existing_model = await self.get_asr_model(context, model_uuid, include_secret=True) + if existing_model is None: + raise WorkspaceNotFoundError('Model not found') + await _assert_cloud_managed_provider_mutable(self.ap, context, existing_model['provider_uuid']) + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.delete(persistence_model.ASRModel).where(persistence_model.ASRModel.uuid == model_uuid), + persistence_model.ASRModel, + context, + ) + ) + if getattr(result, 'rowcount', None) == 0: + raise WorkspaceNotFoundError('Model not found') + await self.ap.model_mgr.remove_asr_model(context, model_uuid) + + async def test_asr_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None: + """Test an ASR model""" + require_workspace_uuid(context) + runtime_asr_model: model_requester.RuntimeASRModel | None = None + + if model_uuid != '_': + if await self.get_asr_model(context, model_uuid) is None: + raise WorkspaceNotFoundError('Model not found') + runtime_asr_model = await self.ap.model_mgr.get_asr_model_by_uuid(context, model_uuid) + else: + runtime_asr_model = await self.ap.model_mgr.init_temporary_runtime_asr_model( + context, + model_data, + ) + + await runtime_asr_model.provider.invoke_asr( + model=runtime_asr_model, + audio=_generate_test_wav(), + mime='audio/wav', + execution_context=runtime_asr_model.execution_context, + ) + + +class TTSModelsService: + ap: app.Application + + def __init__(self, ap: app.Application) -> None: + self.ap = ap + + async def get_tts_models(self, context: TenantContext, include_secret: bool = False) -> list[dict]: + """Get all TTS models with provider info""" + result = await self.ap.persistence_mgr.execute_async( + scope_statement(sqlalchemy.select(persistence_model.TTSModel), persistence_model.TTSModel, context) + ) + models = result.all() + + providers_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ModelProvider), persistence_model.ModelProvider, context + ) + ) + providers = {p.uuid: p for p in providers_result.all()} + + models_list = [] + for model in models: + model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.TTSModel, model) + provider = providers.get(model.provider_uuid) + if provider: + provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider) + provider_dict = _parse_provider_api_keys(provider_dict) + model_dict['provider'] = provider_dict + if not include_secret: + model_dict = _redact_model_secrets(model_dict) + models_list.append(model_dict) + + return models_list + + async def get_tts_models_by_provider( + self, + context: TenantContext, + provider_uuid: str, + *, + include_secret: bool = False, + ) -> list[dict]: + """Get TTS models by provider UUID""" + await _require_workspace_provider(self.ap, context, provider_uuid) + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.TTSModel).where( + persistence_model.TTSModel.provider_uuid == provider_uuid + ), + persistence_model.TTSModel, + context, + ) + ) + models = result.all() + serialized = [self.ap.persistence_mgr.serialize_model(persistence_model.TTSModel, m) for m in models] + return serialized if include_secret else [_redact_model_secrets(model) for model in serialized] + + async def create_tts_model( + self, + context: TenantContext, + model_data: dict, + preserve_uuid: bool = False, + ) -> str: + """Create a new TTS model""" + model_data = model_data.copy() + if not preserve_uuid: + model_data['uuid'] = str(uuid.uuid4()) + model_data['workspace_uuid'] = require_workspace_uuid(context) + if 'extra_args' in model_data: + model_data['extra_args'] = restore_secret_placeholders(model_data['extra_args']) + + if 'provider' in model_data: + provider_data = model_data.pop('provider') + if provider_data.get('uuid'): + model_data['provider_uuid'] = provider_data['uuid'] + else: + provider_uuid = await self.ap.provider_service.find_or_create_provider( + context, + requester=provider_data.get('requester', ''), + base_url=provider_data.get('base_url', ''), + api_keys=provider_data.get('api_keys', []), + ) + model_data['provider_uuid'] = provider_uuid + + await _require_workspace_provider(self.ap, context, model_data['provider_uuid']) + await _assert_cloud_managed_provider_mutable(self.ap, context, model_data['provider_uuid']) + await _validate_provider_supports(self.ap, context, model_data['provider_uuid'], 'tts') + + await self.ap.persistence_mgr.execute_async(sqlalchemy.insert(persistence_model.TTSModel).values(**model_data)) + + runtime_provider = await _require_runtime_provider(self.ap, context, model_data['provider_uuid']) + runtime_tts_model = await self.ap.model_mgr.load_tts_model_with_provider( + context, + persistence_model.TTSModel(**model_data), + runtime_provider, + ) + await self.ap.model_mgr.cache_tts_model(context, runtime_tts_model) + + return model_data['uuid'] + + async def get_tts_model( + self, + context: TenantContext, + model_uuid: str, + include_secret: bool = False, + ) -> dict | None: + """Get a single TTS model with provider info""" + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.TTSModel).where(persistence_model.TTSModel.uuid == model_uuid), + persistence_model.TTSModel, + context, + ) + ) + model = result.first() + if model is None: + return None + + model_dict = self.ap.persistence_mgr.serialize_model(persistence_model.TTSModel, model) + + provider_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(persistence_model.ModelProvider).where( + persistence_model.ModelProvider.uuid == model.provider_uuid + ), + persistence_model.ModelProvider, + context, + ) + ) + provider = provider_result.first() + if provider: + provider_dict = self.ap.persistence_mgr.serialize_model(persistence_model.ModelProvider, provider) + provider_dict = _parse_provider_api_keys(provider_dict) + model_dict['provider'] = provider_dict + + if not include_secret: + model_dict = _redact_model_secrets(model_dict) + + return model_dict + + async def update_tts_model( + self, + context: TenantContext, + model_uuid: str, + model_data: dict, + ) -> None: + """Update an existing TTS model""" + existing_model = await self.get_tts_model(context, model_uuid, include_secret=True) + if existing_model is None: + raise WorkspaceNotFoundError('Model not found') + await _assert_cloud_managed_provider_mutable(self.ap, context, existing_model['provider_uuid']) + model_data = model_data.copy() + model_data.pop('uuid', None) + model_data.pop('workspace_uuid', None) + if 'extra_args' in model_data: + model_data['extra_args'] = restore_secret_placeholders( + model_data['extra_args'], + existing_model.get('extra_args', {}), + ) + + if 'provider' in model_data: + provider_data = model_data.pop('provider') + if provider_data.get('uuid'): + model_data['provider_uuid'] = provider_data['uuid'] + else: + provider_uuid = await self.ap.provider_service.find_or_create_provider( + context, + requester=provider_data.get('requester', ''), + base_url=provider_data.get('base_url', ''), + api_keys=provider_data.get('api_keys', []), + ) + model_data['provider_uuid'] = provider_uuid + + provider_uuid = model_data.get('provider_uuid', existing_model['provider_uuid']) + await _require_workspace_provider(self.ap, context, provider_uuid) + await _assert_cloud_managed_provider_mutable(self.ap, context, provider_uuid) + await _validate_provider_supports(self.ap, context, provider_uuid, 'tts') + + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.update(persistence_model.TTSModel) + .where(persistence_model.TTSModel.uuid == model_uuid) + .values(**model_data), + persistence_model.TTSModel, + context, + ) + ) + if getattr(result, 'rowcount', None) == 0: + raise WorkspaceNotFoundError('Model not found') + + await self.ap.model_mgr.remove_tts_model(context, model_uuid) + runtime_provider = await _require_runtime_provider(self.ap, context, provider_uuid) + runtime_tts_model = await self.ap.model_mgr.load_tts_model_with_provider( + context, + persistence_model.TTSModel( + **_runtime_model_data( + model_uuid, + { + key: value + for key, value in {**existing_model, **model_data, 'provider_uuid': provider_uuid}.items() + if key not in {'provider', 'created_at', 'updated_at'} + }, + ) + ), + runtime_provider, + ) + await self.ap.model_mgr.cache_tts_model(context, runtime_tts_model) + + async def delete_tts_model(self, context: TenantContext, model_uuid: str) -> None: + """Delete a TTS model""" + if _is_cloud_runtime(self.ap): + existing_model = await self.get_tts_model(context, model_uuid, include_secret=True) + if existing_model is None: + raise WorkspaceNotFoundError('Model not found') + await _assert_cloud_managed_provider_mutable(self.ap, context, existing_model['provider_uuid']) + result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.delete(persistence_model.TTSModel).where(persistence_model.TTSModel.uuid == model_uuid), + persistence_model.TTSModel, + context, + ) + ) + if getattr(result, 'rowcount', None) == 0: + raise WorkspaceNotFoundError('Model not found') + await self.ap.model_mgr.remove_tts_model(context, model_uuid) + + async def test_tts_model(self, context: TenantContext, model_uuid: str, model_data: dict) -> None: + """Test a TTS model""" + require_workspace_uuid(context) + runtime_tts_model: model_requester.RuntimeTTSModel | None = None + + if model_uuid != '_': + if await self.get_tts_model(context, model_uuid) is None: + raise WorkspaceNotFoundError('Model not found') + runtime_tts_model = await self.ap.model_mgr.get_tts_model_by_uuid(context, model_uuid) + else: + runtime_tts_model = await self.ap.model_mgr.init_temporary_runtime_tts_model( + context, + model_data, + ) + + model_extra = ( + getattr(runtime_tts_model, 'extra_args', None) + or getattr(runtime_tts_model.model_entity, 'extra_args', {}) + or {} + ) + voice = model_extra.get('voice') + extra_args: dict[str, typing.Any] = {'response_format': 'mp3'} + if voice: + extra_args['voice'] = voice + await runtime_tts_model.provider.invoke_tts( + model=runtime_tts_model, + text='Hello, this is a test.', + extra_args=extra_args, + execution_context=runtime_tts_model.execution_context, + ) diff --git a/src/langbot/pkg/api/http/service/pipeline.py b/src/langbot/pkg/api/http/service/pipeline.py index 934cf97f9..d362743aa 100644 --- a/src/langbot/pkg/api/http/service/pipeline.py +++ b/src/langbot/pkg/api/http/service/pipeline.py @@ -20,6 +20,7 @@ from .tenant import TenantContext, require_workspace_uuid, scope_statement default_stage_order = [ 'GroupRespondRuleCheckStage', # 群响应规则检查 'BanSessionCheckStage', # 封禁会话检查 + 'SpeechRecognitionStage', # 语音识别(ASR)阶段 'PreContentFilterStage', # 内容过滤前置阶段 'PreProcessor', # 预处理器 'RequireRateLimitOccupancy', # 请求速率限制占用 @@ -27,6 +28,7 @@ default_stage_order = [ 'ReleaseRateLimitOccupancy', # 释放速率限制占用 'PostContentFilterStage', # 内容过滤后置阶段 'ResponseWrapper', # 响应包装器 + 'TextToSpeechStage', # 文本转语音阶段 'LongTextProcessStage', # 长文本处理 'SendResponseBackStage', # 发送响应 ] @@ -105,6 +107,7 @@ class PipelineService: self.ap.pipeline_config_meta_trigger, self.ap.pipeline_config_meta_safety, ai_metadata, + self.ap.pipeline_config_meta_speech, self.ap.pipeline_config_meta_output, ] diff --git a/src/langbot/pkg/api/http/service/pipeline_migration.py b/src/langbot/pkg/api/http/service/pipeline_migration.py index b4ec9b502..c574d32f3 100644 --- a/src/langbot/pkg/api/http/service/pipeline_migration.py +++ b/src/langbot/pkg/api/http/service/pipeline_migration.py @@ -28,7 +28,7 @@ from ....agent.runner.config_resolver import RunnerConfigResolver from ....agent.runner import config_schema from ....agent.runner.resource_builder import AgentResourceBuilder from ....agent.runner.resource_policy import ResourcePolicyProjector -from ....entity.persistence.model import LLMModel, RerankModel, EmbeddingModel +from ....entity.persistence.model import LLMModel, RerankModel, EmbeddingModel, ASRModel, TTSModel from ....entity.persistence.rag import KnowledgeBase from ....entity.persistence.mcp import MCPServer from ....core.taskmgr import TaskContext @@ -681,6 +681,8 @@ class PipelineMigrationService: 'llm-model-selector', 'embedding-model-selector', 'rerank-model-selector', + 'asr-model-selector', + 'tts-model-selector', ) and isinstance(value, str) ) @@ -768,9 +770,19 @@ class PipelineMigrationService: permissions = getattr(descriptor, 'permissions', None) for model_type, model_uuid in config_schema.iter_config_model_refs(descriptor, runner_config): allowed = set(getattr(permissions, 'models', [])) - if not (allowed & ({'rerank'} if model_type == 'rerank' else {'invoke', 'stream'})): + needed = ( + {'rerank'} + if model_type == 'rerank' + else ({'asr'} if model_type == 'asr' else ({'tts'} if model_type == 'tts' else {'invoke', 'stream'})) + ) + if not (allowed & needed): raise MigrationError('runner_resource_unavailable') - await require_row(RerankModel if model_type == 'rerank' else LLMModel, model_uuid) + target_cls = ( + RerankModel + if model_type == 'rerank' + else (ASRModel if model_type == 'asr' else (TTSModel if model_type == 'tts' else LLMModel)) + ) + await require_row(target_cls, model_uuid) for field in config_schema.iter_schema_items(descriptor, {'embedding-model-selector'}): value = runner_config.get(field['name']) if value and value not in config_schema.NONE_SENTINELS: diff --git a/src/langbot/pkg/api/http/service/provider.py b/src/langbot/pkg/api/http/service/provider.py index b5909b404..c86dcfa27 100644 --- a/src/langbot/pkg/api/http/service/provider.py +++ b/src/langbot/pkg/api/http/service/provider.py @@ -203,6 +203,8 @@ class ModelProviderService: (persistence_model.LLMModel, 'LLM', 'remove_llm_model'), (persistence_model.EmbeddingModel, 'Embedding', 'remove_embedding_model'), (persistence_model.RerankModel, 'Rerank', 'remove_rerank_model'), + (persistence_model.ASRModel, 'ASR', 'remove_asr_model'), + (persistence_model.TTSModel, 'TTS', 'remove_tts_model'), ) deleted_models: list[tuple[str, list[str]]] = [] async with persistence.tenant_uow(workspace_uuid): @@ -335,7 +337,35 @@ class ModelProviderService: ) rerank_count = rerank_result.scalar() or 0 - return {'llm_count': llm_count, 'embedding_count': embedding_count, 'rerank_count': rerank_count} + asr_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(sqlalchemy.func.count()) + .select_from(persistence_model.ASRModel) + .where(persistence_model.ASRModel.provider_uuid == provider_uuid), + persistence_model.ASRModel, + workspace_uuid, + ) + ) + asr_count = asr_result.scalar() or 0 + + tts_result = await self.ap.persistence_mgr.execute_async( + scope_statement( + sqlalchemy.select(sqlalchemy.func.count()) + .select_from(persistence_model.TTSModel) + .where(persistence_model.TTSModel.provider_uuid == provider_uuid), + persistence_model.TTSModel, + workspace_uuid, + ) + ) + tts_count = tts_result.scalar() or 0 + + return { + 'llm_count': llm_count, + 'embedding_count': embedding_count, + 'rerank_count': rerank_count, + 'asr_count': asr_count, + 'tts_count': tts_count, + } async def find_or_create_provider( self, @@ -439,9 +469,19 @@ class ModelProviderService: if rerank_service is not None else [] ) + asr_service = getattr(self.ap, 'asr_models_service', None) + asr_models = ( + await asr_service.get_asr_models_by_provider(context, provider_uuid) if asr_service is not None else [] + ) + tts_service = getattr(self.ap, 'tts_models_service', None) + tts_models = ( + await tts_service.get_tts_models_by_provider(context, provider_uuid) if tts_service is not None else [] + ) existing_llm_names = {model['name'] for model in llm_models} existing_embedding_names = {model['name'] for model in embedding_models} existing_rerank_names = {model['name'] for model in rerank_models} + existing_asr_names = {model['name'] for model in asr_models} + existing_tts_names = {model['name'] for model in tts_models} filtered_models = [] for model in scanned_models: @@ -470,6 +510,10 @@ class ModelProviderService: if scanned_type == 'embedding' else model_name in existing_rerank_names if scanned_type == 'rerank' + else model_name in existing_asr_names + if scanned_type == 'asr' + else model_name in existing_tts_names + if scanned_type == 'tts' else model_name in existing_llm_names ), } diff --git a/src/langbot/pkg/api/mcp/server.py b/src/langbot/pkg/api/mcp/server.py index 967a964ca..12c725eaa 100644 --- a/src/langbot/pkg/api/mcp/server.py +++ b/src/langbot/pkg/api/mcp/server.py @@ -583,6 +583,16 @@ class LangBotMCPServer: context = _authorized(Permission.RESOURCE_VIEW) return _dump(await ap.embedding_models_service.get_embedding_models(context, include_secret=False)) + @mcp.tool(description='List all configured ASR (speech recognition) models.') + async def list_asr_models() -> str: + context = _authorized(Permission.RESOURCE_VIEW) + return _dump(await ap.asr_models_service.get_asr_models(context, include_secret=False)) + + @mcp.tool(description='List all configured TTS (text-to-speech) models.') + async def list_tts_models() -> str: + context = _authorized(Permission.RESOURCE_VIEW) + return _dump(await ap.tts_models_service.get_tts_models(context, include_secret=False)) + @mcp.tool(description='List all model providers (OpenAI-compatible, Anthropic, etc.).') async def list_model_providers() -> str: context = _authorized(Permission.RESOURCE_VIEW) diff --git a/src/langbot/pkg/core/app.py b/src/langbot/pkg/core/app.py index 9f84ee10f..cc05321f9 100644 --- a/src/langbot/pkg/core/app.py +++ b/src/langbot/pkg/core/app.py @@ -118,6 +118,7 @@ class Application: pipeline_config_meta_safety: config_mgr.ConfigManager = None pipeline_config_meta_ai: config_mgr.ConfigManager = None pipeline_config_meta_output: config_mgr.ConfigManager = None + pipeline_config_meta_speech: config_mgr.ConfigManager = None # ========================= @@ -179,6 +180,10 @@ class Application: rerank_models_service: model_service.RerankModelsService = None + asr_models_service: model_service.ASRModelsService = None + + tts_models_service: model_service.TTSModelsService = None + provider_service: provider_service.ModelProviderService = None pipeline_service: pipeline_service.PipelineService = None diff --git a/src/langbot/pkg/core/stages/build_app.py b/src/langbot/pkg/core/stages/build_app.py index c42dd769e..94f073a6b 100644 --- a/src/langbot/pkg/core/stages/build_app.py +++ b/src/langbot/pkg/core/stages/build_app.py @@ -111,6 +111,12 @@ class BuildAppStage(stage.BootingStage): rerank_models_service_inst = model_service.RerankModelsService(ap) ap.rerank_models_service = rerank_models_service_inst + asr_models_service_inst = model_service.ASRModelsService(ap) + ap.asr_models_service = asr_models_service_inst + + tts_models_service_inst = model_service.TTSModelsService(ap) + ap.tts_models_service = tts_models_service_inst + provider_service_inst = provider_service.ModelProviderService(ap) ap.provider_service = provider_service_inst diff --git a/src/langbot/pkg/core/stages/load_config.py b/src/langbot/pkg/core/stages/load_config.py index b41e2accd..eac589a34 100644 --- a/src/langbot/pkg/core/stages/load_config.py +++ b/src/langbot/pkg/core/stages/load_config.py @@ -345,4 +345,5 @@ class LoadConfigStage(stage.BootingStage): ap.pipeline_config_meta_trigger = await load_resource_yaml_template_data('metadata/pipeline/trigger.yaml') ap.pipeline_config_meta_safety = await load_resource_yaml_template_data('metadata/pipeline/safety.yaml') ap.pipeline_config_meta_ai = await load_resource_yaml_template_data('metadata/pipeline/ai.yaml') + ap.pipeline_config_meta_speech = await load_resource_yaml_template_data('metadata/pipeline/speech.yaml') ap.pipeline_config_meta_output = await load_resource_yaml_template_data('metadata/pipeline/output.yaml') diff --git a/src/langbot/pkg/entity/persistence/model.py b/src/langbot/pkg/entity/persistence/model.py index 1e133475a..645b7f158 100644 --- a/src/langbot/pkg/entity/persistence/model.py +++ b/src/langbot/pkg/entity/persistence/model.py @@ -163,3 +163,71 @@ class RerankModel(Base): sqlalchemy.Index('ix_rerank_models_workspace_provider', 'workspace_uuid', 'provider_uuid'), sqlalchemy.Index('ix_rerank_models_workspace_name', 'workspace_uuid', 'name'), ) + + +class ASRModel(Base): + """ASR model""" + + __tablename__ = 'asr_models' + + uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True) + workspace_uuid = sqlalchemy.Column( + sqlalchemy.String(36), + sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'), + nullable=False, + ) + name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + provider_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={}) + prefered_ranking = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0) + created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now()) + updated_at = sqlalchemy.Column( + sqlalchemy.DateTime, + nullable=False, + server_default=sqlalchemy.func.now(), + onupdate=sqlalchemy.func.now(), + ) + + __table_args__ = ( + sqlalchemy.ForeignKeyConstraint( + ['workspace_uuid', 'provider_uuid'], + ['model_providers.workspace_uuid', 'model_providers.uuid'], + name='fk_asr_models_workspace_provider', + ), + sqlalchemy.Index('ix_asr_models_workspace_provider', 'workspace_uuid', 'provider_uuid'), + sqlalchemy.Index('ix_asr_models_workspace_name', 'workspace_uuid', 'name'), + ) + + +class TTSModel(Base): + """TTS model""" + + __tablename__ = 'tts_models' + + uuid = sqlalchemy.Column(sqlalchemy.String(255), primary_key=True, unique=True) + workspace_uuid = sqlalchemy.Column( + sqlalchemy.String(36), + sqlalchemy.ForeignKey('workspaces.uuid', ondelete='CASCADE'), + nullable=False, + ) + name = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + provider_uuid = sqlalchemy.Column(sqlalchemy.String(255), nullable=False) + extra_args = sqlalchemy.Column(sqlalchemy.JSON, nullable=False, default={}) + prefered_ranking = sqlalchemy.Column(sqlalchemy.Integer, nullable=False, default=0) + created_at = sqlalchemy.Column(sqlalchemy.DateTime, nullable=False, server_default=sqlalchemy.func.now()) + updated_at = sqlalchemy.Column( + sqlalchemy.DateTime, + nullable=False, + server_default=sqlalchemy.func.now(), + onupdate=sqlalchemy.func.now(), + ) + + __table_args__ = ( + sqlalchemy.ForeignKeyConstraint( + ['workspace_uuid', 'provider_uuid'], + ['model_providers.workspace_uuid', 'model_providers.uuid'], + name='fk_tts_models_workspace_provider', + ), + sqlalchemy.Index('ix_tts_models_workspace_provider', 'workspace_uuid', 'provider_uuid'), + sqlalchemy.Index('ix_tts_models_workspace_name', 'workspace_uuid', 'name'), + ) diff --git a/src/langbot/pkg/i18n/locales/en_US.json b/src/langbot/pkg/i18n/locales/en_US.json index dd07ff255..d5ae9f4b4 100644 --- a/src/langbot/pkg/i18n/locales/en_US.json +++ b/src/langbot/pkg/i18n/locales/en_US.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.", "Option {number}": "Option {number}", "Confirmation needed": "Confirmation needed", - "Session reset successfully": "Session reset successfully" + "Session reset successfully": "Session reset successfully", + "Voice transcription failed.": "Voice transcription failed.", + "Voice message exceeds maximum duration.": "Voice message exceeds maximum duration.", + "Voice synthesis failed.": "Voice synthesis failed." } diff --git a/src/langbot/pkg/i18n/locales/es_ES.json b/src/langbot/pkg/i18n/locales/es_ES.json index 116aa09a8..1aa2b69c2 100644 --- a/src/langbot/pkg/i18n/locales/es_ES.json +++ b/src/langbot/pkg/i18n/locales/es_ES.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "No se pudo iniciar el acceso por dispositivo. Activa el inicio de sesión con código de dispositivo en la configuración de ChatGPT.", "Option {number}": "Opción {number}", "Confirmation needed": "Se requiere confirmación", - "Session reset successfully": "Sesión restablecida" + "Session reset successfully": "Sesión restablecida", + "Voice transcription failed.": "Error en la transcripción de voz.", + "Voice message exceeds maximum duration.": "El mensaje de voz excede la duración máxima.", + "Voice synthesis failed.": "Error en la síntesis de voz." } diff --git a/src/langbot/pkg/i18n/locales/ja_JP.json b/src/langbot/pkg/i18n/locales/ja_JP.json index 292bb1b56..153eccb25 100644 --- a/src/langbot/pkg/i18n/locales/ja_JP.json +++ b/src/langbot/pkg/i18n/locales/ja_JP.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "ChatGPT のデバイスログインを開始できません。ChatGPT の設定でデバイスコードログインを有効にしてください。", "Option {number}": "選択肢 {number}", "Confirmation needed": "確認が必要です", - "Session reset successfully": "セッションをリセットしました" + "Session reset successfully": "セッションをリセットしました", + "Voice transcription failed.": "音声の文字起こしに失敗しました。", + "Voice message exceeds maximum duration.": "音声メッセージが最大時間を超えています。", + "Voice synthesis failed.": "音声合成に失敗しました。" } diff --git a/src/langbot/pkg/i18n/locales/ru_RU.json b/src/langbot/pkg/i18n/locales/ru_RU.json index ed9529018..b9a87de23 100644 --- a/src/langbot/pkg/i18n/locales/ru_RU.json +++ b/src/langbot/pkg/i18n/locales/ru_RU.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "Не удалось начать вход с устройства. Включите вход по коду устройства в настройках ChatGPT.", "Option {number}": "Вариант {number}", "Confirmation needed": "Требуется подтверждение", - "Session reset successfully": "Сессия сброшена" + "Session reset successfully": "Сессия сброшена", + "Voice transcription failed.": "Не удалось распознать голосовое сообщение.", + "Voice message exceeds maximum duration.": "Длительность голосового сообщения превышает максимум.", + "Voice synthesis failed.": "Сбой синтеза речи." } diff --git a/src/langbot/pkg/i18n/locales/th_TH.json b/src/langbot/pkg/i18n/locales/th_TH.json index ff40282cd..134809789 100644 --- a/src/langbot/pkg/i18n/locales/th_TH.json +++ b/src/langbot/pkg/i18n/locales/th_TH.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "ไม่สามารถเริ่มเข้าสู่ระบบด้วยอุปกรณ์ได้ โปรดเปิดการเข้าสู่ระบบด้วยรหัสอุปกรณ์ในการตั้งค่า ChatGPT", "Option {number}": "ตัวเลือก {number}", "Confirmation needed": "ต้องยืนยัน", - "Session reset successfully": "รีเซ็ตเซสชันแล้ว" + "Session reset successfully": "รีเซ็ตเซสชันแล้ว", + "Voice transcription failed.": "การแปลงเสียงเป็นข้อความล้มเหลว", + "Voice message exceeds maximum duration.": "ข้อความเสียงมีความยาวเกินกำหนดสูงสุด", + "Voice synthesis failed.": "การสังเคราะห์เสียงล้มเหลว" } diff --git a/src/langbot/pkg/i18n/locales/vi_VN.json b/src/langbot/pkg/i18n/locales/vi_VN.json index 39b3110a8..5ebe612e5 100644 --- a/src/langbot/pkg/i18n/locales/vi_VN.json +++ b/src/langbot/pkg/i18n/locales/vi_VN.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "Không thể bắt đầu đăng nhập thiết bị ChatGPT. Hãy bật đăng nhập bằng mã thiết bị trong cài đặt ChatGPT.", "Option {number}": "Lựa chọn {number}", "Confirmation needed": "Cần xác nhận", - "Session reset successfully": "Đã đặt lại phiên" + "Session reset successfully": "Đã đặt lại phiên", + "Voice transcription failed.": "Chuyển đổi giọng nói thành văn bản thất bại.", + "Voice message exceeds maximum duration.": "Tin nhắn thoại vượt quá thời lượng tối đa.", + "Voice synthesis failed.": "Tổng hợp giọng nói thất bại." } diff --git a/src/langbot/pkg/i18n/locales/zh_Hans.json b/src/langbot/pkg/i18n/locales/zh_Hans.json index bf507bd74..abd258dc2 100644 --- a/src/langbot/pkg/i18n/locales/zh_Hans.json +++ b/src/langbot/pkg/i18n/locales/zh_Hans.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "无法启动 ChatGPT 设备登录。请在 ChatGPT 设置中启用设备代码登录。", "Option {number}": "选项 {number}", "Confirmation needed": "需要确认", - "Session reset successfully": "会话已重置" + "Session reset successfully": "会话已重置", + "Voice transcription failed.": "语音转写失败。", + "Voice message exceeds maximum duration.": "语音消息超出最大时长。", + "Voice synthesis failed.": "语音合成失败。" } diff --git a/src/langbot/pkg/i18n/locales/zh_Hant.json b/src/langbot/pkg/i18n/locales/zh_Hant.json index 15f5e90c3..6cfe3a887 100644 --- a/src/langbot/pkg/i18n/locales/zh_Hant.json +++ b/src/langbot/pkg/i18n/locales/zh_Hant.json @@ -110,5 +110,8 @@ "Unable to start ChatGPT device login. Enable device code login in ChatGPT settings.": "無法啟動 ChatGPT 裝置登入。請在 ChatGPT 設定中啟用裝置代碼登入。", "Option {number}": "選項 {number}", "Confirmation needed": "需要確認", - "Session reset successfully": "對話已重設" + "Session reset successfully": "對話已重設", + "Voice transcription failed.": "語音轉寫失敗。", + "Voice message exceeds maximum duration.": "語音訊息超過最大時長。", + "Voice synthesis failed.": "語音合成失敗。" } diff --git a/src/langbot/pkg/persistence/alembic/versions/0043_speech_models.py b/src/langbot/pkg/persistence/alembic/versions/0043_speech_models.py new file mode 100644 index 000000000..c662adb86 --- /dev/null +++ b/src/langbot/pkg/persistence/alembic/versions/0043_speech_models.py @@ -0,0 +1,183 @@ +"""Add ASR and TTS models and speech pipeline stages. + +Revision ID: 0043_speech_models +Revises: 0042_sqlite_incremental_vacuum +""" + +import json +from alembic import op +import sqlalchemy as sa + +revision = '0043_speech_models' +down_revision = '0042_sqlite_incremental_vacuum' +branch_labels = None +depends_on = None + +DEFAULT_SPEECH_CONFIG = { + 'speech-recognition': { + 'enabled': False, + 'model': '', + 'max-duration': 60, + 'keep-original-voice': False, + 'on-failure': 'ignore', + }, + 'text-to-speech': { + 'enabled': False, + 'model': '', + 'trigger': 'voice-input-only', + 'output': 'voice-only', + 'max-text-length': 1000, + 'voice': '', + 'speed': 1.0, + }, +} + + +def upgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + existing_tables = set(inspector.get_table_names()) + model_parents_exist = {'workspaces', 'model_providers'} <= existing_tables + + if model_parents_exist and 'asr_models' not in existing_tables: + op.create_table( + 'asr_models', + sa.Column('uuid', sa.String(255), primary_key=True, unique=True), + sa.Column( + 'workspace_uuid', + sa.String(36), + sa.ForeignKey('workspaces.uuid', ondelete='CASCADE'), + nullable=False, + ), + sa.Column('name', sa.String(255), nullable=False), + sa.Column('provider_uuid', sa.String(255), nullable=False), + sa.Column('extra_args', sa.JSON, nullable=False, server_default='{}'), + sa.Column('prefered_ranking', sa.Integer, nullable=False, server_default='0'), + sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.ForeignKeyConstraint( + ['workspace_uuid', 'provider_uuid'], + ['model_providers.workspace_uuid', 'model_providers.uuid'], + name='fk_asr_models_workspace_provider', + ), + ) + op.create_index( + 'ix_asr_models_workspace_provider', + 'asr_models', + ['workspace_uuid', 'provider_uuid'], + ) + op.create_index( + 'ix_asr_models_workspace_name', + 'asr_models', + ['workspace_uuid', 'name'], + ) + + if model_parents_exist and 'tts_models' not in existing_tables: + op.create_table( + 'tts_models', + sa.Column('uuid', sa.String(255), primary_key=True, unique=True), + sa.Column( + 'workspace_uuid', + sa.String(36), + sa.ForeignKey('workspaces.uuid', ondelete='CASCADE'), + nullable=False, + ), + sa.Column('name', sa.String(255), nullable=False), + sa.Column('provider_uuid', sa.String(255), nullable=False), + sa.Column('extra_args', sa.JSON, nullable=False, server_default='{}'), + sa.Column('prefered_ranking', sa.Integer, nullable=False, server_default='0'), + sa.Column('created_at', sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.Column('updated_at', sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.ForeignKeyConstraint( + ['workspace_uuid', 'provider_uuid'], + ['model_providers.workspace_uuid', 'model_providers.uuid'], + name='fk_tts_models_workspace_provider', + ), + ) + op.create_index( + 'ix_tts_models_workspace_provider', + 'tts_models', + ['workspace_uuid', 'provider_uuid'], + ) + op.create_index( + 'ix_tts_models_workspace_name', + 'tts_models', + ['workspace_uuid', 'name'], + ) + + if conn.dialect.name == 'postgresql': + for tbl in ('asr_models', 'tts_models'): + if not sa.inspect(conn).has_table(tbl): + continue + op.execute(f'ALTER TABLE {tbl} ENABLE ROW LEVEL SECURITY') + op.execute(f'ALTER TABLE {tbl} FORCE ROW LEVEL SECURITY') + op.execute(f'DROP POLICY IF EXISTS langbot_workspace_isolation ON {tbl}') + op.execute(f"""CREATE POLICY langbot_workspace_isolation ON {tbl} + USING (workspace_uuid::text = NULLIF(current_setting('langbot.workspace_uuid', true), '')) + WITH CHECK (workspace_uuid::text = NULLIF(current_setting('langbot.workspace_uuid', true), ''))""") + + # Migrate legacy_pipelines stages and config + if 'legacy_pipelines' in existing_tables: + pipeline_columns = {col['name'] for col in inspector.get_columns('legacy_pipelines')} + if {'stages', 'config', 'uuid'} <= pipeline_columns: + metadata = sa.MetaData() + pipelines_tbl = sa.Table('legacy_pipelines', metadata, autoload_with=conn) + rows = conn.execute( + sa.select(pipelines_tbl.c.uuid, pipelines_tbl.c.stages, pipelines_tbl.c.config) + ).fetchall() + + for row in rows: + p_uuid, stages_raw, config_raw = row[0], row[1], row[2] + stages = stages_raw if isinstance(stages_raw, list) else json.loads(stages_raw or '[]') + config = config_raw if isinstance(config_raw, dict) else json.loads(config_raw or '{}') + + stages_changed = False + config_changed = False + + if 'SpeechRecognitionStage' not in stages: + # Place after BanSessionCheckStage or GroupRespondRuleCheckStage, and before PreContentFilterStage + if 'BanSessionCheckStage' in stages: + idx = stages.index('BanSessionCheckStage') + 1 + stages.insert(idx, 'SpeechRecognitionStage') + elif 'GroupRespondRuleCheckStage' in stages: + idx = stages.index('GroupRespondRuleCheckStage') + 1 + stages.insert(idx, 'SpeechRecognitionStage') + elif 'PreContentFilterStage' in stages: + idx = stages.index('PreContentFilterStage') + stages.insert(idx, 'SpeechRecognitionStage') + else: + stages.append('SpeechRecognitionStage') + stages_changed = True + + if 'TextToSpeechStage' not in stages: + # Place after ResponseWrapper, before LongTextProcessStage + if 'ResponseWrapper' in stages: + idx = stages.index('ResponseWrapper') + 1 + stages.insert(idx, 'TextToSpeechStage') + elif 'LongTextProcessStage' in stages: + idx = stages.index('LongTextProcessStage') + stages.insert(idx, 'TextToSpeechStage') + else: + stages.append('TextToSpeechStage') + stages_changed = True + + if 'speech' not in config: + config['speech'] = DEFAULT_SPEECH_CONFIG.copy() + config_changed = True + + if stages_changed or config_changed: + conn.execute( + sa.update(pipelines_tbl) + .where(pipelines_tbl.c.uuid == p_uuid) + .values(stages=stages, config=config) + ) + + +def downgrade() -> None: + conn = op.get_bind() + inspector = sa.inspect(conn) + existing_tables = set(inspector.get_table_names()) + if 'tts_models' in existing_tables: + op.drop_table('tts_models') + if 'asr_models' in existing_tables: + op.drop_table('asr_models') diff --git a/src/langbot/pkg/persistence/mgr.py b/src/langbot/pkg/persistence/mgr.py index 445af890a..50a24981d 100644 --- a/src/langbot/pkg/persistence/mgr.py +++ b/src/langbot/pkg/persistence/mgr.py @@ -70,6 +70,8 @@ _ALEMBIC_TENANT_TABLES = { 'llm_models', 'embedding_models', 'rerank_models', + 'asr_models', + 'tts_models', 'legacy_pipelines', 'pipeline_run_records', 'pipeline_migration_snapshots', diff --git a/src/langbot/pkg/persistence/tenant_uow.py b/src/langbot/pkg/persistence/tenant_uow.py index 51974d66a..4821fa597 100644 --- a/src/langbot/pkg/persistence/tenant_uow.py +++ b/src/langbot/pkg/persistence/tenant_uow.py @@ -59,6 +59,8 @@ TENANT_TABLE_COLUMNS: dict[str, str] = { 'llm_models': 'workspace_uuid', 'embedding_models': 'workspace_uuid', 'rerank_models': 'workspace_uuid', + 'asr_models': 'workspace_uuid', + 'tts_models': 'workspace_uuid', 'legacy_pipelines': 'workspace_uuid', 'pipeline_run_records': 'workspace_uuid', 'pipeline_migration_snapshots': 'workspace_uuid', diff --git a/src/langbot/pkg/pipeline/pipelinemgr.py b/src/langbot/pkg/pipeline/pipelinemgr.py index 194f61478..baf31619b 100644 --- a/src/langbot/pkg/pipeline/pipelinemgr.py +++ b/src/langbot/pkg/pipeline/pipelinemgr.py @@ -39,6 +39,7 @@ from . import ( wrapper, preproc, ratelimit, + speech, ) importutil.import_modules_in_pkgs( @@ -52,6 +53,7 @@ importutil.import_modules_in_pkgs( wrapper, preproc, ratelimit, + speech, ] ) @@ -915,6 +917,7 @@ class PipelineManager: getattr(self.ap, 'pipeline_config_meta_trigger', {'name': 'trigger', 'stages': []}), getattr(self.ap, 'pipeline_config_meta_safety', {'name': 'safety', 'stages': []}), getattr(self.ap, 'pipeline_config_meta_ai', {'name': 'ai', 'stages': []}), + getattr(self.ap, 'pipeline_config_meta_speech', {'name': 'speech', 'stages': []}), getattr(self.ap, 'pipeline_config_meta_output', {'name': 'output', 'stages': []}), ) diff --git a/src/langbot/pkg/pipeline/speech/__init__.py b/src/langbot/pkg/pipeline/speech/__init__.py new file mode 100644 index 000000000..9d48db4f9 --- /dev/null +++ b/src/langbot/pkg/pipeline/speech/__init__.py @@ -0,0 +1 @@ +from __future__ import annotations diff --git a/src/langbot/pkg/pipeline/speech/speech.py b/src/langbot/pkg/pipeline/speech/speech.py new file mode 100644 index 000000000..6ad99aa23 --- /dev/null +++ b/src/langbot/pkg/pipeline/speech/speech.py @@ -0,0 +1,308 @@ +from __future__ import annotations + +import base64 +import logging +import re +import typing + +import langbot_plugin.api.entities.builtin.pipeline.query as pipeline_query +import langbot_plugin.api.entities.builtin.platform.message as platform_message + +from ...i18n import pipeline_notice +from ...utils import audio as audio_utils +from .. import entities, stage +from ..pool import get_query_execution_context + +logger = logging.getLogger(__name__) + + +def strip_markdown(text: str) -> str: + """Strip markdown formatting from text for TTS input.""" + # Remove code blocks + text = re.sub(r'```[\s\S]*?```', '', text) + # Remove inline code + text = re.sub(r'`([^`]+)`', r'\1', text) + # Remove images: ![alt](url) -> '' + text = re.sub(r'!\[([^\]]*)\]\([^\)]+\)', '', text) + # Replace links: [text](url) -> text + text = re.sub(r'\[([^\]]+)\]\([^\)]+\)', r'\1', text) + # Remove headers (# Heading) + text = re.sub(r'^\s*#{1,6}\s*', '', text, flags=re.MULTILINE) + # Remove bold/italic (**text**, *text*, __text__, _text_) + text = re.sub(r'(\*\*|__)(.*?)\1', r'\2', text) + text = re.sub(r'(\*|_)(.*?)\1', r'\2', text) + # Remove strikethrough (~~text~~) + text = re.sub(r'~~(.*?)~~', r'\1', text) + # Remove blockquotes + text = re.sub(r'^\s*>\s*', '', text, flags=re.MULTILINE) + # Remove horizontal rules + text = re.sub(r'^\s*[-*_]{3,}\s*$', '', text, flags=re.MULTILINE) + # Normalize whitespace + text = re.sub(r'\n{3,}', '\n\n', text).strip() + return text + + +@stage.stage_class('SpeechRecognitionStage') +class SpeechRecognitionStage(stage.PipelineStage): + """Speech Recognition (ASR) Stage + + Transcribes incoming voice messages into plain text and updates the message chain. + """ + + async def initialize(self, pipeline_config: dict): + pass + + async def process( + self, + query: pipeline_query.Query, + stage_inst_name: str, + ) -> entities.StageProcessResult: + config = query.pipeline_config.get('speech', {}).get('speech-recognition', {}) + if not config.get('enable', False): + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + model_uuid = config.get('model') + if not model_uuid: + self.ap.logger.debug('SpeechRecognitionStage enabled but no model configured; skipping.') + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + voice_components = [comp for comp in query.message_chain if isinstance(comp, platform_message.Voice)] + if not voice_components: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + query.variables['_input_was_voice'] = True + primary_voice = voice_components[0] + + max_duration = config.get('max-duration', 60) + if primary_voice.length and primary_voice.length > max_duration: + self.ap.logger.info( + f'Voice message duration ({primary_voice.length}s) exceeds max-duration ({max_duration}s).' + ) + if config.get('on-failure', 'notify') == 'notify': + notice = pipeline_notice( + query, + 'voice-duration-exceeded-hint', + 'Voice message exceeds maximum duration.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.INTERRUPT, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + try: + raw_bytes, source_format = await audio_utils.extract_voice_bytes(primary_voice) + audio_bytes = raw_bytes + target_format = source_format if source_format in ('wav', 'mp3', 'ogg', 'm4a', 'flac') else 'wav' + if source_format != target_format and audio_utils.is_ffmpeg_available(): + audio_bytes = await audio_utils.transcode_audio(raw_bytes, target_format, source_format) + except Exception as exc: + self.ap.logger.error(f'Failed to extract or transcode voice bytes: {exc}') + if config.get('on-failure', 'notify') == 'notify': + notice = pipeline_notice( + query, + 'voice-transcribe-failed-hint', + 'Voice transcription failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.INTERRUPT, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + execution_context = get_query_execution_context(query) + asr_model = await self.ap.model_mgr.get_asr_model_by_uuid(execution_context, model_uuid) + if asr_model is None: + self.ap.logger.error(f'ASR model {model_uuid} not found.') + if config.get('on-failure', 'notify') == 'notify': + notice = pipeline_notice( + query, + 'voice-transcribe-failed-hint', + 'Voice transcription failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.INTERRUPT, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + try: + transcript = await asr_model.provider.invoke_asr( + model=asr_model, + audio=audio_bytes, + mime=f'audio/{target_format}', + execution_context=asr_model.execution_context, + ) + transcript = (transcript or '').strip() + except Exception as exc: + self.ap.logger.error(f'ASR model invocation failed: {exc}') + if config.get('on-failure', 'notify') == 'notify': + notice = pipeline_notice( + query, + 'voice-transcribe-failed-hint', + 'Voice transcription failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.INTERRUPT, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + query.variables['voice_transcript'] = transcript + + keep_original = config.get('keep-original-voice', False) + new_chain = [] + for comp in query.message_chain: + if isinstance(comp, platform_message.Voice): + if keep_original: + new_chain.append(comp) + new_chain.append(platform_message.Plain(text=transcript)) + else: + new_chain.append(comp) + query.message_chain = platform_message.MessageChain(new_chain) + + quote = getattr(query, 'quote', None) + if quote and getattr(quote, 'origin', None): + new_origin = [] + for comp in quote.origin: + if isinstance(comp, platform_message.Voice): + if keep_original: + new_origin.append(comp) + new_origin.append(platform_message.Plain(text=transcript)) + else: + new_origin.append(comp) + quote.origin = platform_message.MessageChain(new_origin) + + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + +@stage.stage_class('TextToSpeechStage') +class TextToSpeechStage(stage.PipelineStage): + """Text to Speech Stage + + Synthesizes outgoing response text into voice and attaches/replaces with Voice component. + """ + + async def initialize(self, pipeline_config: dict): + pass + + async def process( + self, + query: pipeline_query.Query, + stage_inst_name: str, + ) -> entities.StageProcessResult: + config = query.pipeline_config.get('speech', {}).get('text-to-speech', {}) + if not config.get('enable', False): + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + model_uuid = config.get('model') + if not model_uuid: + self.ap.logger.debug('TextToSpeechStage enabled but no model configured; skipping.') + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + trigger = config.get('trigger', 'voice-input-only') + if trigger == 'never': + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + if trigger == 'voice-input-only' and not query.variables.get('_input_was_voice', False): + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + # Skip command messages, tool call announcements, or streaming chunks that are not final + if query.resp_messages: + last_resp = query.resp_messages[-1] + if getattr(last_resp, 'role', None) == 'command': + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + if getattr(last_resp, 'tool_calls', None): + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + if hasattr(last_resp, 'is_final') and not last_resp.is_final: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + if not query.resp_message_chain: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + plain_texts = [comp.text for comp in query.resp_message_chain[-1] if isinstance(comp, platform_message.Plain)] + full_text = ''.join(plain_texts).strip() + if not full_text: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + clean_text = strip_markdown(full_text) + if not clean_text: + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + max_text_length = config.get('max-text-length', 500) + if len(clean_text) > max_text_length: + self.ap.logger.info( + f'Response text length ({len(clean_text)}) exceeds max-text-length ({max_text_length}); falling back to text.' + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + execution_context = get_query_execution_context(query) + tts_model = await self.ap.model_mgr.get_tts_model_by_uuid(execution_context, model_uuid) + if tts_model is None: + self.ap.logger.warning(f'TTS model {model_uuid} not found; falling back to text.') + if config.get('on-failure', 'fallback') == 'notify': + notice = pipeline_notice( + query, + 'voice-synthesis-failed-hint', + 'Voice synthesis failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.CONTINUE, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + model_extra = getattr(tts_model, 'extra_args', None) + if model_extra is None and hasattr(tts_model, 'model_entity') and tts_model.model_entity is not None: + model_extra = getattr(tts_model.model_entity, 'extra_args', {}) + model_extra = model_extra or {} + voice_param = config.get('voice') or model_extra.get('voice') + extra_args: dict[str, typing.Any] = {'response_format': 'mp3'} + if voice_param: + extra_args['voice'] = voice_param + + try: + audio_bytes, mime = await tts_model.provider.invoke_tts( + model=tts_model, + text=clean_text, + extra_args=extra_args, + execution_context=tts_model.execution_context, + ) + except Exception as exc: + self.ap.logger.error(f'TTS invocation failed: {exc}; falling back to text.') + if config.get('on-failure', 'fallback') == 'notify': + notice = pipeline_notice( + query, + 'voice-synthesis-failed-hint', + 'Voice synthesis failed.', + ) + return entities.StageProcessResult( + result_type=entities.ResultType.CONTINUE, + new_query=query, + user_notice=notice, + ) + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) + + b64_audio = base64.b64encode(audio_bytes).decode('ascii') + audio_duration = audio_utils.estimate_audio_duration( + audio_bytes, mime.split('/')[-1] if '/' in mime else 'unknown' + ) + voice_comp = platform_message.Voice( + base64=f'data:{mime};base64,{b64_audio}', + length=int(round(audio_duration)) if audio_duration > 0 else None, + ) + + output_mode = config.get('output', 'voice-only') + if output_mode == 'voice-only': + non_plain_components = [ + comp for comp in query.resp_message_chain[-1] if not isinstance(comp, platform_message.Plain) + ] + query.resp_message_chain[-1] = platform_message.MessageChain([voice_comp] + non_plain_components) + else: + query.resp_message_chain[-1].append(voice_comp) + + return entities.StageProcessResult(result_type=entities.ResultType.CONTINUE, new_query=query) diff --git a/src/langbot/pkg/platform/adapters/aiocqhttp/message_converter.py b/src/langbot/pkg/platform/adapters/aiocqhttp/message_converter.py index a8e2fe2a2..106a3b397 100644 --- a/src/langbot/pkg/platform/adapters/aiocqhttp/message_converter.py +++ b/src/langbot/pkg/platform/adapters/aiocqhttp/message_converter.py @@ -99,7 +99,18 @@ class AiocqhttpMessageConverter(abstract_platform_adapter.AbstractMessageConvert if file_arg: target.append(aiocqhttp.MessageSegment.image(file_arg)) elif isinstance(component, platform_message.Voice): - file_arg = AiocqhttpMessageConverter._file_arg(component) + file_arg = None + from ....utils import audio as audio_utils + + if audio_utils.is_silk_available() and audio_utils.is_ffmpeg_available(): + try: + voice_bytes, src_fmt = await audio_utils.extract_voice_bytes(component) + silk_bytes = await audio_utils.transcode_audio(voice_bytes, 'silk', src_fmt) + file_arg = f'base64://{base64.b64encode(silk_bytes).decode("ascii")}' + except Exception: + pass + if not file_arg: + file_arg = AiocqhttpMessageConverter._file_arg(component) if file_arg: target.append(aiocqhttp.MessageSegment.record(file_arg)) elif isinstance(component, platform_message.File): diff --git a/src/langbot/pkg/platform/adapters/dingtalk/adapter.py b/src/langbot/pkg/platform/adapters/dingtalk/adapter.py index 60a14087d..6dd0f55df 100644 --- a/src/langbot/pkg/platform/adapters/dingtalk/adapter.py +++ b/src/langbot/pkg/platform/adapters/dingtalk/adapter.py @@ -167,7 +167,11 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo incoming_message = message_source.source_platform_object.incoming_message markdown_enabled = self.config.get('markdown_card', False) text_message = platform_message.MessageChain( - [part for part in message if not isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if not isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) content, at = await DingTalkMessageConverter.yiri2target(text_message, markdown_enabled) raw = await self.bot.send_message(content, incoming_message, at) @@ -180,7 +184,7 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo async def _send_attachments(self, message, incoming_message): failures = [] for part in message: - if isinstance(part, (platform_message.Image, platform_message.File)): + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)): try: await self.bot.send_attachment(part, incoming_message) except Exception as exc: @@ -207,7 +211,11 @@ class DingTalkAdapter(DingTalkAPIMixin, abstract_platform_adapter.AbstractPlatfo markdown_enabled = self.config.get('markdown_card', False) text_message = platform_message.MessageChain( - [part for part in message if not isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if not isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) content, _ = await DingTalkMessageConverter.yiri2target(text_message, markdown_enabled) card_instance, card_instance_id = self.card_instance_id_dict[message_id] diff --git a/src/langbot/pkg/platform/adapters/dingtalk/message_converter.py b/src/langbot/pkg/platform/adapters/dingtalk/message_converter.py index 253eed8d1..c9090d13a 100644 --- a/src/langbot/pkg/platform/adapters/dingtalk/message_converter.py +++ b/src/langbot/pkg/platform/adapters/dingtalk/message_converter.py @@ -125,10 +125,11 @@ class DingTalkMessageConverter(abstract_platform_adapter.AbstractMessageConverte if event.file: components.append(platform_message.File(url=event.file, name=event.name or 'file')) if event.audio: - if event.content and event.type == 'audio': - components.append(platform_message.Plain(text=event.content)) - else: - components.append(platform_message.Voice(base64=event.audio)) + # Keep the audio so the pipeline's speech recognition setting decides how it is transcribed. + components.append(platform_message.Voice(base64=event.audio, length=event.audio_duration or None)) + elif event.type == 'audio' and event.content: + # The audio could not be downloaded; fall back to DingTalk's own transcription. + components.append(platform_message.Plain(text=event.content)) quote = DingTalkMessageConverter._quote_component(event) if quote: diff --git a/src/langbot/pkg/platform/adapters/discord/message_converter.py b/src/langbot/pkg/platform/adapters/discord/message_converter.py index 82bb5aa6c..f8faedfa5 100644 --- a/src/langbot/pkg/platform/adapters/discord/message_converter.py +++ b/src/langbot/pkg/platform/adapters/discord/message_converter.py @@ -66,7 +66,35 @@ class DiscordMessageConverter: elements.extend(DiscordMessageConverter._text_components(message.content)) for attachment in message.attachments: - if DiscordMessageConverter._is_image_attachment(attachment): + if DiscordMessageConverter._is_audio_attachment(attachment): + audio_data = b'' + mime_type = getattr(attachment, 'content_type', None) or 'audio/ogg' + try: + audio_data, downloaded_mime = await DiscordMessageConverter._download(attachment.url) + if downloaded_mime: + mime_type = downloaded_mime + except Exception: + pass + from ....utils import audio as audio_utils + + detected_fmt = audio_utils.detect_audio_format(audio_data) if audio_data else 'unknown' + duration = getattr(attachment, 'duration_secs', None) + if duration is not None: + duration = int(round(duration)) + elif audio_data: + duration = int(round(audio_utils.estimate_audio_duration(audio_data, detected_fmt))) + + import base64 as _b64 + + b64_val = _b64.b64encode(audio_data).decode('utf-8') if audio_data else '' + elements.append( + platform_message.Voice( + url=attachment.url, + base64=f'data:{mime_type};base64,{b64_val}' if b64_val else '', + length=duration, + ) + ) + elif DiscordMessageConverter._is_image_attachment(attachment): elements.append(platform_message.Image(url=attachment.url)) else: elements.append( @@ -134,22 +162,22 @@ class DiscordMessageConverter: @staticmethod async def _load_voice(element: platform_message.Voice) -> tuple[bytes | None, str]: - filename = f'{uuid.uuid4()}.mp3' - if element.base64: - header, _, payload = element.base64.partition(',') - data = payload or header - for ext in ('wav', 'mp3', 'ogg', 'm4a', 'aac', 'flac', 'opus', 'webm'): - if ext in header: - filename = f'{uuid.uuid4()}.{ext}' - break - return await asyncio.to_thread(_decode_discord_base64_limited, data), filename - if element.url: - data, _ = await DiscordMessageConverter._download(element.url) - return data, filename - return None, filename + from ....utils import audio as audio_utils + + raw_bytes = await DiscordMessageConverter._load_file(element) + fmt = audio_utils.detect_audio_format(raw_bytes) if raw_bytes else 'unknown' + if fmt in ('silk', 'amr') and audio_utils.is_ffmpeg_available() and raw_bytes: + try: + raw_bytes = await audio_utils.transcode_audio(raw_bytes, 'mp3', fmt) + fmt = 'mp3' + except Exception: + pass + ext = fmt if fmt != 'unknown' else 'mp3' + filename = getattr(element, 'name', '') or f'{uuid.uuid4()}.{ext}' + return raw_bytes, filename @staticmethod - async def _load_file(element: platform_message.File) -> bytes | None: + async def _load_file(element: platform_message.File | platform_message.Voice) -> bytes | None: if element.base64: return await asyncio.to_thread(_decode_discord_base64_limited, element.base64) if element.url: @@ -169,7 +197,20 @@ class DiscordMessageConverter: @staticmethod def _is_image_attachment(attachment: discord.Attachment) -> bool: - content_type = attachment.content_type or '' - return content_type.startswith('image/') or attachment.filename.lower().endswith( + content_type = getattr(attachment, 'content_type', '') or '' + filename = getattr(attachment, 'filename', '') or '' + return content_type.startswith('image/') or filename.lower().endswith( ('.png', '.jpg', '.jpeg', '.gif', '.webp') ) + + @staticmethod + def _is_audio_attachment(attachment: discord.Attachment) -> bool: + content_type = getattr(attachment, 'content_type', '') or '' + filename = (getattr(attachment, 'filename', '') or '').lower() + flags = getattr(attachment, 'flags', None) + is_voice = getattr(flags, 'voice', False) if flags else False + return ( + is_voice + or content_type.startswith('audio/') + or filename.endswith(('.mp3', '.ogg', '.oga', '.opus', '.wav', '.m4a', '.aac', '.flac', '.weba')) + ) diff --git a/src/langbot/pkg/platform/adapters/lark/message_converter.py b/src/langbot/pkg/platform/adapters/lark/message_converter.py index 40db36bde..797ac1af3 100644 --- a/src/langbot/pkg/platform/adapters/lark/message_converter.py +++ b/src/langbot/pkg/platform/adapters/lark/message_converter.py @@ -132,6 +132,13 @@ class LarkMessageConverter(abstract_platform_adapter.AbstractMessageConverter): elif isinstance(msg, platform_message.Voice): data = await LarkMessageConverter._get_component_bytes(msg) if data: + from ....utils import audio as audio_utils + + if audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'opus') + except Exception: + pass duration = int(msg.length * 1000) if msg.length else None file_key = await LarkMessageConverter.upload_file_to_lark( data, api_client, file_type='opus', file_name='voice.opus', duration=duration diff --git a/src/langbot/pkg/platform/adapters/telegram/adapter.py b/src/langbot/pkg/platform/adapters/telegram/adapter.py index 5d6519267..4f1664726 100644 --- a/src/langbot/pkg/platform/adapters/telegram/adapter.py +++ b/src/langbot/pkg/platform/adapters/telegram/adapter.py @@ -239,6 +239,14 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo filename = component.get('filename', 'file') args['document'] = telegram.InputFile(doc, filename=filename) sent_messages.append(await self.bot.send_document(**args)) + elif component_type == 'voice': + voice = component.get('voice') + if voice is None: + continue + args['voice'] = telegram.InputFile(voice, filename='voice.ogg') + if component.get('duration'): + args['duration'] = component['duration'] + sent_messages.append(await self.bot.send_voice(**args)) return platform_events.MessageResult( message_id=sent_messages[-1].message_id if sent_messages else None, @@ -290,6 +298,14 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo filename = component.get('filename', 'file') args['document'] = telegram.InputFile(doc, filename=filename) sent_messages.append(await self.bot.send_document(**args)) + elif component_type == 'voice': + voice = component.get('voice') + if voice is None: + continue + args['voice'] = telegram.InputFile(voice, filename='voice.ogg') + if component.get('duration'): + args['duration'] = component['duration'] + sent_messages.append(await self.bot.send_voice(**args)) return platform_events.MessageResult( message_id=sent_messages[-1].message_id if sent_messages else None, @@ -382,7 +398,11 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo # through the regular reply path, including attachment-only completions. if is_final and not bot_message.tool_calls: attachments = platform_message.MessageChain( - [part for part in message if isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) if attachments: await self.reply_message(message_source, attachments, quote_origin=quote_origin) @@ -425,6 +445,8 @@ class TelegramAdapter(TelegramAPIMixin, abstract_platform_adapter.AbstractPlatfo if is_final and bot_message.tool_calls is None: # Finalise: send the real message, discard the draft args = self._build_message_args(chat_id, content, message_thread_id) + if quote_origin and effective_message: + args['reply_to_message_id'] = effective_message.id try: await self.bot.send_message(**args) except telegram.error.BadRequest as exc: diff --git a/src/langbot/pkg/platform/adapters/wecombot/adapter.py b/src/langbot/pkg/platform/adapters/wecombot/adapter.py index 90ee727e7..09f7c8e16 100644 --- a/src/langbot/pkg/platform/adapters/wecombot/adapter.py +++ b/src/langbot/pkg/platform/adapters/wecombot/adapter.py @@ -137,26 +137,88 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo _iter_media_components = staticmethod(LegacyWecomBotAdapter._iter_media_components) @staticmethod - async def _send_media(bot, req_id: str, item: dict) -> bool: + async def _send_media(bot, req_id: str, item: dict, config: dict | None = None) -> bool: """Require upload and delivery acknowledgements instead of dropping attachments.""" kind = item['type'] payload = item.get('base64') or '' - if payload.startswith('data:'): - payload = payload.split(',', 1)[-1] - try: - data = base64.b64decode(payload, validate=True) - except ValueError as exc: - raise ValueError(f'Invalid {kind} attachment data') from exc + data = b'' + if payload: + if payload.startswith('data:'): + payload = payload.split(',', 1)[-1] + try: + data = base64.b64decode(payload, validate=False) + except Exception as exc: + raise ValueError(f'Invalid {kind} attachment data') from exc + elif item.get('path'): + with open(item['path'], 'rb') as f: + data = f.read() + if not data: raise ValueError(f'{kind} attachment has no data to upload') - result = await bot.upload_media(data, item.get('name') or f'attachment.{kind}', media_type=kind) - media_id = result.get('media_id') if isinstance(result, dict) else None + + media_id = None + delivery_kind = kind + filename = item.get('name') + voice_mode = (config or {}).get('voice_delivery', 'voice') + + if kind == 'voice': + from langbot.pkg.utils import audio as audio_utils + + fmt = audio_utils.detect_audio_format(data) + duration = audio_utils.estimate_audio_duration(data, fmt) if data else 0 + + if voice_mode == 'file' or (duration and duration > 60): + delivery_kind = 'file' + if fmt in ('silk', 'amr') and audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'mp3', fmt) + fmt = 'mp3' + except Exception: + pass + filename = filename or f'voice.{fmt if fmt != "unknown" else "mp3"}' + else: + # WeCom native voice message ONLY supports SILK or AMR format. + # If audio is MP3/WAV/etc. (e.g. from TTS), transcode it to SILK + # so the WeCom client can play the native voice bubble. + if fmt != 'silk' and fmt != 'amr': + if audio_utils.is_silk_available() and audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'silk', fmt) + fmt = 'silk' + except Exception: + pass + + if fmt == 'silk': + delivery_kind = 'voice' + filename = filename or 'voice.silk' + elif fmt == 'amr': + delivery_kind = 'voice' + filename = filename or 'voice.amr' + elif fmt in ('mp3', 'wav', 'ogg', 'flac'): + delivery_kind = 'file' + filename = filename or f'voice.{fmt}' + else: + delivery_kind = 'voice' + filename = filename or 'voice.silk' + else: + filename = filename or f'attachment.{kind}' + + result = await bot.upload_media(data, filename, media_type=delivery_kind) + if isinstance(result, dict): + media_id = result.get('media_id') + if not media_id and delivery_kind == 'voice': + # Fall back to uploading as audio file if voice-type upload is not accepted + delivery_kind = 'file' + fallback_filename = filename.rsplit('.', 1)[0] + '.mp3' if filename else 'voice.mp3' + result = await bot.upload_media(data, fallback_filename, media_type='file') + if isinstance(result, dict): + media_id = result.get('media_id') if not media_id: raise RuntimeError(f'WeCom {kind} upload failed: no media ID returned') - ack = await getattr(bot, f'reply_{kind}')(req_id, media_id) + ack = await getattr(bot, f'reply_{delivery_kind}')(req_id, media_id) if ack is None or ack.get('errcode', 0) != 0: code = ack.get('errcode') if isinstance(ack, dict) else 'no acknowledgement' - raise RuntimeError(f'WeCom {kind} delivery failed: {code}') + raise RuntimeError(f'WeCom {delivery_kind} delivery failed: {code}') return True async def send_message( @@ -225,7 +287,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo if not finished: raw = await self.bot.reply_text(event.get('req_id'), content) for item in self._iter_media_components(items): - await self._send_media(self.bot, event.get('req_id'), item) + await self._send_media(self.bot, event.get('req_id'), item, self.config) else: raw = await self.bot.set_message(event.message_id, content) return platform_events.MessageResult(message_id=event.message_id, raw={'result': raw}) @@ -253,7 +315,7 @@ class WecomBotAdapter(WecomBotAPIMixin, abstract_platform_adapter.AbstractPlatfo if is_final: if not self.config.get('enable-webhook', False) and event.get('req_id'): for item in self._iter_media_components(items): - await self._send_media(self.bot, event.get('req_id'), item) + await self._send_media(self.bot, event.get('req_id'), item, self.config) elif not success: await self.bot.set_message(event.message_id, content) return {'stream': success} diff --git a/src/langbot/pkg/platform/adapters/wecombot/manifest.yaml b/src/langbot/pkg/platform/adapters/wecombot/manifest.yaml index d5b7c1eeb..5b3f18c5f 100644 --- a/src/langbot/pkg/platform/adapters/wecombot/manifest.yaml +++ b/src/langbot/pkg/platform/adapters/wecombot/manifest.yaml @@ -170,6 +170,29 @@ spec: type: boolean required: false default: true + - name: voice_delivery + label: + en_US: Voice Delivery Mode + zh_Hans: 语音发送模式 + zh_Hant: 語音傳送模式 + description: + en_US: Choose how voice responses are delivered. 'voice' sends native voice messages (requires SILK transcoding). 'file' sends audio files. + zh_Hans: 选择语音回复的发送方式。'voice' 为企微原生语音条(默认自动转码为 SILK),'file' 为音频文件卡片。 + zh_Hant: 選擇語音回覆的傳送方式。'voice' 為企微原生語音條(預設自動轉碼為 SILK),'file' 為音訊檔案卡片。 + type: select + options: + - label: + en_US: Native Voice (Recommended) + zh_Hans: 原生语音条(推荐) + zh_Hant: 原生語音條(推薦) + value: voice + - label: + en_US: Audio File + zh_Hans: 音频文件 + zh_Hant: 音訊檔案 + value: file + required: false + default: voice supported_events: - message.received diff --git a/src/langbot/pkg/platform/adapters/wecombot/message_converter.py b/src/langbot/pkg/platform/adapters/wecombot/message_converter.py index 95ce52202..9c13e9c5e 100644 --- a/src/langbot/pkg/platform/adapters/wecombot/message_converter.py +++ b/src/langbot/pkg/platform/adapters/wecombot/message_converter.py @@ -112,13 +112,46 @@ class WecomBotMessageConverter(abstract_platform_adapter.AbstractMessageConverte voice_payload = voice_info.get('base64') or voice_info.get('url') if not voice_payload: return + length = voice_info.get('length') + import base64 + from langbot.pkg.utils import audio as audio_utils + if voice_info.get('base64') and not voice_payload.startswith('data:'): - voice_payload = f'data:audio/mpeg;base64,{voice_info.get("base64")}' + try: + raw_bytes = base64.b64decode(voice_info.get('base64'), validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + if fmt in ('amr', 'silk') and audio_utils.is_ffmpeg_available(): + try: + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + except Exception: + voice_payload = f'data:audio/{fmt};base64,{voice_info.get("base64")}' + else: + mime = 'audio/mpeg' if fmt == 'mp3' else ('audio/wav' if fmt == 'wav' else f'audio/{fmt}') + voice_payload = f'data:{mime};base64,{voice_info.get("base64")}' + except Exception: + voice_payload = f'data:audio/mpeg;base64,{voice_info.get("base64")}' + elif voice_payload.startswith('data:'): + header, _, b64_body = voice_payload.partition(',') + if ('amr' in header or 'silk' in header or 'octet-stream' in header) and audio_utils.is_ffmpeg_available(): + try: + raw_bytes = base64.b64decode(b64_body, validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if fmt in ('amr', 'silk'): + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + except Exception: + pass + try: if voice_payload.startswith('data:'): - components.append(platform_message.Voice(base64=voice_payload)) + components.append(platform_message.Voice(base64=voice_payload, length=length)) else: - components.append(platform_message.Voice(url=voice_payload)) + components.append(platform_message.Voice(url=voice_payload, length=length)) except Exception: components.append(platform_message.Unknown(text='[voice message unsupported]')) diff --git a/src/langbot/pkg/platform/sources/dingtalk.py b/src/langbot/pkg/platform/sources/dingtalk.py index a8fd2349d..7e3efcbcd 100644 --- a/src/langbot/pkg/platform/sources/dingtalk.py +++ b/src/langbot/pkg/platform/sources/dingtalk.py @@ -96,11 +96,11 @@ class DingTalkMessageConverter(abstract_platform_adapter.AbstractMessageConverte if event.file: yiri_msg_list.append(platform_message.File(url=event.file, name=event.name)) if event.audio: - # 优先使用钉钉自带的语音转写文字(recognition字段) - if event.content and event.type == 'audio': - yiri_msg_list.append(platform_message.Plain(text=event.content)) - else: - yiri_msg_list.append(platform_message.Voice(base64=event.audio)) + # Keep the audio so the pipeline's speech recognition setting decides how it is transcribed. + yiri_msg_list.append(platform_message.Voice(base64=event.audio, length=event.audio_duration or None)) + elif event.type == 'audio' and event.content: + # The audio could not be downloaded; fall back to DingTalk's own transcription. + yiri_msg_list.append(platform_message.Plain(text=event.content)) # Handle quoted/replied message - extract content as top-level components # so that plugins like FileReader can process them the same way as direct messages @@ -614,12 +614,16 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): markdown_enabled = self.config.get('markdown_card', False) text_message = platform_message.MessageChain( - [part for part in message if not isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if not isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) content, at = await DingTalkMessageConverter.yiri2target(text_message, markdown_enabled) await self.bot.send_message(content, incoming_message, at) for part in message: - if isinstance(part, (platform_message.Image, platform_message.File)): + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)): await self.bot.send_attachment(part, incoming_message) async def reply_message_chunk( @@ -648,7 +652,11 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if (msg_seq - 1) % 8 == 0 or is_final: markdown_enabled = self.config.get('markdown_card', False) text_message = platform_message.MessageChain( - [part for part in message if not isinstance(part, (platform_message.Image, platform_message.File))] + [ + part + for part in message + if not isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] ) content, at = await DingTalkMessageConverter.yiri2target(text_message, markdown_enabled) if not content and bot_message.content: @@ -721,7 +729,7 @@ class DingTalkAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): self.card_instance_id_dict.pop(message_id, None) event = await DingTalkEventConverter.yiri2target(message_source) for part in message: - if isinstance(part, (platform_message.Image, platform_message.File)): + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)): await self.bot.send_attachment(part, event.incoming_message) async def send_message(self, target_type: str, target_id: str, message: platform_message.MessageChain): diff --git a/src/langbot/pkg/platform/sources/discord.py b/src/langbot/pkg/platform/sources/discord.py index 1d9a29234..610d51482 100644 --- a/src/langbot/pkg/platform/sources/discord.py +++ b/src/langbot/pkg/platform/sources/discord.py @@ -659,33 +659,15 @@ class DiscordMessageConverter(abstract_platform_adapter.AbstractMessageConverter text_string += ele.text elif isinstance(ele, platform_message.Voice): file_bytes = None - filename = f'{uuid.uuid4()}.mp3' + filename = getattr(ele, 'name', '') or '' if ele.base64: - if ele.base64.startswith('data:'): - data_header = ele.base64.split(',')[0] - if 'wav' in data_header: - filename = f'{uuid.uuid4()}.wav' - elif 'mp3' in data_header: - filename = f'{uuid.uuid4()}.mp3' - elif 'ogg' in data_header: - filename = f'{uuid.uuid4()}.ogg' - elif 'm4a' in data_header: - filename = f'{uuid.uuid4()}.m4a' - elif 'aac' in data_header: - filename = f'{uuid.uuid4()}.aac' - elif 'flac' in data_header: - filename = f'{uuid.uuid4()}.flac' - elif 'alac' in data_header: - filename = f'{uuid.uuid4()}.alac' - elif 'opus' in data_header: - filename = f'{uuid.uuid4()}.opus' - elif 'webm' in data_header: - filename = f'{uuid.uuid4()}.webm' - file_bytes = await asyncio.to_thread( _decode_discord_base64_limited, ele.base64, ) + elif getattr(ele, 'path', None): + with open(ele.path, 'rb') as f: + file_bytes = await asyncio.to_thread(f.read) elif ele.url: session = httpclient.get_session() async with session.get(ele.url) as response: @@ -694,6 +676,19 @@ class DiscordMessageConverter(abstract_platform_adapter.AbstractMessageConverter max_bytes=_MAX_DISCORD_MEDIA_BYTES, ) if file_bytes: + from langbot.pkg.utils import audio as audio_utils + + fmt = audio_utils.detect_audio_format(file_bytes) + # If format is silk or amr, Discord client cannot decode it; transcode to mp3 + if fmt in ('silk', 'amr') and audio_utils.is_ffmpeg_available(): + try: + file_bytes = await audio_utils.transcode_audio(file_bytes, 'mp3', fmt) + fmt = 'mp3' + except Exception: + pass + if not filename: + ext = fmt if fmt != 'unknown' else 'mp3' + filename = f'{uuid.uuid4()}.{ext}' files.append(discord.File(fp=io.BytesIO(file_bytes), filename=filename)) elif isinstance(ele, platform_message.File): file_bytes = None @@ -767,16 +762,68 @@ class DiscordMessageConverter(abstract_platform_adapter.AbstractMessageConverter for attachment in message.attachments: session = httpclient.get_session(trust_env=True) async with session.get(attachment.url) as response: - image_data = await httpclient.read_limited( + media_data = await httpclient.read_limited( response, max_bytes=_MAX_DISCORD_MEDIA_BYTES, ) - image_base64 = (await asyncio.to_thread(base64.b64encode, image_data)).decode('utf-8') - image_format = response.headers['Content-Type'] - element_list.append( - platform_message.Image(url=attachment.url, base64=f'data:{image_format};base64,{image_base64}') + media_base64 = (await asyncio.to_thread(base64.b64encode, media_data)).decode('utf-8') + content_type = response.headers.get('Content-Type') or getattr(attachment, 'content_type', None) or '' + filename = (getattr(attachment, 'filename', '') or '').lower() + flags = getattr(attachment, 'flags', None) + is_voice_flag = getattr(flags, 'voice', False) if flags else False + + from langbot.pkg.utils import audio as audio_utils + + detected_fmt = audio_utils.detect_audio_format(media_data) + is_audio = ( + is_voice_flag + or content_type.startswith('audio/') + or any( + filename.endswith(ext) + for ext in ('.mp3', '.ogg', '.oga', '.opus', '.wav', '.m4a', '.aac', '.flac', '.weba') + ) + or detected_fmt in ('wav', 'mp3', 'ogg', 'opus', 'flac', 'silk', 'amr') ) + if is_audio: + mime = ( + content_type + if content_type.startswith('audio/') + else ( + 'audio/ogg' + if detected_fmt in ('ogg', 'opus') + else f'audio/{detected_fmt if detected_fmt != "unknown" else "mpeg"}' + ) + ) + duration = getattr(attachment, 'duration_secs', None) + if duration is not None: + duration = int(round(duration)) + elif media_data: + duration = int(round(audio_utils.estimate_audio_duration(media_data, detected_fmt))) + element_list.append( + platform_message.Voice( + url=attachment.url, + base64=f'data:{mime};base64,{media_base64}', + length=duration, + ) + ) + elif content_type.startswith('image/') or any( + filename.endswith(ext) for ext in ('.png', '.jpg', '.jpeg', '.gif', '.webp', '.bmp', '.svg') + ): + mime = content_type if content_type.startswith('image/') else 'image/png' + element_list.append( + platform_message.Image(url=attachment.url, base64=f'data:{mime};base64,{media_base64}') + ) + else: + mime = content_type or 'application/octet-stream' + element_list.append( + platform_message.File( + url=attachment.url, + base64=f'data:{mime};base64,{media_base64}', + name=getattr(attachment, 'filename', 'file'), + ) + ) + return platform_message.MessageChain(element_list) diff --git a/src/langbot/pkg/platform/sources/lark.py b/src/langbot/pkg/platform/sources/lark.py index 4dc600c8f..b38fb9331 100644 --- a/src/langbot/pkg/platform/sources/lark.py +++ b/src/langbot/pkg/platform/sources/lark.py @@ -600,6 +600,13 @@ class LarkMessageConverter(abstract_platform_adapter.AbstractMessageConverter): elif isinstance(msg, platform_message.Voice): data = await LarkMessageConverter._get_media_bytes(msg) if data: + from ...utils import audio as audio_utils + + if audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'opus') + except Exception: + pass duration = int(msg.length * 1000) if msg.length else None file_key = await LarkMessageConverter.upload_file_to_lark( data, api_client, file_type='opus', file_name='voice.opus', duration=duration diff --git a/src/langbot/pkg/platform/sources/openclaw_weixin.py b/src/langbot/pkg/platform/sources/openclaw_weixin.py index b4c9dc270..3b4751eef 100644 --- a/src/langbot/pkg/platform/sources/openclaw_weixin.py +++ b/src/langbot/pkg/platform/sources/openclaw_weixin.py @@ -12,6 +12,7 @@ import asyncio import base64 import contextlib import os +import logging import traceback import typing @@ -39,6 +40,7 @@ import langbot_plugin.api.entities.builtin.platform.message as platform_message from langbot.pkg.api.http.context import ExecutionContext _MAX_OPENCLAW_COMPONENT_BYTES = 10 * 1024 * 1024 +logger = logging.getLogger('langbot.openclaw-weixin') class OpenClawWeixinMessageConverter(abstract_platform_adapter.AbstractMessageConverter): @@ -130,31 +132,32 @@ class OpenClawWeixinMessageConverter(abstract_platform_adapter.AbstractMessageCo components.append(platform_message.Unknown(text='[Image]')) elif item.type == MessageItem.VOICE and item.voice_item: - # Voice with speech-to-text: use the transcribed text - if item.voice_item.text: + if hasattr(item.voice_item, '_downloaded_bytes') and item.voice_item._downloaded_bytes: + b64 = ( + await asyncio.to_thread( + base64.b64encode, + item.voice_item._downloaded_bytes, + ) + ).decode('utf-8') + playtime = item.voice_item.playtime or 0 + length_sec = int(round(playtime / 1000.0)) if playtime > 0 else 0 + if length_sec == 0: + from langbot.pkg.utils import audio as audio_utils + + dur = audio_utils.estimate_audio_duration(item.voice_item._downloaded_bytes, 'silk') + length_sec = int(round(dur)) if dur > 0 else 0 + + components.append( + platform_message.Voice( + base64=f'data:audio/silk;base64,{b64}', + length=length_sec, + ) + ) + elif item.voice_item.text: components.append(platform_message.Plain(text=item.voice_item.text)) else: components.append(platform_message.Unknown(text='[Voice]')) - # TODO: enable after full testing - # elif item.type == MessageItem.VOICE and item.voice_item: - # if item.voice_item.text: - # components.append(platform_message.Plain(text=item.voice_item.text)) - # elif hasattr(item.voice_item, '_downloaded_bytes') and item.voice_item._downloaded_bytes: - # b64 = base64.b64encode(item.voice_item._downloaded_bytes).decode('utf-8') - # components.append( - # platform_message.Voice( - # base64=b64, - # length=item.voice_item.playtime or 0, - # ) - # ) - # else: - # components.append( - # platform_message.Voice( - # length=item.voice_item.playtime or 0, - # ) - # ) - elif item.type == MessageItem.FILE and item.file_item: components.append(platform_message.Unknown(text=f'[File: {item.file_item.file_name or ""}]')) @@ -389,7 +392,123 @@ class OpenClawWeixinAdapter(abstract_platform_adapter.AbstractMessagePlatformAda elif isinstance(component, platform_message.Voice): voice_bytes = await self._get_component_bytes(component) if voice_bytes: - await self.client.send_voice(target_id, voice_bytes, component.length or 0, context_token) + from langbot.pkg.utils import audio as audio_utils + + audio_fmt = audio_utils.detect_audio_format(voice_bytes) + delivery_mode = self.config.get('voice_delivery', 'file') + + if delivery_mode == 'voice': + duration_sec = component.length or audio_utils.estimate_audio_duration( + voice_bytes, audio_fmt + ) + playtime_ms = int(duration_sec * 1000) if duration_sec > 0 else 0 + + if audio_fmt == 'silk': + if playtime_ms <= 0: + silk_dur = audio_utils.estimate_audio_duration(voice_bytes, 'silk') + playtime_ms = int(silk_dur * 1000) if silk_dur > 0 else 1000 + + await self.client.send_voice( + target_id, + voice_bytes, + playtime=playtime_ms, + encode_type=6, + sample_rate=24000, + bits_per_sample=16, + context_token=context_token, + ) + elif audio_utils.is_silk_available() and audio_utils.is_ffmpeg_available(): + silk_bytes = None + try: + silk_bytes = await audio_utils.transcode_audio(voice_bytes, 'silk', audio_fmt) + except Exception as exc: + logger.warning( + f'Failed to transcode voice to SILK: {exc}; falling back to audio file.' + ) + await self.logger.warning( + f'Failed to transcode voice to SILK: {exc}; gracefully falling back to audio file.' + ) + + if silk_bytes: + try: + if playtime_ms <= 0: + silk_dur = audio_utils.estimate_audio_duration(silk_bytes, 'silk') + playtime_ms = int(silk_dur * 1000) if silk_dur > 0 else 1000 + + await self.client.send_voice( + target_id, + silk_bytes, + playtime=playtime_ms, + encode_type=6, + sample_rate=24000, + bits_per_sample=16, + context_token=context_token, + ) + logger.info( + f'Successfully sent SILK voice message to {target_id} (playtime={playtime_ms}ms)' + ) + except Exception as exc: + logger.warning( + f'Failed to send WeChat voice: {exc}; falling back to audio file.' + ) + await self.logger.warning( + f'Failed to send WeChat voice: {exc}; gracefully falling back to audio file.' + ) + # Deliver as playable MP3 file + out_bytes = voice_bytes + if audio_fmt != 'mp3' and audio_utils.is_ffmpeg_available(): + try: + out_bytes = await audio_utils.transcode_audio( + voice_bytes, 'mp3', audio_fmt + ) + except Exception: + pass + await self.client.send_file( + target_id, + out_bytes, + 'voice.mp3', + context_token, + ) + else: + out_bytes = voice_bytes + if audio_fmt != 'mp3' and audio_utils.is_ffmpeg_available(): + try: + out_bytes = await audio_utils.transcode_audio(voice_bytes, 'mp3', audio_fmt) + except Exception: + pass + await self.client.send_file( + target_id, + out_bytes, + 'voice.mp3', + context_token, + ) + else: + out_bytes = voice_bytes + if audio_fmt != 'mp3' and audio_utils.is_ffmpeg_available(): + try: + out_bytes = await audio_utils.transcode_audio(voice_bytes, 'mp3', audio_fmt) + except Exception: + pass + await self.client.send_file( + target_id, + out_bytes, + 'voice.mp3', + context_token, + ) + else: + # Standard OpenClaw behavior: deliver as playable MP3 audio file card + out_bytes = voice_bytes + if audio_fmt != 'mp3' and audio_utils.is_ffmpeg_available(): + try: + out_bytes = await audio_utils.transcode_audio(voice_bytes, 'mp3', audio_fmt) + except Exception: + pass + await self.client.send_file( + target_id, + out_bytes, + 'voice.mp3', + context_token, + ) elif isinstance(component, platform_message.Forward): for node in component.node_list: @@ -426,6 +545,8 @@ class OpenClawWeixinAdapter(abstract_platform_adapter.AbstractMessagePlatformAda path_val = getattr(component, 'path', None) if b64_val: + if b64_val.startswith('data:'): + _, _, b64_val = b64_val.partition(',') max_encoded_chars = 4 * ((_MAX_OPENCLAW_COMPONENT_BYTES + 2) // 3) + 4 if len(b64_val) > max_encoded_chars: raise ValueError('OpenClaw media exceeds the size limit') @@ -618,17 +739,17 @@ class OpenClawWeixinAdapter(abstract_platform_adapter.AbstractMessagePlatformAda img_bytes = await self.client.download_media(item.image_item.media) item.image_item._downloaded_bytes = img_bytes + elif item.type == MessageItem.VOICE and item.voice_item and item.voice_item.media: + if item.voice_item.media.encrypt_query_param and item.voice_item.media.aes_key: + voice_bytes = await self.client.download_media(item.voice_item.media) + item.voice_item._downloaded_bytes = voice_bytes + # TODO: enable after full testing # elif item.type == MessageItem.FILE and item.file_item and item.file_item.media: # if item.file_item.media.encrypt_query_param and item.file_item.media.aes_key: # file_bytes = await self.client.download_media(item.file_item.media) # item.file_item._downloaded_bytes = file_bytes # - # elif item.type == MessageItem.VOICE and item.voice_item and item.voice_item.media: - # if item.voice_item.media.encrypt_query_param and item.voice_item.media.aes_key: - # voice_bytes = await self.client.download_media(item.voice_item.media) - # item.voice_item._downloaded_bytes = voice_bytes - # # elif item.type == MessageItem.VIDEO and item.video_item and item.video_item.media: # if item.video_item.media.encrypt_query_param and item.video_item.media.aes_key: # video_bytes = await self.client.download_media(item.video_item.media) diff --git a/src/langbot/pkg/platform/sources/openclaw_weixin.yaml b/src/langbot/pkg/platform/sources/openclaw_weixin.yaml index 963b6da7a..5335cb925 100644 --- a/src/langbot/pkg/platform/sources/openclaw_weixin.yaml +++ b/src/langbot/pkg/platform/sources/openclaw_weixin.yaml @@ -82,6 +82,29 @@ spec: type: integer required: false default: 35 + - name: voice_delivery + label: + en_US: Voice Delivery Format + zh_Hans: 语音发送方式 + zh_Hant: 語音發送方式 + description: + en_US: Choose whether to send audio as a playable audio file card (recommended, as WeChat personal bots often silently drop native voice bubbles) or attempt native SILK voice bubble. + zh_Hans: 选择以可播放音频文件卡片发送(推荐,微信官方个人助手常静默丢弃机器人的原生语音条),或尝试原生 SILK 语音条。 + zh_Hant: 選擇以可播放音訊檔案卡片發送(推薦,微信官方個人助手常靜默丟棄機器人的原生語音條),或嘗試原生 SILK 語音條。 + type: select + options: + - label: + en_US: Playable Audio File (Recommended) + zh_Hans: 可播放音频文件(推荐,稳定投递) + zh_Hant: 可播放音訊檔案(推薦,穩定投遞) + value: file + - label: + en_US: Native Voice Bubble (Experimental) + zh_Hans: 原生语音条(实验性,可能被微信平台静默拦截) + zh_Hant: 原生語音條(實驗性,可能被微信平台靜默攔截) + value: voice + required: false + default: file execution: python: path: ./openclaw_weixin.py diff --git a/src/langbot/pkg/platform/sources/telegram.py b/src/langbot/pkg/platform/sources/telegram.py index 92aa91705..794e71299 100644 --- a/src/langbot/pkg/platform/sources/telegram.py +++ b/src/langbot/pkg/platform/sources/telegram.py @@ -116,6 +116,24 @@ class TelegramMessageConverter(abstract_platform_adapter.AbstractMessageConverte photo_bytes, _mime_type = await component.get_bytes() components.append({'type': 'photo', 'photo': photo_bytes}) + elif isinstance(component, platform_message.Voice): + from ...utils import audio as audio_utils + + voice_bytes, src_fmt = await audio_utils.extract_voice_bytes(component) + if audio_utils.is_ffmpeg_available(): + try: + voice_bytes = await audio_utils.transcode_audio(voice_bytes, 'ogg', src_fmt) + except Exception: + pass + duration = component.length + if duration is not None: + duration = int(round(duration)) + elif voice_bytes: + try: + duration = int(round(audio_utils.estimate_audio_duration(voice_bytes, 'ogg'))) + except Exception: + duration = None + components.append({'type': 'voice', 'voice': voice_bytes, 'duration': duration}) elif isinstance(component, platform_message.File): file_bytes = None @@ -542,6 +560,14 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): filename = component.get('filename', 'file') args['document'] = telegram.InputFile(doc, filename=filename) await self.bot.send_document(**args) + elif component_type == 'voice': + voice = component.get('voice') + if voice is None: + continue + args['voice'] = telegram.InputFile(voice, filename='voice.ogg') + if component.get('duration'): + args['duration'] = component['duration'] + await self.bot.send_voice(**args) async def reply_message( self, @@ -552,28 +578,53 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): assert isinstance(message_source.source_platform_object, Update) components = await TelegramMessageConverter.yiri2target(message, self.bot) + update = message_source.source_platform_object + chat_id = update.effective_chat.id + effective_message = getattr(update, 'effective_message', None) or getattr(update, 'message', None) + message_thread_id = getattr(effective_message, 'message_thread_id', None) if effective_message else None + reply_to_message_id = effective_message.id if (quote_origin and effective_message) else None + + sent_messages = [] for component in components: - if component['type'] == 'text': - if self.config['markdown_card'] is True: - content = telegramify_markdown.markdownify( - content=component['text'], - ) - else: - content = component['text'] - args = { - 'chat_id': message_source.source_platform_object.effective_chat.id, - 'text': content, - } - if self.config['markdown_card'] is True: + component_type = component.get('type') + args = {'chat_id': chat_id} + + if message_thread_id: + args['message_thread_id'] = message_thread_id + + if reply_to_message_id: + args['reply_to_message_id'] = reply_to_message_id + + if component_type == 'text': + content = component.get('text', '') + if self.config.get('markdown_card', False): + content = telegramify_markdown.markdownify(content=content) args['parse_mode'] = 'MarkdownV2' + args['text'] = content + sent_messages.append(await self.bot.send_message(**args)) + elif component_type == 'photo': + photo = component.get('photo') + if photo is None: + continue + args['photo'] = telegram.InputFile(photo) + sent_messages.append(await self.bot.send_photo(**args)) + elif component_type == 'document': + doc = component.get('document') + if doc is None: + continue + filename = component.get('filename', 'file') + args['document'] = telegram.InputFile(doc, filename=filename) + sent_messages.append(await self.bot.send_document(**args)) + elif component_type == 'voice': + voice = component.get('voice') + if voice is None: + continue + args['voice'] = telegram.InputFile(voice, filename='voice.ogg') + if component.get('duration'): + args['duration'] = component['duration'] + sent_messages.append(await self.bot.send_voice(**args)) - if message_source.source_platform_object.message.message_thread_id: - args['message_thread_id'] = message_source.source_platform_object.message.message_thread_id - - if quote_origin: - args['reply_to_message_id'] = message_source.source_platform_object.message.id - - await self.bot.send_message(**args) + return sent_messages def _process_markdown(self, text: str) -> str: if self.config.get('markdown_card', False): @@ -589,7 +640,7 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): return args async def _delete_group_stream_message(self, chat_mode: str, chat_id: int, stream_id: int | None): - if chat_mode != 'group' or stream_id is None: + if chat_mode == 'private' or stream_id is None: return try: await self.bot.delete_message(chat_id=chat_id, message_id=stream_id) @@ -639,14 +690,29 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if message_id not in self.msg_stream_id: return + # Media cannot be edited into a text stream. Deliver terminal attachments + # through the regular reply path, including attachment-only completions. + if is_final and not getattr(bot_message, 'tool_calls', None): + attachments = platform_message.MessageChain( + [ + part + for part in message + if isinstance(part, (platform_message.Image, platform_message.File, platform_message.Voice)) + ] + ) + if attachments: + await self.reply_message(message_source, attachments, quote_origin=quote_origin) + stream_state = self.msg_stream_id[message_id] chat_mode, stream_id = stream_state[:2] has_visible_content = len(stream_state) > 2 and stream_state[2] components = await TelegramMessageConverter.yiri2target(message, self.bot) if not components or components[0]['type'] != 'text': - if is_final and bot_message.tool_calls is None: - self.msg_stream_id.pop(message_id) + if is_final and getattr(bot_message, 'tool_calls', None) is None: + if not has_visible_content: + await self._delete_group_stream_message(chat_mode, chat_id, stream_id) + self.msg_stream_id.pop(message_id, None) return content = components[0]['text'] @@ -685,6 +751,8 @@ class TelegramAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if is_final and bot_message.tool_calls is None: # Finalise: send the real message, discard the draft args = self._build_message_args(chat_id, content, message_thread_id) + if quote_origin and effective_message: + args['reply_to_message_id'] = effective_message.id try: await self.bot.send_message(**args) except telegram.error.BadRequest as exc: diff --git a/src/langbot/pkg/platform/sources/wecombot.py b/src/langbot/pkg/platform/sources/wecombot.py index 92d58da7d..18459a002 100644 --- a/src/langbot/pkg/platform/sources/wecombot.py +++ b/src/langbot/pkg/platform/sources/wecombot.py @@ -98,10 +98,47 @@ class WecomBotMessageConverter(abstract_platform_adapter.AbstractMessageConverte if voice_info: voice_payload = voice_info.get('base64') or voice_info.get('url') if voice_payload: + length = voice_info.get('length') + from langbot.pkg.utils import audio as audio_utils + if voice_info.get('base64') and not voice_payload.startswith('data:'): - voice_payload = f'data:audio/mpeg;base64,{voice_info.get("base64")}' + try: + raw_bytes = base64.b64decode(voice_info.get('base64'), validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + if fmt in ('amr', 'silk') and audio_utils.is_ffmpeg_available(): + try: + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + except Exception: + voice_payload = f'data:audio/{fmt};base64,{voice_info.get("base64")}' + else: + mime = 'audio/mpeg' if fmt == 'mp3' else ('audio/wav' if fmt == 'wav' else f'audio/{fmt}') + voice_payload = f'data:{mime};base64,{voice_info.get("base64")}' + except Exception: + voice_payload = f'data:audio/mpeg;base64,{voice_info.get("base64")}' + elif voice_payload.startswith('data:'): + header, _, b64_body = voice_payload.partition(',') + if ( + 'amr' in header or 'silk' in header or 'octet-stream' in header + ) and audio_utils.is_ffmpeg_available(): + try: + raw_bytes = base64.b64decode(b64_body, validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if fmt in ('amr', 'silk'): + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + except Exception: + pass + try: - yiri_msg_list.append(platform_message.Voice(base64=voice_payload)) + if voice_payload.startswith('data:'): + yiri_msg_list.append(platform_message.Voice(base64=voice_payload, length=length)) + else: + yiri_msg_list.append(platform_message.Voice(url=voice_payload, length=length)) except Exception: try: voice_kwargs = {'url': voice_payload} @@ -194,10 +231,53 @@ class WecomBotMessageConverter(abstract_platform_adapter.AbstractMessageConverte if quote_voice: voice_payload = quote_voice.get('base64') or quote_voice.get('url') if voice_payload: + length = quote_voice.get('length') + from langbot.pkg.utils import audio as audio_utils + if quote_voice.get('base64') and not voice_payload.startswith('data:'): - voice_payload = f'data:audio/mpeg;base64,{quote_voice.get("base64")}' + try: + raw_bytes = base64.b64decode(quote_voice.get('base64'), validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + if fmt in ('amr', 'silk') and audio_utils.is_ffmpeg_available(): + try: + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = ( + f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + ) + except Exception: + voice_payload = f'data:audio/{fmt};base64,{quote_voice.get("base64")}' + else: + mime = ( + 'audio/mpeg' if fmt == 'mp3' else ('audio/wav' if fmt == 'wav' else f'audio/{fmt}') + ) + voice_payload = f'data:{mime};base64,{quote_voice.get("base64")}' + except Exception: + voice_payload = f'data:audio/mpeg;base64,{quote_voice.get("base64")}' + elif voice_payload.startswith('data:'): + header, _, b64_body = voice_payload.partition(',') + if ( + 'amr' in header or 'silk' in header or 'octet-stream' in header + ) and audio_utils.is_ffmpeg_available(): + try: + raw_bytes = base64.b64decode(b64_body, validate=False) + fmt = audio_utils.detect_audio_format(raw_bytes) + if fmt in ('amr', 'silk'): + if length is None and raw_bytes: + length = int(round(audio_utils.estimate_audio_duration(raw_bytes, fmt))) + mp3_bytes = audio_utils.transcode_audio_sync(raw_bytes, 'mp3', fmt) + voice_payload = ( + f'data:audio/mpeg;base64,{base64.b64encode(mp3_bytes).decode("utf-8")}' + ) + except Exception: + pass + try: - yiri_msg_list.append(platform_message.Voice(base64=voice_payload)) + if voice_payload.startswith('data:'): + yiri_msg_list.append(platform_message.Voice(base64=voice_payload, length=length)) + else: + yiri_msg_list.append(platform_message.Voice(url=voice_payload, length=length)) except Exception: try: voice_kwargs = {'url': voice_payload} @@ -394,6 +474,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): bot, req_id: str, item: dict, + config: dict | None = None, ) -> bool: """Upload *item* to the WeCom AI Bot CDN and send it as a media reply. @@ -407,30 +488,87 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): if upload is None: return False b64_text = item.get('base64') or '' - if not b64_text: - return False - if b64_text.startswith('data:') and ',' in b64_text: - b64_text = b64_text.split(',', 1)[1] - try: - data = base64.b64decode(b64_text, validate=False) - except Exception: - return False + data = b'' + if b64_text: + if b64_text.startswith('data:') and ',' in b64_text: + b64_text = b64_text.split(',', 1)[1] + try: + data = base64.b64decode(b64_text, validate=False) + except Exception: + return False + elif item.get('path'): + with open(item['path'], 'rb') as f: + data = f.read() if not data: return False + + delivery_kind = kind + filename = item.get('name') + voice_mode = (config or {}).get('voice_delivery', 'voice') + + if kind == 'voice': + from langbot.pkg.utils import audio as audio_utils + + fmt = audio_utils.detect_audio_format(data) + duration = audio_utils.estimate_audio_duration(data, fmt) if data else 0 + + if voice_mode == 'file' or (duration and duration > 60): + delivery_kind = 'file' + if fmt in ('silk', 'amr') and audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'mp3', fmt) + fmt = 'mp3' + except Exception: + pass + filename = filename or f'voice.{fmt if fmt != "unknown" else "mp3"}' + else: + if fmt != 'silk' and fmt != 'amr': + if audio_utils.is_silk_available() and audio_utils.is_ffmpeg_available(): + try: + data = await audio_utils.transcode_audio(data, 'silk', fmt) + fmt = 'silk' + except Exception: + pass + + if fmt == 'silk': + delivery_kind = 'voice' + filename = filename or 'voice.silk' + elif fmt == 'amr': + delivery_kind = 'voice' + filename = filename or 'voice.amr' + elif fmt in ('mp3', 'wav', 'ogg', 'flac'): + delivery_kind = 'file' + filename = filename or f'voice.{fmt}' + else: + delivery_kind = 'voice' + filename = filename or 'voice.silk' + else: + filename = filename or f'attachment.{kind}' + try: - upload_result = await upload(data, item.get('name') or f'attachment.{kind}', media_type=kind) + upload_result = await upload(data, filename, media_type=delivery_kind) except Exception: return False media_id = getattr(upload_result, 'media_id', None) or ( isinstance(upload_result, dict) and upload_result.get('media_id') ) + if not media_id and delivery_kind == 'voice': + delivery_kind = 'file' + fallback_filename = filename.rsplit('.', 1)[0] + '.mp3' if filename else 'voice.mp3' + try: + upload_result = await upload(data, fallback_filename, media_type='file') + media_id = getattr(upload_result, 'media_id', None) or ( + isinstance(upload_result, dict) and upload_result.get('media_id') + ) + except Exception: + return False if not media_id: return False reply_fn = { 'image': getattr(bot, 'reply_image', None), 'file': getattr(bot, 'reply_file', None), 'voice': getattr(bot, 'reply_voice', None), - }.get(kind) + }.get(delivery_kind) if reply_fn is None: return False try: @@ -480,7 +618,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): else: await self.bot.set_message(event.message_id, text) for item in self._iter_media_components(items): - await self._send_media(self.bot, req_id, item) + await self._send_media(self.bot, req_id, item, config=self.config) else: await self.bot.set_message(event.message_id, text) @@ -555,7 +693,7 @@ class WecomBotAdapter(abstract_platform_adapter.AbstractMessagePlatformAdapter): event = message_source.source_platform_object req_id = event.get('req_id', '') for item in self._iter_media_components(items): - await self._send_media(self.bot, req_id, item) + await self._send_media(self.bot, req_id, item, config=self.config) return {'stream': success} else: success = await self.bot.push_stream_chunk(msg_id, text, is_final=is_final) diff --git a/src/langbot/pkg/platform/sources/wecombot.yaml b/src/langbot/pkg/platform/sources/wecombot.yaml index 51961d357..d4bf3366b 100644 --- a/src/langbot/pkg/platform/sources/wecombot.yaml +++ b/src/langbot/pkg/platform/sources/wecombot.yaml @@ -148,6 +148,29 @@ spec: type: boolean required: false default: true + - name: voice_delivery + label: + en_US: Voice Delivery Mode + zh_Hans: 语音发送模式 + zh_Hant: 語音傳送模式 + description: + en_US: Choose how voice responses are delivered. 'voice' sends native voice messages (requires SILK transcoding). 'file' sends audio files. + zh_Hans: 选择语音回复的发送方式。'voice' 为企微原生语音条(默认自动转码为 SILK),'file' 为音频文件卡片。 + zh_Hant: 選擇語音回覆的傳送方式。'voice' 為企微原生語音條(預設自動轉碼為 SILK),'file' 為音訊檔案卡片。 + type: select + options: + - label: + en_US: Native Voice (Recommended) + zh_Hans: 原生语音条(推荐) + zh_Hant: 原生語音條(推薦) + value: voice + - label: + en_US: Audio File + zh_Hans: 音频文件 + zh_Hant: 音訊檔案 + value: file + required: false + default: voice execution: python: path: ./wecombot.py diff --git a/src/langbot/pkg/provider/modelmgr/modelmgr.py b/src/langbot/pkg/provider/modelmgr/modelmgr.py index e13464a9c..e0713f732 100644 --- a/src/langbot/pkg/provider/modelmgr/modelmgr.py +++ b/src/langbot/pkg/provider/modelmgr/modelmgr.py @@ -40,6 +40,8 @@ class ModelManager: llm_model_dict: dict[_CacheKey, requester.RuntimeLLMModel] embedding_model_dict: dict[_CacheKey, requester.RuntimeEmbeddingModel] rerank_model_dict: dict[_CacheKey, requester.RuntimeRerankModel] + asr_model_dict: dict[_CacheKey, requester.RuntimeASRModel] + tts_model_dict: dict[_CacheKey, requester.RuntimeTTSModel] requester_components: list[engine.Component] requester_dict: dict[str, type[requester.ProviderAPIRequester]] @@ -50,6 +52,8 @@ class ModelManager: self.llm_model_dict = {} self.embedding_model_dict = {} self.rerank_model_dict = {} + self.asr_model_dict = {} + self.tts_model_dict = {} self.requester_components = [] self.requester_dict = {} self._scope_generations: dict[tuple[str, str], int] = {} @@ -57,6 +61,8 @@ class ModelManager: self._llm_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} self._embedding_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} self._rerank_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} + self._asr_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} + self._tts_keys_by_scope: dict[tuple[str, str], set[_CacheKey]] = {} def _cache_index(self, cache: dict) -> dict[tuple[str, str], set[_CacheKey]]: if cache is self.provider_dict: @@ -67,6 +73,10 @@ class ModelManager: return self._embedding_keys_by_scope if cache is self.rerank_model_dict: return self._rerank_keys_by_scope + if cache is self.asr_model_dict: + return self._asr_keys_by_scope + if cache is self.tts_model_dict: + return self._tts_keys_by_scope raise ValueError('Unknown model runtime cache') def _cache_set(self, cache: dict, key: _CacheKey, value: object) -> None: @@ -89,6 +99,8 @@ class ModelManager: self._llm_keys_by_scope, self._embedding_keys_by_scope, self._rerank_keys_by_scope, + self._asr_keys_by_scope, + self._tts_keys_by_scope, ) ): self._scope_generations.pop(scope, None) @@ -113,6 +125,8 @@ class ModelManager: (self.llm_model_dict, self._llm_keys_by_scope), (self.embedding_model_dict, self._embedding_keys_by_scope), (self.rerank_model_dict, self._rerank_keys_by_scope), + (self.asr_model_dict, self._asr_keys_by_scope), + (self.tts_model_dict, self._tts_keys_by_scope), ): for key in index.pop(scope, ()): removed = cache.pop(key, None) @@ -156,6 +170,8 @@ class ModelManager: self._llm_keys_by_scope, self._embedding_keys_by_scope, self._rerank_keys_by_scope, + self._asr_keys_by_scope, + self._tts_keys_by_scope, ) ): self._scope_generations.pop(scope, None) @@ -168,11 +184,15 @@ class ModelManager: self.llm_model_dict = {} self.embedding_model_dict = {} self.rerank_model_dict = {} + self.asr_model_dict = {} + self.tts_model_dict = {} self._scope_generations = {} self._provider_keys_by_scope = {} self._llm_keys_by_scope = {} self._embedding_keys_by_scope = {} self._rerank_keys_by_scope = {} + self._asr_keys_by_scope = {} + self._tts_keys_by_scope = {} await self._close_runtime_providers(providers) @staticmethod @@ -336,11 +356,15 @@ class ModelManager: self.llm_model_dict = {} self.embedding_model_dict = {} self.rerank_model_dict = {} + self.asr_model_dict = {} + self.tts_model_dict = {} self._scope_generations = {} self._provider_keys_by_scope = {} self._llm_keys_by_scope = {} self._embedding_keys_by_scope = {} self._rerank_keys_by_scope = {} + self._asr_keys_by_scope = {} + self._tts_keys_by_scope = {} list_bindings = getattr(self.ap.workspace_service, 'list_active_execution_bindings', None) tenant_uow = getattr(self.ap.persistence_mgr, 'tenant_uow', None) @@ -412,6 +436,18 @@ class ModelManager: self._build_rerank_model, context_for, ) + await self._load_model_kind( + persistence_model.ASRModel, + self.asr_model_dict, + self._build_asr_model, + context_for, + ) + await self._load_model_kind( + persistence_model.TTSModel, + self.tts_model_dict, + self._build_tts_model, + context_for, + ) async def _load_model_kind(self, entity_type, cache: dict, builder, context_for) -> None: result = await self.ap.persistence_mgr.execute_async(sqlalchemy.select(entity_type)) @@ -480,6 +516,18 @@ class ModelManager: self.rerank_model_dict, self._build_rerank_model, ) + await self._load_workspace_model_kind( + context, + persistence_model.ASRModel, + self.asr_model_dict, + self._build_asr_model, + ) + await self._load_workspace_model_kind( + context, + persistence_model.TTSModel, + self.tts_model_dict, + self._build_tts_model, + ) async def _load_workspace_model_kind(self, context, entity_type, cache: dict, builder) -> None: result = await self.ap.persistence_mgr.execute_async( @@ -716,6 +764,68 @@ class ModelManager: ) return self._build_rerank_model(execution_context, model_entity, runtime_provider) + async def init_temporary_runtime_asr_model( + self, + context: TenantContext, + model_info: dict, + ) -> requester.RuntimeASRModel: + execution_context = await self.resolve_execution_context(context) + provider_info = {**model_info.get('provider', {}), 'workspace_uuid': execution_context.workspace_uuid} + provider_uuid = model_info.get('provider_uuid') or provider_info.get('uuid') + provider_entity = persistence_model.ModelProvider(**provider_info) + if provider_uuid and not provider_entity.requester: + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(persistence_model.ModelProvider).where( + persistence_model.ModelProvider.workspace_uuid == execution_context.workspace_uuid, + persistence_model.ModelProvider.uuid == provider_uuid, + ) + ) + saved_provider = result.first() + if saved_provider is not None: + provider_entity = self._coerce_provider(saved_provider, execution_context) + elif model_info.get('provider_uuid'): + raise WorkspaceNotFoundError('Provider not found') + runtime_provider = await self._build_provider(execution_context, provider_entity) + model_entity = persistence_model.ASRModel( + workspace_uuid=execution_context.workspace_uuid, + uuid=model_info.get('uuid', ''), + name=model_info.get('name', ''), + provider_uuid=runtime_provider.provider_entity.uuid, + extra_args=model_info.get('extra_args', {}), + ) + return self._build_asr_model(execution_context, model_entity, runtime_provider) + + async def init_temporary_runtime_tts_model( + self, + context: TenantContext, + model_info: dict, + ) -> requester.RuntimeTTSModel: + execution_context = await self.resolve_execution_context(context) + provider_info = {**model_info.get('provider', {}), 'workspace_uuid': execution_context.workspace_uuid} + provider_uuid = model_info.get('provider_uuid') or provider_info.get('uuid') + provider_entity = persistence_model.ModelProvider(**provider_info) + if provider_uuid and not provider_entity.requester: + result = await self.ap.persistence_mgr.execute_async( + sqlalchemy.select(persistence_model.ModelProvider).where( + persistence_model.ModelProvider.workspace_uuid == execution_context.workspace_uuid, + persistence_model.ModelProvider.uuid == provider_uuid, + ) + ) + saved_provider = result.first() + if saved_provider is not None: + provider_entity = self._coerce_provider(saved_provider, execution_context) + elif model_info.get('provider_uuid'): + raise WorkspaceNotFoundError('Provider not found') + runtime_provider = await self._build_provider(execution_context, provider_entity) + model_entity = persistence_model.TTSModel( + workspace_uuid=execution_context.workspace_uuid, + uuid=model_info.get('uuid', ''), + name=model_info.get('name', ''), + provider_uuid=runtime_provider.provider_entity.uuid, + extra_args=model_info.get('extra_args', {}), + ) + return self._build_tts_model(execution_context, model_entity, runtime_provider) + @staticmethod def _coerce_provider( provider_info: persistence_model.ModelProvider | sqlalchemy.Row | dict, @@ -906,6 +1016,34 @@ class ModelManager: provider=provider, ) + def _build_asr_model( + self, + context: ExecutionContext, + model_info: persistence_model.ASRModel | sqlalchemy.Row, + provider: requester.RuntimeProvider, + ) -> requester.RuntimeASRModel: + model_entity = self._coerce_model(model_info, persistence_model.ASRModel) + self._validate_model_provider(context, model_entity, provider) + return requester.RuntimeASRModel( + execution_context=context, + model_entity=model_entity, + provider=provider, + ) + + def _build_tts_model( + self, + context: ExecutionContext, + model_info: persistence_model.TTSModel | sqlalchemy.Row, + provider: requester.RuntimeProvider, + ) -> requester.RuntimeTTSModel: + model_entity = self._coerce_model(model_info, persistence_model.TTSModel) + self._validate_model_provider(context, model_entity, provider) + return requester.RuntimeTTSModel( + execution_context=context, + model_entity=model_entity, + provider=provider, + ) + async def load_llm_model_with_provider( self, context: TenantContext, @@ -933,6 +1071,24 @@ class ModelManager: execution_context = await self.resolve_execution_context(context) return self._build_rerank_model(execution_context, model_info, provider) + async def load_asr_model_with_provider( + self, + context: TenantContext, + model_info: persistence_model.ASRModel | sqlalchemy.Row, + provider: requester.RuntimeProvider, + ) -> requester.RuntimeASRModel: + execution_context = await self.resolve_execution_context(context) + return self._build_asr_model(execution_context, model_info, provider) + + async def load_tts_model_with_provider( + self, + context: TenantContext, + model_info: persistence_model.TTSModel | sqlalchemy.Row, + provider: requester.RuntimeProvider, + ) -> requester.RuntimeTTSModel: + execution_context = await self.resolve_execution_context(context) + return self._build_tts_model(execution_context, model_info, provider) + async def cache_llm_model(self, context: TenantContext, model: requester.RuntimeLLMModel) -> None: execution_context = await self.resolve_execution_context(context) self._ensure_same_scope(execution_context, model.execution_context, resource='LLM model') @@ -967,6 +1123,26 @@ class ModelManager: model, ) + async def cache_asr_model(self, context: TenantContext, model: requester.RuntimeASRModel) -> None: + execution_context = await self.resolve_execution_context(context) + self._ensure_same_scope(execution_context, model.execution_context, resource='ASR model') + self._observe_execution_context(execution_context) + self._cache_set( + self.asr_model_dict, + self._cache_key(execution_context, model.model_entity.uuid), + model, + ) + + async def cache_tts_model(self, context: TenantContext, model: requester.RuntimeTTSModel) -> None: + execution_context = await self.resolve_execution_context(context) + self._ensure_same_scope(execution_context, model.execution_context, resource='TTS model') + self._observe_execution_context(execution_context) + self._cache_set( + self.tts_model_dict, + self._cache_key(execution_context, model.model_entity.uuid), + model, + ) + async def get_model_by_uuid(self, context: TenantContext, model_uuid: str) -> requester.RuntimeLLMModel: execution_context = await self.resolve_execution_context(context) model = self.llm_model_dict.get(self._cache_key(execution_context, model_uuid)) @@ -999,6 +1175,30 @@ class ModelManager: self._ensure_same_scope(execution_context, model.execution_context, resource='Rerank model') return model + async def get_asr_model_by_uuid( + self, + context: TenantContext, + model_uuid: str, + ) -> requester.RuntimeASRModel: + execution_context = await self.resolve_execution_context(context) + model = self.asr_model_dict.get(self._cache_key(execution_context, model_uuid)) + if model is None: + raise ValueError(f'ASR model {model_uuid} not found') + self._ensure_same_scope(execution_context, model.execution_context, resource='ASR model') + return model + + async def get_tts_model_by_uuid( + self, + context: TenantContext, + model_uuid: str, + ) -> requester.RuntimeTTSModel: + execution_context = await self.resolve_execution_context(context) + model = self.tts_model_dict.get(self._cache_key(execution_context, model_uuid)) + if model is None: + raise ValueError(f'TTS model {model_uuid} not found') + self._ensure_same_scope(execution_context, model.execution_context, resource='TTS model') + return model + async def remove_llm_model(self, context: TenantContext, model_uuid: str) -> None: execution_context = await self.resolve_execution_context(context) self._cache_pop( @@ -1020,6 +1220,20 @@ class ModelManager: self._cache_key(execution_context, model_uuid), ) + async def remove_asr_model(self, context: TenantContext, model_uuid: str) -> None: + execution_context = await self.resolve_execution_context(context) + self._cache_pop( + self.asr_model_dict, + self._cache_key(execution_context, model_uuid), + ) + + async def remove_tts_model(self, context: TenantContext, model_uuid: str) -> None: + execution_context = await self.resolve_execution_context(context) + self._cache_pop( + self.tts_model_dict, + self._cache_key(execution_context, model_uuid), + ) + def get_available_requesters_info(self, model_type: str) -> list[dict]: if model_type: return [ diff --git a/src/langbot/pkg/provider/modelmgr/requester.py b/src/langbot/pkg/provider/modelmgr/requester.py index 41dd62a8c..f03a1f93e 100644 --- a/src/langbot/pkg/provider/modelmgr/requester.py +++ b/src/langbot/pkg/provider/modelmgr/requester.py @@ -75,7 +75,7 @@ class RuntimeProvider: def _validate_invocation( self, - model: RuntimeLLMModel | RuntimeEmbeddingModel | RuntimeRerankModel, + model: RuntimeLLMModel | RuntimeEmbeddingModel | RuntimeRerankModel | RuntimeASRModel | RuntimeTTSModel, execution_context: ExecutionContext, ) -> None: _ensure_same_execution_scope(self.execution_context, execution_context, resource='Provider invocation') @@ -380,6 +380,74 @@ class RuntimeProvider: except Exception as monitor_err: self.requester.ap.logger.error(f'[Monitoring] Failed to record rerank call: {monitor_err}') + async def invoke_asr( + self, + model: RuntimeASRModel, + audio: bytes, + mime: str, + extra_args: dict[str, typing.Any] = {}, + *, + execution_context: ExecutionContext, + ) -> str: + """Bridge method for invoking ASR with monitoring""" + self._validate_invocation(model, execution_context) + start_time = time.time() + status = 'success' + + try: + result = await self.requester.invoke_asr( + model=model, + audio=audio, + mime=mime, + extra_args=extra_args, + ) + return result + except Exception: + status = 'error' + raise + finally: + duration_ms = int((time.time() - start_time) * 1000) + try: + self.requester.ap.logger.debug( + f'[ASR] model={model.model_entity.name} bytes={len(audio)} mime={mime} ' + f'duration={duration_ms}ms status={status}' + ) + except Exception as monitor_err: + self.requester.ap.logger.error(f'[Monitoring] Failed to record ASR call: {monitor_err}') + + async def invoke_tts( + self, + model: RuntimeTTSModel, + text: str, + extra_args: dict[str, typing.Any] = {}, + *, + execution_context: ExecutionContext, + ) -> tuple[bytes, str]: + """Bridge method for invoking TTS with monitoring""" + self._validate_invocation(model, execution_context) + start_time = time.time() + status = 'success' + + try: + result = await self.requester.invoke_tts( + model=model, + text=text, + extra_args=extra_args, + ) + return result + except Exception: + status = 'error' + raise + finally: + duration_ms = int((time.time() - start_time) * 1000) + try: + self.requester.ap.logger.debug( + f'[TTS] model={model.model_entity.name} text_len={len(text)} ' + f'duration={duration_ms}ms status={status}' + ) + except Exception as monitor_err: + self.requester.ap.logger.error(f'[Monitoring] Failed to record TTS call: {monitor_err}') + class RuntimeLLMModel: """运行时模型""" @@ -461,6 +529,80 @@ class RuntimeRerankModel: self.provider = provider +class RuntimeASRModel: + """运行时 ASR 模型""" + + model_entity: persistence_model.ASRModel + """模型数据""" + + provider: RuntimeProvider + """提供商实例""" + + def __init__( + self, + execution_context: ExecutionContext, + model_entity: persistence_model.ASRModel, + provider: RuntimeProvider, + ): + _ensure_same_execution_scope(provider.execution_context, execution_context, resource='ASR model') + if model_entity.workspace_uuid != execution_context.workspace_uuid: + raise WorkspaceInvariantError('ASR model belongs to another Workspace') + if model_entity.provider_uuid != provider.provider_entity.uuid: + raise WorkspaceInvariantError('ASR model references another provider') + self.execution_context = execution_context + self.model_entity = model_entity + self.provider = provider + + @property + def uuid(self) -> str: + return self.model_entity.uuid + + @property + def name(self) -> str: + return self.model_entity.name + + @property + def extra_args(self) -> dict[str, typing.Any]: + return self.model_entity.extra_args or {} + + +class RuntimeTTSModel: + """运行时 TTS 模型""" + + model_entity: persistence_model.TTSModel + """模型数据""" + + provider: RuntimeProvider + """提供商实例""" + + def __init__( + self, + execution_context: ExecutionContext, + model_entity: persistence_model.TTSModel, + provider: RuntimeProvider, + ): + _ensure_same_execution_scope(provider.execution_context, execution_context, resource='TTS model') + if model_entity.workspace_uuid != execution_context.workspace_uuid: + raise WorkspaceInvariantError('TTS model belongs to another Workspace') + if model_entity.provider_uuid != provider.provider_entity.uuid: + raise WorkspaceInvariantError('TTS model references another provider') + self.execution_context = execution_context + self.model_entity = model_entity + self.provider = provider + + @property + def uuid(self) -> str: + return self.model_entity.uuid + + @property + def name(self) -> str: + return self.model_entity.name + + @property + def extra_args(self) -> dict[str, typing.Any]: + return self.model_entity.extra_args or {} + + class ProviderAPIRequester(metaclass=abc.ABCMeta): """Provider API请求器""" @@ -606,3 +748,41 @@ class ProviderAPIRequester(metaclass=abc.ABCMeta): typing.List[dict]: [{"index": int, "relevance_score": float}, ...] """ raise NotImplementedError('This requester does not support rerank') + + async def invoke_asr( + self, + model: RuntimeASRModel, + audio: bytes, + mime: str, + extra_args: dict[str, typing.Any] = {}, + ) -> str: + """调用 ASR API + + Args: + model (RuntimeASRModel): 使用的模型信息 + audio (bytes): 音频字节数据 + mime (str): 音频 mime 类型 + extra_args (dict[str, typing.Any], optional): 额外参数. Defaults to {}. + + Returns: + str: 转写文本 + """ + raise NotImplementedError('This requester does not support asr') + + async def invoke_tts( + self, + model: RuntimeTTSModel, + text: str, + extra_args: dict[str, typing.Any] = {}, + ) -> tuple[bytes, str]: + """调用 TTS API + + Args: + model (RuntimeTTSModel): 使用的模型信息 + text (str): 待合成文本 + extra_args (dict[str, typing.Any], optional): 额外参数. Defaults to {}. + + Returns: + tuple[bytes, str]: (音频字节数据, mime 类型) + """ + raise NotImplementedError('This requester does not support tts') diff --git a/src/langbot/pkg/provider/modelmgr/requesters/302aichatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/302aichatcmpl.yaml index 3cfec1982..6cdeaac18 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/302aichatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/302aichatcmpl.yaml @@ -28,6 +28,8 @@ spec: - llm - text-embedding - rerank + - asr + - tts provider_category: maas execution: python: diff --git a/src/langbot/pkg/provider/modelmgr/requesters/chatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/chatcmpl.yaml index 526721b0e..72a1118dd 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/chatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/chatcmpl.yaml @@ -27,6 +27,8 @@ spec: support_type: - llm - text-embedding + - asr + - tts provider_category: manufacturer execution: python: diff --git a/src/langbot/pkg/provider/modelmgr/requesters/groqchatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/groqchatcmpl.yaml index d51367473..85997daec 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/groqchatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/groqchatcmpl.yaml @@ -26,4 +26,5 @@ spec: alias: "groq Groq 高速 llama mixtral 推理加速 lpu" support_type: - llm + - asr provider_category: manufacturer diff --git a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py index 16eb776c4..2becbd5bf 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py +++ b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.py @@ -19,7 +19,7 @@ from urllib.parse import urlsplit from ..request_policy import current_call, RequestPolicy, error_metadata import litellm -from litellm import acompletion, aembedding, arerank +from litellm import acompletion, aembedding, arerank, atranscription, aspeech from litellm.llms.custom_httpx.http_handler import AsyncHTTPHandler from litellm.secret_managers.main import get_secret, get_secret_bool @@ -182,6 +182,17 @@ class LiteLLMRequester(requester.ProviderAPIRequester): _EMBEDDING_MODEL_HINTS = ('embedding', 'embed', 'bge-', 'e5-', 'm3e', 'gte-', 'text-embedding') _RERANK_MODEL_HINTS = ('rerank', 're-rank', 're_rank') + _ASR_MODEL_HINTS = ( + 'whisper', + 'sensevoice', + 'asr', + 'paraformer', + 'conformer', + 'speech-to-text', + 'transcription', + 'stt', + ) + _TTS_MODEL_HINTS = ('tts', 'text-to-speech', 'speech-synthesis', 'cosyvoice', 'voice-clone', 'sambert', 'chattts') _QWEN_DEDICATED_THINKING_MODELS = frozenset( { 'qwen3.7-max-preview', @@ -786,12 +797,27 @@ class LiteLLMRequester(requester.ProviderAPIRequester): def _reasoning_level(self, model: requester.RuntimeLLMModel) -> str: return reasoning.normalize_reasoning_config(self._reasoning_config_value(model))['level'] - def _infer_model_type(self, model_id: str) -> str: + def _infer_model_type(self, model_id: str, model_payload: dict[str, typing.Any] | None = None) -> str: + if model_payload: + explicit_type = (model_payload.get('type') or model_payload.get('task') or '').lower() + if explicit_type in ('asr', 'audio_transcription', 'speech-to-text', 'stt'): + return 'asr' + if explicit_type in ('tts', 'text_to_speech', 'text-to-speech', 'speech-synthesis'): + return 'tts' + if explicit_type in ('embedding', 'text-embedding'): + return 'embedding' + if explicit_type in ('rerank', 'reranking'): + return 'rerank' + normalized_id = (model_id or '').lower() if any(kw in normalized_id for kw in self._RERANK_MODEL_HINTS): return 'rerank' if any(kw in normalized_id for kw in self._EMBEDDING_MODEL_HINTS): return 'embedding' + if any(kw in normalized_id for kw in self._ASR_MODEL_HINTS): + return 'asr' + if any(kw in normalized_id for kw in self._TTS_MODEL_HINTS): + return 'tts' return 'llm' def _enrich_scanned_model( @@ -799,7 +825,7 @@ class LiteLLMRequester(requester.ProviderAPIRequester): model_id: str, model_payload: dict[str, typing.Any] | None = None, ) -> dict[str, typing.Any]: - model_type = self._infer_model_type(model_id) + model_type = self._infer_model_type(model_id, model_payload) scanned_model: dict[str, typing.Any] = { 'id': model_id, 'name': model_id, @@ -1231,11 +1257,15 @@ class LiteLLMRequester(requester.ProviderAPIRequester): mappings = ( (litellm.ContextWindowExceededError, 'context_length', t('Context length exceeded'), False), (litellm.AuthenticationError, 'authentication', t('Invalid API key'), False), + (getattr(litellm, 'PermissionDeniedError', ()), 'permission_denied', t('Permission denied'), False), (litellm.NotFoundError, 'not_found', t('Invalid model or endpoint'), False), (litellm.RateLimitError, 'rate_limit', t('Rate limit exceeded or insufficient balance'), True), (litellm.Timeout, 'timeout', t('Request timed out'), True), (litellm.APIConnectionError, 'connection', t('Connection failed'), True), (litellm.BadRequestError, 'bad_request', t('Invalid request parameters'), False), + (getattr(litellm, 'InternalServerError', ()), 'internal_error', t('Internal server error'), True), + (getattr(litellm, 'BadGatewayError', ()), 'bad_gateway', t('Internal server error'), True), + (getattr(litellm, 'ServiceUnavailableError', ()), 'service_unavailable', t('Internal server error'), True), ) for exception_type, code, message, retryable in mappings: if isinstance(e, exception_type): @@ -1247,9 +1277,38 @@ class LiteLLMRequester(requester.ProviderAPIRequester): code, retryable = 'quota_exceeded', False raise errors.RequesterError(message, code=code, retryable=retryable, **metadata) from e status = metadata['status_code'] + code = 'upstream_error' + if status in (500, 502, 503): + message = t('Internal server error') + code = 'internal_error' + elif status == 504: + message = t('Request timed out') + code = 'timeout' + elif status == 401: + message = t('Invalid API key') + code = 'authentication' + elif status == 403: + message = t('Permission denied') + code = 'permission_denied' + elif status == 404: + message = t('Invalid model or endpoint') + code = 'not_found' + elif status == 429: + message = t('Rate limit exceeded or insufficient balance') + code = 'rate_limit' + elif status == 400: + message = t('Invalid request parameters') + code = 'bad_request' + elif status is not None: + message = f'{t("API error")} (HTTP {status})' + elif isinstance(e, (litellm.APIError, getattr(litellm, 'OpenAIError', ()))): + message = t('API error') + else: + message = t('Unknown error') + raise errors.RequesterError( - t('API error') if isinstance(e, litellm.APIError) else t('Unknown error'), - code='upstream_error', + message, + code=code, retryable=status in (408, 409, 429, 500, 502, 503, 504), **metadata, ) from e @@ -1881,6 +1940,123 @@ class LiteLLMRequester(requester.ProviderAPIRequester): return results + async def invoke_asr( + self, + model: requester.RuntimeASRModel, + audio: bytes, + mime: str, + extra_args: dict[str, typing.Any] = {}, + ) -> str: + """Invoke speech recognition (ASR) and return transcript.""" + model_name = self._build_litellm_model_name(model.model_entity.name) + api_key = model.provider.token_mgr.get_token() + + ext = 'wav' + if 'mp3' in mime or 'mpeg' in mime: + ext = 'mp3' + elif 'ogg' in mime: + ext = 'ogg' + elif 'flac' in mime: + ext = 'flac' + elif 'm4a' in mime or 'mp4' in mime: + ext = 'm4a' + elif 'wav' in mime: + ext = 'wav' + filename = f'audio.{ext}' + + args: dict[str, typing.Any] = { + 'model': model_name, + 'file': (filename, audio, mime), + 'api_key': api_key, + 'response_format': 'json', + } + self._build_common_args(args, include_retry_params=False) + + provider = self._get_custom_llm_provider() + if provider: + args['custom_llm_provider'] = provider + + if model.model_entity.extra_args: + args.update(model.model_entity.extra_args) + args.update(extra_args) + + try: + response = await atranscription(**args) + if hasattr(response, 'text'): + return response.text or '' + if isinstance(response, dict): + return response.get('text', '') + return str(response) + except errors.RequesterError: + raise + except Exception as e: + self._handle_litellm_error(e) + + async def invoke_tts( + self, + model: requester.RuntimeTTSModel, + text: str, + extra_args: dict[str, typing.Any] = {}, + ) -> tuple[bytes, str]: + """Invoke text-to-speech and return (audio_bytes, mime_type).""" + model_name = self._build_litellm_model_name(model.model_entity.name) + api_key = model.provider.token_mgr.get_token() + + combined_extra = {**(model.model_entity.extra_args or {}), **extra_args} + voice = combined_extra.pop('voice', None) + response_format = combined_extra.pop('response_format', None) or combined_extra.pop('format', None) or 'mp3' + + args: dict[str, typing.Any] = { + 'model': model_name, + 'input': text, + 'response_format': response_format, + 'api_key': api_key, + } + if voice: + args['voice'] = voice + self._build_common_args(args, include_retry_params=False) + + provider = self._get_custom_llm_provider() + if provider: + args['custom_llm_provider'] = provider + + args.update(combined_extra) + + try: + response = await aspeech(**args) + content = b'' + if hasattr(response, 'content'): + content = response.content + elif hasattr(response, 'read'): + res = response.read() + if asyncio.iscoroutine(res): + content = await res + else: + content = res + elif isinstance(response, bytes): + content = response + elif hasattr(response, 'response') and hasattr(response.response, 'content'): + content = response.response.content + + # Determine mime type + mime = 'audio/mpeg' + if response_format == 'wav': + mime = 'audio/wav' + elif response_format in ('ogg', 'opus'): + mime = 'audio/ogg' + elif response_format == 'flac': + mime = 'audio/flac' + elif response_format == 'aac': + mime = 'audio/aac' + elif hasattr(response, 'response') and hasattr(response.response, 'headers'): + mime = response.response.headers.get('content-type', mime) + + return content, mime + except errors.RequesterError: + raise + except Exception as e: + self._handle_litellm_error(e) + async def scan_models(self, api_key: str | None = None) -> dict[str, typing.Any]: """Scan models supported by the provider.""" import httpx diff --git a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.yaml b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.yaml index 1d5452d50..79b8c3a72 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/litellmchat.yaml @@ -58,6 +58,8 @@ spec: - llm - text-embedding - rerank + - asr + - tts provider_category: unified execution: python: diff --git a/src/langbot/pkg/provider/modelmgr/requesters/newapichatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/newapichatcmpl.yaml index 694af440e..3fa339720 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/newapichatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/newapichatcmpl.yaml @@ -28,6 +28,8 @@ spec: - llm - text-embedding - rerank + - asr + - tts provider_category: maas execution: python: diff --git a/src/langbot/pkg/provider/modelmgr/requesters/siliconflowchatcmpl.yaml b/src/langbot/pkg/provider/modelmgr/requesters/siliconflowchatcmpl.yaml index b4e5d736e..d0ac01dfe 100644 --- a/src/langbot/pkg/provider/modelmgr/requesters/siliconflowchatcmpl.yaml +++ b/src/langbot/pkg/provider/modelmgr/requesters/siliconflowchatcmpl.yaml @@ -28,6 +28,8 @@ spec: - llm - text-embedding - rerank + - asr + - tts provider_category: maas execution: python: diff --git a/src/langbot/pkg/utils/audio.py b/src/langbot/pkg/utils/audio.py new file mode 100644 index 000000000..b56fa7c97 --- /dev/null +++ b/src/langbot/pkg/utils/audio.py @@ -0,0 +1,358 @@ +from __future__ import annotations + +import asyncio +import base64 +import importlib.util +import os +import shutil +import subprocess +import sys +import tempfile +import typing + +if typing.TYPE_CHECKING: + import langbot_plugin.api.entities.builtin.platform.message as platform_message + + +class AudioDependencyError(RuntimeError): + """Raised when an audio transcoding tool or optional dependency is missing or fails.""" + + pass + + +def _safe_tempdir() -> str | None: + """Return a valid temporary directory path even if TMPDIR was pointed to a deleted directory.""" + current = tempfile.gettempdir() + if not os.path.isdir(current): + tempfile.tempdir = None + for candidate in ('/tmp', '/var/tmp'): + if os.path.isdir(candidate): + tempfile.tempdir = candidate + return candidate + current = tempfile.gettempdir() + return current if os.path.isdir(current) else None + + +def get_ffmpeg_cmd() -> str: + """Get the path to the ffmpeg executable.""" + cmd = shutil.which('ffmpeg') + if cmd: + return cmd + venv_ffmpeg = os.path.join(os.path.dirname(sys.executable), 'ffmpeg') + if os.path.isfile(venv_ffmpeg) and os.access(venv_ffmpeg, os.X_OK): + return venv_ffmpeg + try: + import imageio_ffmpeg + + return imageio_ffmpeg.get_ffmpeg_exe() + except (ImportError, Exception): + return 'ffmpeg' + + +def is_ffmpeg_available() -> bool: + """Check if the ffmpeg command line tool is installed and available.""" + if shutil.which('ffmpeg') is not None: + return True + venv_ffmpeg = os.path.join(os.path.dirname(sys.executable), 'ffmpeg') + if os.path.isfile(venv_ffmpeg) and os.access(venv_ffmpeg, os.X_OK): + return True + try: + import imageio_ffmpeg + + return bool(imageio_ffmpeg.get_ffmpeg_exe()) + except (ImportError, Exception): + return False + + +def is_silk_available() -> bool: + """Check if the pilk library is installed.""" + return importlib.util.find_spec('pilk') is not None + + +def detect_audio_format(data: bytes) -> str: + """Detect audio format from magic bytes.""" + if not data or len(data) < 4: + return 'unknown' + + if data.startswith(b'#!AMR'): + return 'amr' + + if data.startswith(b'\x02#!SILK_V3') or data.startswith(b'#!SILK_V3'): + return 'silk' + + if data.startswith(b'RIFF') and len(data) >= 12 and data[8:12] == b'WAVE': + return 'wav' + + if data.startswith(b'OggS'): + if b'OpusHead' in data[:100]: + return 'opus' + return 'ogg' + + if data.startswith(b'ID3'): + return 'mp3' + + if len(data) >= 2 and data[0] == 0xFF and (data[1] & 0xE0) == 0xE0: + return 'mp3' + + if data.startswith(b'fLaC'): + return 'flac' + + if len(data) >= 8 and data[4:8] == b'ftyp': + return 'm4a' + + return 'unknown' + + +def transcode_audio_sync( + audio_bytes: bytes, + target_format: str, + source_format: str | None = None, +) -> bytes: + """Synchronously transcode audio bytes to target_format.""" + target_format = target_format.lower().lstrip('.') + if target_format == 'opus': + target_format = 'ogg' + + if not source_format or source_format == 'unknown': + source_format = detect_audio_format(audio_bytes) + + if source_format == 'opus': + source_format = 'ogg' + + if source_format == target_format and source_format != 'unknown': + return audio_bytes + + # Handle SILK decoding + if source_format == 'silk': + if not is_silk_available(): + raise AudioDependencyError( + "Transcoding SILK audio requires 'pilk'. Install it via 'pip install pilk' or 'pip install langbot[audio]'." + ) + if not is_ffmpeg_available(): + raise AudioDependencyError("Transcoding requires 'ffmpeg' which is not installed on the system.") + + import pilk + + with tempfile.TemporaryDirectory(dir=_safe_tempdir()) as tmpdir: + silk_path = os.path.join(tmpdir, 'input.silk') + pcm_path = os.path.join(tmpdir, 'decoded.pcm') + with open(silk_path, 'wb') as f: + f.write(audio_bytes) + + try: + pilk.decode(silk_path, pcm_path) + except Exception as exc: + raise AudioDependencyError(f'Failed to decode SILK audio: {exc}') from exc + + # Convert PCM to target format via ffmpeg + cmd = [get_ffmpeg_cmd(), '-y', '-f', 's16le', '-ar', '24000', '-ac', '1', '-i', pcm_path] + if target_format == 'mp3': + cmd.extend(['-f', 'mp3']) + elif target_format == 'ogg': + cmd.extend(['-c:a', 'libopus', '-f', 'ogg']) + elif target_format == 'wav': + cmd.extend(['-f', 'wav']) + elif target_format == 'amr': + cmd.extend(['-ar', '8000', '-ac', '1', '-c:a', 'libopencore_amrnb', '-f', 'amr']) + else: + cmd.extend(['-f', target_format]) + cmd.append('pipe:1') + + proc = subprocess.run(cmd, capture_output=True) + if proc.returncode != 0: + raise AudioDependencyError( + f'ffmpeg conversion failed from silk(pcm) to {target_format}: {proc.stderr.decode("utf-8", errors="replace")}' + ) + return proc.stdout + + # Handle SILK encoding + if target_format == 'silk': + if not is_silk_available(): + raise AudioDependencyError( + "Transcoding to SILK requires 'pilk'. Install it via 'pip install pilk' or 'pip install langbot[audio]'." + ) + if not is_ffmpeg_available(): + raise AudioDependencyError("Transcoding requires 'ffmpeg' which is not installed on the system.") + + import pilk + + # Convert source audio to PCM s16le, 24000Hz, mono + cmd = [get_ffmpeg_cmd(), '-y'] + if source_format and source_format != 'unknown': + cmd.extend(['-f', source_format]) + cmd.extend(['-i', 'pipe:0', '-f', 's16le', '-ar', '24000', '-ac', '1', 'pipe:1']) + + proc = subprocess.run(cmd, input=audio_bytes, capture_output=True) + if proc.returncode != 0: + raise AudioDependencyError( + f'ffmpeg conversion failed from {source_format} to pcm: {proc.stderr.decode("utf-8", errors="replace")}' + ) + pcm_bytes = proc.stdout + + with tempfile.TemporaryDirectory(dir=_safe_tempdir()) as tmpdir: + pcm_path = os.path.join(tmpdir, 'input.pcm') + silk_path = os.path.join(tmpdir, 'output.silk') + with open(pcm_path, 'wb') as f: + f.write(pcm_bytes) + + try: + pilk.encode(pcm_path, silk_path, pcm_rate=24000, tencent=True) + except Exception as exc: + raise AudioDependencyError(f'Failed to encode SILK audio: {exc}') from exc + + with open(silk_path, 'rb') as f: + return f.read() + + # Standard ffmpeg transcoding + if not is_ffmpeg_available(): + raise AudioDependencyError("Transcoding requires 'ffmpeg' which is not installed on the system.") + + cmd = [get_ffmpeg_cmd(), '-y'] + if source_format and source_format != 'unknown': + cmd.extend(['-f', source_format]) + cmd.extend(['-i', 'pipe:0']) + + if target_format == 'mp3': + cmd.extend(['-f', 'mp3']) + elif target_format == 'ogg': + cmd.extend(['-c:a', 'libopus', '-f', 'ogg']) + elif target_format == 'wav': + cmd.extend(['-f', 'wav']) + elif target_format == 'amr': + cmd.extend(['-ar', '8000', '-ac', '1', '-c:a', 'libopencore_amrnb', '-f', 'amr']) + else: + cmd.extend(['-f', target_format]) + cmd.append('pipe:1') + + proc = subprocess.run(cmd, input=audio_bytes, capture_output=True) + if proc.returncode != 0: + raise AudioDependencyError( + f'ffmpeg conversion failed from {source_format} to {target_format}: {proc.stderr.decode("utf-8", errors="replace")}' + ) + return proc.stdout + + +async def transcode_audio( + audio_bytes: bytes, + target_format: str, + source_format: str | None = None, +) -> bytes: + """Asynchronously transcode audio bytes to target_format.""" + return await asyncio.to_thread(transcode_audio_sync, audio_bytes, target_format, source_format) + + +async def extract_voice_bytes(voice: platform_message.Voice) -> tuple[bytes, str]: + """Extract raw bytes and audio format from a platform_message.Voice component.""" + if voice.base64: + b64_str = voice.base64 + format_hint = 'unknown' + if b64_str.startswith('data:'): + header, _, b64_str = b64_str.partition(',') + if ';' in header: + mime_part = header[5:].split(';')[0] + if '/' in mime_part: + format_hint = mime_part.split('/')[-1] + raw_bytes = await asyncio.to_thread(base64.b64decode, b64_str) + detected = detect_audio_format(raw_bytes) + if detected != 'unknown': + return raw_bytes, detected + # Only trust audio subtypes ffmpeg can parse; generic hints such as + # "octet-stream" would make ffmpeg reject the input format. + format_hint = {'mpeg': 'mp3', 'x-wav': 'wav', 'wave': 'wav', 'x-m4a': 'm4a', 'mp4': 'm4a'}.get( + format_hint, format_hint + ) + if format_hint not in ('wav', 'mp3', 'ogg', 'opus', 'amr', 'silk', 'flac', 'm4a', 'aac', 'webm'): + format_hint = 'unknown' + return raw_bytes, format_hint + + if voice.path: + with open(voice.path, 'rb') as f: + raw_bytes = await asyncio.to_thread(f.read) + detected = detect_audio_format(raw_bytes) + return raw_bytes, detected + + if voice.url: + from ...http import client as httpclient + + session = httpclient.get_session() + async with session.get(voice.url) as response: + raw_bytes = await response.read() + detected = detect_audio_format(raw_bytes) + return raw_bytes, detected + + raise ValueError('Voice component contains no base64, path, or url') + + +def estimate_audio_duration(data: bytes, format_hint: str = 'unknown') -> float: + """Estimate or calculate audio duration in seconds from raw bytes.""" + if not data: + return 0.0 + + fmt = detect_audio_format(data) + if fmt == 'unknown': + fmt = format_hint.lower().lstrip('.') + + if fmt == 'wav' and len(data) >= 44: + try: + byte_rate = int.from_bytes(data[28:32], byteorder='little') + if byte_rate > 0: + pos = 12 + while pos + 8 <= len(data): + chunk_id = data[pos : pos + 4] + chunk_size = int.from_bytes(data[pos + 4 : pos + 8], byteorder='little') + if chunk_id == b'data': + return max(0.1, round(chunk_size / byte_rate, 2)) + pos += 8 + chunk_size + return max(0.1, round((len(data) - 44) / byte_rate, 2)) + except Exception: + pass + + if fmt == 'mp3': + offset = 0 + if data.startswith(b'ID3') and len(data) >= 10: + size = ((data[6] & 0x7F) << 21) | ((data[7] & 0x7F) << 14) | ((data[8] & 0x7F) << 7) | (data[9] & 0x7F) + offset = 10 + size + audio_data = data[offset:] + if audio_data: + for i in range(min(4096, len(audio_data) - 4)): + if audio_data[i] == 0xFF and (audio_data[i + 1] & 0xE0) == 0xE0: + bitrate_idx = (audio_data[i + 2] >> 4) & 15 + bitrates = [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 0] + if 0 < bitrate_idx < len(bitrates) and bitrates[bitrate_idx] > 0: + kbps = bitrates[bitrate_idx] + return max(0.1, round(len(audio_data) * 8 / (kbps * 1000), 2)) + return max(0.1, round(len(audio_data) * 8 / 128000, 2)) + + if fmt == 'silk': + header_len = 10 if data.startswith(b'\x02#!SILK_V3') else (9 if data.startswith(b'#!SILK_V3') else 0) + pos = header_len + frames = 0 + while pos + 2 <= len(data): + size = int.from_bytes(data[pos : pos + 2], byteorder='little') + if size <= 0 or pos + 2 + size > len(data): + break + frames += 1 + pos += 2 + size + if frames > 0: + return max(0.1, round(frames * 0.02, 2)) + + if fmt == 'amr': + body_len = max(0, len(data) - 6) + return max(0.1, round((body_len / 32) * 0.02, 2)) + + if fmt in ('ogg', 'opus'): + # The granule position of the last Ogg page is the total sample count. + last_page = data.rfind(b'OggS') + if last_page >= 0 and last_page + 14 <= len(data): + granule = int.from_bytes(data[last_page + 6 : last_page + 14], byteorder='little') + sample_rate = 0 + if fmt == 'opus': + sample_rate = 48000 + else: + vorbis = data.find(b'\x01vorbis') + if vorbis >= 0 and vorbis + 16 <= len(data): + sample_rate = int.from_bytes(data[vorbis + 12 : vorbis + 16], byteorder='little') + if sample_rate > 0 and 0 < granule < 2**62: + return max(0.1, round(granule / sample_rate, 2)) + + return 0.0 diff --git a/src/langbot/templates/default-pipeline-config.json b/src/langbot/templates/default-pipeline-config.json index b692c3e0e..83fe401f6 100644 --- a/src/langbot/templates/default-pipeline-config.json +++ b/src/langbot/templates/default-pipeline-config.json @@ -43,6 +43,24 @@ }, "runner_config": {} }, + "speech": { + "speech-recognition": { + "enable": false, + "model": "", + "max-duration": 60, + "keep-original-voice": false, + "on-failure": "notify" + }, + "text-to-speech": { + "enable": false, + "model": "", + "trigger": "voice-input-only", + "output": "voice-only", + "max-text-length": 500, + "voice": "", + "on-failure": "fallback" + } + }, "output": { "long-text-processing": { "threshold": 1000, diff --git a/src/langbot/templates/metadata/pipeline/speech.yaml b/src/langbot/templates/metadata/pipeline/speech.yaml new file mode 100644 index 000000000..38d484bd4 --- /dev/null +++ b/src/langbot/templates/metadata/pipeline/speech.yaml @@ -0,0 +1,209 @@ +name: speech +label: + en_US: Speech + zh_Hans: 语音 +stages: + - name: speech-recognition + label: + en_US: Speech Recognition (ASR) + zh_Hans: 语音识别 (ASR) + description: + en_US: Transcribe incoming voice messages to text + zh_Hans: 将收到的语音消息识别为文本 + config: + - name: enable + label: + en_US: Enable + zh_Hans: 启用 + type: boolean + required: true + default: false + - name: model + label: + en_US: Model + zh_Hans: 模型 + type: asr-model-selector + required: false + default: "" + show_if: + field: enable + operator: eq + value: true + - name: max-duration + label: + en_US: Maximum Duration (seconds) + zh_Hans: 最大语音时长(秒) + description: + en_US: Voice messages longer than this limit will not be processed + zh_Hans: 超过此时长的语音消息将不会被转写 + type: integer + required: true + default: 60 + show_if: + field: enable + operator: eq + value: true + - name: keep-original-voice + label: + en_US: Keep Original Voice + zh_Hans: 保留原始语音 + description: + en_US: Keep the original Voice component in the message chain alongside transcribed text + zh_Hans: 在消息链中保留原始语音组件(与转写后的文本共存) + type: boolean + required: true + default: false + show_if: + field: enable + operator: eq + value: true + - name: on-failure + label: + en_US: On Failure + zh_Hans: 失败处理 + description: + en_US: Action to take when transcription fails + zh_Hans: 语音识别失败时的处理策略 + type: select + required: true + default: notify + options: + - name: notify + label: + en_US: Notify User + zh_Hans: 提示用户 + - name: ignore + label: + en_US: Ignore + zh_Hans: 忽略 + show_if: + field: enable + operator: eq + value: true + - name: text-to-speech + label: + en_US: Text to Speech (TTS) + zh_Hans: 文本转语音 (TTS) + description: + en_US: "Synthesize outgoing response text to voice. Note: Native WeChat voice messages require SILK transcoding (ffmpeg & pilk); if not installed, the system gracefully degrades to audio file attachments or text." + zh_Hans: "将回复的文本内容转换为语音。注:微信原生语音气泡依赖 SILK 转码环境(ffmpeg 与 pilk);若未安装,系统将自动优雅降级为音频文件或文本。" + config: + - name: enable + label: + en_US: Enable + zh_Hans: 启用 + type: boolean + required: true + default: false + - name: model + label: + en_US: Model + zh_Hans: 模型 + type: tts-model-selector + required: false + default: "" + show_if: + field: enable + operator: eq + value: true + - name: trigger + label: + en_US: Trigger Mode + zh_Hans: 触发模式 + description: + en_US: When to convert response text to voice + zh_Hans: 何时将回复文本转换为语音 + type: select + required: true + default: voice-input-only + options: + - name: voice-input-only + label: + en_US: Voice Input Only + zh_Hans: 仅语音输入时 + - name: always + label: + en_US: Always + zh_Hans: 总是转换为语音 + - name: never + label: + en_US: Never + zh_Hans: 从不转换为语音 + show_if: + field: enable + operator: eq + value: true + - name: output + label: + en_US: Output Mode + zh_Hans: 输出模式 + description: + en_US: "How to output synthesized voice. If platform-native voice transcoding (e.g. SILK for WeChat) is unavailable, output gracefully degrades to an audio file." + zh_Hans: "合成语音的输出形式。若平台原生语音转码环境(如微信 SILK)未安装,将自动优雅降级为音频文件。" + type: select + required: true + default: voice-only + options: + - name: voice-only + label: + en_US: Voice Only + zh_Hans: 仅语音 + - name: voice-and-text + label: + en_US: Voice and Text + zh_Hans: 语音与文本并存 + show_if: + field: enable + operator: eq + value: true + - name: max-text-length + label: + en_US: Maximum Text Length + zh_Hans: 最大文本长度 + description: + en_US: Text longer than this limit will fall back to text output + zh_Hans: 超过此时长的文本将降级为纯文本输出 + type: integer + required: true + default: 500 + show_if: + field: enable + operator: eq + value: true + - name: voice + label: + en_US: Voice + zh_Hans: 发音人 / 音色 + description: + en_US: Optional voice parameter to pass to the TTS model + zh_Hans: 传给 TTS 模型的发音人音色参数(可选) + type: string + required: false + default: "" + show_if: + field: enable + operator: eq + value: true + - name: on-failure + label: + en_US: On Failure + zh_Hans: 失败处理 + description: + en_US: Action to take when synthesis fails + zh_Hans: 语音合成失败时的处理策略 + type: select + required: true + default: fallback + options: + - name: fallback + label: + en_US: Fallback to Text + zh_Hans: 降级为纯文本 + - name: notify + label: + en_US: Notify User and Fallback + zh_Hans: 提示用户并降级 + show_if: + field: enable + operator: eq + value: true diff --git a/tests/e2e/test_startup.py b/tests/e2e/test_startup.py index 939effc6b..5e63a6b48 100644 --- a/tests/e2e/test_startup.py +++ b/tests/e2e/test_startup.py @@ -243,7 +243,7 @@ class TestMinimalStartupNoLLM: payload = response.json() assert payload['code'] == 0 configs = payload['data']['configs'] - assert [config['name'] for config in configs] == ['trigger', 'safety', 'ai', 'output'] + assert [config['name'] for config in configs] == ['trigger', 'safety', 'ai', 'speech', 'output'] ai = configs[2] runner = next(stage for stage in ai['stages'] if stage['name'] == 'runner') assert any(item['name'] == 'id' for item in runner['config']) diff --git a/tests/integration/api/test_providers.py b/tests/integration/api/test_providers.py index cc01466b4..b7627bd27 100644 --- a/tests/integration/api/test_providers.py +++ b/tests/integration/api/test_providers.py @@ -102,7 +102,7 @@ def fake_provider_app(): app.provider_service.update_provider = AsyncMock(return_value={}) app.provider_service.delete_provider = AsyncMock() app.provider_service.get_provider_model_counts = AsyncMock( - return_value={'llm_count': 2, 'embedding_count': 1, 'rerank_count': 0} + return_value={'llm_count': 2, 'embedding_count': 1, 'rerank_count': 0, 'asr_count': 1, 'tts_count': 0} ) # LLM model service @@ -271,10 +271,13 @@ class TestProviderEndpoints: assert called.await_args.args[0].workspace_uuid == 'workspace-test' assert called.await_args.args[1:] == ('test-provider-uuid',) assert called.await_args.kwargs == {} - assert {key: provider_data[key] for key in ('llm_count', 'embedding_count', 'rerank_count')} == { + count_keys = ('llm_count', 'embedding_count', 'rerank_count', 'asr_count', 'tts_count') + assert {key: provider_data[key] for key in count_keys} == { 'llm_count': 2, 'embedding_count': 1, 'rerank_count': 0, + 'asr_count': 1, + 'tts_count': 0, } diff --git a/tests/integration/persistence/test_migrations.py b/tests/integration/persistence/test_migrations.py index cbe7a8cba..63081abc1 100644 --- a/tests/integration/persistence/test_migrations.py +++ b/tests/integration/persistence/test_migrations.py @@ -122,6 +122,60 @@ class TestSQLiteMigrationBaseline: class TestSQLiteMigrationUpgrade: """Tests for upgrade to head workflow.""" + @pytest.mark.asyncio + @pytest.mark.parametrize('with_model_parents', [False, True]) + async def test_speech_upgrade_preserves_pipeline_and_model_constraints(self, sqlite_engine, with_model_parents): + metadata = sqlalchemy.MetaData() + if with_model_parents: + sqlalchemy.Table('workspaces', metadata, sqlalchemy.Column('uuid', sqlalchemy.String(36), primary_key=True)) + sqlalchemy.Table( + 'model_providers', + metadata, + sqlalchemy.Column('uuid', sqlalchemy.String(255), primary_key=True), + sqlalchemy.Column('workspace_uuid', sqlalchemy.String(36), nullable=False), + sqlalchemy.UniqueConstraint('workspace_uuid', 'uuid'), + ) + pipelines = sqlalchemy.Table( + 'legacy_pipelines', + metadata, + sqlalchemy.Column('uuid', sqlalchemy.String(255), primary_key=True), + sqlalchemy.Column('stages', sqlalchemy.JSON), + sqlalchemy.Column('config', sqlalchemy.JSON), + ) + stages = ['GroupRespondRuleCheckStage', 'PreContentFilterStage', 'ResponseWrapper', 'LongTextProcessStage'] + async with sqlite_engine.begin() as conn: + await conn.run_sync(metadata.create_all) + await conn.execute( + pipelines.insert().values(uuid='pipeline', stages=stages, config={'ai': {'model': 'llm'}}) + ) + await run_alembic_stamp(sqlite_engine, '0042_sqlite_incremental_vacuum') + await run_alembic_upgrade(sqlite_engine, '0043_speech_models') + async with sqlite_engine.connect() as conn: + pipeline = (await conn.execute(pipelines.select())).one() + assert pipeline.stages == [ + 'GroupRespondRuleCheckStage', + 'SpeechRecognitionStage', + 'PreContentFilterStage', + 'ResponseWrapper', + 'TextToSpeechStage', + 'LongTextProcessStage', + ] + assert pipeline.config['ai'] == {'model': 'llm'} + assert pipeline.config['speech']['speech-recognition']['enabled'] is False + assert pipeline.config['speech']['text-to-speech']['enabled'] is False + for table_name in ('asr_models', 'tts_models'): + assert ( + await conn.run_sync(lambda sync: sqlalchemy.inspect(sync).has_table(table_name)) + is with_model_parents + ) + if with_model_parents: + foreign_keys = await conn.run_sync( + lambda sync: sqlalchemy.inspect(sync).get_foreign_keys(table_name) + ) + provider_key = next(key for key in foreign_keys if key['referred_table'] == 'model_providers') + assert provider_key['constrained_columns'] == ['workspace_uuid', 'provider_uuid'] + assert provider_key['referred_columns'] == ['workspace_uuid', 'uuid'] + @pytest.mark.asyncio async def test_execution_filter_index_round_trip_preserves_history(self, sqlite_engine): from langbot.pkg.entity.persistence.monitoring import MonitoringMessage diff --git a/tests/unit_tests/api/service/test_pipeline_service.py b/tests/unit_tests/api/service/test_pipeline_service.py index 6f324c9b8..30f2507d2 100644 --- a/tests/unit_tests/api/service/test_pipeline_service.py +++ b/tests/unit_tests/api/service/test_pipeline_service.py @@ -76,6 +76,7 @@ class TestPipelineServiceGetPipelineMetadata: ap.pipeline_config_meta_trigger = {'trigger': {}} ap.pipeline_config_meta_safety = {'safety': {}} ap.pipeline_config_meta_ai = {'ai': {}} + ap.pipeline_config_meta_speech = {'speech': {}} ap.pipeline_config_meta_output = {'output': {}} service = PipelineService(ap) @@ -86,11 +87,12 @@ class TestPipelineServiceGetPipelineMetadata: ) # Verify - assert len(result) == 4 + assert len(result) == 5 assert 'trigger' in result[0] assert 'safety' in result[1] assert 'ai' in result[2] - assert 'output' in result[3] + assert 'speech' in result[3] + assert 'output' in result[4] class TestPipelineServiceGetPipelines: diff --git a/tests/unit_tests/api/service/test_provider_cascade.py b/tests/unit_tests/api/service/test_provider_cascade.py index 971dafc3b..75f7091cc 100644 --- a/tests/unit_tests/api/service/test_provider_cascade.py +++ b/tests/unit_tests/api/service/test_provider_cascade.py @@ -13,7 +13,15 @@ from sqlalchemy.ext.asyncio import create_async_engine from langbot.pkg.api.http.context import ExecutionContext from langbot.pkg.api.http.controller.groups.provider.providers import ModelProvidersRouterGroup from langbot.pkg.api.http.service.provider import ModelProviderService -from langbot.pkg.entity.persistence.model import CodexCredential, EmbeddingModel, LLMModel, ModelProvider, RerankModel +from langbot.pkg.entity.persistence.model import ( + ASRModel, + TTSModel, + CodexCredential, + EmbeddingModel, + LLMModel, + ModelProvider, + RerankModel, +) from langbot.pkg.entity.persistence.user import User from langbot.pkg.entity.persistence.metadata import WorkspaceMetadata from langbot.pkg.entity.persistence.workspace import Workspace @@ -22,7 +30,7 @@ from langbot.pkg.provider.modelmgr.modelmgr import ModelManager from langbot.pkg.workspace.errors import WorkspaceNotFoundError pytestmark = pytest.mark.asyncio -MODEL_TYPES = (LLMModel, EmbeddingModel, RerankModel) +MODEL_TYPES = (LLMModel, EmbeddingModel, RerankModel, ASRModel, TTSModel) TABLES = (*MODEL_TYPES, CodexCredential, ModelProvider) @@ -100,6 +108,8 @@ async def deletion(tmp_path): manager.llm_model_dict, manager.embedding_model_dict, manager.rerank_model_dict, + manager.asr_model_dict, + manager.tts_model_dict, ), ): for index in range(2): @@ -121,6 +131,8 @@ async def deletion(tmp_path): manager.llm_model_dict, manager.embedding_model_dict, manager.rerank_model_dict, + manager.asr_model_dict, + manager.tts_model_dict, ) ] try: @@ -148,6 +160,8 @@ def assert_caches_unchanged(deletion): deletion.manager.llm_model_dict, deletion.manager.embedding_model_dict, deletion.manager.rerank_model_dict, + deletion.manager.asr_model_dict, + deletion.manager.tts_model_dict, ) ] @@ -167,6 +181,8 @@ async def test_cascade_deletes_all_model_types_and_credentials_after_commit(dele deletion.manager.llm_model_dict, deletion.manager.embedding_model_dict, deletion.manager.rerank_model_dict, + deletion.manager.asr_model_dict, + deletion.manager.tts_model_dict, ): assert all(not key[-1].startswith('target') for key in cache) assert any(key[1] == 'b' for key in cache) diff --git a/tests/unit_tests/api/service/test_provider_service.py b/tests/unit_tests/api/service/test_provider_service.py index 62612892d..391cd34b8 100644 --- a/tests/unit_tests/api/service/test_provider_service.py +++ b/tests/unit_tests/api/service/test_provider_service.py @@ -21,7 +21,14 @@ from unittest.mock import AsyncMock, Mock from types import SimpleNamespace from langbot.pkg.api.http.service.provider import ModelProviderService -from langbot.pkg.entity.persistence.model import ModelProvider, LLMModel, EmbeddingModel, RerankModel +from langbot.pkg.entity.persistence.model import ( + ModelProvider, + LLMModel, + EmbeddingModel, + RerankModel, + ASRModel, + TTSModel, +) from langbot.pkg.workspace.errors import WorkspaceNotFoundError @@ -412,12 +419,12 @@ class TestModelProviderServiceUpdateProvider: class TestModelProviderServiceDeleteProvider: """Fast guard coverage; real transaction/cache behavior is in test_provider_cascade.""" - @pytest.mark.parametrize('label', ['LLM', 'Embedding', 'Rerank', None]) + @pytest.mark.parametrize('label', ['LLM', 'Embedding', 'Rerank', 'ASR', 'TTS', None]) async def test_delete_provider_requires_no_references(self, label): provider_result = Mock() provider_result.first.return_value = SimpleNamespace(requester='openai') results = [provider_result] - for model_label in ('LLM', 'Embedding', 'Rerank'): + for model_label in ('LLM', 'Embedding', 'Rerank', 'ASR', 'TTS'): result = Mock() result.scalars.return_value = ['model'] if label == model_label else [] results.append(result) @@ -462,6 +469,8 @@ class TestModelProviderServiceGetProviderModelCounts: LLMModel.__tablename__: (LLMModel, llm_result), EmbeddingModel.__tablename__: (EmbeddingModel, embedding_result), RerankModel.__tablename__: (RerankModel, rerank_result), + ASRModel.__tablename__: (ASRModel, Mock(scalar=Mock(return_value=4))), + TTSModel.__tablename__: (TTSModel, Mock(scalar=Mock(return_value=5))), } async def mock_execute(query): @@ -485,6 +494,8 @@ class TestModelProviderServiceGetProviderModelCounts: assert result['llm_count'] == 3 assert result['embedding_count'] == 2 assert result['rerank_count'] == 1 + assert result['asr_count'] == 4 + assert result['tts_count'] == 5 async def test_get_model_counts_zero_counts(self): """Returns zero counts when no models.""" @@ -507,6 +518,8 @@ class TestModelProviderServiceGetProviderModelCounts: assert result['llm_count'] == 0 assert result['embedding_count'] == 0 assert result['rerank_count'] == 0 + assert result['asr_count'] == 0 + assert result['tts_count'] == 0 class TestModelProviderServiceFindOrCreateProvider: diff --git a/tests/unit_tests/api/test_provider_controller_secrets.py b/tests/unit_tests/api/test_provider_controller_secrets.py index 73d9fe990..c156bfddc 100644 --- a/tests/unit_tests/api/test_provider_controller_secrets.py +++ b/tests/unit_tests/api/test_provider_controller_secrets.py @@ -62,7 +62,7 @@ async def _create_client(role: str): application.provider_service = SimpleNamespace( get_providers=AsyncMock(side_effect=get_providers), get_provider=AsyncMock(side_effect=get_provider), - get_provider_model_counts=AsyncMock(return_value={'llm_count': 1, 'embedding_count': 1, 'rerank_count': 1}), + get_provider_model_counts=AsyncMock(return_value={'llm_count': 1, 'embedding_count': 1, 'rerank_count': 1, 'asr_count': 1, 'tts_count': 1}), ) def model_service(list_name: str, get_name: str): @@ -135,7 +135,7 @@ async def test_default_preserves_secrets_and_explicit_false_redacts_high_permiss default_value = default_data['data'].get('providers', default_data['data'].get('models'))[0] false_value = false_data['data'].get('providers', false_data['data'].get('models'))[0] expected = ( - {**RAW_PROVIDER, 'llm_count': 1, 'embedding_count': 1, 'rerank_count': 1} + {**RAW_PROVIDER, 'llm_count': 1, 'embedding_count': 1, 'rerank_count': 1, 'asr_count': 1, 'tts_count': 1} if endpoint.endswith('/providers') else RAW_MODEL ) diff --git a/tests/unit_tests/api/test_resource_secret_permissions.py b/tests/unit_tests/api/test_resource_secret_permissions.py index 9e4e31d98..ffd604f30 100644 --- a/tests/unit_tests/api/test_resource_secret_permissions.py +++ b/tests/unit_tests/api/test_resource_secret_permissions.py @@ -94,7 +94,7 @@ async def _create_client(role: str): application.provider_service = SimpleNamespace( get_providers=AsyncMock(side_effect=get_providers), - get_provider_model_counts=AsyncMock(return_value={'llm_count': 0, 'embedding_count': 0, 'rerank_count': 0}), + get_provider_model_counts=AsyncMock(return_value={'llm_count': 0, 'embedding_count': 0, 'rerank_count': 0, 'asr_count': 0, 'tts_count': 0}), ) async def get_knowledge_bases(_context, *, include_secret=False): diff --git a/tests/unit_tests/pipeline/test_speech_stages.py b/tests/unit_tests/pipeline/test_speech_stages.py new file mode 100644 index 000000000..4812d057d --- /dev/null +++ b/tests/unit_tests/pipeline/test_speech_stages.py @@ -0,0 +1,406 @@ +from __future__ import annotations + +import base64 +import io +from unittest.mock import AsyncMock, Mock +import wave + +import langbot_plugin.api.entities.builtin.platform.message as platform_message +import pytest + +from langbot.pkg.pipeline.entities import ResultType +from langbot.pkg.pipeline.speech import speech as speech_module + + +def _create_wav_bytes() -> bytes: + buf = io.BytesIO() + with wave.open(buf, 'wb') as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b'\x00' * 3200) + return buf.getvalue() + + +@pytest.mark.asyncio +async def test_asr_disabled_continues(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': False, + 'model': 'asr-model-uuid', + } + } + } + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + assert result.result_type == ResultType.CONTINUE + + +@pytest.mark.asyncio +async def test_asr_no_voice_continues(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + } + } + } + sample_query.message_chain = platform_message.MessageChain([platform_message.Plain(text='hello')]) + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + assert result.result_type == ResultType.CONTINUE + + +@pytest.mark.asyncio +async def test_asr_max_duration_exceeded_notify_stops(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'max-duration': 30, + 'on-failure': 'notify', + } + } + } + sample_query.message_chain = platform_message.MessageChain( + [platform_message.Voice(base64='dummy', length=45)] + ) + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + assert result.result_type == ResultType.INTERRUPT + assert result.user_notice == 'Voice message exceeds maximum duration.' + + +@pytest.mark.asyncio +async def test_asr_max_duration_exceeded_ignore_continues(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'max-duration': 30, + 'on-failure': 'ignore', + } + } + } + sample_query.message_chain = platform_message.MessageChain( + [platform_message.Voice(base64='dummy', length=45)] + ) + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + assert result.result_type == ResultType.CONTINUE + + +@pytest.mark.asyncio +async def test_asr_success_replaces_voice(mock_app, sample_query): + wav_bytes = _create_wav_bytes() + b64_str = base64.b64encode(wav_bytes).decode('ascii') + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'keep-original-voice': False, + 'max-duration': 60, + } + } + } + sample_query.message_chain = platform_message.MessageChain([platform_message.Voice(base64=b64_str)]) + + mock_asr_model = Mock() + mock_asr_model.execution_context = Mock() + mock_asr_model.provider = Mock() + mock_asr_model.provider.invoke_asr = AsyncMock(return_value='Transcribed voice message') + mock_app.model_mgr.get_asr_model_by_uuid = AsyncMock(return_value=mock_asr_model) + + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + + assert result.result_type == ResultType.CONTINUE + assert sample_query.variables['_input_was_voice'] is True + assert sample_query.variables['voice_transcript'] == 'Transcribed voice message' + assert len(sample_query.message_chain) == 1 + assert isinstance(sample_query.message_chain[0], platform_message.Plain) + assert sample_query.message_chain[0].text == 'Transcribed voice message' + + +@pytest.mark.asyncio +async def test_asr_success_keep_original_voice(mock_app, sample_query): + wav_bytes = _create_wav_bytes() + b64_str = base64.b64encode(wav_bytes).decode('ascii') + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'keep-original-voice': True, + 'max-duration': 60, + } + } + } + sample_query.message_chain = platform_message.MessageChain([platform_message.Voice(base64=b64_str)]) + + mock_asr_model = Mock() + mock_asr_model.execution_context = Mock() + mock_asr_model.provider = Mock() + mock_asr_model.provider.invoke_asr = AsyncMock(return_value='Transcribed voice message') + mock_app.model_mgr.get_asr_model_by_uuid = AsyncMock(return_value=mock_asr_model) + + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.message_chain) == 2 + assert isinstance(sample_query.message_chain[0], platform_message.Voice) + assert isinstance(sample_query.message_chain[1], platform_message.Plain) + assert sample_query.message_chain[1].text == 'Transcribed voice message' + + +@pytest.mark.asyncio +async def test_asr_failure_notify_stops(mock_app, sample_query): + wav_bytes = _create_wav_bytes() + b64_str = base64.b64encode(wav_bytes).decode('ascii') + sample_query.pipeline_config = { + 'speech': { + 'speech-recognition': { + 'enable': True, + 'model': 'asr-model-uuid', + 'on-failure': 'notify', + } + } + } + sample_query.message_chain = platform_message.MessageChain([platform_message.Voice(base64=b64_str)]) + + mock_asr_model = Mock() + mock_asr_model.execution_context = Mock() + mock_asr_model.provider = Mock() + mock_asr_model.provider.invoke_asr = AsyncMock(side_effect=RuntimeError('Transcription API error')) + mock_app.model_mgr.get_asr_model_by_uuid = AsyncMock(return_value=mock_asr_model) + + stage = speech_module.SpeechRecognitionStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'SpeechRecognitionStage') + + assert result.result_type == ResultType.INTERRUPT + assert result.user_notice == 'Voice transcription failed.' + + +@pytest.mark.asyncio +async def test_tts_disabled_continues(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': False, + 'model': 'tts-model-uuid', + } + } + } + sample_query.resp_message_chain = [platform_message.MessageChain([platform_message.Plain(text='Hello')])] + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + assert result.result_type == ResultType.CONTINUE + + +@pytest.mark.asyncio +async def test_tts_voice_input_only_skips_when_not_voice(mock_app, sample_query): + sample_query.variables['_input_was_voice'] = False + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'voice-input-only', + } + } + } + sample_query.resp_message_chain = [platform_message.MessageChain([platform_message.Plain(text='Hello')])] + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.resp_message_chain[-1]) == 1 + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + + +@pytest.mark.asyncio +async def test_tts_voice_omitted_when_not_configured(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'always', + 'output': 'voice-only', + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='Hello without voice')]) + ] + + mock_tts_model = Mock() + mock_tts_model.execution_context = Mock() + mock_tts_model.extra_args = {} + mock_tts_model.model_entity = Mock(extra_args={}) + mock_tts_model.provider = Mock() + mock_tts_model.provider.invoke_tts = AsyncMock(return_value=(b'fake-mp3', 'audio/mp3')) + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + extra_args = mock_tts_model.provider.invoke_tts.call_args[1]['extra_args'] + assert 'voice' not in extra_args + + +@pytest.mark.asyncio +async def test_tts_failure_on_failure_fallback_and_notify(mock_app, sample_query): + # Test on-failure=fallback + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'always', + 'on-failure': 'fallback', + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='Fallback test')]) + ] + + mock_tts_model = Mock() + mock_tts_model.execution_context = Mock() + mock_tts_model.extra_args = {} + mock_tts_model.model_entity = Mock(extra_args={}) + mock_tts_model.provider = Mock() + mock_tts_model.provider.invoke_tts = AsyncMock(side_effect=RuntimeError('TTS provider error')) + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + assert not result.user_notice + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + + # Test on-failure=notify + sample_query.pipeline_config['speech']['text-to-speech']['on-failure'] = 'notify' + result = await stage.process(sample_query, 'TextToSpeechStage') + assert result.result_type == ResultType.CONTINUE + assert result.user_notice == 'Voice synthesis failed.' + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + + +@pytest.mark.asyncio +async def test_tts_voice_only_success(mock_app, sample_query): + sample_query.variables['_input_was_voice'] = True + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'voice-input-only', + 'output': 'voice-only', + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='**Hello** [world](https://example.com)!')]) + ] + + mock_tts_model = Mock() + mock_tts_model.execution_context = Mock() + mock_tts_model.extra_args = {'voice': 'alloy'} + mock_tts_model.provider = Mock() + mock_tts_model.provider.invoke_tts = AsyncMock(return_value=(b'fake-mp3-audio-content', 'audio/mp3')) + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.resp_message_chain[-1]) == 1 + voice_comp = sample_query.resp_message_chain[-1][0] + assert isinstance(voice_comp, platform_message.Voice) + assert voice_comp.base64.startswith('data:audio/mp3;base64,') + mock_tts_model.provider.invoke_tts.assert_called_once() + assert mock_tts_model.provider.invoke_tts.call_args[1]['text'] == 'Hello world!' + + +@pytest.mark.asyncio +async def test_tts_voice_and_text_success(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'always', + 'output': 'voice-and-text', + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='Response with both voice and text')]) + ] + + mock_tts_model = Mock() + mock_tts_model.execution_context = Mock() + mock_tts_model.extra_args = {} + mock_tts_model.provider = Mock() + mock_tts_model.provider.invoke_tts = AsyncMock(return_value=(b'fake-audio', 'audio/mp3')) + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.resp_message_chain[-1]) == 2 + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + assert isinstance(sample_query.resp_message_chain[-1][1], platform_message.Voice) + + +@pytest.mark.asyncio +async def test_tts_max_text_length_fallback(mock_app, sample_query): + sample_query.pipeline_config = { + 'speech': { + 'text-to-speech': { + 'enable': True, + 'model': 'tts-model-uuid', + 'trigger': 'always', + 'max-text-length': 10, + } + } + } + sample_query.resp_message_chain = [ + platform_message.MessageChain([platform_message.Plain(text='This is a long response text exceeding 10 chars')]) + ] + + mock_tts_model = Mock() + mock_app.model_mgr.get_tts_model_by_uuid = AsyncMock(return_value=mock_tts_model) + + stage = speech_module.TextToSpeechStage(mock_app) + await stage.initialize(sample_query.pipeline_config) + result = await stage.process(sample_query, 'TextToSpeechStage') + + assert result.result_type == ResultType.CONTINUE + assert len(sample_query.resp_message_chain[-1]) == 1 + assert isinstance(sample_query.resp_message_chain[-1][0], platform_message.Plain) + + diff --git a/tests/unit_tests/platform/test_dingtalk_api.py b/tests/unit_tests/platform/test_dingtalk_api.py index 91803baac..12cecff0d 100644 --- a/tests/unit_tests/platform/test_dingtalk_api.py +++ b/tests/unit_tests/platform/test_dingtalk_api.py @@ -1,9 +1,12 @@ """Tests for DingTalk API payload helpers.""" +import base64 import json from contextlib import asynccontextmanager from unittest.mock import AsyncMock +import pytest + from langbot.libs.dingtalk_api.api import DingTalkClient, _stringify_card_param_map from langbot.pkg.utils import httpclient @@ -82,3 +85,97 @@ async def test_create_card_embeds_layout_config_as_template_parameter(monkeypatc } } assert original_params == {'content': 'hello'} + + +def _voice_client(): + from langbot.libs.dingtalk_api.api import DingTalkClient + + client = DingTalkClient.__new__(DingTalkClient) + client.logger = None + client._upload_media = AsyncMock(return_value='media-1') + client._send_robot_message = AsyncMock(return_value={'ok': True}) + return client + + +@pytest.mark.asyncio +async def test_send_voice_transcodes_to_ogg_and_sends_sample_audio(monkeypatch): + from langbot.pkg.utils import audio as audio_utils + from langbot_plugin.api.entities.builtin.platform.message import Voice + + transcode = AsyncMock(return_value=b'OggS' + b'\x00' * 32) + monkeypatch.setattr(audio_utils, 'transcode_audio', transcode) + client = _voice_client() + voice = Voice(base64='data:audio/mpeg;base64,' + base64.b64encode(b'ID3' + b'\x00' * 32).decode(), length=5) + + await client.send_voice(voice, 'incoming') + + transcode.assert_awaited_once() + assert transcode.await_args.args[1] == 'ogg' + client._upload_media.assert_awaited_once() + name, _body, media_type = client._upload_media.await_args.args + assert (name, media_type) == ('voice.ogg', 'voice') + msg_key, params, incoming = client._send_robot_message.await_args.args + assert msg_key == 'sampleAudio' + assert params == {'mediaId': 'media-1', 'duration': '5000'} + assert incoming == 'incoming' + + +@pytest.mark.asyncio +async def test_send_voice_over_limit_is_sent_as_file(monkeypatch): + from langbot.pkg.utils import audio as audio_utils + from langbot_plugin.api.entities.builtin.platform.message import Voice + + monkeypatch.setattr(audio_utils, 'transcode_audio', AsyncMock()) + client = _voice_client() + voice = Voice(base64='data:audio/ogg;base64,' + base64.b64encode(b'OggS' + b'\x00' * 32).decode(), length=90) + + await client.send_voice(voice, 'incoming') + + audio_utils.transcode_audio.assert_not_awaited() + assert client._upload_media.await_args.args[2] == 'file' + msg_key, params, _ = client._send_robot_message.await_args.args + assert msg_key == 'sampleFile' + assert params == {'mediaId': 'media-1', 'fileName': 'voice.ogg', 'fileType': 'ogg'} + + +@pytest.mark.asyncio +async def test_send_voice_falls_back_to_file_when_transcode_fails(monkeypatch): + from langbot.pkg.utils import audio as audio_utils + from langbot_plugin.api.entities.builtin.platform.message import Voice + + monkeypatch.setattr(audio_utils, 'transcode_audio', AsyncMock(side_effect=RuntimeError('no ffmpeg'))) + client = _voice_client() + voice = Voice(base64='data:audio/mpeg;base64,' + base64.b64encode(b'ID3' + b'\x00' * 32).decode(), length=3) + + await client.send_voice(voice, 'incoming') + + assert client._upload_media.await_args.args[2] == 'file' + msg_key, params, _ = client._send_robot_message.await_args.args + assert msg_key == 'sampleFile' + assert params['fileName'] == 'voice.mp3' + + +def test_estimate_audio_duration_reads_ogg_opus_granule_position(): + from langbot.pkg.utils import audio as audio_utils + + def page(granule: int, payload: bytes) -> bytes: + return b'OggS\x00\x00' + granule.to_bytes(8, 'little') + b'\x00' * 12 + payload + + data = page(0, b'OpusHead' + b'\x00' * 11) + page(0, b'OpusTags') + page(48000 * 3, b'\x00' * 8) + + assert audio_utils.detect_audio_format(data) == 'opus' + assert audio_utils.estimate_audio_duration(data) == 3.0 + + +@pytest.mark.asyncio +async def test_extract_voice_bytes_ignores_non_audio_mime_hint(): + from langbot.pkg.utils import audio as audio_utils + from langbot_plugin.api.entities.builtin.platform.message import Voice + + voice = Voice(base64='data:application/octet-stream;base64,' + base64.b64encode(b'unrecognized').decode()) + _, fmt = await audio_utils.extract_voice_bytes(voice) + assert fmt == 'unknown' + + voice = Voice(base64='data:audio/mpeg;base64,' + base64.b64encode(b'unrecognized').decode()) + _, fmt = await audio_utils.extract_voice_bytes(voice) + assert fmt == 'mp3' diff --git a/tests/unit_tests/platform/test_dingtalk_eba_adapter.py b/tests/unit_tests/platform/test_dingtalk_eba_adapter.py index 74d5a1c04..015aa2fe7 100644 --- a/tests/unit_tests/platform/test_dingtalk_eba_adapter.py +++ b/tests/unit_tests/platform/test_dingtalk_eba_adapter.py @@ -559,3 +559,47 @@ async def test_dingtalk_subscription_retry_does_not_repeat_successful_members(): assert (await handler.process(message))[0] == 200 assert (await handler.process(message))[0] == 200 assert [call.args[0].member.id for call in adapter._dispatch_eba_event.await_args_list] == ['a', 'b', 'b'] + + +@pytest.mark.asyncio +async def test_dingtalk_inbound_voice_keeps_audio_for_pipeline_speech_recognition(): + event = dingtalk_event( + conversation='OneToOneMessage', + msg_type='audio', + content='DingTalk recognition text', + audio='data:audio/ogg;base64,AAAA', + ) + event['AudioDuration'] = 4 + chain = await DingTalkMessageConverter.target2yiri(event, 'LangBot') + + voices = [part for part in chain if isinstance(part, platform_message.Voice)] + assert len(voices) == 1 + assert voices[0].base64 == 'data:audio/ogg;base64,AAAA' + assert voices[0].length == 4 + assert not any(isinstance(part, platform_message.Plain) for part in chain) + + +@pytest.mark.asyncio +async def test_dingtalk_inbound_voice_falls_back_to_recognition_without_audio(): + event = dingtalk_event(conversation='OneToOneMessage', msg_type='audio', content='DingTalk recognition text') + chain = await DingTalkMessageConverter.target2yiri(event, 'LangBot') + + assert not any(isinstance(part, platform_message.Voice) for part in chain) + plains = [part for part in chain if isinstance(part, platform_message.Plain)] + assert [part.text for part in plains] == ['DingTalk recognition text'] + + +@pytest.mark.asyncio +async def test_dingtalk_reply_sends_voice_as_attachment_not_text(): + adapter = make_adapter() + adapter.bot.send_attachment = AsyncMock() + voice = platform_message.Voice(base64='data:audio/mpeg;base64,AAAA', length=3) + message = platform_message.MessageChain([platform_message.Plain(text='hello'), voice]) + + source_event = await DingTalkEventConverter.target2yiri(dingtalk_event(), 'LangBot') + await adapter.reply_message(source_event, message) + + content = adapter.bot.send_message.await_args.args[0] + assert content == 'hello' + adapter.bot.send_attachment.assert_awaited_once() + assert adapter.bot.send_attachment.await_args.args[0] is voice diff --git a/tests/unit_tests/platform/test_discord_eba_adapter.py b/tests/unit_tests/platform/test_discord_eba_adapter.py index 0e776bf1a..db86c1d21 100644 --- a/tests/unit_tests/platform/test_discord_eba_adapter.py +++ b/tests/unit_tests/platform/test_discord_eba_adapter.py @@ -181,6 +181,38 @@ async def test_discord_message_converter_maps_mentions_and_text_to_target(): assert files == [] +@pytest.mark.asyncio +async def test_discord_message_converter_maps_audio_attachments_to_voice(monkeypatch): + wav_header = b'RIFF$\x00\x00\x00WAVEfmt \x10\x00\x00\x00\x01\x00\x01\x00\x80>\x00\x00\x00}\x00\x00\x02\x00\x10\x00data\x00\x00\x00\x00' + attachment = SimpleNamespace( + id=999, + filename='voice-message.ogg', + url='https://cdn.discordapp.com/attachments/1/2/voice-message.ogg', + size=len(wav_header), + content_type='audio/ogg', + flags=SimpleNamespace(voice=True), + duration_secs=4.5, + ) + message = SimpleNamespace( + id=123, + content='Voice test', + created_at=datetime.datetime.now(), + attachments=[attachment], + ) + monkeypatch.setattr( + DiscordMessageConverter, + '_download', + AsyncMock(return_value=(wav_header, 'audio/ogg')), + ) + + chain = await DiscordMessageConverter.target2yiri(message) + voices = [c for c in chain if isinstance(c, platform_message.Voice)] + assert len(voices) == 1 + assert voices[0].url == attachment.url + assert voices[0].length == 4 + assert voices[0].base64.startswith('data:audio/ogg;base64,') + + def test_discord_message_converter_splits_discord_mentions(): components = DiscordMessageConverter._text_components('hi <@123> and @everyone') diff --git a/tests/unit_tests/platform/test_openclaw_weixin_client.py b/tests/unit_tests/platform/test_openclaw_weixin_client.py index 11f0779ab..7762dec2a 100644 --- a/tests/unit_tests/platform/test_openclaw_weixin_client.py +++ b/tests/unit_tests/platform/test_openclaw_weixin_client.py @@ -37,3 +37,68 @@ async def test_upload_media_rejects_oversized_input_before_network_access(): 'recipient', 3, ) + + +def test_voice_item_serialization_includes_encode_type_and_playtime(): + from langbot.libs.openclaw_weixin_api.client import _message_item_to_dict + from langbot.libs.openclaw_weixin_api.types import CDNMedia, MessageItem, VoiceItem + + media = CDNMedia(encrypt_query_param='param', aes_key='key', encrypt_type=1) + item = MessageItem( + type=MessageItem.VOICE, + voice_item=VoiceItem( + media=media, + playtime=4500, + encode_type=6, + bits_per_sample=16, + sample_rate=24000, + size=1024, + ), + ) + d = _message_item_to_dict(item) + + assert d['type'] == MessageItem.VOICE + assert 'voice_item' in d + assert d['voice_item']['playtime'] == 4500 + assert d['voice_item']['encode_type'] == 6 + assert d['voice_item']['bits_per_sample'] == 16 + assert d['voice_item']['sample_rate'] == 24000 + assert d['voice_item']['size'] == 1024 + assert d['voice_item']['media']['encrypt_query_param'] == 'param' + + +@pytest.mark.asyncio +async def test_inbound_voice_conversion_to_voice_component(): + import base64 + import langbot_plugin.api.entities.builtin.platform.message as platform_message + from langbot.libs.openclaw_weixin_api.types import CDNMedia, MessageItem, VoiceItem, WeixinMessage + from langbot.pkg.platform.sources.openclaw_weixin import OpenClawWeixinMessageConverter + + fake_silk_data = b'\x02#!SILK_V3' + b'\x05\x00abcde' * 50 + item = MessageItem( + type=MessageItem.VOICE, + voice_item=VoiceItem( + media=CDNMedia(encrypt_query_param='param', aes_key='key'), + playtime=1000, + _downloaded_bytes=fake_silk_data, + ), + ) + msg = WeixinMessage( + message_id=1, + from_user_id='user1', + to_user_id='bot1', + item_list=[item], + ) + + converter = OpenClawWeixinMessageConverter() + chain = await converter.target2yiri(msg) + + assert len(chain) == 1 + voice = chain[0] + assert isinstance(voice, platform_message.Voice) + assert voice.length == 1 + assert voice.base64.startswith('data:audio/silk;base64,') + b64_payload = voice.base64.split(',', 1)[1] + assert base64.b64decode(b64_payload) == fake_silk_data + + diff --git a/tests/unit_tests/platform/test_openclaw_weixin_tenancy.py b/tests/unit_tests/platform/test_openclaw_weixin_tenancy.py index 8a2f53e9b..cbc236b15 100644 --- a/tests/unit_tests/platform/test_openclaw_weixin_tenancy.py +++ b/tests/unit_tests/platform/test_openclaw_weixin_tenancy.py @@ -1,3 +1,4 @@ +import base64 from types import SimpleNamespace from unittest.mock import AsyncMock, Mock @@ -26,6 +27,8 @@ def make_adapter(*, execution_context: ExecutionContext | None): ap=app, execution_context=execution_context, warning=AsyncMock(), + info=AsyncMock(), + error=AsyncMock(), ) adapter = OpenClawWeixinAdapter.model_construct( config={'token': 'refreshed-token'}, @@ -93,3 +96,41 @@ async def test_component_base64_decode_is_bounded(monkeypatch): with pytest.raises(ValueError, match='exceeds'): await OpenClawWeixinAdapter._get_component_bytes(component) + + +@pytest.mark.asyncio +async def test_component_base64_strips_data_uri_prefix(): + raw_payload = b'hello world audio payload' + encoded = base64.b64encode(raw_payload).decode('ascii') + component = platform_message.Voice(base64=f'data:audio/mpeg;base64,{encoded}') + + result = await OpenClawWeixinAdapter._get_component_bytes(component) + assert result == raw_payload + + +@pytest.mark.asyncio +async def test_voice_send_gracefully_degrades_to_file_when_silk_unavailable(monkeypatch): + adapter, app, logger = make_adapter(execution_context=None) + adapter.client = AsyncMock() + + # Emulate SILK or ffmpeg unavailable + monkeypatch.setattr('langbot.pkg.utils.audio.is_silk_available', lambda: False) + monkeypatch.setattr('langbot.pkg.utils.audio.is_ffmpeg_available', lambda: False) + + raw_mp3 = b'ID3\x04\x00\x00\x00\x00\x00\x00\xff\xfb\x90\x00fake-mp3-audio' + encoded = base64.b64encode(raw_mp3).decode('ascii') + voice_comp = platform_message.Voice(base64=f'data:audio/mpeg;base64,{encoded}', length=3) + + msg = platform_message.MessageChain([voice_comp]) + await adapter.send_message('friend', 'target-wxid', msg) + + # Should NOT call send_voice (which WeChat mobile would silently drop for MP3) + adapter.client.send_voice.assert_not_awaited() + # Should call send_file as graceful degradation + adapter.client.send_file.assert_awaited_once_with( + 'target-wxid', + raw_mp3, + 'voice.mp3', + '', + ) + diff --git a/tests/unit_tests/platform/test_telegram_adapter.py b/tests/unit_tests/platform/test_telegram_adapter.py index cf661d638..daeef1799 100644 --- a/tests/unit_tests/platform/test_telegram_adapter.py +++ b/tests/unit_tests/platform/test_telegram_adapter.py @@ -367,3 +367,123 @@ async def test_telegram_text_field_does_not_show_action_buttons(): assert args['text'].startswith('[人工介入]') assert 'us_input (paragraph)' in args['text'] assert adapter._form_action_titles == {} + + +@pytest.mark.asyncio +async def test_telegram_source_reply_message_sends_text_and_voice(): + import base64 as _b64 + import telegram as _tg + + bot = MagicMock() + bot.send_message = AsyncMock() + bot.send_voice = AsyncMock() + adapter = TelegramAdapter.model_construct(bot=bot, config={}, msg_stream_id={}, seq=1, listeners={}) + + update = _tg.Update.de_json( + { + 'update_id': 1000, + 'message': { + 'message_id': 99, + 'date': 1700000000, + 'chat': {'id': 123, 'type': 'private'}, + 'from': {'id': 1, 'is_bot': False, 'first_name': 'User'}, + 'text': 'hi', + }, + }, + None, + ) + event = platform_events.FriendMessage( + sender=platform_entities.Friend(id='user-1', nickname='', remark=''), + message_chain=platform_message.MessageChain([]), + source_platform_object=update, + ) + + # Coexist mode: text and voice + chain = platform_message.MessageChain( + [ + platform_message.Plain(text='Hello'), + platform_message.Voice( + base64='data:audio/ogg;base64,' + _b64.b64encode(b'OggS-voice').decode('utf-8'), + length=6, + ), + ] + ) + await adapter.reply_message(event, chain, quote_origin=True) + bot.send_message.assert_awaited_once() + bot.send_voice.assert_awaited_once() + assert bot.send_message.await_args.kwargs['reply_to_message_id'] == 99 + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 99 + assert bot.send_voice.await_args.kwargs['duration'] == 6 + assert bot.send_voice.await_args.kwargs['voice'].filename == 'voice.ogg' + + # Voice only mode + bot.send_message.reset_mock() + bot.send_voice.reset_mock() + voice_only_chain = platform_message.MessageChain( + [ + platform_message.Voice( + base64='data:audio/ogg;base64,' + _b64.b64encode(b'OggS-voice-only').decode('utf-8'), + length=4, + ), + ] + ) + await adapter.reply_message(event, voice_only_chain, quote_origin=True) + bot.send_message.assert_not_awaited() + bot.send_voice.assert_awaited_once() + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 99 + assert bot.send_voice.await_args.kwargs['duration'] == 4 + + +@pytest.mark.asyncio +async def test_telegram_source_reply_message_chunk_delivers_voice(): + import base64 as _b64 + import telegram as _tg + + bot = MagicMock() + bot.send_message = AsyncMock() + bot.send_voice = AsyncMock() + bot.send_message_draft = AsyncMock() + adapter = TelegramAdapter.model_construct(bot=bot, config={}, msg_stream_id={}, seq=1, listeners={}) + adapter.msg_stream_id['stream_1'] = ('private', 101, True) + + update = _tg.Update.de_json( + { + 'update_id': 1000, + 'message': { + 'message_id': 99, + 'date': 1700000000, + 'chat': {'id': 123, 'type': 'private'}, + 'from': {'id': 1, 'is_bot': False, 'first_name': 'User'}, + 'text': 'hi', + }, + }, + None, + ) + event = platform_events.FriendMessage( + sender=platform_entities.Friend(id='user-1', nickname='', remark=''), + message_chain=platform_message.MessageChain([]), + source_platform_object=update, + ) + + chunk = SimpleNamespace(resp_message_id='stream_1', msg_sequence=1, tool_calls=None) + chain = platform_message.MessageChain( + [ + platform_message.Plain(text='Streaming text'), + platform_message.Voice( + base64='data:audio/ogg;base64,' + _b64.b64encode(b'OggS-voice').decode('utf-8'), + length=5, + ), + ] + ) + + # Not final chunk: no voice sent yet + await adapter.reply_message_chunk(event, chunk, chain, is_final=False) + bot.send_voice.assert_not_awaited() + + # Final chunk: voice sent and message finalized + await adapter.reply_message_chunk(event, chunk, chain, quote_origin=True, is_final=True) + bot.send_voice.assert_awaited_once() + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 99 + assert bot.send_voice.await_args.kwargs['duration'] == 5 + bot.send_message.assert_awaited_once() + assert 'stream_1' not in adapter.msg_stream_id diff --git a/tests/unit_tests/platform/test_telegram_eba_adapter.py b/tests/unit_tests/platform/test_telegram_eba_adapter.py index 3c2a0a15f..f3808679a 100644 --- a/tests/unit_tests/platform/test_telegram_eba_adapter.py +++ b/tests/unit_tests/platform/test_telegram_eba_adapter.py @@ -531,6 +531,7 @@ async def test_telegram_reply_message_sends_text_image_and_file_components(): send_message=AsyncMock(), send_photo=AsyncMock(), send_document=AsyncMock(), + send_voice=AsyncMock(), ) object.__setattr__(adapter, 'bot', bot) update = make_update({'message': base_message_payload(message_id=88)}) @@ -552,6 +553,10 @@ async def test_telegram_reply_message_sends_text_image_and_file_components(): size=4, base64='data:text/plain;base64,' + base64.b64encode(b'test').decode('utf-8'), ), + platform_message.Voice( + base64='data:audio/ogg;base64,' + base64.b64encode(b'OggS-voice').decode('utf-8'), + length=5, + ), ] ), quote_origin=True, @@ -560,11 +565,15 @@ async def test_telegram_reply_message_sends_text_image_and_file_components(): bot.send_message.assert_awaited_once() bot.send_photo.assert_awaited_once() bot.send_document.assert_awaited_once() + bot.send_voice.assert_awaited_once() assert bot.send_message.await_args.kwargs['reply_to_message_id'] == 88 assert bot.send_photo.await_args.kwargs['reply_to_message_id'] == 88 assert bot.send_photo.await_args.kwargs['photo'].input_file_content == b'image-bytes' assert bot.send_document.await_args.kwargs['document'].filename == 'test.txt' assert bot.send_document.await_args.kwargs['document'].input_file_content == b'test' + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 88 + assert bot.send_voice.await_args.kwargs['duration'] == 5 + assert bot.send_voice.await_args.kwargs['voice'].filename == 'voice.ogg' @pytest.mark.asyncio @@ -683,6 +692,7 @@ async def test_stream_completion_delivers_attachments_once(with_text): edit_message_text=AsyncMock(), send_photo=AsyncMock(return_value=SimpleNamespace(message_id=901)), send_document=AsyncMock(return_value=SimpleNamespace(message_id=902)), + send_voice=AsyncMock(return_value=SimpleNamespace(message_id=903)), ) object.__setattr__(adapter, 'bot', bot) adapter.msg_stream_id['stream'] = ('message', 999, True) @@ -696,6 +706,10 @@ async def test_stream_completion_delivers_attachments_once(with_text): platform_message.File( name='result.txt', base64='data:text/plain;base64,' + base64.b64encode(b'result').decode() ), + platform_message.Voice( + base64='data:audio/ogg;base64,' + base64.b64encode(b'OggS-voice').decode(), + length=3, + ), ] ) chunk = SimpleNamespace(resp_message_id='stream', msg_sequence=2, tool_calls=None) @@ -703,12 +717,16 @@ async def test_stream_completion_delivers_attachments_once(with_text): await adapter.reply_message_chunk(source, chunk, chain, is_final=False) bot.send_photo.assert_not_awaited() bot.send_document.assert_not_awaited() + bot.send_voice.assert_not_awaited() await adapter.reply_message_chunk(source, chunk, chain, quote_origin=True, is_final=True) await adapter.reply_message_chunk(source, chunk, chain, quote_origin=True, is_final=True) bot.send_photo.assert_awaited_once() bot.send_document.assert_awaited_once() + bot.send_voice.assert_awaited_once() assert bot.send_document.await_args.kwargs['document'].filename == 'result.txt' assert bot.send_photo.await_args.kwargs['reply_to_message_id'] == 88 + assert bot.send_voice.await_args.kwargs['reply_to_message_id'] == 88 + assert bot.send_voice.await_args.kwargs['duration'] == 3 assert 'stream' not in adapter.msg_stream_id diff --git a/tests/unit_tests/platform/test_wecombot_media.py b/tests/unit_tests/platform/test_wecombot_media.py index 8bbff53e1..4bd3f9429 100644 --- a/tests/unit_tests/platform/test_wecombot_media.py +++ b/tests/unit_tests/platform/test_wecombot_media.py @@ -51,6 +51,12 @@ class Bot: async def reply_image(self, req_id, media_id): self.calls.append(('reply_image', req_id, media_id)) + async def reply_voice(self, req_id, media_id): + self.calls.append(('reply_voice', req_id, media_id)) + + async def reply_file(self, req_id, media_id): + self.calls.append(('reply_file', req_id, media_id)) + async def send_message(self, target_id, content): self.calls.append(('send_message', target_id, content)) @@ -106,6 +112,23 @@ async def test_reply_message_uploads_and_replies_image_media(): ] +@pytest.mark.asyncio +async def test_reply_message_uploads_and_replies_voice_media(): + bot = Bot() + adapter = make_adapter(bot) + silk_data = b'\x02#!SILK_V3test' + silk_b64 = base64.b64encode(silk_data).decode('utf-8') + chain = platform_message.MessageChain([platform_message.Voice(base64=f'data:audio/silk;base64,{silk_b64}')]) + + items = await WecomBotMessageConverter.yiri2target(chain) + await adapter._send_media(bot, 'req-1', items[0]) + + assert bot.calls == [ + ('upload_media', 'voice', 'voice.silk', silk_data), + ('reply_voice', 'req-1', 'media-1'), + ] + + @pytest.mark.asyncio async def test_send_message_sends_text_and_skips_proactive_image(): bot = Bot() diff --git a/tests/unit_tests/provider/test_litellmchat.py b/tests/unit_tests/provider/test_litellmchat.py index b5dff4b34..5f88aed19 100644 --- a/tests/unit_tests/provider/test_litellmchat.py +++ b/tests/unit_tests/provider/test_litellmchat.py @@ -12,6 +12,7 @@ import pytest from unittest.mock import Mock, AsyncMock, patch import litellm +import httpx from langbot.pkg.provider.modelmgr.requesters import litellmchat from langbot.pkg.provider.modelmgr import errors @@ -813,6 +814,33 @@ class TestHandleLiteLLMError: assert 'Context length exceeded' in str(exc_info.value) + def test_internal_server_error(self): + """Test InternalServerError translation""" + requester = litellmchat.LiteLLMRequester(ap=Mock(), config={}) + + error = litellm.InternalServerError(message='server error', model='gpt-4o', llm_provider='openai') + + with pytest.raises(errors.RequesterError) as exc_info: + requester._handle_litellm_error(error) + + assert 'Internal server error' in str(exc_info.value) + assert exc_info.value.retryable + + def test_permission_denied_error(self): + """Test PermissionDeniedError translation""" + requester = litellmchat.LiteLLMRequester(ap=Mock(), config={}) + + response = httpx.Response(403, request=httpx.Request('POST', 'https://example.com')) + error = litellm.PermissionDeniedError( + message='forbidden', model='gpt-4o', llm_provider='openai', response=response + ) + + with pytest.raises(errors.RequesterError) as exc_info: + requester._handle_litellm_error(error) + + assert 'Permission denied' in str(exc_info.value) + assert not exc_info.value.retryable + def test_unknown_error(self): """Test unknown error translation""" requester = litellmchat.LiteLLMRequester(ap=Mock(), config={}) diff --git a/tests/unit_tests/test_audio_utils.py b/tests/unit_tests/test_audio_utils.py new file mode 100644 index 000000000..9bab3d4f8 --- /dev/null +++ b/tests/unit_tests/test_audio_utils.py @@ -0,0 +1,95 @@ +from __future__ import annotations + +import io +import unittest +from unittest.mock import patch +import wave + +import langbot_plugin.api.entities.builtin.platform.message as platform_message + +from langbot.pkg.utils import audio as audio_utils + + +def _create_sample_wav() -> bytes: + buf = io.BytesIO() + with wave.open(buf, 'wb') as wav: + wav.setnchannels(1) + wav.setsampwidth(2) + wav.setframerate(16000) + wav.writeframes(b'\x00' * 3200) + return buf.getvalue() + + +class TestAudioUtils(unittest.IsolatedAsyncioTestCase): + def test_detect_audio_format(self): + sample_wav = _create_sample_wav() + self.assertEqual(audio_utils.detect_audio_format(sample_wav), 'wav') + self.assertEqual(audio_utils.detect_audio_format(b'#!AMR\n12345'), 'amr') + self.assertEqual(audio_utils.detect_audio_format(b'#!SILK_V3\n'), 'silk') + self.assertEqual(audio_utils.detect_audio_format(b'\x02#!SILK_V3\n'), 'silk') + self.assertEqual(audio_utils.detect_audio_format(b'OggS\x00\x02...OpusHead...'), 'opus') + self.assertEqual(audio_utils.detect_audio_format(b'OggS\x00\x02...'), 'ogg') + self.assertEqual(audio_utils.detect_audio_format(b'ID3\x03\x00\x00'), 'mp3') + self.assertEqual(audio_utils.detect_audio_format(b'\xff\xfb\x90\x64'), 'mp3') + self.assertEqual(audio_utils.detect_audio_format(b'fLaC\x00\x00'), 'flac') + self.assertEqual(audio_utils.detect_audio_format(b''), 'unknown') + self.assertEqual(audio_utils.detect_audio_format(b'12'), 'unknown') + + def test_is_ffmpeg_and_silk_available(self): + with patch.object(audio_utils.shutil, 'which', return_value='/usr/bin/ffmpeg'): + self.assertTrue(audio_utils.is_ffmpeg_available()) + with ( + patch.object(audio_utils.shutil, 'which', return_value=None), + patch.object(audio_utils.os.path, 'isfile', return_value=False), + patch.dict('sys.modules', {'imageio_ffmpeg': None}), + ): + self.assertFalse(audio_utils.is_ffmpeg_available()) + with patch.object(audio_utils.importlib.util, 'find_spec', return_value=object()): + self.assertTrue(audio_utils.is_silk_available()) + with patch.object(audio_utils.importlib.util, 'find_spec', return_value=None): + self.assertFalse(audio_utils.is_silk_available()) + + @unittest.skipUnless(audio_utils.is_ffmpeg_available(), 'ffmpeg is not installed') + async def test_transcode_audio_wav_to_mp3_and_ogg(self): + sample_wav = _create_sample_wav() + mp3_bytes = await audio_utils.transcode_audio(sample_wav, 'mp3', 'wav') + self.assertTrue(len(mp3_bytes) > 0) + self.assertEqual(audio_utils.detect_audio_format(mp3_bytes), 'mp3') + + ogg_bytes = await audio_utils.transcode_audio(sample_wav, 'ogg', 'wav') + self.assertTrue(len(ogg_bytes) > 0) + fmt = audio_utils.detect_audio_format(ogg_bytes) + self.assertIn(fmt, ('ogg', 'opus')) + + async def test_transcode_same_format_returns_input(self): + sample_wav = _create_sample_wav() + result = await audio_utils.transcode_audio(sample_wav, 'wav', 'wav') + self.assertEqual(result, sample_wav) + + def test_transcode_missing_ffmpeg_raises_error(self): + sample_wav = _create_sample_wav() + with patch.object(audio_utils, 'is_ffmpeg_available', return_value=False): + with self.assertRaises(audio_utils.AudioDependencyError): + audio_utils.transcode_audio_sync(sample_wav, 'mp3', 'wav') + + def test_transcode_missing_silk_raises_error(self): + sample_wav = _create_sample_wav() + with patch.object(audio_utils, 'is_silk_available', return_value=False): + with self.assertRaises(audio_utils.AudioDependencyError): + audio_utils.transcode_audio_sync(sample_wav, 'silk', 'wav') + + async def test_extract_voice_bytes_from_base64(self): + import base64 + + sample_wav = _create_sample_wav() + b64_raw = base64.b64encode(sample_wav).decode('ascii') + voice_raw = platform_message.Voice(base64=b64_raw) + data, fmt = await audio_utils.extract_voice_bytes(voice_raw) + self.assertEqual(data, sample_wav) + self.assertEqual(fmt, 'wav') + + data_uri = f'data:audio/wav;base64,{b64_raw}' + voice_uri = platform_message.Voice(base64=data_uri) + data2, fmt2 = await audio_utils.extract_voice_bytes(voice_uri) + self.assertEqual(data2, sample_wav) + self.assertEqual(fmt2, 'wav') diff --git a/tests/unit_tests/test_speech_models_api.py b/tests/unit_tests/test_speech_models_api.py new file mode 100644 index 000000000..dda0d8496 --- /dev/null +++ b/tests/unit_tests/test_speech_models_api.py @@ -0,0 +1,279 @@ +from __future__ import annotations + +import unittest +from unittest.mock import AsyncMock, Mock, patch + +from langbot.pkg.api.http.context import ExecutionContext +from langbot.pkg.entity.persistence.model import ModelProvider, ASRModel, TTSModel +from langbot.pkg.provider.modelmgr.modelmgr import ModelManager +from langbot.pkg.provider.modelmgr.requester import RuntimeProvider, RuntimeASRModel, RuntimeTTSModel +from langbot.pkg.provider.modelmgr.requesters.litellmchat import LiteLLMRequester + + +class TestSpeechModels(unittest.IsolatedAsyncioTestCase): + async def test_model_manager_asr_and_tts_cache_and_get(self): + ap = Mock() + ap.logger = Mock() + ap.workspace_service = Mock() + ap.workspace_service.get_execution_binding = AsyncMock( + return_value=Mock( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + ) + mgr = ModelManager(ap) + + context = ExecutionContext( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + + provider_entity = ModelProvider( + uuid='prov-1', + name='Test Provider', + requester='openai', + base_url='https://api.openai.com/v1', + api_keys=['sk-test'], + workspace_uuid='ws-1', + ) + runtime_provider = RuntimeProvider( + execution_context=context, + provider_entity=provider_entity, + token_mgr=Mock(), + requester=Mock(), + ) + + asr_entity = ASRModel( + uuid='asr-1', + name='whisper-1', + provider_uuid='prov-1', + workspace_uuid='ws-1', + ) + tts_entity = TTSModel( + uuid='tts-1', + name='tts-1', + provider_uuid='prov-1', + workspace_uuid='ws-1', + ) + + runtime_asr = await mgr.load_asr_model_with_provider(context, asr_entity, runtime_provider) + await mgr.cache_asr_model(context, runtime_asr) + + runtime_tts = await mgr.load_tts_model_with_provider(context, tts_entity, runtime_provider) + await mgr.cache_tts_model(context, runtime_tts) + + retrieved_asr = await mgr.get_asr_model_by_uuid(context, 'asr-1') + self.assertIsNotNone(retrieved_asr) + self.assertEqual(retrieved_asr.model_entity.uuid, 'asr-1') + self.assertEqual(retrieved_asr.model_entity.name, 'whisper-1') + + retrieved_tts = await mgr.get_tts_model_by_uuid(context, 'tts-1') + self.assertIsNotNone(retrieved_tts) + self.assertEqual(retrieved_tts.model_entity.uuid, 'tts-1') + self.assertEqual(retrieved_tts.model_entity.name, 'tts-1') + + await mgr.remove_asr_model(context, 'asr-1') + with self.assertRaises(ValueError): + await mgr.get_asr_model_by_uuid(context, 'asr-1') + + await mgr.remove_tts_model(context, 'tts-1') + with self.assertRaises(ValueError): + await mgr.get_tts_model_by_uuid(context, 'tts-1') + + async def test_litellm_requester_invoke_asr(self): + ap = Mock() + requester = LiteLLMRequester(ap, {'base_url': 'https://api.openai.com/v1'}) + + context = ExecutionContext( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + + provider_entity = ModelProvider( + uuid='prov-1', + name='OpenAI', + requester='openai', + base_url='https://api.openai.com/v1', + api_keys=['sk-test-key'], + workspace_uuid='ws-1', + ) + runtime_provider = RuntimeProvider( + execution_context=context, + provider_entity=provider_entity, + token_mgr=Mock(get_token=Mock(return_value='sk-test-key')), + requester=requester, + ) + asr_entity = ASRModel( + uuid='asr-1', + name='whisper-1', + provider_uuid='prov-1', + workspace_uuid='ws-1', + ) + runtime_asr = RuntimeASRModel( + execution_context=context, + model_entity=asr_entity, + provider=runtime_provider, + ) + + mock_resp = Mock(text='Hello world transcription') + + with patch('langbot.pkg.provider.modelmgr.requesters.litellmchat.atranscription', new_callable=AsyncMock) as mock_atranscription: + mock_atranscription.return_value = mock_resp + + result = await requester.invoke_asr( + model=runtime_asr, + audio=b'fake-wav-bytes', + mime='audio/wav', + ) + + self.assertEqual(result, 'Hello world transcription') + mock_atranscription.assert_called_once() + call_kwargs = mock_atranscription.call_args[1] + self.assertEqual(call_kwargs['model'], 'whisper-1') + self.assertEqual(call_kwargs['api_key'], 'sk-test-key') + self.assertEqual(call_kwargs['api_base'], 'https://api.openai.com/v1') + self.assertEqual(call_kwargs['file'], ('audio.wav', b'fake-wav-bytes', 'audio/wav')) + + async def test_litellm_requester_invoke_tts(self): + ap = Mock() + requester = LiteLLMRequester(ap, {}) + + context = ExecutionContext( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + + provider_entity = ModelProvider( + uuid='prov-1', + name='OpenAI', + requester='openai', + base_url='https://api.openai.com/v1', + api_keys=['sk-test-key'], + workspace_uuid='ws-1', + ) + runtime_provider = RuntimeProvider( + execution_context=context, + provider_entity=provider_entity, + token_mgr=Mock(get_token=Mock(return_value='sk-test-key')), + requester=requester, + ) + tts_entity = TTSModel( + uuid='tts-1', + name='tts-1', + provider_uuid='prov-1', + workspace_uuid='ws-1', + extra_args={'voice': 'alloy'}, + ) + runtime_tts = RuntimeTTSModel( + execution_context=context, + model_entity=tts_entity, + provider=runtime_provider, + ) + + mock_resp = Mock(content=b'fake-mp3-bytes', spec=['content']) + + with patch('langbot.pkg.provider.modelmgr.requesters.litellmchat.aspeech', new_callable=AsyncMock) as mock_aspeech: + mock_aspeech.return_value = mock_resp + + content, mime = await requester.invoke_tts( + model=runtime_tts, + text='Hello, world!', + extra_args={'voice': 'alloy', 'response_format': 'mp3'}, + ) + + self.assertEqual(content, b'fake-mp3-bytes') + self.assertEqual(mime, 'audio/mpeg') + mock_aspeech.assert_called_once() + call_kwargs = mock_aspeech.call_args[1] + self.assertEqual(call_kwargs['model'], 'tts-1') + self.assertEqual(call_kwargs['input'], 'Hello, world!') + self.assertEqual(call_kwargs['voice'], 'alloy') + self.assertEqual(call_kwargs['response_format'], 'mp3') + self.assertEqual(call_kwargs['api_key'], 'sk-test-key') + + # Test without voice configured + tts_entity_no_voice = TTSModel( + uuid='tts-2', + name='CosyVoice2', + provider_uuid='prov-1', + workspace_uuid='ws-1', + extra_args={}, + ) + runtime_tts_no_voice = RuntimeTTSModel( + execution_context=context, + model_entity=tts_entity_no_voice, + provider=runtime_provider, + ) + with patch('langbot.pkg.provider.modelmgr.requesters.litellmchat.aspeech', new_callable=AsyncMock) as mock_aspeech_no_voice: + mock_aspeech_no_voice.return_value = mock_resp + await requester.invoke_tts( + model=runtime_tts_no_voice, + text='Hello without voice', + extra_args={}, + ) + mock_aspeech_no_voice.assert_called_once() + call_kwargs_no_voice = mock_aspeech_no_voice.call_args[1] + self.assertNotIn('voice', call_kwargs_no_voice) + + async def test_init_temporary_runtime_tts_model_by_provider_uuid(self): + ap = Mock() + ap.logger = Mock() + ap.workspace_service = Mock() + ap.workspace_service.get_execution_binding = AsyncMock( + return_value=Mock( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + ) + ap.persistence_mgr = Mock() + saved_prov = ModelProvider( + uuid='prov-1', + name='Test Provider', + requester='new-api-chat-completions', + base_url='https://api.openai.com/v1', + api_keys=['sk-test'], + workspace_uuid='ws-1', + ) + ap.persistence_mgr.execute_async = AsyncMock(return_value=Mock(first=Mock(return_value=saved_prov))) + + mgr = ModelManager(ap) + context = ExecutionContext( + instance_uuid='inst-1', + workspace_uuid='ws-1', + placement_generation=1, + ) + + with patch.object(mgr, '_build_provider', new_callable=AsyncMock) as mock_build_prov: + mock_runtime_prov = Mock(provider_entity=saved_prov, execution_context=context) + mock_build_prov.return_value = mock_runtime_prov + + model = await mgr.init_temporary_runtime_tts_model( + context, + { + 'name': 'gpt-4o-mini-tts', + 'provider_uuid': 'prov-1', + 'extra_args': {'voice': 'alloy'}, + }, + ) + + self.assertEqual(model.name, 'gpt-4o-mini-tts') + self.assertEqual(model.uuid, '') + self.assertEqual(model.extra_args, {'voice': 'alloy'}) + mock_build_prov.assert_called_once() + + def test_litellm_requester_infer_model_type_speech(self): + req = LiteLLMRequester('openai', {'base_url': 'https://api.openai.com/v1'}) + self.assertEqual(req._infer_model_type('whisper-1'), 'asr') + self.assertEqual(req._infer_model_type('FunAudioLLM/SenseVoiceSmall'), 'asr') + self.assertEqual(req._infer_model_type('tts-1'), 'tts') + self.assertEqual(req._infer_model_type('tts-1-hd'), 'tts') + self.assertEqual(req._infer_model_type('FunAudioLLM/CosyVoice2-0.5B'), 'tts') + self.assertEqual(req._infer_model_type('text-embedding-3-small'), 'embedding') + self.assertEqual(req._infer_model_type('bge-reranker-v2-m3'), 'rerank') + self.assertEqual(req._infer_model_type('gpt-4o'), 'llm') + diff --git a/uv.lock b/uv.lock index 410790882..6ba1a6d82 100644 --- a/uv.lock +++ b/uv.lock @@ -2199,6 +2199,9 @@ dependencies = [ ] [package.optional-dependencies] +audio = [ + { name = "pilk" }, +] seekdb = [ { name = "pylibseekdb", marker = "(platform_machine == 'arm64' and sys_platform == 'darwin') or sys_platform == 'linux'" }, { name = "pyseekdb" }, @@ -2259,6 +2262,7 @@ requires-dist = [ { name = "openai", specifier = ">1.0.0" }, { name = "pandas", specifier = ">=2.2.2" }, { name = "pgvector", specifier = ">=0.4.1" }, + { name = "pilk", marker = "extra == 'audio'", specifier = ">=0.1.0" }, { name = "pillow", specifier = ">=13.0.0" }, { name = "pip", specifier = ">=26.2.0" }, { name = "pre-commit", specifier = ">=4.2.0" }, @@ -2297,7 +2301,7 @@ requires-dist = [ { name = "websockets", specifier = ">=15.0.1" }, { name = "xxhash", specifier = ">=3.5.0" }, ] -provides-extras = ["seekdb"] +provides-extras = ["audio", "seekdb"] [package.metadata.requires-dev] dev = [ @@ -3958,6 +3962,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5a/26/6cee8a1ce8c43625ec561aff19df07f9776b7525d9002c86bceb3e0ac970/pgvector-0.4.2-py3-none-any.whl", hash = "sha256:549d45f7a18593783d5eec609ea1684a724ba8405c4cb182a0b2b08aeff04e08", size = 27441, upload-time = "2025-12-05T01:07:16.536Z" }, ] +[[package]] +name = "pilk" +version = "0.2.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/bb/938dd697b6bc2d851ffec4ffe82ed20078e58bd0ef049e84f4d038d6f991/pilk-0.2.4.tar.gz", hash = "sha256:d4a1bcf93dc6ef5e95e0cfd728ed4ef4d49f9c0476d70816fecbe456cc762e7f", size = 226451, upload-time = "2023-05-26T03:32:34.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ff/06/80cac61bc7f791bcaae552ba90fb868f7af7503ef1c74e423cc20fca1a53/pilk-0.2.4-cp311-cp311-win_amd64.whl", hash = "sha256:a6692096607e8d77d348aeec00df633788c85b527aa83cbd803ddf508936db7f", size = 127024, upload-time = "2023-05-26T03:32:24.524Z" }, +] + [[package]] name = "pillow" version = "13.0.0" diff --git a/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx b/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx index b6edb70dd..e6c59b4ee 100644 --- a/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx +++ b/web/src/app/home/bots/components/bot-session/BotSessionMonitor.tsx @@ -575,7 +575,7 @@ const BotSessionMonitor = forwardRef< case 'Voice': { const voice = component as Voice; - const voiceUrl = voice.url || (voice.base64 ? voice.base64 : ''); + const voiceUrl = voice.base64 || voice.url || ''; if (!voiceUrl) { return ( ([]); const [embeddingModels, setEmbeddingModels] = useState([]); const [rerankModels, setRerankModels] = useState([]); + const [asrModels, setAsrModels] = useState([]); + const [ttsModels, setTtsModels] = useState([]); const [knowledgeBases, setKnowledgeBases] = useState([]); const [bots, setBots] = useState([]); const [tools, setTools] = useState([]); @@ -167,6 +171,8 @@ export default function DynamicFormItemComponent({ DynamicFormItemType.LLM_MODEL_SELECTOR, DynamicFormItemType.EMBEDDING_MODEL_SELECTOR, DynamicFormItemType.RERANK_MODEL_SELECTOR, + DynamicFormItemType.ASR_MODEL_SELECTOR, + DynamicFormItemType.TTS_MODEL_SELECTOR, DynamicFormItemType.MODEL_FALLBACK_SELECTOR, ].includes(config.type); const { @@ -244,12 +250,36 @@ export default function DynamicFormItemComponent({ }); }; + const fetchAsrModels = () => { + httpClient + .getProviderASRModels() + .then((resp) => { + setAsrModels(resp.models.filter(hasUsableUuid)); + }) + .catch((err) => { + toast.error('Failed to load ASR models: ' + err.msg); + }); + }; + + const fetchTtsModels = () => { + httpClient + .getProviderTTSModels() + .then((resp) => { + setTtsModels(resp.models.filter(hasUsableUuid)); + }) + .catch((err) => { + toast.error('Failed to load TTS models: ' + err.msg); + }); + }; + const handleModelsDialogChange = (open: boolean) => { setModelsDialogOpen(open); if (!open) { fetchLlmModels(); fetchEmbeddingModels(); fetchRerankModels(); + fetchAsrModels(); + fetchTtsModels(); } }; @@ -326,6 +356,18 @@ export default function DynamicFormItemComponent({ } }, [config.type]); + useEffect(() => { + if (config.type === DynamicFormItemType.ASR_MODEL_SELECTOR) { + fetchAsrModels(); + } + }, [config.type]); + + useEffect(() => { + if (config.type === DynamicFormItemType.TTS_MODEL_SELECTOR) { + fetchTtsModels(); + } + }, [config.type]); + useEffect(() => { if (config.type === DynamicFormItemType.MODEL_FALLBACK_SELECTOR) { fetchLlmModels(); @@ -1060,6 +1102,112 @@ export default function DynamicFormItemComponent({ ); + case DynamicFormItemType.ASR_MODEL_SELECTOR: + const groupedAsrModels = asrModels.reduce( + (acc, model) => { + const providerName = model.provider?.name || 'Unknown'; + if (!acc[providerName]) acc[providerName] = []; + acc[providerName].push(model); + return acc; + }, + {} as Record, + ); + for (const [providerName, models] of Object.entries(groupedAsrModels)) { + if ( + models[0]?.provider?.requester === LANGBOT_MODELS_PROVIDER_REQUESTER + ) { + groupedAsrModels[providerName] = sortModelsByCatalog( + models, + langbotModelMetadata, + ); + } + } + + return ( +
+ +
+ ); + + case DynamicFormItemType.TTS_MODEL_SELECTOR: + const groupedTtsModels = ttsModels.reduce( + (acc, model) => { + const providerName = model.provider?.name || 'Unknown'; + if (!acc[providerName]) acc[providerName] = []; + acc[providerName].push(model); + return acc; + }, + {} as Record, + ); + for (const [providerName, models] of Object.entries(groupedTtsModels)) { + if ( + models[0]?.provider?.requester === LANGBOT_MODELS_PROVIDER_REQUESTER + ) { + groupedTtsModels[providerName] = sortModelsByCatalog( + models, + langbotModelMetadata, + ); + } + } + + return ( +
+ +
+ ); + case DynamicFormItemType.MODEL_FALLBACK_SELECTOR: { // Separate space models from regular models const fbSpaceModels = sortModelsByCatalog( diff --git a/web/src/app/home/components/dynamic-form/DynamicFormSaveValues.ts b/web/src/app/home/components/dynamic-form/DynamicFormSaveValues.ts index 2ddc942d6..f229796d2 100644 --- a/web/src/app/home/components/dynamic-form/DynamicFormSaveValues.ts +++ b/web/src/app/home/components/dynamic-form/DynamicFormSaveValues.ts @@ -21,6 +21,8 @@ const STRING_FIELD_TYPES = new Set([ 'llm-model-selector', 'embedding-model-selector', 'rerank-model-selector', + 'asr-model-selector', + 'tts-model-selector', 'knowledge-base-selector', 'bot-selector', ]); diff --git a/web/src/app/home/components/models-dialog/ModelsPanel.tsx b/web/src/app/home/components/models-dialog/ModelsPanel.tsx index 789082b0b..091b3ea44 100644 --- a/web/src/app/home/components/models-dialog/ModelsPanel.tsx +++ b/web/src/app/home/components/models-dialog/ModelsPanel.tsx @@ -213,17 +213,22 @@ export default function ModelsPanel({ setLoadingProviders((prev) => new Set(prev).add(providerUuid)); } try { - const [llmResp, embeddingResp, rerankResp] = await Promise.all([ - httpClient.getProviderLLMModels(providerUuid), - httpClient.getProviderEmbeddingModels(providerUuid), - httpClient.getProviderRerankModels(providerUuid), - ]); + const [llmResp, embeddingResp, rerankResp, asrResp, ttsResp] = + await Promise.all([ + httpClient.getProviderLLMModels(providerUuid), + httpClient.getProviderEmbeddingModels(providerUuid), + httpClient.getProviderRerankModels(providerUuid), + httpClient.getProviderASRModels(providerUuid), + httpClient.getProviderTTSModels(providerUuid), + ]); setProviderModels((prev) => ({ ...prev, [providerUuid]: { llm: llmResp.models, embedding: embeddingResp.models, rerank: rerankResp.models, + asr: asrResp.models, + tts: ttsResp.models, }, })); } catch (err) { @@ -320,12 +325,24 @@ export default function ModelsPanel({ provider_uuid: providerUuid, extra_args: extraArgsObj, } as never); - } else { + } else if (modelType === 'rerank') { await httpClient.createProviderRerankModel({ name, provider_uuid: providerUuid, extra_args: extraArgsObj, } as never); + } else if (modelType === 'asr') { + await httpClient.createProviderASRModel({ + name, + provider_uuid: providerUuid, + extra_args: extraArgsObj, + } as never); + } else { + await httpClient.createProviderTTSModel({ + name, + provider_uuid: providerUuid, + extra_args: extraArgsObj, + } as never); } setAddModelPopoverOpen(null); loadProviderModels(providerUuid, true); @@ -379,12 +396,24 @@ export default function ModelsPanel({ provider_uuid: providerUuid, extra_args: {}, } as never); - } else { + } else if (effectiveType === 'rerank') { await httpClient.createProviderRerankModel({ name: item.model.name, provider_uuid: providerUuid, extra_args: {}, } as never); + } else if (effectiveType === 'asr') { + await httpClient.createProviderASRModel({ + name: item.model.name, + provider_uuid: providerUuid, + extra_args: {}, + } as never); + } else { + await httpClient.createProviderTTSModel({ + name: item.model.name, + provider_uuid: providerUuid, + extra_args: {}, + } as never); } } setAddModelPopoverOpen(null); @@ -436,12 +465,24 @@ export default function ModelsPanel({ provider_uuid: providerUuid, extra_args: extraArgsObj, } as never); - } else { + } else if (modelType === 'rerank') { await httpClient.updateProviderRerankModel(modelId, { name, provider_uuid: providerUuid, extra_args: extraArgsObj, } as never); + } else if (modelType === 'asr') { + await httpClient.updateProviderASRModel(modelId, { + name, + provider_uuid: providerUuid, + extra_args: extraArgsObj, + } as never); + } else { + await httpClient.updateProviderTTSModel(modelId, { + name, + provider_uuid: providerUuid, + extra_args: extraArgsObj, + } as never); } setEditModelPopoverOpen(null); loadProviderModels(providerUuid, true); @@ -463,8 +504,12 @@ export default function ModelsPanel({ await httpClient.deleteProviderLLMModel(modelId); } else if (modelType === 'embedding') { await httpClient.deleteProviderEmbeddingModel(modelId); - } else { + } else if (modelType === 'rerank') { await httpClient.deleteProviderRerankModel(modelId); + } else if (modelType === 'asr') { + await httpClient.deleteProviderASRModel(modelId); + } else { + await httpClient.deleteProviderTTSModel(modelId); } toast.success(t('models.deleteSuccess')); loadProviderModels(providerUuid, true); @@ -515,7 +560,7 @@ export default function ModelsPanel({ provider: providerData, extra_args: extraArgsObj, } as never); - } else { + } else if (modelType === 'rerank') { await httpClient.testRerankModel('_', { uuid: '', name, @@ -523,6 +568,22 @@ export default function ModelsPanel({ provider: providerData, extra_args: extraArgsObj, } as never); + } else if (modelType === 'asr') { + await httpClient.testASRModel('_', { + uuid: '', + name, + provider_uuid: providerUuid, + provider: providerData, + extra_args: extraArgsObj, + } as never); + } else { + await httpClient.testTTSModel('_', { + uuid: '', + name, + provider_uuid: providerUuid, + provider: providerData, + extra_args: extraArgsObj, + } as never); } const duration = Date.now() - startTime; setTestResult({ success: true, duration }); diff --git a/web/src/app/home/components/models-dialog/components/AddModelPopover.tsx b/web/src/app/home/components/models-dialog/components/AddModelPopover.tsx index bbf71c6a2..ffc97b5c9 100644 --- a/web/src/app/home/components/models-dialog/components/AddModelPopover.tsx +++ b/web/src/app/home/components/models-dialog/components/AddModelPopover.tsx @@ -4,6 +4,8 @@ import { MessageSquareText, Cpu, ArrowUpDown, + Mic, + Volume2, Eye, Wrench, BrainCircuit, @@ -93,8 +95,10 @@ export default function AddModelPopover({ llm: 'llm', embedding: 'text-embedding', rerank: 'rerank', + asr: 'asr', + tts: 'tts', }; - const allTabs: ModelType[] = ['llm', 'embedding', 'rerank']; + const allTabs: ModelType[] = ['llm', 'embedding', 'rerank', 'asr', 'tts']; // When supportTypes is undefined (unknown requester), show all tabs for // backward compatibility. Otherwise only show supported tabs. const visibleTabs: ModelType[] = supportTypes @@ -284,7 +288,7 @@ export default function AddModelPopover({ )} {!(trigger && initialMode === 'scan') && visibleTabs.length > 1 && ( {visibleTabs.includes('llm') && ( - - - {t('models.chat')} + + + {t('models.chat')} )} {visibleTabs.includes('embedding') && ( - - - {t('models.embedding')} + + + {t('models.embedding')} )} {visibleTabs.includes('rerank') && ( - - - {t('models.rerank')} + + + {t('models.rerank')} + + )} + {visibleTabs.includes('asr') && ( + + + + {t('models.asrTab', { defaultValue: 'ASR' })} + + + )} + {visibleTabs.includes('tts') && ( + + + + {t('models.ttsTab', { defaultValue: 'TTS' })} + )} @@ -531,7 +571,11 @@ export default function AddModelPopover({ ? t('models.chat') : model.type === 'embedding' ? t('models.embedding') - : t('models.rerank')} + : model.type === 'rerank' + ? t('models.rerank') + : model.type === 'asr' + ? t('models.asr') + : t('models.tts')} diff --git a/web/src/app/home/components/models-dialog/components/ModelItem.tsx b/web/src/app/home/components/models-dialog/components/ModelItem.tsx index aa2f47041..4a6881b64 100644 --- a/web/src/app/home/components/models-dialog/components/ModelItem.tsx +++ b/web/src/app/home/components/models-dialog/components/ModelItem.tsx @@ -211,7 +211,11 @@ export default function ModelItem({ ? t('models.chat') : modelType === 'embedding' ? t('models.embedding') - : t('models.rerank')} + : modelType === 'rerank' + ? t('models.rerank') + : modelType === 'asr' + ? t('models.asr') + : t('models.tts')} {modelType === 'llm' && (model as LLMModel).abilities?.includes('vision') && ( diff --git a/web/src/app/home/components/models-dialog/components/ProviderCard.tsx b/web/src/app/home/components/models-dialog/components/ProviderCard.tsx index 2ba418c56..07457b08a 100644 --- a/web/src/app/home/components/models-dialog/components/ProviderCard.tsx +++ b/web/src/app/home/components/models-dialog/components/ProviderCard.tsx @@ -153,15 +153,23 @@ export default function ProviderCard({ useState(false); const [addModelMode, setAddModelMode] = useState<'manual' | 'scan'>('manual'); - const canDelete = - !isLangBotModels && - (provider.llm_count || 0) === 0 && - (provider.embedding_count || 0) === 0 && - (provider.rerank_count || 0) === 0; + const modelsCountFromLoaded = models + ? (models.llm?.length || 0) + + (models.embedding?.length || 0) + + (models.rerank?.length || 0) + + (models.asr?.length || 0) + + (models.tts?.length || 0) + : undefined; const totalModels = - (provider.llm_count || 0) + - (provider.embedding_count || 0) + - (provider.rerank_count || 0); + modelsCountFromLoaded !== undefined + ? modelsCountFromLoaded + : (provider.llm_count || 0) + + (provider.embedding_count || 0) + + (provider.rerank_count || 0) + + (provider.asr_count || 0) + + (provider.tts_count || 0); + + const canDelete = !isLangBotModels && totalModels === 0; return ( @@ -601,9 +609,117 @@ export default function ProviderCard({ onResetTestResult={onResetTestResult} /> ))} + {(models.asr || []).map((model) => ( + onDeleteModel(model.uuid, 'asr')} + onUpdateModel={( + name, + abilities, + extraArgs, + reasoningConfig, + ) => + onUpdateModel( + model.uuid, + 'asr', + name, + abilities, + extraArgs, + reasoningConfig, + ) + } + onTestModel={( + name, + abilities, + extraArgs, + reasoningConfig, + ) => + onTestModel( + name, + 'asr', + abilities, + extraArgs, + reasoningConfig, + ) + } + isSubmitting={isSubmitting} + isTesting={isTesting} + testResult={testResult} + onResetTestResult={onResetTestResult} + /> + ))} + {(models.tts || []).map((model) => ( + onDeleteModel(model.uuid, 'tts')} + onUpdateModel={( + name, + abilities, + extraArgs, + reasoningConfig, + ) => + onUpdateModel( + model.uuid, + 'tts', + name, + abilities, + extraArgs, + reasoningConfig, + ) + } + onTestModel={( + name, + abilities, + extraArgs, + reasoningConfig, + ) => + onTestModel( + name, + 'tts', + abilities, + extraArgs, + reasoningConfig, + ) + } + isSubmitting={isSubmitting} + isTesting={isTesting} + testResult={testResult} + onResetTestResult={onResetTestResult} + /> + ))} {models.llm.length === 0 && models.embedding.length === 0 && - models.rerank.length === 0 && ( + models.rerank.length === 0 && + (!models.asr || models.asr.length === 0) && + (!models.tts || models.tts.length === 0) && (

{t('models.noModels')}

diff --git a/web/src/app/home/components/models-dialog/types.ts b/web/src/app/home/components/models-dialog/types.ts index 6b4eaf614..48754afbd 100644 --- a/web/src/app/home/components/models-dialog/types.ts +++ b/web/src/app/home/components/models-dialog/types.ts @@ -2,6 +2,8 @@ import { LLMModel, EmbeddingModel, RerankModel, + ASRModel, + TTSModel, ModelProvider, ProviderScanDebugInfo, ScannedProviderModel, @@ -15,7 +17,7 @@ export type ExtraArg = { value: string; }; -export type ModelType = 'llm' | 'embedding' | 'rerank'; +export type ModelType = 'llm' | 'embedding' | 'rerank' | 'asr' | 'tts'; export const DEFAULT_REASONING_CONFIG: ReasoningConfig = { level: 'provider_default', @@ -25,6 +27,8 @@ export interface ProviderModels { llm: LLMModel[]; embedding: EmbeddingModel[]; rerank: RerankModel[]; + asr: ASRModel[]; + tts: TTSModel[]; } export type TestResult = @@ -41,7 +45,7 @@ export type ScanModelsResult = { }; export interface ModelItemProps { - model: LLMModel | EmbeddingModel; + model: LLMModel | EmbeddingModel | RerankModel | ASRModel | TTSModel; modelType: ModelType; providerUuid: string; isLangBotModels: boolean; diff --git a/web/src/app/home/pipelines/components/debug-dialog/DebugDialog.tsx b/web/src/app/home/pipelines/components/debug-dialog/DebugDialog.tsx index 4b8c7be09..1b7ef0557 100644 --- a/web/src/app/home/pipelines/components/debug-dialog/DebugDialog.tsx +++ b/web/src/app/home/pipelines/components/debug-dialog/DebugDialog.tsx @@ -646,7 +646,7 @@ export default function DebugDialog({ case 'Voice': { const voice = component as Voice; - const voiceUrl = voice.url || (voice.base64 ? voice.base64 : ''); + const voiceUrl = voice.base64 || voice.url || ''; if (!voiceUrl) { return [{t('pipelines.debugDialog.voice')}]; diff --git a/web/src/app/home/pipelines/components/pipeline-form/PipelineFormComponent.tsx b/web/src/app/home/pipelines/components/pipeline-form/PipelineFormComponent.tsx index a401e41f7..b5cd5bc44 100644 --- a/web/src/app/home/pipelines/components/pipeline-form/PipelineFormComponent.tsx +++ b/web/src/app/home/pipelines/components/pipeline-form/PipelineFormComponent.tsx @@ -60,6 +60,7 @@ import { Info, Brain, Zap, + Mic, Shield, FileOutput, Puzzle, @@ -170,6 +171,7 @@ const PipelineFormComponent = forwardRef< }), ai: z.record(z.string(), z.any()), trigger: z.record(z.string(), z.any()), + speech: z.record(z.string(), z.any()).optional(), safety: z.record(z.string(), z.any()), output: z.record(z.string(), z.any()), }) @@ -181,6 +183,7 @@ const PipelineFormComponent = forwardRef< }), ai: z.record(z.string(), z.any()).optional(), trigger: z.record(z.string(), z.any()).optional(), + speech: z.record(z.string(), z.any()).optional(), safety: z.record(z.string(), z.any()).optional(), output: z.record(z.string(), z.any()).optional(), }); @@ -191,6 +194,7 @@ const PipelineFormComponent = forwardRef< basic: Info, ai: Brain, trigger: Zap, + speech: Mic, safety: Shield, output: FileOutput, extensions: Puzzle, @@ -213,6 +217,11 @@ const PipelineFormComponent = forwardRef< name: 'trigger', icon: SECTION_ICONS.trigger, }, + { + label: t('pipelines.speechProcessing'), + name: 'speech', + icon: SECTION_ICONS.speech, + }, { label: t('pipelines.safetyControls'), name: 'safety', @@ -243,6 +252,7 @@ const PipelineFormComponent = forwardRef< const guideSteps = useMemo(() => { const sections = [ 'trigger', + 'speech', 'ai', 'output', 'safety', @@ -286,7 +296,7 @@ const PipelineFormComponent = forwardRef< }, ]; }, [t, debugGuideEnabled, monitoringGuideEnabled]); - const primarySectionNames = ['trigger', 'ai', 'output']; + const primarySectionNames = ['trigger', 'speech', 'ai', 'output']; const primarySections = primarySectionNames .map((name) => formLabelList.find((section) => section.name === name)) .filter((section): section is SectionItem => Boolean(section)); @@ -302,6 +312,8 @@ const PipelineFormComponent = forwardRef< useState(); const [triggerConfigTabSchema, setTriggerConfigTabSchema] = useState(); + const [speechConfigTabSchema, setSpeechConfigTabSchema] = + useState(); const [safetyConfigTabSchema, setSafetyConfigTabSchema] = useState(); const [outputConfigTabSchema, setOutputConfigTabSchema] = @@ -371,6 +383,8 @@ const PipelineFormComponent = forwardRef< setAIConfigTabSchema(config); } else if (config.name === 'trigger') { setTriggerConfigTabSchema(config); + } else if (config.name === 'speech') { + setSpeechConfigTabSchema(config); } else if (config.name === 'safety') { setSafetyConfigTabSchema(config); } else if (config.name === 'output') { @@ -404,6 +418,7 @@ const PipelineFormComponent = forwardRef< }, ai: resp.pipeline.config.ai, trigger: resp.pipeline.config.trigger, + speech: resp.pipeline.config.speech, safety: resp.pipeline.config.safety, output: resp.pipeline.config.output, }; @@ -528,6 +543,7 @@ const PipelineFormComponent = forwardRef< const realConfig = { ai: values.ai, trigger: values.trigger, + speech: values.speech, safety: values.safety, output: values.output, }; @@ -902,7 +918,7 @@ const PipelineFormComponent = forwardRef<