diff --git a/app/client/api.ts b/app/client/api.ts index 7a242ea99..a3da99ba7 100644 --- a/app/client/api.ts +++ b/app/client/api.ts @@ -39,6 +39,7 @@ export interface MultimodalContent { export interface RequestMessage { role: MessageRole; content: string | MultimodalContent[]; + audio_url?: string; } export interface LLMConfig { diff --git a/app/components/chat.module.scss b/app/components/chat.module.scss index 73542fc67..597299994 100644 --- a/app/components/chat.module.scss +++ b/app/components/chat.module.scss @@ -430,7 +430,7 @@ .chat-message-item { box-sizing: border-box; - max-width: 100%; + max-width: 300px; margin-top: 10px; border-radius: 10px; background-color: rgba(0, 0, 0, 0.05); @@ -443,6 +443,10 @@ transition: all ease 0.3s; } +.audio-message { + min-width: 350px; +} + .chat-message-item-image { width: 100%; margin-top: 10px; @@ -471,6 +475,10 @@ border: rgba($color: #888, $alpha: 0.2) 1px solid; } +.chat-message-item-audio { + margin-top: 10px; + width: 100%; +} @media only screen and (max-width: 600px) { $calc-image-width: calc(100vw/3*2/var(--image-count)); @@ -519,7 +527,7 @@ background-color: var(--second); &:hover { - min-width: 0; + //min-width: 350px; } } @@ -693,4 +701,4 @@ .shortcut-key span { font-size: 12px; color: var(--black); -} \ No newline at end of file +} diff --git a/app/components/chat.tsx b/app/components/chat.tsx index 81858bc8a..1f8a13e4b 100644 --- a/app/components/chat.tsx +++ b/app/components/chat.tsx @@ -1121,6 +1121,15 @@ function _Chat() { ); }; + const updateMessageAudio = (msgId?: string, audio_url?: string) => { + chatStore.updateCurrentSession( + (session) => + (session.messages = session.messages.map((m) => + m.id === msgId ? { ...m, audio_url } : m, + )), + ); + }; + const onDelete = (msgId: string) => { deleteMessage(msgId); }; @@ -1197,7 +1206,7 @@ function _Chat() { const accessStore = useAccessStore(); const [speechStatus, setSpeechStatus] = useState(false); const [speechLoading, setSpeechLoading] = useState(false); - async function openaiSpeech(text: string) { + async function openaiSpeech(text: string): Promise { if (speechStatus) { ttsPlayer.stop(); setSpeechStatus(false); @@ -1230,11 +1239,12 @@ function _Chat() { try { const waveFile = arrayBufferToWav(audioBuffer); const audioFile = new Blob([waveFile], { type: "audio/wav" }); - const url = uploadAudio(audioFile); + const audioUrl: string = await uploadImageRemote(audioFile); await ttsPlayer.play(audioBuffer, () => { setSpeechStatus(false); }); + return audioUrl; } catch (e) { console.error("[OpenAI Speech]", e); showToast(prettyObject(e)); @@ -1510,53 +1520,6 @@ function _Chat() { setAttachImages(images); } - async function uploadAudio(file: Blob) { - const audioUrl: string = await uploadImageRemote(file); - console.log("audioUrl: ", audioUrl); - //const images: string[] = []; - //images.push(...attachImages); - - //images.push( - // ...(await new Promise((res, rej) => { - // const fileInput = document.createElement("input"); - // fileInput.type = "file"; - // fileInput.accept = - // "image/png, image/jpeg, image/webp, image/heic, image/heif"; - // fileInput.multiple = true; - // fileInput.onchange = (event: any) => { - // setUploading(true); - // const files = event.target.files; - // const imagesData: string[] = []; - // for (let i = 0; i < files.length; i++) { - // const file = event.target.files[i]; - // uploadImageRemote(file) - // .then((dataUrl) => { - // imagesData.push(dataUrl); - // if ( - // imagesData.length === 3 || - // imagesData.length === files.length - // ) { - // setUploading(false); - // res(imagesData); - // } - // }) - // .catch((e) => { - // setUploading(false); - // rej(e); - // }); - // } - // }; - // fileInput.click(); - // })), - //); - - //const imagesLength = images.length; - //if (imagesLength > 3) { - // images.splice(3, imagesLength - 3); - //} - //setAttachImages(images); - } - // 快捷键 shortcut keys const [showShortcutKeyModal, setShowShortcutKeyModal] = useState(false); @@ -1845,9 +1808,12 @@ function _Chat() { ) } - onClick={() => - openaiSpeech(getMessageTextContent(message)) - } + onClick={async () => { + const url = await openaiSpeech( + getMessageTextContent(message), + ); + updateMessageAudio(message.id, url); + }} /> )} @@ -1881,7 +1847,11 @@ function _Chat() { ))} )} -
+
)} + {message.audio_url && ( + + )}
diff --git a/app/styles/globals.scss b/app/styles/globals.scss index e8c10de3f..fa7c0f2f2 100644 --- a/app/styles/globals.scss +++ b/app/styles/globals.scss @@ -399,3 +399,12 @@ pre { .copyable { user-select: text; } + +audio { + height: 35px; +} + +audio::-webkit-media-controls-play-button, + audio::-webkit-media-controls-panel { + background-color: none; + } diff --git a/package.json b/package.json index 8696f83b5..4c468e2ff 100644 --- a/package.json +++ b/package.json @@ -31,8 +31,8 @@ "html-to-image": "^1.11.11", "idb-keyval": "^6.2.1", "lodash-es": "^4.17.21", - "mermaid": "^10.6.1", "markdown-to-txt": "^2.0.1", + "mermaid": "^10.6.1", "nanoid": "^5.0.3", "next": "^14.1.1", "node-fetch": "^3.3.1", @@ -80,4 +80,4 @@ "lint-staged/yaml": "^2.2.2" }, "packageManager": "yarn@1.22.19" -} \ No newline at end of file +}