Merge 85e70f559d into 108069a0c6

app/components/chat.module.scss

@@ -443,6 +443,10 @@
   transition: all ease 0.3s;
 }
 
+.audio-message {
+  min-width: 350px;
+}
+
 .chat-message-item-image {
   width: 100%;
   margin-top: 10px;
@@ -471,6 +475,10 @@
   border: rgba($color: #888, $alpha: 0.2) 1px solid;
 }
 
+.chat-message-item-audio {
+  margin-top: 10px;
+  width: 100%;
+}
 
 @media only screen and (max-width: 600px) {
   $calc-image-width: calc(100vw/3*2/var(--image-count));
@@ -519,7 +527,7 @@
   background-color: var(--second);
 
   &:hover {
-    min-width: 0;
+    //min-width: 350px;
   }
 }
 
@@ -693,4 +701,4 @@
 .shortcut-key span {
   font-size: 12px;
   color: var(--black);
-}
+}

app/components/chat.tsx

@@ -116,7 +116,7 @@ import { useAllModels } from "../utils/hooks";
 import { MultimodalContent } from "../client/api";
 
 import { ClientApi } from "../client/api";
-import { createTTSPlayer } from "../utils/audio";
+import { createTTSPlayer, arrayBufferToWav } from "../utils/audio";
 import { MsEdgeTTS, OUTPUT_FORMAT } from "../utils/ms_edge_tts";
 
 import { isEmpty } from "lodash-es";
@@ -1132,6 +1132,14 @@ function _Chat() {
     );
   };
 
+  const updateMessageAudio = (msgId?: string, audio_url?: string) => {
+    chatStore.updateCurrentSession((session) => {
+      session.messages = session.messages.map((m) =>
+        m.id === msgId ? { ...m, audio_url } : m,
+      );
+    });
+  };
+
   const onDelete = (msgId: string) => {
     deleteMessage(msgId);
   };
@@ -1208,7 +1216,7 @@ function _Chat() {
   const accessStore = useAccessStore();
   const [speechStatus, setSpeechStatus] = useState(false);
   const [speechLoading, setSpeechLoading] = useState(false);
-  async function openaiSpeech(text: string) {
+  async function openaiSpeech(text: string): Promise<string | undefined> {
     if (speechStatus) {
       ttsPlayer.stop();
       setSpeechStatus(false);
@@ -1238,16 +1246,22 @@ function _Chat() {
         });
       }
       setSpeechStatus(true);
-      ttsPlayer
-        .play(audioBuffer, () => {
+      try {
+        const waveFile = arrayBufferToWav(audioBuffer);
+        const audioFile = new Blob([waveFile], { type: "audio/wav" });
+
+        const audioUrl: string = await uploadImageRemote(audioFile);
+        await ttsPlayer.play(audioBuffer, () => {
           setSpeechStatus(false);
-        })
-        .catch((e) => {
-          console.error("[OpenAI Speech]", e);
-          showToast(prettyObject(e));
-          setSpeechStatus(false);
-        })
-        .finally(() => setSpeechLoading(false));
+        });
+        return audioUrl;
+      } catch (e) {
+        console.error("[Speech Error]", e);
+        showToast(prettyObject(e));
+        setSpeechStatus(false);
+      } finally {
+        setSpeechLoading(false);
+      }
     }
   }
 
@@ -1810,9 +1824,12 @@ function _Chat() {
                                       <SpeakIcon />
                                     )
                                   }
-                                  onClick={() =>
-                                    openaiSpeech(getMessageTextContent(message))
-                                  }
+                                  onClick={async () => {
+                                    const url = await openaiSpeech(
+                                      getMessageTextContent(message),
+                                    );
+                                    updateMessageAudio(message.id, url);
+                                  }}
                                 />
                               )}
                             </>
@@ -1847,7 +1864,11 @@ function _Chat() {
                       ))}
                     </div>
                   )}
-                  <div className={styles["chat-message-item"]}>
+                  <div
+                    className={`${styles["chat-message-item"]} ${
+                      message.audio_url ? styles["audio-message"] : ""
+                    }`}
+                  >
                     <Markdown
                       key={message.streaming ? "loading" : "done"}
                       content={getMessageTextContent(message)}
@@ -1896,6 +1917,16 @@ function _Chat() {
                         })}
                       </div>
                     )}
+                    {message.audio_url && (
+                      <audio
+                        preload="auto"
+                        controls
+                        className={styles["chat-message-item-audio"]}
+                      >
+                        <source type="audio/mp3" src={message.audio_url} />
+                        Sorry, your browser does not support HTML5 audio.
+                      </audio>
+                    )}
                   </div>
 
                   <div className={styles["chat-message-action-date"]}>