From 52a3f13f1eaee4b2ff4473542712e360777f138c Mon Sep 17 00:00:00 2001
From: RockYang <yangjian102621@gmail.com>
Date: Sat, 12 Oct 2024 19:07:29 +0800
Subject: [PATCH] add voice chat test case

---
 web/package-lock.json             |  54 ++++--
 web/package.json                  |   1 +
 web/src/components/FileSelect.vue |   5 +-
 web/src/views/Test.vue            | 271 +++++++++++++++++++++++++++++-
 web/src/views/Test1.vue           | 110 ++++++++++++
 5 files changed, 421 insertions(+), 20 deletions(-)
 create mode 100644 web/src/views/Test1.vue

diff --git a/web/package-lock.json b/web/package-lock.json
index 92bf98f4..5ff1a255 100644
--- a/web/package-lock.json
+++ b/web/package-lock.json
@@ -9,6 +9,7 @@
       "version": "0.1.0",
       "dependencies": {
         "@element-plus/icons-vue": "^2.1.0",
+        "@openai/realtime-api-beta": "github:openai/openai-realtime-api-beta",
         "axios": "^0.27.2",
         "clipboard": "^2.0.11",
         "compressorjs": "^1.2.1",
@@ -27,7 +28,6 @@
         "markmap-view": "^0.16.0",
         "md-editor-v3": "^2.2.1",
         "memfs": "^4.9.3",
-        "mitt": "^3.0.1",
         "pinia": "^2.1.4",
         "qrcode": "^1.5.3",
         "qs": "^6.11.1",
@@ -2022,6 +2022,33 @@
         "node": ">= 8"
       }
     },
+    "node_modules/@openai/realtime-api-beta": {
+      "version": "0.0.0",
+      "resolved": "git+ssh://git@github.com/openai/openai-realtime-api-beta.git#339e9553a757ef1cf8c767272fc750c1e62effbb",
+      "dependencies": {
+        "ws": "^8.18.0"
+      }
+    },
+    "node_modules/@openai/realtime-api-beta/node_modules/ws": {
+      "version": "8.18.0",
+      "resolved": "https://registry.npmmirror.com/ws/-/ws-8.18.0.tgz",
+      "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@polka/url": {
       "version": "1.0.0-next.21",
       "resolved": "https://registry.npmmirror.com/@polka/url/-/url-1.0.0-next.21.tgz",
@@ -8769,11 +8796,6 @@
         "node": ">=8"
       }
     },
-    "node_modules/mitt": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz",
-      "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw=="
-    },
     "node_modules/mj-context-menu": {
       "version": "0.6.1",
       "resolved": "https://registry.npmmirror.com/mj-context-menu/-/mj-context-menu-0.6.1.tgz",
@@ -14196,6 +14218,21 @@
         "fastq": "^1.6.0"
       }
     },
+    "@openai/realtime-api-beta": {
+      "version": "git+ssh://git@github.com/openai/openai-realtime-api-beta.git#339e9553a757ef1cf8c767272fc750c1e62effbb",
+      "from": "@openai/realtime-api-beta@github:openai/openai-realtime-api-beta",
+      "requires": {
+        "ws": "^8.18.0"
+      },
+      "dependencies": {
+        "ws": {
+          "version": "8.18.0",
+          "resolved": "https://registry.npmmirror.com/ws/-/ws-8.18.0.tgz",
+          "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
+          "requires": {}
+        }
+      }
+    },
     "@polka/url": {
       "version": "1.0.0-next.21",
       "resolved": "https://registry.npmmirror.com/@polka/url/-/url-1.0.0-next.21.tgz",
@@ -19700,11 +19737,6 @@
         "yallist": "^4.0.0"
       }
     },
-    "mitt": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz",
-      "integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw=="
-    },
     "mj-context-menu": {
       "version": "0.6.1",
       "resolved": "https://registry.npmmirror.com/mj-context-menu/-/mj-context-menu-0.6.1.tgz",
diff --git a/web/package.json b/web/package.json
index 984ca9b5..74c864cf 100644
--- a/web/package.json
+++ b/web/package.json
@@ -9,6 +9,7 @@
   },
   "dependencies": {
     "@element-plus/icons-vue": "^2.1.0",
+    "@openai/realtime-api-beta": "github:openai/openai-realtime-api-beta",
     "axios": "^0.27.2",
     "clipboard": "^2.0.11",
     "compressorjs": "^1.2.1",
diff --git a/web/src/components/FileSelect.vue b/web/src/components/FileSelect.vue
index 83e7d937..fc59da5a 100644
--- a/web/src/components/FileSelect.vue
+++ b/web/src/components/FileSelect.vue
@@ -70,7 +70,6 @@ const props = defineProps({
 });
 const emits = defineEmits(['selected']);
 const show = ref(false)
-const fileList = ref([])
 const scrollbarRef = ref(null)
 const fileData = reactive({
   items:[],
@@ -116,7 +115,7 @@ const afterRead = (file) => {
   formData.append('file', file.file, file.name);
   // 执行上传操作
   httpPost('/api/upload', formData).then((res) => {
-    fileList.value.unshift(res.data)
+    fileData.items.unshift(res.data)
     ElMessage.success({message: "上传成功", duration: 500})
   }).catch((e) => {
     ElMessage.error('图片上传失败:' + e.message)
@@ -125,7 +124,7 @@ const afterRead = (file) => {
 
 const removeFile = (file) => {
   httpGet('/api/upload/remove?id=' + file.id).then(() => {
-    fileList.value = removeArrayItem(fileList.value, file, (v1, v2) => {
+    fileData.items = removeArrayItem(fileData.items, file, (v1, v2) => {
       return v1.id === v2.id
     })
     ElMessage.success("文件删除成功！")
diff --git a/web/src/views/Test.vue b/web/src/views/Test.vue
index b29761ad..6bac1ba5 100644
--- a/web/src/views/Test.vue
+++ b/web/src/views/Test.vue
@@ -1,16 +1,275 @@
 <template>
-  <div>
-   {{data}}
+  <div class="video-call-container">
+    <div class="wave-animation">
+      <div v-for="i in 5" :key="i" class="wave-ellipse"></div>
+    </div>
+    <!-- 其余部分保持不变 -->
+    <div class="voice-indicators">
+      <div class="voice-indicator left">
+        <div v-for="i in 3" :key="i" class="bar"></div>
+      </div>
+      <div class="voice-indicator right">
+        <canvas id="canvas" width="600" height="200"></canvas>
+      </div>
+    </div>
+    <div class="call-controls">
+      <button class="call-button hangup" @click="hangUp">
+        <span class="icon">×</span>
+      </button>
+      <button class="call-button answer" @click="answer">
+        <span class="icon">□</span>
+      </button>
+    </div>
   </div>
 </template>
 
 <script setup>
-import {onMounted, ref} from "vue";
+// Script 部分保持不变
+import {ref, onMounted, onUnmounted} from 'vue';
 
-const data = ref('abc')
+const leftVoiceActive = ref(false);
+const rightVoiceActive = ref(false);
+
+const animateVoice = () => {
+  leftVoiceActive.value = Math.random() > 0.5;
+  rightVoiceActive.value = Math.random() > 0.5;
+};
+
+let voiceInterval;
 
 onMounted(() => {
-  
-})
+  voiceInterval = setInterval(animateVoice, 500);
 
+  const flag = ref(false)
+  const canvas = document.getElementById('canvas');
+  const ctx = canvas.getContext('2d');
+
+  async function setupAudioProcessing() {
+    try {
+      const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+      const audioContext = new (window.AudioContext || window.webkitAudioContext)();
+      const analyser = audioContext.createAnalyser();
+      const source = audioContext.createMediaStreamSource(stream);
+      source.connect(analyser);
+      analyser.fftSize = 256;
+      const bufferLength = analyser.frequencyBinCount;
+      const dataArray = new Uint8Array(bufferLength);
+
+      function draw() {
+        analyser.getByteFrequencyData(dataArray);
+
+       if (!flag.value) {
+         // ctx.clearRect(0, 0, canvas.width, canvas.height);
+         // requestAnimationFrame(draw);
+         return;
+       }
+
+        ctx.clearRect(0, 0, canvas.width, canvas.height);
+
+        const barWidth = (canvas.width / bufferLength) * 5;
+        let x = 0;
+
+        for (let i = 0; i < bufferLength; i++) {
+          const barHeight = dataArray[i] / 2;
+
+          ctx.fillStyle = 'rgba(173, 216, 230, 0.7)'; // 淡蓝色
+          ctx.fillRect(x, canvas.height - barHeight, barWidth, barHeight);
+
+          x += barWidth + 2;
+        }
+
+        requestAnimationFrame(draw);
+      }
+
+      draw();
+    } catch (err) {
+      console.error('获取麦克风权限失败:', err);
+    }
+  }
+
+  function setupSpeechRecognition() {
+    const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
+    if (!SpeechRecognition) {
+      alert('您的浏览器不支持语音识别功能');
+      return;
+    }
+
+    const recognition = new SpeechRecognition();
+    recognition.continuous = true;
+    recognition.interimResults = true;
+    recognition.lang = 'zh-CN';
+
+    recognition.onresult = (event) => {
+      const transcript = Array.from(event.results)
+          .map(result => result[0])
+          .map(result => result.transcript)
+          .join('');
+
+      document.getElementById('transcript').textContent = '识别到的文本: ' + transcript;
+      flag.value = true
+    };
+
+    recognition.onerror = (event) => {
+      console.error('语音识别错误：', event.error);
+      flag.value = false
+    };
+
+    recognition.start();
+  }
+
+
+  setupAudioProcessing();
+  setupSpeechRecognition()
+});
+
+onUnmounted(() => {
+  clearInterval(voiceInterval);
+});
+
+const hangUp = () => {
+  console.log('Call hung up');
+};
+
+const answer = () => {
+  console.log('Call answered');
+};
 </script>
+
+<style scoped>
+.video-call-container {
+  background: linear-gradient(to right, #2c3e50, #4a5568, #6b46c1);
+  height: 100vh;
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  align-items: center;
+  padding: 2rem;
+}
+
+canvas {
+  background-color: transparent;
+}
+
+.wave-animation {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  gap: 10px;
+}
+
+.wave-ellipse {
+  width: 40px;
+  height: 40px;
+  background-color: white;
+  border-radius: 20px;
+  animation: wave 0.8s infinite ease-in-out;
+}
+
+.wave-ellipse:nth-child(odd) {
+  height: 60px;
+}
+
+.wave-ellipse:nth-child(even) {
+  height: 80px;
+}
+
+@keyframes wave {
+  0%, 100% {
+    transform: scaleY(0.8);
+  }
+  50% {
+    transform: scaleY(1.2);
+  }
+}
+
+.wave-ellipse:nth-child(2) {
+  animation-delay: 0.1s;
+}
+
+.wave-ellipse:nth-child(3) {
+  animation-delay: 0.2s;
+}
+
+.wave-ellipse:nth-child(4) {
+  animation-delay: 0.3s;
+}
+
+.wave-ellipse:nth-child(5) {
+  animation-delay: 0.4s;
+}
+
+/* 其余样式保持不变 */
+.voice-indicators {
+  display: flex;
+  justify-content: space-between;
+  width: 100%;
+}
+
+.voice-indicator {
+  display: flex;
+  align-items: flex-end;
+}
+
+.bar {
+  width: 10px;
+  height: 20px;
+  background-color: #3498db;
+  margin: 0 2px;
+  transition: height 0.2s ease;
+}
+
+.voice-indicator.left .bar:nth-child(1) {
+  height: 15px;
+}
+
+.voice-indicator.left .bar:nth-child(2) {
+  height: 25px;
+}
+
+.voice-indicator.left .bar:nth-child(3) {
+  height: 20px;
+}
+
+.voice-indicator.right .bar:nth-child(1) {
+  height: 20px;
+}
+
+.voice-indicator.right .bar:nth-child(2) {
+  height: 10px;
+}
+
+.voice-indicator.right .bar:nth-child(3) {
+  height: 30px;
+}
+
+.call-controls {
+  display: flex;
+  justify-content: center;
+  gap: 2rem;
+}
+
+.call-button {
+  width: 60px;
+  height: 60px;
+  border-radius: 50%;
+  border: none;
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  font-size: 24px;
+  color: white;
+  cursor: pointer;
+}
+
+.hangup {
+  background-color: #e74c3c;
+}
+
+.answer {
+  background-color: #2ecc71;
+}
+
+.icon {
+  font-size: 28px;
+}
+</style>
\ No newline at end of file
diff --git a/web/src/views/Test1.vue b/web/src/views/Test1.vue
new file mode 100644
index 00000000..db588902
--- /dev/null
+++ b/web/src/views/Test1.vue
@@ -0,0 +1,110 @@
+<template>
+  <div>
+   {{data}}
+    <div>
+      <canvas ref="clientCanvasRef" />
+    </div>
+    <el-button type="primary" @click="sendMessage">连接电话</el-button>
+  </div>
+</template>
+
+<script setup>
+import {nextTick, onMounted, ref} from "vue";
+import Storage from "good-storage";
+
+const data = ref('abc')
+import { RealtimeClient } from '@openai/realtime-api-beta';
+
+const client = new RealtimeClient({
+  url: "wss://api.geekai.pro/v1/realtime",
+  apiKey: "sk-Gc5cEzDzGQLIqxWA9d62089350F3454bB359C4A3Fa21B3E4",
+  dangerouslyAllowAPIKeyInBrowser: true,
+});
+
+// Can set parameters ahead of connecting, either separately or all at once
+client.updateSession({ instructions: 'You are a great, upbeat friend.' });
+client.updateSession({ voice: 'alloy' });
+client.updateSession({
+  turn_detection: 'disabled', // or 'server_vad'
+  input_audio_transcription: { model: 'whisper-1' },
+});
+
+// Set up event handling
+client.on('conversation.updated', ({ item, delta }) => {
+  console.info('conversation.updated', item, delta)
+  switch (item.type) {
+    case 'message':
+      // system, user, or assistant message (item.role)
+      localStorage.setItem("chat_data", JSON.stringify(Array.from(item.formatted.audio)))
+        console.log("语言消息")
+      break;
+    case 'function_call':
+      // always a function call from the model
+      break;
+    case 'function_call_output':
+      // always a response from the user / application
+      break;
+  }
+  if (delta) {
+     console.info(delta.audio)
+    //localStorage.setItem("chat_data", JSON.stringify(Array.from(delta.audio)))
+    playPCM16(delta.audio, 24000);
+    // Only one of the following will be populated for any given event
+    // delta.audio = Int16Array, audio added
+    // delta.transcript = string, transcript added
+    // delta.arguments = string, function arguments added
+  }
+});
+
+client.on('conversation.item.appended', ({ item }) => {
+  if (item.role === 'assistant') {
+    playPCM16(item.formatted.audio, 24000);
+  }
+});
+
+const speaker = ref(null)
+// 假设 PCM16 数据已经存储在一个 Int16Array 中
+function playPCM16(pcm16Array, sampleRate = 44100) {
+  // 创建 AudioContext
+  const audioContext = new (window.AudioContext || window.webkitAudioContext)();
+
+  // 将 Int16Array 转换为 Float32Array (Web Audio API 使用 Float32)
+  let float32Array = new Float32Array(pcm16Array.length);
+  for (let i = 0; i < pcm16Array.length; i++) {
+    float32Array[i] = pcm16Array[i] / 32768; // Int16 转换为 Float32
+  }
+
+  // 创建 AudioBuffer
+  const audioBuffer = audioContext.createBuffer(1, float32Array.length, sampleRate); // 单声道
+  audioBuffer.getChannelData(0).set(float32Array); // 设置音频数据
+
+  // 创建 AudioBufferSourceNode 并播放音频
+  const source = audioContext.createBufferSource();
+  source.buffer = audioBuffer;
+  source.connect(audioContext.destination); // 连接到扬声器
+  source.start(); // 播放
+  speaker.value = source
+
+}
+
+onMounted(() => {
+  // Connect to Realtime API
+  // client.connect().then(res => {
+  //   if (res) {
+  //     console.log("连接成功!")
+  //   }
+  // }).catch(e => {
+  //   console.log(e)
+  // })
+
+
+})
+
+const sendMessage = () => {
+  const data = JSON.parse(localStorage.getItem("chat_data"))
+  playPCM16(data, 24000)
+  setTimeout(() => {speaker.value.stop()}, 5000)
+  // client.sendUserMessageContent([{ type: 'input_text', text: `你好，请用四川话给我讲一个笑话?` }]);
+}
+
+</script>