mirror of
https://github.com/yangjian102621/geekai.git
synced 2025-09-17 16:56:38 +08:00
add voice chat test case
This commit is contained in:
RockYang
parent
a678a11c33
commit
52a3f13f1e
54
web/package-lock.json
generated
54
web/package-lock.json
generated
@ -9,6 +9,7 @@
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@element-plus/icons-vue": "^2.1.0",
|
||||
"@openai/realtime-api-beta": "github:openai/openai-realtime-api-beta",
|
||||
"axios": "^0.27.2",
|
||||
"clipboard": "^2.0.11",
|
||||
"compressorjs": "^1.2.1",
|
||||
@ -27,7 +28,6 @@
|
||||
"markmap-view": "^0.16.0",
|
||||
"md-editor-v3": "^2.2.1",
|
||||
"memfs": "^4.9.3",
|
||||
"mitt": "^3.0.1",
|
||||
"pinia": "^2.1.4",
|
||||
"qrcode": "^1.5.3",
|
||||
"qs": "^6.11.1",
|
||||
@ -2022,6 +2022,33 @@
|
||||
"node": ">= 8"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/realtime-api-beta": {
|
||||
"version": "0.0.0",
|
||||
"resolved": "git+ssh://git@github.com/openai/openai-realtime-api-beta.git#339e9553a757ef1cf8c767272fc750c1e62effbb",
|
||||
"dependencies": {
|
||||
"ws": "^8.18.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@openai/realtime-api-beta/node_modules/ws": {
|
||||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmmirror.com/ws/-/ws-8.18.0.tgz",
|
||||
"integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
},
|
||||
"node_modules/@polka/url": {
|
||||
"version": "1.0.0-next.21",
|
||||
"resolved": "https://registry.npmmirror.com/@polka/url/-/url-1.0.0-next.21.tgz",
|
||||
@ -8769,11 +8796,6 @@
|
||||
"node": ">=8"
|
||||
}
|
||||
},
|
||||
"node_modules/mitt": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz",
|
||||
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw=="
|
||||
},
|
||||
"node_modules/mj-context-menu": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmmirror.com/mj-context-menu/-/mj-context-menu-0.6.1.tgz",
|
||||
@ -14196,6 +14218,21 @@
|
||||
"fastq": "^1.6.0"
|
||||
}
|
||||
},
|
||||
"@openai/realtime-api-beta": {
|
||||
"version": "git+ssh://git@github.com/openai/openai-realtime-api-beta.git#339e9553a757ef1cf8c767272fc750c1e62effbb",
|
||||
"from": "@openai/realtime-api-beta@github:openai/openai-realtime-api-beta",
|
||||
"requires": {
|
||||
"ws": "^8.18.0"
|
||||
},
|
||||
"dependencies": {
|
||||
"ws": {
|
||||
"version": "8.18.0",
|
||||
"resolved": "https://registry.npmmirror.com/ws/-/ws-8.18.0.tgz",
|
||||
"integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==",
|
||||
"requires": {}
|
||||
}
|
||||
}
|
||||
},
|
||||
"@polka/url": {
|
||||
"version": "1.0.0-next.21",
|
||||
"resolved": "https://registry.npmmirror.com/@polka/url/-/url-1.0.0-next.21.tgz",
|
||||
@ -19700,11 +19737,6 @@
|
||||
"yallist": "^4.0.0"
|
||||
}
|
||||
},
|
||||
"mitt": {
|
||||
"version": "3.0.1",
|
||||
"resolved": "https://registry.npmmirror.com/mitt/-/mitt-3.0.1.tgz",
|
||||
"integrity": "sha512-vKivATfr97l2/QBCYAkXYDbrIWPM2IIKEl7YPhjCvKlG3kE2gm+uBo6nEXK3M5/Ffh/FLpKExzOQ3JJoJGFKBw=="
|
||||
},
|
||||
"mj-context-menu": {
|
||||
"version": "0.6.1",
|
||||
"resolved": "https://registry.npmmirror.com/mj-context-menu/-/mj-context-menu-0.6.1.tgz",
|
||||
|
||||
@ -9,6 +9,7 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"@element-plus/icons-vue": "^2.1.0",
|
||||
"@openai/realtime-api-beta": "github:openai/openai-realtime-api-beta",
|
||||
"axios": "^0.27.2",
|
||||
"clipboard": "^2.0.11",
|
||||
"compressorjs": "^1.2.1",
|
||||
|
||||
@ -70,7 +70,6 @@ const props = defineProps({
|
||||
});
|
||||
const emits = defineEmits(['selected']);
|
||||
const show = ref(false)
|
||||
const fileList = ref([])
|
||||
const scrollbarRef = ref(null)
|
||||
const fileData = reactive({
|
||||
items:[],
|
||||
@ -116,7 +115,7 @@ const afterRead = (file) => {
|
||||
formData.append('file', file.file, file.name);
|
||||
// 执行上传操作
|
||||
httpPost('/api/upload', formData).then((res) => {
|
||||
fileList.value.unshift(res.data)
|
||||
fileData.items.unshift(res.data)
|
||||
ElMessage.success({message: "上传成功", duration: 500})
|
||||
}).catch((e) => {
|
||||
ElMessage.error('图片上传失败:' + e.message)
|
||||
@ -125,7 +124,7 @@ const afterRead = (file) => {
|
||||
|
||||
const removeFile = (file) => {
|
||||
httpGet('/api/upload/remove?id=' + file.id).then(() => {
|
||||
fileList.value = removeArrayItem(fileList.value, file, (v1, v2) => {
|
||||
fileData.items = removeArrayItem(fileData.items, file, (v1, v2) => {
|
||||
return v1.id === v2.id
|
||||
})
|
||||
ElMessage.success("文件删除成功！")
|
||||
|
||||
@ -1,16 +1,275 @@
|
||||
<template>
|
||||
<div>
|
||||
{{data}}
|
||||
<div class="video-call-container">
|
||||
<div class="wave-animation">
|
||||
<div v-for="i in 5" :key="i" class="wave-ellipse"></div>
|
||||
</div>
|
||||
<!-- 其余部分保持不变 -->
|
||||
<div class="voice-indicators">
|
||||
<div class="voice-indicator left">
|
||||
<div v-for="i in 3" :key="i" class="bar"></div>
|
||||
</div>
|
||||
<div class="voice-indicator right">
|
||||
<canvas id="canvas" width="600" height="200"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
<div class="call-controls">
|
||||
<button class="call-button hangup" @click="hangUp">
|
||||
<span class="icon">×</span>
|
||||
</button>
|
||||
<button class="call-button answer" @click="answer">
|
||||
<span class="icon">□</span>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import {onMounted, ref} from "vue";
|
||||
// Script 部分保持不变
|
||||
import {ref, onMounted, onUnmounted} from 'vue';
|
||||
|
||||
const data = ref('abc')
|
||||
const leftVoiceActive = ref(false);
|
||||
const rightVoiceActive = ref(false);
|
||||
|
||||
const animateVoice = () => {
|
||||
leftVoiceActive.value = Math.random() > 0.5;
|
||||
rightVoiceActive.value = Math.random() > 0.5;
|
||||
};
|
||||
|
||||
let voiceInterval;
|
||||
|
||||
onMounted(() => {
|
||||
|
||||
})
|
||||
voiceInterval = setInterval(animateVoice, 500);
|
||||
|
||||
const flag = ref(false)
|
||||
const canvas = document.getElementById('canvas');
|
||||
const ctx = canvas.getContext('2d');
|
||||
|
||||
async function setupAudioProcessing() {
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
|
||||
const audioContext = new (window.AudioContext || window.webkitAudioContext)();
|
||||
const analyser = audioContext.createAnalyser();
|
||||
const source = audioContext.createMediaStreamSource(stream);
|
||||
source.connect(analyser);
|
||||
analyser.fftSize = 256;
|
||||
const bufferLength = analyser.frequencyBinCount;
|
||||
const dataArray = new Uint8Array(bufferLength);
|
||||
|
||||
function draw() {
|
||||
analyser.getByteFrequencyData(dataArray);
|
||||
|
||||
if (!flag.value) {
|
||||
// ctx.clearRect(0, 0, canvas.width, canvas.height);
|
||||
// requestAnimationFrame(draw);
|
||||
return;
|
||||
}
|
||||
|
||||
ctx.clearRect(0, 0, canvas.width, canvas.height);
|
||||
|
||||
const barWidth = (canvas.width / bufferLength) * 5;
|
||||
let x = 0;
|
||||
|
||||
for (let i = 0; i < bufferLength; i++) {
|
||||
const barHeight = dataArray[i] / 2;
|
||||
|
||||
ctx.fillStyle = 'rgba(173, 216, 230, 0.7)'; // 淡蓝色
|
||||
ctx.fillRect(x, canvas.height - barHeight, barWidth, barHeight);
|
||||
|
||||
x += barWidth + 2;
|
||||
}
|
||||
|
||||
requestAnimationFrame(draw);
|
||||
}
|
||||
|
||||
draw();
|
||||
} catch (err) {
|
||||
console.error('获取麦克风权限失败:', err);
|
||||
}
|
||||
}
|
||||
|
||||
function setupSpeechRecognition() {
|
||||
const SpeechRecognition = window.SpeechRecognition || window.webkitSpeechRecognition;
|
||||
if (!SpeechRecognition) {
|
||||
alert('您的浏览器不支持语音识别功能');
|
||||
return;
|
||||
}
|
||||
|
||||
const recognition = new SpeechRecognition();
|
||||
recognition.continuous = true;
|
||||
recognition.interimResults = true;
|
||||
recognition.lang = 'zh-CN';
|
||||
|
||||
recognition.onresult = (event) => {
|
||||
const transcript = Array.from(event.results)
|
||||
.map(result => result[0])
|
||||
.map(result => result.transcript)
|
||||
.join('');
|
||||
|
||||
document.getElementById('transcript').textContent = '识别到的文本: ' + transcript;
|
||||
flag.value = true
|
||||
};
|
||||
|
||||
recognition.onerror = (event) => {
|
||||
console.error('语音识别错误：', event.error);
|
||||
flag.value = false
|
||||
};
|
||||
|
||||
recognition.start();
|
||||
}
|
||||
|
||||
|
||||
setupAudioProcessing();
|
||||
setupSpeechRecognition()
|
||||
});
|
||||
|
||||
onUnmounted(() => {
|
||||
clearInterval(voiceInterval);
|
||||
});
|
||||
|
||||
const hangUp = () => {
|
||||
console.log('Call hung up');
|
||||
};
|
||||
|
||||
const answer = () => {
|
||||
console.log('Call answered');
|
||||
};
|
||||
</script>
|
||||
|
||||
<style scoped>
|
||||
.video-call-container {
|
||||
background: linear-gradient(to right, #2c3e50, #4a5568, #6b46c1);
|
||||
height: 100vh;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
justify-content: space-between;
|
||||
align-items: center;
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
canvas {
|
||||
background-color: transparent;
|
||||
}
|
||||
|
||||
.wave-animation {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
gap: 10px;
|
||||
}
|
||||
|
||||
.wave-ellipse {
|
||||
width: 40px;
|
||||
height: 40px;
|
||||
background-color: white;
|
||||
border-radius: 20px;
|
||||
animation: wave 0.8s infinite ease-in-out;
|
||||
}
|
||||
|
||||
.wave-ellipse:nth-child(odd) {
|
||||
height: 60px;
|
||||
}
|
||||
|
||||
.wave-ellipse:nth-child(even) {
|
||||
height: 80px;
|
||||
}
|
||||
|
||||
@keyframes wave {
|
||||
0%, 100% {
|
||||
transform: scaleY(0.8);
|
||||
}
|
||||
50% {
|
||||
transform: scaleY(1.2);
|
||||
}
|
||||
}
|
||||
|
||||
.wave-ellipse:nth-child(2) {
|
||||
animation-delay: 0.1s;
|
||||
}
|
||||
|
||||
.wave-ellipse:nth-child(3) {
|
||||
animation-delay: 0.2s;
|
||||
}
|
||||
|
||||
.wave-ellipse:nth-child(4) {
|
||||
animation-delay: 0.3s;
|
||||
}
|
||||
|
||||
.wave-ellipse:nth-child(5) {
|
||||
animation-delay: 0.4s;
|
||||
}
|
||||
|
||||
/* 其余样式保持不变 */
|
||||
.voice-indicators {
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.voice-indicator {
|
||||
display: flex;
|
||||
align-items: flex-end;
|
||||
}
|
||||
|
||||
.bar {
|
||||
width: 10px;
|
||||
height: 20px;
|
||||
background-color: #3498db;
|
||||
margin: 0 2px;
|
||||
transition: height 0.2s ease;
|
||||
}
|
||||
|
||||
.voice-indicator.left .bar:nth-child(1) {
|
||||
height: 15px;
|
||||
}
|
||||
|
||||
.voice-indicator.left .bar:nth-child(2) {
|
||||
height: 25px;
|
||||
}
|
||||
|
||||
.voice-indicator.left .bar:nth-child(3) {
|
||||
height: 20px;
|
||||
}
|
||||
|
||||
.voice-indicator.right .bar:nth-child(1) {
|
||||
height: 20px;
|
||||
}
|
||||
|
||||
.voice-indicator.right .bar:nth-child(2) {
|
||||
height: 10px;
|
||||
}
|
||||
|
||||
.voice-indicator.right .bar:nth-child(3) {
|
||||
height: 30px;
|
||||
}
|
||||
|
||||
.call-controls {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
gap: 2rem;
|
||||
}
|
||||
|
||||
.call-button {
|
||||
width: 60px;
|
||||
height: 60px;
|
||||
border-radius: 50%;
|
||||
border: none;
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
align-items: center;
|
||||
font-size: 24px;
|
||||
color: white;
|
||||
cursor: pointer;
|
||||
}
|
||||
|
||||
.hangup {
|
||||
background-color: #e74c3c;
|
||||
}
|
||||
|
||||
.answer {
|
||||
background-color: #2ecc71;
|
||||
}
|
||||
|
||||
.icon {
|
||||
font-size: 28px;
|
||||
}
|
||||
</style>
|
||||
110
web/src/views/Test1.vue
Normal file
110
web/src/views/Test1.vue
Normal file
@ -0,0 +1,110 @@
|
||||
<template>
|
||||
<div>
|
||||
{{data}}
|
||||
<div>
|
||||
<canvas ref="clientCanvasRef" />
|
||||
</div>
|
||||
<el-button type="primary" @click="sendMessage">连接电话</el-button>
|
||||
</div>
|
||||
</template>
|
||||
|
||||
<script setup>
|
||||
import {nextTick, onMounted, ref} from "vue";
|
||||
import Storage from "good-storage";
|
||||
|
||||
const data = ref('abc')
|
||||
import { RealtimeClient } from '@openai/realtime-api-beta';
|
||||
|
||||
const client = new RealtimeClient({
|
||||
url: "wss://api.geekai.pro/v1/realtime",
|
||||
apiKey: "sk-Gc5cEzDzGQLIqxWA9d62089350F3454bB359C4A3Fa21B3E4",
|
||||
dangerouslyAllowAPIKeyInBrowser: true,
|
||||
});
|
||||
|
||||
// Can set parameters ahead of connecting, either separately or all at once
|
||||
client.updateSession({ instructions: 'You are a great, upbeat friend.' });
|
||||
client.updateSession({ voice: 'alloy' });
|
||||
client.updateSession({
|
||||
turn_detection: 'disabled', // or 'server_vad'
|
||||
input_audio_transcription: { model: 'whisper-1' },
|
||||
});
|
||||
|
||||
// Set up event handling
|
||||
client.on('conversation.updated', ({ item, delta }) => {
|
||||
console.info('conversation.updated', item, delta)
|
||||
switch (item.type) {
|
||||
case 'message':
|
||||
// system, user, or assistant message (item.role)
|
||||
localStorage.setItem("chat_data", JSON.stringify(Array.from(item.formatted.audio)))
|
||||
console.log("语言消息")
|
||||
break;
|
||||
case 'function_call':
|
||||
// always a function call from the model
|
||||
break;
|
||||
case 'function_call_output':
|
||||
// always a response from the user / application
|
||||
break;
|
||||
}
|
||||
if (delta) {
|
||||
console.info(delta.audio)
|
||||
//localStorage.setItem("chat_data", JSON.stringify(Array.from(delta.audio)))
|
||||
playPCM16(delta.audio, 24000);
|
||||
// Only one of the following will be populated for any given event
|
||||
// delta.audio = Int16Array, audio added
|
||||
// delta.transcript = string, transcript added
|
||||
// delta.arguments = string, function arguments added
|
||||
}
|
||||
});
|
||||
|
||||
client.on('conversation.item.appended', ({ item }) => {
|
||||
if (item.role === 'assistant') {
|
||||
playPCM16(item.formatted.audio, 24000);
|
||||
}
|
||||
});
|
||||
|
||||
const speaker = ref(null)
|
||||
// 假设 PCM16 数据已经存储在一个 Int16Array 中
|
||||
function playPCM16(pcm16Array, sampleRate = 44100) {
|
||||
// 创建 AudioContext
|
||||
const audioContext = new (window.AudioContext || window.webkitAudioContext)();
|
||||
|
||||
// 将 Int16Array 转换为 Float32Array (Web Audio API 使用 Float32)
|
||||
let float32Array = new Float32Array(pcm16Array.length);
|
||||
for (let i = 0; i < pcm16Array.length; i++) {
|
||||
float32Array[i] = pcm16Array[i] / 32768; // Int16 转换为 Float32
|
||||
}
|
||||
|
||||
// 创建 AudioBuffer
|
||||
const audioBuffer = audioContext.createBuffer(1, float32Array.length, sampleRate); // 单声道
|
||||
audioBuffer.getChannelData(0).set(float32Array); // 设置音频数据
|
||||
|
||||
// 创建 AudioBufferSourceNode 并播放音频
|
||||
const source = audioContext.createBufferSource();
|
||||
source.buffer = audioBuffer;
|
||||
source.connect(audioContext.destination); // 连接到扬声器
|
||||
source.start(); // 播放
|
||||
speaker.value = source
|
||||
|
||||
}
|
||||
|
||||
onMounted(() => {
|
||||
// Connect to Realtime API
|
||||
// client.connect().then(res => {
|
||||
// if (res) {
|
||||
// console.log("连接成功!")
|
||||
// }
|
||||
// }).catch(e => {
|
||||
// console.log(e)
|
||||
// })
|
||||
|
||||
|
||||
})
|
||||
|
||||
const sendMessage = () => {
|
||||
const data = JSON.parse(localStorage.getItem("chat_data"))
|
||||
playPCM16(data, 24000)
|
||||
setTimeout(() => {speaker.value.stop()}, 5000)
|
||||
// client.sendUserMessageContent([{ type: 'input_text', text: `你好，请用四川话给我讲一个笑话?` }]);
|
||||
}
|
||||
|
||||
</script>
|
||||
Loading…
Reference in New Issue
Block a user