From 155c56f502ddac98ae7d863f77bacb39bd73a094 Mon Sep 17 00:00:00 2001
From: RockYang <yangjian102621@gmail.com>
Date: Tue, 15 Oct 2024 19:25:18 +0800
Subject: [PATCH] integrated openai realtime console

---
 web/src/App.vue                               |   2 +-
 web/src/components/Calling.vue                | 130 ++++
 web/src/components/Conversation .vue          | 161 ++++
 web/src/components/FileList.vue               |   2 +-
 web/src/lib/wavtools/index.js                 |   6 +
 .../wavtools/lib/analysis/audio_analysis.js   | 203 +++++
 .../lib/wavtools/lib/analysis/constants.js    |  60 ++
 web/src/lib/wavtools/lib/wav_packer.js        | 113 +++
 web/src/lib/wavtools/lib/wav_recorder.js      | 548 +++++++++++++
 web/src/lib/wavtools/lib/wav_stream_player.js | 160 ++++
 .../wavtools/lib/worklets/audio_processor.js  | 214 ++++++
 .../wavtools/lib/worklets/stream_processor.js |  96 +++
 web/src/utils/conversation_config.js          |  16 +
 web/src/utils/wav_player.js                   |  27 +
 web/src/utils/wav_renderer.js                 |  81 ++
 web/src/views/Test.vue                        | 720 +++++++++++-------
 web/src/views/Test2.vue                       |   2 +-
 web/src/views/Test3.vue                       | 184 +++++
 18 files changed, 2444 insertions(+), 281 deletions(-)
 create mode 100644 web/src/components/Calling.vue
 create mode 100644 web/src/components/Conversation .vue
 create mode 100644 web/src/lib/wavtools/index.js
 create mode 100644 web/src/lib/wavtools/lib/analysis/audio_analysis.js
 create mode 100644 web/src/lib/wavtools/lib/analysis/constants.js
 create mode 100644 web/src/lib/wavtools/lib/wav_packer.js
 create mode 100644 web/src/lib/wavtools/lib/wav_recorder.js
 create mode 100644 web/src/lib/wavtools/lib/wav_stream_player.js
 create mode 100644 web/src/lib/wavtools/lib/worklets/audio_processor.js
 create mode 100644 web/src/lib/wavtools/lib/worklets/stream_processor.js
 create mode 100644 web/src/utils/conversation_config.js
 create mode 100644 web/src/utils/wav_player.js
 create mode 100644 web/src/utils/wav_renderer.js
 create mode 100644 web/src/views/Test3.vue

diff --git a/web/src/App.vue b/web/src/App.vue
index ea30e4bf..cbde045c 100644
--- a/web/src/App.vue
+++ b/web/src/App.vue
@@ -116,7 +116,7 @@ html, body {
     margin 0;
 
     .el-dialog__body {
-      max-height 80vh
+      //max-height 80vh
       overflow-y auto
     }
   }
diff --git a/web/src/components/Calling.vue b/web/src/components/Calling.vue
new file mode 100644
index 00000000..e7b6c724
--- /dev/null
+++ b/web/src/components/Calling.vue
@@ -0,0 +1,130 @@
+<template>
+  <!--拨号组件-->
+  <el-container class="calling-container" :style="{height: height}">
+    <div class="phone-container">
+      <div class="signal"></div>
+      <div class="signal"></div>
+      <div class="signal"></div>
+      <div class="phone"></div>
+    </div>
+    <div class="status-text">{{ text }}</div>
+  </el-container>
+</template>
+
+<script setup>
+import {onMounted, ref} from "vue";
+
+const fullText = "正在接通中...";
+const text = ref("")
+let index = 0;
+const props = defineProps({
+  height: {
+    type: String,
+    default: '100vh'
+  }
+})
+
+function typeText() {
+  if (index < fullText.length) {
+    text.value += fullText[index];
+    index++;
+    setTimeout(typeText, 300); // 每300毫秒显示一个字
+  } else {
+    setTimeout(() => {
+      text.value = '';
+      index = 0;
+      typeText();
+    }, 1000); // 等待1秒后重新开始
+  }
+}
+
+onMounted(() => {
+  typeText()
+})
+
+</script>
+
+<style scoped lang="stylus">
+
+.calling-container {
+  background-color: #000;
+  display: flex;
+  flex-direction: column;
+  justify-content: center;
+  align-items: center;
+  margin: 0;
+  overflow: hidden;
+  font-family: Arial, sans-serif;
+  width 100vw
+
+  .phone-container {
+    position: relative;
+    width: 200px;
+    height: 200px;
+  }
+
+  .phone {
+    position: absolute;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+    width: 60px;
+    height: 60px;
+    background-color: #00ffcc;
+    mask: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'%3E%3Cpath d='M20 15.5c-1.25 0-2.45-.2-3.57-.57a1.02 1.02 0 0 0-1.02.24l-2.2 2.2a15.074 15.074 0 0 1-6.59-6.59l2.2-2.2c.27-.27.35-.68.24-1.02a11.36 11.36 0 0 1-.57-3.57c0-.55-.45-1-1-1H4c-.55 0-1 .45-1 1 0 9.39 7.61 17 17 17 .55 0 1-.45 1-1v-3.5c0-.55-.45-1-1-1zM5.03 5h1.5c.07.89.22 1.76.46 2.59l-1.2 1.2c-.41-1.2-.67-2.47-.76-3.79zM19 18.97c-1.32-.09-2.59-.35-3.8-.75l1.2-1.2c.85.24 1.72.39 2.6.45v1.5z'/%3E%3C/svg%3E") no-repeat 50% 50%;
+    mask-size: cover;
+    -webkit-mask: url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' viewBox='0 0 24 24'%3E%3Cpath d='M20 15.5c-1.25 0-2.45-.2-3.57-.57a1.02 1.02 0 0 0-1.02.24l-2.2 2.2a15.074 15.074 0 0 1-6.59-6.59l2.2-2.2c.27-.27.35-.68.24-1.02a11.36 11.36 0 0 1-.57-3.57c0-.55-.45-1-1-1H4c-.55 0-1 .45-1 1 0 9.39 7.61 17 17 17 .55 0 1-.45 1-1v-3.5c0-.55-.45-1-1-1zM5.03 5h1.5c.07.89.22 1.76.46 2.59l-1.2 1.2c-.41-1.2-.67-2.47-.76-3.79zM19 18.97c-1.32-.09-2.59-.35-3.8-.75l1.2-1.2c.85.24 1.72.39 2.6.45v1.5z'/%3E%3C/svg%3E") no-repeat 50% 50%;
+    -webkit-mask-size: cover;
+    animation: shake 0.5s ease-in-out infinite;
+  }
+
+  .signal {
+    position: absolute;
+    top: 50%;
+    left: 50%;
+    transform: translate(-50%, -50%);
+    width: 100px;
+    height: 100px;
+    border: 2px dashed #00ffcc;
+    border-radius: 50%;
+    opacity: 0;
+    animation: signal 2s linear infinite;
+  }
+
+  .signal:nth-child(2) {
+    animation-delay: 0.5s;
+  }
+
+  .signal:nth-child(3) {
+    animation-delay: 1s;
+  }
+
+  .status-text {
+    color: #00ffcc;
+    font-size: 18px;
+    margin-top: 20px;
+    height: 1.2em;
+    overflow: hidden;
+  }
+
+  @keyframes shake {
+    0%, 100% { transform: translate(-50%, -50%) rotate(0deg); }
+    25% { transform: translate(-52%, -48%) rotate(-5deg); }
+    75% { transform: translate(-48%, -52%) rotate(5deg); }
+  }
+
+  @keyframes signal {
+    0% {
+      width: 60px;
+      height: 60px;
+      opacity: 1;
+    }
+    100% {
+      width: 200px;
+      height: 200px;
+      opacity: 0;
+    }
+  }
+}
+
+</style>
\ No newline at end of file
diff --git a/web/src/components/Conversation .vue b/web/src/components/Conversation .vue
new file mode 100644
index 00000000..8b8e0736
--- /dev/null
+++ b/web/src/components/Conversation .vue	
@@ -0,0 +1,161 @@
+<template>
+  <!--语音通话组件-->
+  <div class="video-call-container" :style="{height: height}">
+    <div class="wave-container">
+      <div class="wave-animation">
+        <div v-for="i in 5" :key="i" class="wave-ellipse"></div>
+      </div>
+    </div>
+    <!-- 其余部分保持不变 -->
+    <div class="voice-indicators">
+      <div class="voice-indicator left">
+        <canvas ref="canvasClientRef" width="600" height="200"></canvas>
+      </div>
+      <div class="voice-indicator right">
+        <canvas ref="canvasServerRef" width="600" height="200"></canvas>
+      </div>
+    </div>
+    <div class="call-controls">
+      <button class="call-button hangup" @click="hangUp">
+        <i class="iconfont icon-hung-up"></i>
+      </button>
+    </div>
+  </div>
+</template>
+
+<script setup>
+import {onMounted, onUnmounted, ref} from "vue";
+
+const leftVoiceActive = ref(false);
+const rightVoiceActive = ref(false);
+const props = defineProps({
+  height: {
+    type: String,
+    default: '100vh'
+  }
+})
+const emits = defineEmits(['hangUp']);
+
+const animateVoice = () => {
+  leftVoiceActive.value = Math.random() > 0.5;
+  rightVoiceActive.value = Math.random() > 0.5;
+};
+
+let voiceInterval;
+
+onMounted(() => {
+  voiceInterval = setInterval(animateVoice, 500);
+});
+
+onUnmounted(() => {
+  clearInterval(voiceInterval);
+});
+
+const hangUp = () => {
+  console.log('Call hung up');
+  emits('hangUp')
+};
+
+</script>
+
+<style scoped lang="stylus">
+
+.video-call-container {
+  background: linear-gradient(to right, #2c3e50, #4a5568, #6b46c1);
+  display: flex;
+  flex-direction: column;
+  justify-content: space-between;
+  align-items: center;
+  padding: 0;
+  width 100vw
+
+  .wave-container {
+    padding 3rem
+    .wave-animation {
+      display: flex;
+      justify-content: center;
+      align-items: center;
+      gap: 10px;
+    }
+  }
+
+
+  .wave-ellipse {
+    width: 40px;
+    height: 40px;
+    background-color: white;
+    border-radius: 20px;
+    animation: wave 0.8s infinite ease-in-out;
+  }
+
+  .wave-ellipse:nth-child(odd) {
+    height: 60px;
+  }
+
+  .wave-ellipse:nth-child(even) {
+    height: 80px;
+  }
+
+  @keyframes wave {
+    0%, 100% {
+      transform: scaleY(0.8);
+    }
+    50% {
+      transform: scaleY(1.2);
+    }
+  }
+
+  .wave-ellipse:nth-child(2) {
+    animation-delay: 0.1s;
+  }
+
+  .wave-ellipse:nth-child(3) {
+    animation-delay: 0.2s;
+  }
+
+  .wave-ellipse:nth-child(4) {
+    animation-delay: 0.3s;
+  }
+
+  .wave-ellipse:nth-child(5) {
+    animation-delay: 0.4s;
+  }
+
+  .call-controls {
+    display: flex;
+    justify-content: center;
+    gap: 3rem;
+    padding 3rem
+
+    .call-button {
+      width: 60px;
+      height: 60px;
+      border-radius: 50%;
+      border: none;
+      display: flex;
+      justify-content: center;
+      align-items: center;
+      font-size: 24px;
+      color: white;
+      cursor: pointer;
+
+      .iconfont {
+        font-size 24px
+      }
+    }
+    .hangup {
+      background-color: #e74c3c;
+    }
+
+    .answer {
+      background-color: #2ecc71;
+    }
+
+    .icon {
+      font-size: 28px;
+    }
+  }
+
+}
+
+</style>
\ No newline at end of file
diff --git a/web/src/components/FileList.vue b/web/src/components/FileList.vue
index 8bf96897..2819902a 100644
--- a/web/src/components/FileList.vue
+++ b/web/src/components/FileList.vue
@@ -1,6 +1,6 @@
 <template>
   <el-container class="chat-file-list">
-    <div v-for="file in fileList">
+    <div v-for="file in fileList" :key="file.url">
       <div class="image" v-if="isImage(file.ext)">
         <el-image :src="file.url" fit="cover"/>
         <div class="action">
diff --git a/web/src/lib/wavtools/index.js b/web/src/lib/wavtools/index.js
new file mode 100644
index 00000000..71238942
--- /dev/null
+++ b/web/src/lib/wavtools/index.js
@@ -0,0 +1,6 @@
+import { WavPacker } from './lib/wav_packer.js';
+import { AudioAnalysis } from './lib/analysis/audio_analysis.js';
+import { WavStreamPlayer } from './lib/wav_stream_player.js';
+import { WavRecorder } from './lib/wav_recorder.js';
+
+export { AudioAnalysis, WavPacker, WavStreamPlayer, WavRecorder };
diff --git a/web/src/lib/wavtools/lib/analysis/audio_analysis.js b/web/src/lib/wavtools/lib/analysis/audio_analysis.js
new file mode 100644
index 00000000..4af34d54
--- /dev/null
+++ b/web/src/lib/wavtools/lib/analysis/audio_analysis.js
@@ -0,0 +1,203 @@
+import {
+  noteFrequencies,
+  noteFrequencyLabels,
+  voiceFrequencies,
+  voiceFrequencyLabels,
+} from './constants.js';
+
+/**
+ * Output of AudioAnalysis for the frequency domain of the audio
+ * @typedef {Object} AudioAnalysisOutputType
+ * @property {Float32Array} values Amplitude of this frequency between {0, 1} inclusive
+ * @property {number[]} frequencies Raw frequency bucket values
+ * @property {string[]} labels Labels for the frequency bucket values
+ */
+
+/**
+ * Analyzes audio for visual output
+ * @class
+ */
+export class AudioAnalysis {
+  /**
+   * Retrieves frequency domain data from an AnalyserNode adjusted to a decibel range
+   * returns human-readable formatting and labels
+   * @param {AnalyserNode} analyser
+   * @param {number} sampleRate
+   * @param {Float32Array} [fftResult]
+   * @param {"frequency"|"music"|"voice"} [analysisType]
+   * @param {number} [minDecibels] default -100
+   * @param {number} [maxDecibels] default -30
+   * @returns {AudioAnalysisOutputType}
+   */
+  static getFrequencies(
+    analyser,
+    sampleRate,
+    fftResult,
+    analysisType = 'frequency',
+    minDecibels = -100,
+    maxDecibels = -30,
+  ) {
+    if (!fftResult) {
+      fftResult = new Float32Array(analyser.frequencyBinCount);
+      analyser.getFloatFrequencyData(fftResult);
+    }
+    const nyquistFrequency = sampleRate / 2;
+    const frequencyStep = (1 / fftResult.length) * nyquistFrequency;
+    let outputValues;
+    let frequencies;
+    let labels;
+    if (analysisType === 'music' || analysisType === 'voice') {
+      const useFrequencies =
+        analysisType === 'voice' ? voiceFrequencies : noteFrequencies;
+      const aggregateOutput = Array(useFrequencies.length).fill(minDecibels);
+      for (let i = 0; i < fftResult.length; i++) {
+        const frequency = i * frequencyStep;
+        const amplitude = fftResult[i];
+        for (let n = useFrequencies.length - 1; n >= 0; n--) {
+          if (frequency > useFrequencies[n]) {
+            aggregateOutput[n] = Math.max(aggregateOutput[n], amplitude);
+            break;
+          }
+        }
+      }
+      outputValues = aggregateOutput;
+      frequencies =
+        analysisType === 'voice' ? voiceFrequencies : noteFrequencies;
+      labels =
+        analysisType === 'voice' ? voiceFrequencyLabels : noteFrequencyLabels;
+    } else {
+      outputValues = Array.from(fftResult);
+      frequencies = outputValues.map((_, i) => frequencyStep * i);
+      labels = frequencies.map((f) => `${f.toFixed(2)} Hz`);
+    }
+    // We normalize to {0, 1}
+    const normalizedOutput = outputValues.map((v) => {
+      return Math.max(
+        0,
+        Math.min((v - minDecibels) / (maxDecibels - minDecibels), 1),
+      );
+    });
+    const values = new Float32Array(normalizedOutput);
+    return {
+      values,
+      frequencies,
+      labels,
+    };
+  }
+
+  /**
+   * Creates a new AudioAnalysis instance for an HTMLAudioElement
+   * @param {HTMLAudioElement} audioElement
+   * @param {AudioBuffer|null} [audioBuffer] If provided, will cache all frequency domain data from the buffer
+   * @returns {AudioAnalysis}
+   */
+  constructor(audioElement, audioBuffer = null) {
+    this.fftResults = [];
+    if (audioBuffer) {
+      /**
+       * Modified from
+       * https://stackoverflow.com/questions/75063715/using-the-web-audio-api-to-analyze-a-song-without-playing
+       *
+       * We do this to populate FFT values for the audio if provided an `audioBuffer`
+       * The reason to do this is that Safari fails when using `createMediaElementSource`
+       * This has a non-zero RAM cost so we only opt-in to run it on Safari, Chrome is better
+       */
+      const { length, sampleRate } = audioBuffer;
+      const offlineAudioContext = new OfflineAudioContext({
+        length,
+        sampleRate,
+      });
+      const source = offlineAudioContext.createBufferSource();
+      source.buffer = audioBuffer;
+      const analyser = offlineAudioContext.createAnalyser();
+      analyser.fftSize = 8192;
+      analyser.smoothingTimeConstant = 0.1;
+      source.connect(analyser);
+      // limit is :: 128 / sampleRate;
+      // but we just want 60fps - cuts ~1s from 6MB to 1MB of RAM
+      const renderQuantumInSeconds = 1 / 60;
+      const durationInSeconds = length / sampleRate;
+      const analyze = (index) => {
+        const suspendTime = renderQuantumInSeconds * index;
+        if (suspendTime < durationInSeconds) {
+          offlineAudioContext.suspend(suspendTime).then(() => {
+            const fftResult = new Float32Array(analyser.frequencyBinCount);
+            analyser.getFloatFrequencyData(fftResult);
+            this.fftResults.push(fftResult);
+            analyze(index + 1);
+          });
+        }
+        if (index === 1) {
+          offlineAudioContext.startRendering();
+        } else {
+          offlineAudioContext.resume();
+        }
+      };
+      source.start(0);
+      analyze(1);
+      this.audio = audioElement;
+      this.context = offlineAudioContext;
+      this.analyser = analyser;
+      this.sampleRate = sampleRate;
+      this.audioBuffer = audioBuffer;
+    } else {
+      const audioContext = new AudioContext();
+      const track = audioContext.createMediaElementSource(audioElement);
+      const analyser = audioContext.createAnalyser();
+      analyser.fftSize = 8192;
+      analyser.smoothingTimeConstant = 0.1;
+      track.connect(analyser);
+      analyser.connect(audioContext.destination);
+      this.audio = audioElement;
+      this.context = audioContext;
+      this.analyser = analyser;
+      this.sampleRate = this.context.sampleRate;
+      this.audioBuffer = null;
+    }
+  }
+
+  /**
+   * Gets the current frequency domain data from the playing audio track
+   * @param {"frequency"|"music"|"voice"} [analysisType]
+   * @param {number} [minDecibels] default -100
+   * @param {number} [maxDecibels] default -30
+   * @returns {AudioAnalysisOutputType}
+   */
+  getFrequencies(
+    analysisType = 'frequency',
+    minDecibels = -100,
+    maxDecibels = -30,
+  ) {
+    let fftResult = null;
+    if (this.audioBuffer && this.fftResults.length) {
+      const pct = this.audio.currentTime / this.audio.duration;
+      const index = Math.min(
+        (pct * this.fftResults.length) | 0,
+        this.fftResults.length - 1,
+      );
+      fftResult = this.fftResults[index];
+    }
+    return AudioAnalysis.getFrequencies(
+      this.analyser,
+      this.sampleRate,
+      fftResult,
+      analysisType,
+      minDecibels,
+      maxDecibels,
+    );
+  }
+
+  /**
+   * Resume the internal AudioContext if it was suspended due to the lack of
+   * user interaction when the AudioAnalysis was instantiated.
+   * @returns {Promise<true>}
+   */
+  async resumeIfSuspended() {
+    if (this.context.state === 'suspended') {
+      await this.context.resume();
+    }
+    return true;
+  }
+}
+
+globalThis.AudioAnalysis = AudioAnalysis;
diff --git a/web/src/lib/wavtools/lib/analysis/constants.js b/web/src/lib/wavtools/lib/analysis/constants.js
new file mode 100644
index 00000000..f14da38e
--- /dev/null
+++ b/web/src/lib/wavtools/lib/analysis/constants.js
@@ -0,0 +1,60 @@
+/**
+ * Constants for help with visualization
+ * Helps map frequency ranges from Fast Fourier Transform
+ * to human-interpretable ranges, notably music ranges and
+ * human vocal ranges.
+ */
+
+// Eighth octave frequencies
+const octave8Frequencies = [
+  4186.01, 4434.92, 4698.63, 4978.03, 5274.04, 5587.65, 5919.91, 6271.93,
+  6644.88, 7040.0, 7458.62, 7902.13,
+];
+
+// Labels for each of the above frequencies
+const octave8FrequencyLabels = [
+  'C',
+  'C#',
+  'D',
+  'D#',
+  'E',
+  'F',
+  'F#',
+  'G',
+  'G#',
+  'A',
+  'A#',
+  'B',
+];
+
+/**
+ * All note frequencies from 1st to 8th octave
+ * in format "A#8" (A#, 8th octave)
+ */
+export const noteFrequencies = [];
+export const noteFrequencyLabels = [];
+for (let i = 1; i <= 8; i++) {
+  for (let f = 0; f < octave8Frequencies.length; f++) {
+    const freq = octave8Frequencies[f];
+    noteFrequencies.push(freq / Math.pow(2, 8 - i));
+    noteFrequencyLabels.push(octave8FrequencyLabels[f] + i);
+  }
+}
+
+/**
+ * Subset of the note frequencies between 32 and 2000 Hz
+ * 6 octave range: C1 to B6
+ */
+const voiceFrequencyRange = [32.0, 2000.0];
+export const voiceFrequencies = noteFrequencies.filter((_, i) => {
+  return (
+    noteFrequencies[i] > voiceFrequencyRange[0] &&
+    noteFrequencies[i] < voiceFrequencyRange[1]
+  );
+});
+export const voiceFrequencyLabels = noteFrequencyLabels.filter((_, i) => {
+  return (
+    noteFrequencies[i] > voiceFrequencyRange[0] &&
+    noteFrequencies[i] < voiceFrequencyRange[1]
+  );
+});
diff --git a/web/src/lib/wavtools/lib/wav_packer.js b/web/src/lib/wavtools/lib/wav_packer.js
new file mode 100644
index 00000000..7146b7fd
--- /dev/null
+++ b/web/src/lib/wavtools/lib/wav_packer.js
@@ -0,0 +1,113 @@
+/**
+ * Raw wav audio file contents
+ * @typedef {Object} WavPackerAudioType
+ * @property {Blob} blob
+ * @property {string} url
+ * @property {number} channelCount
+ * @property {number} sampleRate
+ * @property {number} duration
+ */
+
+/**
+ * Utility class for assembling PCM16 "audio/wav" data
+ * @class
+ */
+export class WavPacker {
+  /**
+   * Converts Float32Array of amplitude data to ArrayBuffer in Int16Array format
+   * @param {Float32Array} float32Array
+   * @returns {ArrayBuffer}
+   */
+  static floatTo16BitPCM(float32Array) {
+    const buffer = new ArrayBuffer(float32Array.length * 2);
+    const view = new DataView(buffer);
+    let offset = 0;
+    for (let i = 0; i < float32Array.length; i++, offset += 2) {
+      let s = Math.max(-1, Math.min(1, float32Array[i]));
+      view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+    }
+    return buffer;
+  }
+
+  /**
+   * Concatenates two ArrayBuffers
+   * @param {ArrayBuffer} leftBuffer
+   * @param {ArrayBuffer} rightBuffer
+   * @returns {ArrayBuffer}
+   */
+  static mergeBuffers(leftBuffer, rightBuffer) {
+    const tmpArray = new Uint8Array(
+      leftBuffer.byteLength + rightBuffer.byteLength
+    );
+    tmpArray.set(new Uint8Array(leftBuffer), 0);
+    tmpArray.set(new Uint8Array(rightBuffer), leftBuffer.byteLength);
+    return tmpArray.buffer;
+  }
+
+  /**
+   * Packs data into an Int16 format
+   * @private
+   * @param {number} size 0 = 1x Int16, 1 = 2x Int16
+   * @param {number} arg value to pack
+   * @returns
+   */
+  _packData(size, arg) {
+    return [
+      new Uint8Array([arg, arg >> 8]),
+      new Uint8Array([arg, arg >> 8, arg >> 16, arg >> 24]),
+    ][size];
+  }
+
+  /**
+   * Packs audio into "audio/wav" Blob
+   * @param {number} sampleRate
+   * @param {{bitsPerSample: number, channels: Array<Float32Array>, data: Int16Array}} audio
+   * @returns {WavPackerAudioType}
+   */
+  pack(sampleRate, audio) {
+    if (!audio?.bitsPerSample) {
+      throw new Error(`Missing "bitsPerSample"`);
+    } else if (!audio?.channels) {
+      throw new Error(`Missing "channels"`);
+    } else if (!audio?.data) {
+      throw new Error(`Missing "data"`);
+    }
+    const { bitsPerSample, channels, data } = audio;
+    const output = [
+      // Header
+      'RIFF',
+      this._packData(
+        1,
+        4 + (8 + 24) /* chunk 1 length */ + (8 + 8) /* chunk 2 length */
+      ), // Length
+      'WAVE',
+      // chunk 1
+      'fmt ', // Sub-chunk identifier
+      this._packData(1, 16), // Chunk length
+      this._packData(0, 1), // Audio format (1 is linear quantization)
+      this._packData(0, channels.length),
+      this._packData(1, sampleRate),
+      this._packData(1, (sampleRate * channels.length * bitsPerSample) / 8), // Byte rate
+      this._packData(0, (channels.length * bitsPerSample) / 8),
+      this._packData(0, bitsPerSample),
+      // chunk 2
+      'data', // Sub-chunk identifier
+      this._packData(
+        1,
+        (channels[0].length * channels.length * bitsPerSample) / 8
+      ), // Chunk length
+      data,
+    ];
+    const blob = new Blob(output, { type: 'audio/mpeg' });
+    const url = URL.createObjectURL(blob);
+    return {
+      blob,
+      url,
+      channelCount: channels.length,
+      sampleRate,
+      duration: data.byteLength / (channels.length * sampleRate * 2),
+    };
+  }
+}
+
+globalThis.WavPacker = WavPacker;
diff --git a/web/src/lib/wavtools/lib/wav_recorder.js b/web/src/lib/wavtools/lib/wav_recorder.js
new file mode 100644
index 00000000..a4f1d045
--- /dev/null
+++ b/web/src/lib/wavtools/lib/wav_recorder.js
@@ -0,0 +1,548 @@
+import { AudioProcessorSrc } from './worklets/audio_processor.js';
+import { AudioAnalysis } from './analysis/audio_analysis.js';
+import { WavPacker } from './wav_packer.js';
+
+/**
+ * Decodes audio into a wav file
+ * @typedef {Object} DecodedAudioType
+ * @property {Blob} blob
+ * @property {string} url
+ * @property {Float32Array} values
+ * @property {AudioBuffer} audioBuffer
+ */
+
+/**
+ * Records live stream of user audio as PCM16 "audio/wav" data
+ * @class
+ */
+export class WavRecorder {
+  /**
+   * Create a new WavRecorder instance
+   * @param {{sampleRate?: number, outputToSpeakers?: boolean, debug?: boolean}} [options]
+   * @returns {WavRecorder}
+   */
+  constructor({
+    sampleRate = 44100,
+    outputToSpeakers = false,
+    debug = false,
+  } = {}) {
+    // Script source
+    this.scriptSrc = AudioProcessorSrc;
+    // Config
+    this.sampleRate = sampleRate;
+    this.outputToSpeakers = outputToSpeakers;
+    this.debug = !!debug;
+    this._deviceChangeCallback = null;
+    this._devices = [];
+    // State variables
+    this.stream = null;
+    this.processor = null;
+    this.source = null;
+    this.node = null;
+    this.recording = false;
+    // Event handling with AudioWorklet
+    this._lastEventId = 0;
+    this.eventReceipts = {};
+    this.eventTimeout = 5000;
+    // Process chunks of audio
+    this._chunkProcessor = () => {};
+    this._chunkProcessorSize = void 0;
+    this._chunkProcessorBuffer = {
+      raw: new ArrayBuffer(0),
+      mono: new ArrayBuffer(0),
+    };
+  }
+
+  /**
+   * Decodes audio data from multiple formats to a Blob, url, Float32Array and AudioBuffer
+   * @param {Blob|Float32Array|Int16Array|ArrayBuffer|number[]} audioData
+   * @param {number} sampleRate
+   * @param {number} fromSampleRate
+   * @returns {Promise<DecodedAudioType>}
+   */
+  static async decode(audioData, sampleRate = 44100, fromSampleRate = -1) {
+    const context = new AudioContext({ sampleRate });
+    let arrayBuffer;
+    let blob;
+    if (audioData instanceof Blob) {
+      if (fromSampleRate !== -1) {
+        throw new Error(
+          `Can not specify "fromSampleRate" when reading from Blob`,
+        );
+      }
+      blob = audioData;
+      arrayBuffer = await blob.arrayBuffer();
+    } else if (audioData instanceof ArrayBuffer) {
+      if (fromSampleRate !== -1) {
+        throw new Error(
+          `Can not specify "fromSampleRate" when reading from ArrayBuffer`,
+        );
+      }
+      arrayBuffer = audioData;
+      blob = new Blob([arrayBuffer], { type: 'audio/wav' });
+    } else {
+      let float32Array;
+      let data;
+      if (audioData instanceof Int16Array) {
+        data = audioData;
+        float32Array = new Float32Array(audioData.length);
+        for (let i = 0; i < audioData.length; i++) {
+          float32Array[i] = audioData[i] / 0x8000;
+        }
+      } else if (audioData instanceof Float32Array) {
+        float32Array = audioData;
+      } else if (audioData instanceof Array) {
+        float32Array = new Float32Array(audioData);
+      } else {
+        throw new Error(
+          `"audioData" must be one of: Blob, Float32Arrray, Int16Array, ArrayBuffer, Array<number>`,
+        );
+      }
+      if (fromSampleRate === -1) {
+        throw new Error(
+          `Must specify "fromSampleRate" when reading from Float32Array, In16Array or Array`,
+        );
+      } else if (fromSampleRate < 3000) {
+        throw new Error(`Minimum "fromSampleRate" is 3000 (3kHz)`);
+      }
+      if (!data) {
+        data = WavPacker.floatTo16BitPCM(float32Array);
+      }
+      const audio = {
+        bitsPerSample: 16,
+        channels: [float32Array],
+        data,
+      };
+      const packer = new WavPacker();
+      const result = packer.pack(fromSampleRate, audio);
+      blob = result.blob;
+      arrayBuffer = await blob.arrayBuffer();
+    }
+    const audioBuffer = await context.decodeAudioData(arrayBuffer);
+    const values = audioBuffer.getChannelData(0);
+    const url = URL.createObjectURL(blob);
+    return {
+      blob,
+      url,
+      values,
+      audioBuffer,
+    };
+  }
+
+  /**
+   * Logs data in debug mode
+   * @param {...any} arguments
+   * @returns {true}
+   */
+  log() {
+    if (this.debug) {
+      this.log(...arguments);
+    }
+    return true;
+  }
+
+  /**
+   * Retrieves the current sampleRate for the recorder
+   * @returns {number}
+   */
+  getSampleRate() {
+    return this.sampleRate;
+  }
+
+  /**
+   * Retrieves the current status of the recording
+   * @returns {"ended"|"paused"|"recording"}
+   */
+  getStatus() {
+    if (!this.processor) {
+      return 'ended';
+    } else if (!this.recording) {
+      return 'paused';
+    } else {
+      return 'recording';
+    }
+  }
+
+  /**
+   * Sends an event to the AudioWorklet
+   * @private
+   * @param {string} name
+   * @param {{[key: string]: any}} data
+   * @param {AudioWorkletNode} [_processor]
+   * @returns {Promise<{[key: string]: any}>}
+   */
+  async _event(name, data = {}, _processor = null) {
+    _processor = _processor || this.processor;
+    if (!_processor) {
+      throw new Error('Can not send events without recording first');
+    }
+    const message = {
+      event: name,
+      id: this._lastEventId++,
+      data,
+    };
+    _processor.port.postMessage(message);
+    const t0 = new Date().valueOf();
+    while (!this.eventReceipts[message.id]) {
+      if (new Date().valueOf() - t0 > this.eventTimeout) {
+        throw new Error(`Timeout waiting for "${name}" event`);
+      }
+      await new Promise((res) => setTimeout(() => res(true), 1));
+    }
+    const payload = this.eventReceipts[message.id];
+    delete this.eventReceipts[message.id];
+    return payload;
+  }
+
+  /**
+   * Sets device change callback, remove if callback provided is `null`
+   * @param {(Array<MediaDeviceInfo & {default: boolean}>): void|null} callback
+   * @returns {true}
+   */
+  listenForDeviceChange(callback) {
+    if (callback === null && this._deviceChangeCallback) {
+      navigator.mediaDevices.removeEventListener(
+        'devicechange',
+        this._deviceChangeCallback,
+      );
+      this._deviceChangeCallback = null;
+    } else if (callback !== null) {
+      // Basically a debounce; we only want this called once when devices change
+      // And we only want the most recent callback() to be executed
+      // if a few are operating at the same time
+      let lastId = 0;
+      let lastDevices = [];
+      const serializeDevices = (devices) =>
+        devices
+          .map((d) => d.deviceId)
+          .sort()
+          .join(',');
+      const cb = async () => {
+        let id = ++lastId;
+        const devices = await this.listDevices();
+        if (id === lastId) {
+          if (serializeDevices(lastDevices) !== serializeDevices(devices)) {
+            lastDevices = devices;
+            callback(devices.slice());
+          }
+        }
+      };
+      navigator.mediaDevices.addEventListener('devicechange', cb);
+      cb();
+      this._deviceChangeCallback = cb;
+    }
+    return true;
+  }
+
+  /**
+   * Manually request permission to use the microphone
+   * @returns {Promise<true>}
+   */
+  async requestPermission() {
+    const permissionStatus = await navigator.permissions.query({
+      name: 'microphone',
+    });
+    if (permissionStatus.state === 'denied') {
+      window.alert('You must grant microphone access to use this feature.');
+    } else if (permissionStatus.state === 'prompt') {
+      try {
+        const stream = await navigator.mediaDevices.getUserMedia({
+          audio: true,
+        });
+        const tracks = stream.getTracks();
+        tracks.forEach((track) => track.stop());
+      } catch (e) {
+        window.alert('You must grant microphone access to use this feature.');
+      }
+    }
+    return true;
+  }
+
+  /**
+   * List all eligible devices for recording, will request permission to use microphone
+   * @returns {Promise<Array<MediaDeviceInfo & {default: boolean}>>}
+   */
+  async listDevices() {
+    if (
+      !navigator.mediaDevices ||
+      !('enumerateDevices' in navigator.mediaDevices)
+    ) {
+      throw new Error('Could not request user devices');
+    }
+    await this.requestPermission();
+    const devices = await navigator.mediaDevices.enumerateDevices();
+    const audioDevices = devices.filter(
+      (device) => device.kind === 'audioinput',
+    );
+    const defaultDeviceIndex = audioDevices.findIndex(
+      (device) => device.deviceId === 'default',
+    );
+    const deviceList = [];
+    if (defaultDeviceIndex !== -1) {
+      let defaultDevice = audioDevices.splice(defaultDeviceIndex, 1)[0];
+      let existingIndex = audioDevices.findIndex(
+        (device) => device.groupId === defaultDevice.groupId,
+      );
+      if (existingIndex !== -1) {
+        defaultDevice = audioDevices.splice(existingIndex, 1)[0];
+      }
+      defaultDevice.default = true;
+      deviceList.push(defaultDevice);
+    }
+    return deviceList.concat(audioDevices);
+  }
+
+  /**
+   * Begins a recording session and requests microphone permissions if not already granted
+   * Microphone recording indicator will appear on browser tab but status will be "paused"
+   * @param {string} [deviceId] if no device provided, default device will be used
+   * @returns {Promise<true>}
+   */
+  async begin(deviceId) {
+    if (this.processor) {
+      throw new Error(
+        `Already connected: please call .end() to start a new session`,
+      );
+    }
+
+    if (
+      !navigator.mediaDevices ||
+      !('getUserMedia' in navigator.mediaDevices)
+    ) {
+      throw new Error('Could not request user media');
+    }
+    try {
+      const config = { audio: true };
+      if (deviceId) {
+        config.audio = { deviceId: { exact: deviceId } };
+      }
+      this.stream = await navigator.mediaDevices.getUserMedia(config);
+    } catch (err) {
+      throw new Error('Could not start media stream');
+    }
+
+    const context = new AudioContext({ sampleRate: this.sampleRate });
+    const source = context.createMediaStreamSource(this.stream);
+    // Load and execute the module script.
+    try {
+      await context.audioWorklet.addModule(this.scriptSrc);
+    } catch (e) {
+      console.error(e);
+      throw new Error(`Could not add audioWorklet module: ${this.scriptSrc}`);
+    }
+    const processor = new AudioWorkletNode(context, 'audio_processor');
+    processor.port.onmessage = (e) => {
+      const { event, id, data } = e.data;
+      if (event === 'receipt') {
+        this.eventReceipts[id] = data;
+      } else if (event === 'chunk') {
+        if (this._chunkProcessorSize) {
+          const buffer = this._chunkProcessorBuffer;
+          this._chunkProcessorBuffer = {
+            raw: WavPacker.mergeBuffers(buffer.raw, data.raw),
+            mono: WavPacker.mergeBuffers(buffer.mono, data.mono),
+          };
+          if (
+            this._chunkProcessorBuffer.mono.byteLength >=
+            this._chunkProcessorSize
+          ) {
+            this._chunkProcessor(this._chunkProcessorBuffer);
+            this._chunkProcessorBuffer = {
+              raw: new ArrayBuffer(0),
+              mono: new ArrayBuffer(0),
+            };
+          }
+        } else {
+          this._chunkProcessor(data);
+        }
+      }
+    };
+
+    const node = source.connect(processor);
+    const analyser = context.createAnalyser();
+    analyser.fftSize = 8192;
+    analyser.smoothingTimeConstant = 0.1;
+    node.connect(analyser);
+    if (this.outputToSpeakers) {
+      // eslint-disable-next-line no-console
+      console.warn(
+        'Warning: Output to speakers may affect sound quality,\n' +
+          'especially due to system audio feedback preventative measures.\n' +
+          'use only for debugging',
+      );
+      analyser.connect(context.destination);
+    }
+
+    this.source = source;
+    this.node = node;
+    this.analyser = analyser;
+    this.processor = processor;
+    return true;
+  }
+
+  /**
+   * Gets the current frequency domain data from the recording track
+   * @param {"frequency"|"music"|"voice"} [analysisType]
+   * @param {number} [minDecibels] default -100
+   * @param {number} [maxDecibels] default -30
+   * @returns {import('./analysis/audio_analysis.js').AudioAnalysisOutputType}
+   */
+  getFrequencies(
+    analysisType = 'frequency',
+    minDecibels = -100,
+    maxDecibels = -30,
+  ) {
+    if (!this.processor) {
+      throw new Error('Session ended: please call .begin() first');
+    }
+    return AudioAnalysis.getFrequencies(
+      this.analyser,
+      this.sampleRate,
+      null,
+      analysisType,
+      minDecibels,
+      maxDecibels,
+    );
+  }
+
+  /**
+   * Pauses the recording
+   * Keeps microphone stream open but halts storage of audio
+   * @returns {Promise<true>}
+   */
+  async pause() {
+    if (!this.processor) {
+      throw new Error('Session ended: please call .begin() first');
+    } else if (!this.recording) {
+      throw new Error('Already paused: please call .record() first');
+    }
+    if (this._chunkProcessorBuffer.raw.byteLength) {
+      this._chunkProcessor(this._chunkProcessorBuffer);
+    }
+    this.log('Pausing ...');
+    await this._event('stop');
+    this.recording = false;
+    return true;
+  }
+
+  /**
+   * Start recording stream and storing to memory from the connected audio source
+   * @param {(data: { mono: Int16Array; raw: Int16Array }) => any} [chunkProcessor]
+   * @param {number} [chunkSize] chunkProcessor will not be triggered until this size threshold met in mono audio
+   * @returns {Promise<true>}
+   */
+  async record(chunkProcessor = () => {}, chunkSize = 8192) {
+    if (!this.processor) {
+      throw new Error('Session ended: please call .begin() first');
+    } else if (this.recording) {
+      throw new Error('Already recording: please call .pause() first');
+    } else if (typeof chunkProcessor !== 'function') {
+      throw new Error(`chunkProcessor must be a function`);
+    }
+    this._chunkProcessor = chunkProcessor;
+    this._chunkProcessorSize = chunkSize;
+    this._chunkProcessorBuffer = {
+      raw: new ArrayBuffer(0),
+      mono: new ArrayBuffer(0),
+    };
+    this.log('Recording ...');
+    await this._event('start');
+    this.recording = true;
+    return true;
+  }
+
+  /**
+   * Clears the audio buffer, empties stored recording
+   * @returns {Promise<true>}
+   */
+  async clear() {
+    if (!this.processor) {
+      throw new Error('Session ended: please call .begin() first');
+    }
+    await this._event('clear');
+    return true;
+  }
+
+  /**
+   * Reads the current audio stream data
+   * @returns {Promise<{meanValues: Float32Array, channels: Array<Float32Array>}>}
+   */
+  async read() {
+    if (!this.processor) {
+      throw new Error('Session ended: please call .begin() first');
+    }
+    this.log('Reading ...');
+    const result = await this._event('read');
+    return result;
+  }
+
+  /**
+   * Saves the current audio stream to a file
+   * @param {boolean} [force] Force saving while still recording
+   * @returns {Promise<import('./wav_packer.js').WavPackerAudioType>}
+   */
+  async save(force = false) {
+    if (!this.processor) {
+      throw new Error('Session ended: please call .begin() first');
+    }
+    if (!force && this.recording) {
+      throw new Error(
+        'Currently recording: please call .pause() first, or call .save(true) to force',
+      );
+    }
+    this.log('Exporting ...');
+    const exportData = await this._event('export');
+    const packer = new WavPacker();
+    const result = packer.pack(this.sampleRate, exportData.audio);
+    return result;
+  }
+
+  /**
+   * Ends the current recording session and saves the result
+   * @returns {Promise<import('./wav_packer.js').WavPackerAudioType>}
+   */
+  async end() {
+    if (!this.processor) {
+      throw new Error('Session ended: please call .begin() first');
+    }
+
+    const _processor = this.processor;
+
+    this.log('Stopping ...');
+    await this._event('stop');
+    this.recording = false;
+    const tracks = this.stream.getTracks();
+    tracks.forEach((track) => track.stop());
+
+    this.log('Exporting ...');
+    const exportData = await this._event('export', {}, _processor);
+
+    this.processor.disconnect();
+    this.source.disconnect();
+    this.node.disconnect();
+    this.analyser.disconnect();
+    this.stream = null;
+    this.processor = null;
+    this.source = null;
+    this.node = null;
+
+    const packer = new WavPacker();
+    const result = packer.pack(this.sampleRate, exportData.audio);
+    return result;
+  }
+
+  /**
+   * Performs a full cleanup of WavRecorder instance
+   * Stops actively listening via microphone and removes existing listeners
+   * @returns {Promise<true>}
+   */
+  async quit() {
+    this.listenForDeviceChange(null);
+    if (this.processor) {
+      await this.end();
+    }
+    return true;
+  }
+}
+
+globalThis.WavRecorder = WavRecorder;
diff --git a/web/src/lib/wavtools/lib/wav_stream_player.js b/web/src/lib/wavtools/lib/wav_stream_player.js
new file mode 100644
index 00000000..500eff6c
--- /dev/null
+++ b/web/src/lib/wavtools/lib/wav_stream_player.js
@@ -0,0 +1,160 @@
+import { StreamProcessorSrc } from './worklets/stream_processor.js';
+import { AudioAnalysis } from './analysis/audio_analysis.js';
+
+/**
+ * Plays audio streams received in raw PCM16 chunks from the browser
+ * @class
+ */
+export class WavStreamPlayer {
+  /**
+   * Creates a new WavStreamPlayer instance
+   * @param {{sampleRate?: number}} options
+   * @returns {WavStreamPlayer}
+   */
+  constructor({ sampleRate = 44100 } = {}) {
+    this.scriptSrc = StreamProcessorSrc;
+    this.sampleRate = sampleRate;
+    this.context = null;
+    this.stream = null;
+    this.analyser = null;
+    this.trackSampleOffsets = {};
+    this.interruptedTrackIds = {};
+  }
+
+  /**
+   * Connects the audio context and enables output to speakers
+   * @returns {Promise<true>}
+   */
+  async connect() {
+    this.context = new AudioContext({ sampleRate: this.sampleRate });
+    if (this.context.state === 'suspended') {
+      await this.context.resume();
+    }
+    try {
+      await this.context.audioWorklet.addModule(this.scriptSrc);
+    } catch (e) {
+      console.error(e);
+      throw new Error(`Could not add audioWorklet module: ${this.scriptSrc}`);
+    }
+    const analyser = this.context.createAnalyser();
+    analyser.fftSize = 8192;
+    analyser.smoothingTimeConstant = 0.1;
+    this.analyser = analyser;
+    return true;
+  }
+
+  /**
+   * Gets the current frequency domain data from the playing track
+   * @param {"frequency"|"music"|"voice"} [analysisType]
+   * @param {number} [minDecibels] default -100
+   * @param {number} [maxDecibels] default -30
+   * @returns {import('./analysis/audio_analysis.js').AudioAnalysisOutputType}
+   */
+  getFrequencies(
+    analysisType = 'frequency',
+    minDecibels = -100,
+    maxDecibels = -30
+  ) {
+    if (!this.analyser) {
+      throw new Error('Not connected, please call .connect() first');
+    }
+    return AudioAnalysis.getFrequencies(
+      this.analyser,
+      this.sampleRate,
+      null,
+      analysisType,
+      minDecibels,
+      maxDecibels
+    );
+  }
+
+  /**
+   * Starts audio streaming
+   * @private
+   * @returns {Promise<true>}
+   */
+  _start() {
+    const streamNode = new AudioWorkletNode(this.context, 'stream_processor');
+    streamNode.connect(this.context.destination);
+    streamNode.port.onmessage = (e) => {
+      const { event } = e.data;
+      if (event === 'stop') {
+        streamNode.disconnect();
+        this.stream = null;
+      } else if (event === 'offset') {
+        const { requestId, trackId, offset } = e.data;
+        const currentTime = offset / this.sampleRate;
+        this.trackSampleOffsets[requestId] = { trackId, offset, currentTime };
+      }
+    };
+    this.analyser.disconnect();
+    streamNode.connect(this.analyser);
+    this.stream = streamNode;
+    return true;
+  }
+
+  /**
+   * Adds 16BitPCM data to the currently playing audio stream
+   * You can add chunks beyond the current play point and they will be queued for play
+   * @param {ArrayBuffer|Int16Array} arrayBuffer
+   * @param {string} [trackId]
+   * @returns {Int16Array}
+   */
+  add16BitPCM(arrayBuffer, trackId = 'default') {
+    if (typeof trackId !== 'string') {
+      throw new Error(`trackId must be a string`);
+    } else if (this.interruptedTrackIds[trackId]) {
+      return;
+    }
+    if (!this.stream) {
+      this._start();
+    }
+    let buffer;
+    if (arrayBuffer instanceof Int16Array) {
+      buffer = arrayBuffer;
+    } else if (arrayBuffer instanceof ArrayBuffer) {
+      buffer = new Int16Array(arrayBuffer);
+    } else {
+      throw new Error(`argument must be Int16Array or ArrayBuffer`);
+    }
+    this.stream.port.postMessage({ event: 'write', buffer, trackId });
+    return buffer;
+  }
+
+  /**
+   * Gets the offset (sample count) of the currently playing stream
+   * @param {boolean} [interrupt]
+   * @returns {{trackId: string|null, offset: number, currentTime: number}}
+   */
+  async getTrackSampleOffset(interrupt = false) {
+    if (!this.stream) {
+      return null;
+    }
+    const requestId = crypto.randomUUID();
+    this.stream.port.postMessage({
+      event: interrupt ? 'interrupt' : 'offset',
+      requestId,
+    });
+    let trackSampleOffset;
+    while (!trackSampleOffset) {
+      trackSampleOffset = this.trackSampleOffsets[requestId];
+      await new Promise((r) => setTimeout(() => r(), 1));
+    }
+    const { trackId } = trackSampleOffset;
+    if (interrupt && trackId) {
+      this.interruptedTrackIds[trackId] = true;
+    }
+    return trackSampleOffset;
+  }
+
+  /**
+   * Strips the current stream and returns the sample offset of the audio
+   * @param {boolean} [interrupt]
+   * @returns {{trackId: string|null, offset: number, currentTime: number}}
+   */
+  async interrupt() {
+    return this.getTrackSampleOffset(true);
+  }
+}
+
+globalThis.WavStreamPlayer = WavStreamPlayer;
diff --git a/web/src/lib/wavtools/lib/worklets/audio_processor.js b/web/src/lib/wavtools/lib/worklets/audio_processor.js
new file mode 100644
index 00000000..61dd7ec9
--- /dev/null
+++ b/web/src/lib/wavtools/lib/worklets/audio_processor.js
@@ -0,0 +1,214 @@
+const AudioProcessorWorklet = `
+class AudioProcessor extends AudioWorkletProcessor {
+
+  constructor() {
+    super();
+    this.port.onmessage = this.receive.bind(this);
+    this.initialize();
+  }
+
+  initialize() {
+    this.foundAudio = false;
+    this.recording = false;
+    this.chunks = [];
+  }
+
+  /**
+   * Concatenates sampled chunks into channels
+   * Format is chunk[Left[], Right[]]
+   */
+  readChannelData(chunks, channel = -1, maxChannels = 9) {
+    let channelLimit;
+    if (channel !== -1) {
+      if (chunks[0] && chunks[0].length - 1 < channel) {
+        throw new Error(
+          \`Channel \${channel} out of range: max \${chunks[0].length}\`
+        );
+      }
+      channelLimit = channel + 1;
+    } else {
+      channel = 0;
+      channelLimit = Math.min(chunks[0] ? chunks[0].length : 1, maxChannels);
+    }
+    const channels = [];
+    for (let n = channel; n < channelLimit; n++) {
+      const length = chunks.reduce((sum, chunk) => {
+        return sum + chunk[n].length;
+      }, 0);
+      const buffers = chunks.map((chunk) => chunk[n]);
+      const result = new Float32Array(length);
+      let offset = 0;
+      for (let i = 0; i < buffers.length; i++) {
+        result.set(buffers[i], offset);
+        offset += buffers[i].length;
+      }
+      channels[n] = result;
+    }
+    return channels;
+  }
+
+  /**
+   * Combines parallel audio data into correct format,
+   * channels[Left[], Right[]] to float32Array[LRLRLRLR...]
+   */
+  formatAudioData(channels) {
+    if (channels.length === 1) {
+      // Simple case is only one channel
+      const float32Array = channels[0].slice();
+      const meanValues = channels[0].slice();
+      return { float32Array, meanValues };
+    } else {
+      const float32Array = new Float32Array(
+        channels[0].length * channels.length
+      );
+      const meanValues = new Float32Array(channels[0].length);
+      for (let i = 0; i < channels[0].length; i++) {
+        const offset = i * channels.length;
+        let meanValue = 0;
+        for (let n = 0; n < channels.length; n++) {
+          float32Array[offset + n] = channels[n][i];
+          meanValue += channels[n][i];
+        }
+        meanValues[i] = meanValue / channels.length;
+      }
+      return { float32Array, meanValues };
+    }
+  }
+
+  /**
+   * Converts 32-bit float data to 16-bit integers
+   */
+  floatTo16BitPCM(float32Array) {
+    const buffer = new ArrayBuffer(float32Array.length * 2);
+    const view = new DataView(buffer);
+    let offset = 0;
+    for (let i = 0; i < float32Array.length; i++, offset += 2) {
+      let s = Math.max(-1, Math.min(1, float32Array[i]));
+      view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
+    }
+    return buffer;
+  }
+
+  /**
+   * Retrieves the most recent amplitude values from the audio stream
+   * @param {number} channel
+   */
+  getValues(channel = -1) {
+    const channels = this.readChannelData(this.chunks, channel);
+    const { meanValues } = this.formatAudioData(channels);
+    return { meanValues, channels };
+  }
+
+  /**
+   * Exports chunks as an audio/wav file
+   */
+  export() {
+    const channels = this.readChannelData(this.chunks);
+    const { float32Array, meanValues } = this.formatAudioData(channels);
+    const audioData = this.floatTo16BitPCM(float32Array);
+    return {
+      meanValues: meanValues,
+      audio: {
+        bitsPerSample: 16,
+        channels: channels,
+        data: audioData,
+      },
+    };
+  }
+
+  receive(e) {
+    const { event, id } = e.data;
+    let receiptData = {};
+    switch (event) {
+      case 'start':
+        this.recording = true;
+        break;
+      case 'stop':
+        this.recording = false;
+        break;
+      case 'clear':
+        this.initialize();
+        break;
+      case 'export':
+        receiptData = this.export();
+        break;
+      case 'read':
+        receiptData = this.getValues();
+        break;
+      default:
+        break;
+    }
+    // Always send back receipt
+    this.port.postMessage({ event: 'receipt', id, data: receiptData });
+  }
+
+  sendChunk(chunk) {
+    const channels = this.readChannelData([chunk]);
+    const { float32Array, meanValues } = this.formatAudioData(channels);
+    const rawAudioData = this.floatTo16BitPCM(float32Array);
+    const monoAudioData = this.floatTo16BitPCM(meanValues);
+    this.port.postMessage({
+      event: 'chunk',
+      data: {
+        mono: monoAudioData,
+        raw: rawAudioData,
+      },
+    });
+  }
+
+  process(inputList, outputList, parameters) {
+    // Copy input to output (e.g. speakers)
+    // Note that this creates choppy sounds with Mac products
+    const sourceLimit = Math.min(inputList.length, outputList.length);
+    for (let inputNum = 0; inputNum < sourceLimit; inputNum++) {
+      const input = inputList[inputNum];
+      const output = outputList[inputNum];
+      const channelCount = Math.min(input.length, output.length);
+      for (let channelNum = 0; channelNum < channelCount; channelNum++) {
+        input[channelNum].forEach((sample, i) => {
+          output[channelNum][i] = sample;
+        });
+      }
+    }
+    const inputs = inputList[0];
+    // There's latency at the beginning of a stream before recording starts
+    // Make sure we actually receive audio data before we start storing chunks
+    let sliceIndex = 0;
+    if (!this.foundAudio) {
+      for (const channel of inputs) {
+        sliceIndex = 0; // reset for each channel
+        if (this.foundAudio) {
+          break;
+        }
+        if (channel) {
+          for (const value of channel) {
+            if (value !== 0) {
+              // find only one non-zero entry in any channel
+              this.foundAudio = true;
+              break;
+            } else {
+              sliceIndex++;
+            }
+          }
+        }
+      }
+    }
+    if (inputs && inputs[0] && this.foundAudio && this.recording) {
+      // We need to copy the TypedArray, because the \`process\`
+      // internals will reuse the same buffer to hold each input
+      const chunk = inputs.map((input) => input.slice(sliceIndex));
+      this.chunks.push(chunk);
+      this.sendChunk(chunk);
+    }
+    return true;
+  }
+}
+
+registerProcessor('audio_processor', AudioProcessor);
+`;
+
+const script = new Blob([AudioProcessorWorklet], {
+  type: 'application/javascript',
+});
+const src = URL.createObjectURL(script);
+export const AudioProcessorSrc = src;
diff --git a/web/src/lib/wavtools/lib/worklets/stream_processor.js b/web/src/lib/wavtools/lib/worklets/stream_processor.js
new file mode 100644
index 00000000..d3c794a8
--- /dev/null
+++ b/web/src/lib/wavtools/lib/worklets/stream_processor.js
@@ -0,0 +1,96 @@
+export const StreamProcessorWorklet = `
+class StreamProcessor extends AudioWorkletProcessor {
+  constructor() {
+    super();
+    this.hasStarted = false;
+    this.hasInterrupted = false;
+    this.outputBuffers = [];
+    this.bufferLength = 128;
+    this.write = { buffer: new Float32Array(this.bufferLength), trackId: null };
+    this.writeOffset = 0;
+    this.trackSampleOffsets = {};
+    this.port.onmessage = (event) => {
+      if (event.data) {
+        const payload = event.data;
+        if (payload.event === 'write') {
+          const int16Array = payload.buffer;
+          const float32Array = new Float32Array(int16Array.length);
+          for (let i = 0; i < int16Array.length; i++) {
+            float32Array[i] = int16Array[i] / 0x8000; // Convert Int16 to Float32
+          }
+          this.writeData(float32Array, payload.trackId);
+        } else if (
+          payload.event === 'offset' ||
+          payload.event === 'interrupt'
+        ) {
+          const requestId = payload.requestId;
+          const trackId = this.write.trackId;
+          const offset = this.trackSampleOffsets[trackId] || 0;
+          this.port.postMessage({
+            event: 'offset',
+            requestId,
+            trackId,
+            offset,
+          });
+          if (payload.event === 'interrupt') {
+            this.hasInterrupted = true;
+          }
+        } else {
+          throw new Error(\`Unhandled event "\${payload.event}"\`);
+        }
+      }
+    };
+  }
+
+  writeData(float32Array, trackId = null) {
+    let { buffer } = this.write;
+    let offset = this.writeOffset;
+    for (let i = 0; i < float32Array.length; i++) {
+      buffer[offset++] = float32Array[i];
+      if (offset >= buffer.length) {
+        this.outputBuffers.push(this.write);
+        this.write = { buffer: new Float32Array(this.bufferLength), trackId };
+        buffer = this.write.buffer;
+        offset = 0;
+      }
+    }
+    this.writeOffset = offset;
+    return true;
+  }
+
+  process(inputs, outputs, parameters) {
+    const output = outputs[0];
+    const outputChannelData = output[0];
+    const outputBuffers = this.outputBuffers;
+    if (this.hasInterrupted) {
+      this.port.postMessage({ event: 'stop' });
+      return false;
+    } else if (outputBuffers.length) {
+      this.hasStarted = true;
+      const { buffer, trackId } = outputBuffers.shift();
+      for (let i = 0; i < outputChannelData.length; i++) {
+        outputChannelData[i] = buffer[i] || 0;
+      }
+      if (trackId) {
+        this.trackSampleOffsets[trackId] =
+          this.trackSampleOffsets[trackId] || 0;
+        this.trackSampleOffsets[trackId] += buffer.length;
+      }
+      return true;
+    } else if (this.hasStarted) {
+      this.port.postMessage({ event: 'stop' });
+      return false;
+    } else {
+      return true;
+    }
+  }
+}
+
+registerProcessor('stream_processor', StreamProcessor);
+`;
+
+const script = new Blob([StreamProcessorWorklet], {
+  type: 'application/javascript',
+});
+const src = URL.createObjectURL(script);
+export const StreamProcessorSrc = src;
diff --git a/web/src/utils/conversation_config.js b/web/src/utils/conversation_config.js
new file mode 100644
index 00000000..bb9a0496
--- /dev/null
+++ b/web/src/utils/conversation_config.js
@@ -0,0 +1,16 @@
+export const instructions = `System settings:
+Tool use: enabled.
+
+Instructions:
+- You are an artificial intelligence agent responsible for helping test realtime voice capabilities
+- Please make sure to respond with a helpful voice via audio
+- Be kind, helpful, and curteous
+- It is okay to ask the user questions
+- Use tools and functions you have available liberally, it is part of the training apparatus
+- Be open to exploration and conversation
+- Remember: this is just for fun and testing!
+
+Personality:
+- Be upbeat and genuine
+- Try speaking quickly as if excited
+`;
diff --git a/web/src/utils/wav_player.js b/web/src/utils/wav_player.js
new file mode 100644
index 00000000..32dc6eff
--- /dev/null
+++ b/web/src/utils/wav_player.js
@@ -0,0 +1,27 @@
+// 播放 PCM16 语音流
+export const playPCM16 = (pcm16Array, sampleRate = 44100) => {
+    try {
+        // 创建 AudioContext
+        const audioContext = new (window.AudioContext || window.webkitAudioContext)();
+
+        // 将 Int16Array 转换为 Float32Array (Web Audio API 使用 Float32)
+        let float32Array = new Float32Array(pcm16Array.length);
+        for (let i = 0; i < pcm16Array.length; i++) {
+            float32Array[i] = pcm16Array[i] / 32768; // Int16 转换为 Float32
+        }
+
+        // 创建 AudioBuffer
+        const audioBuffer = audioContext.createBuffer(1, float32Array.length, sampleRate); // 单声道
+        audioBuffer.getChannelData(0).set(float32Array); // 设置音频数据
+
+        // 创建 AudioBufferSourceNode 并播放音频
+        const source = audioContext.createBufferSource();
+        source.buffer = audioBuffer;
+        source.connect(audioContext.destination); // 连接到扬声器
+        source.start(); // 播放
+        return source
+    } catch (e) {
+        console.warn(e)
+        return null
+    }
+}
diff --git a/web/src/utils/wav_renderer.js b/web/src/utils/wav_renderer.js
new file mode 100644
index 00000000..7a18e128
--- /dev/null
+++ b/web/src/utils/wav_renderer.js
@@ -0,0 +1,81 @@
+const dataMap = new WeakMap();
+
+const normalizeArray = (data, m, downsamplePeaks = false, memoize = false) => {
+  let cache, mKey, dKey;
+  if (memoize) {
+    mKey = m.toString();
+    dKey = downsamplePeaks.toString();
+    cache = dataMap.has(data)? dataMap.get(data) : {};
+    dataMap.set(data, cache);
+    cache[mKey] = cache[mKey] || {};
+    if (cache[mKey][dKey]) {
+      return cache[mKey][dKey];
+    }
+  }
+  const n = data.length;
+  const result = new Array(m);
+  if (m <= n) {
+    // Downsampling
+    result.fill(0);
+    const count = new Array(m).fill(0);
+    for (let i = 0; i < n; i++) {
+      const index = Math.floor(i * (m / n));
+      if (downsamplePeaks) {
+        // take highest result in the set
+        result[index] = Math.max(result[index], Math.abs(data[i]));
+      } else {
+        result[index] += Math.abs(data[i]);
+      }
+      count[index]++;
+    }
+    if (!downsamplePeaks) {
+      for (let i = 0; i < result.length; i++) {
+        result[i] = result[i] / count[i];
+      }
+    }
+  } else {
+    for (let i = 0; i < m; i++) {
+      const index = (i * (n - 1)) / (m - 1);
+      const low = Math.floor(index);
+      const high = Math.ceil(index);
+      const t = index - low;
+      if (high >= n) {
+        result[i] = data[n - 1];
+      } else {
+        result[i] = data[low] * (1 - t) + data[high] * t;
+      }
+    }
+  }
+  if (memoize) {
+    cache[mKey][dKey] = result;
+  }
+  return result;
+};
+
+export const WavRenderer = {
+  drawBars: (canvas, ctx, data, color, pointCount = 0, barWidth = 0, barSpacing = 0, center = false) => {
+    pointCount = Math.floor(
+        Math.min(
+            pointCount,
+            (canvas.width - barSpacing) / (Math.max(barWidth, 1) + barSpacing)
+        )
+    );
+    if (!pointCount) {
+      pointCount = Math.floor(
+          (canvas.width - barSpacing) / (Math.max(barWidth, 1) + barSpacing)
+      );
+    }
+    if (!barWidth) {
+      barWidth = (canvas.width - barSpacing) / pointCount - barSpacing;
+    }
+    const points = normalizeArray(data, pointCount, true);
+    for (let i = 0; i < pointCount; i++) {
+      const amplitude = Math.abs(points[i]);
+      const height = Math.max(1, amplitude * canvas.height);
+      const x = barSpacing + i * (barWidth + barSpacing);
+      const y = center? (canvas.height - height) / 2 : canvas.height - height;
+      ctx.fillStyle = color;
+      ctx.fillRect(x, y, barWidth, height);
+    }
+  },
+};
\ No newline at end of file
diff --git a/web/src/views/Test.vue b/web/src/views/Test.vue
index 14c82748..7d7bfedc 100644
--- a/web/src/views/Test.vue
+++ b/web/src/views/Test.vue
@@ -1,307 +1,471 @@
 <template>
-  <div class="video-call-container">
-    <div class="wave-container">
-      <div class="wave-animation">
-        <div v-for="i in 5" :key="i" class="wave-ellipse"></div>
+  <div data-component="ConsolePage">
+    <div class="content-top">
+      <div class="content-title">
+        <img src="/openai-logomark.svg" alt="OpenAI Logo" />
+        <span>realtime console</span>
       </div>
+
     </div>
-    <!-- 其余部分保持不变 -->
-    <div class="voice-indicators">
-      <div class="voice-indicator left">
-        <canvas ref="canvasClientRef" width="600" height="200"></canvas>
+    <div class="content-main">
+      <div class="content-logs">
+        <div class="content-block events">
+          <div class="visualization">
+            <div class="visualization-entry client">
+              <canvas ref="clientCanvasRef" />
+            </div>
+            <div class="visualization-entry server">
+              <canvas ref="serverCanvasRef" />
+            </div>
+          </div>
+          <div class="content-block-title">events</div>
+          <div class="content-block-body" ref="eventsScrollRef">
+            <template v-if="!realtimeEvents.length">
+              awaiting connection...
+            </template>
+            <template v-else>
+              <div v-for="(realtimeEvent, i) in realtimeEvents" :key="realtimeEvent.event.event_id" class="event">
+                <div class="event-timestamp">
+                  {{ formatTime(realtimeEvent.time) }}
+                </div>
+                <div class="event-details">
+                  <div
+                      class="event-summary"
+                      @click="toggleEventDetails(realtimeEvent.event.event_id)"
+                  >
+                    <div
+                        :class="[
+                        'event-source',
+                        realtimeEvent.event.type === 'error'
+                          ? 'error'
+                          : realtimeEvent.source,
+                      ]"
+                    >
+                      <component :is="realtimeEvent.source === 'client' ? ArrowUp : ArrowDown" />
+                      <span>
+                        {{ realtimeEvent.event.type === 'error'
+                          ? 'error!'
+                          : realtimeEvent.source }}
+                      </span>
+                    </div>
+                    <div class="event-type">
+                      {{ realtimeEvent.event.type }}
+                      {{ realtimeEvent.count ? `(${realtimeEvent.count})` : '' }}
+                    </div>
+                  </div>
+                  <div
+                      v-if="expandedEvents[realtimeEvent.event.event_id]"
+                      class="event-payload"
+                  >
+                    {{ JSON.stringify(realtimeEvent.event, null, 2) }}
+                  </div>
+                </div>
+              </div>
+            </template>
+          </div>
+        </div>
+        <div class="content-block conversation">
+          <div class="content-block-title">conversation</div>
+          <div class="content-block-body" data-conversation-content>
+            <template v-if="!items.length">
+              awaiting connection...
+            </template>
+            <template v-else>
+              <div
+                  v-for="(conversationItem, i) in items"
+                  :key="conversationItem.id"
+                  class="conversation-item"
+              >
+                <div :class="['speaker', conversationItem.role || '']">
+                  <div>
+                    {{
+                      (conversationItem.role || conversationItem.type).replaceAll(
+                          '_',
+                          ' '
+                      )
+                    }}
+                  </div>
+                  <div class="close" @click="deleteConversationItem(conversationItem.id)">
+                    <X />
+                  </div>
+                </div>
+                <div class="speaker-content">
+                  <!-- tool response -->
+                  <div v-if="conversationItem.type === 'function_call_output'">
+                    {{ conversationItem.formatted.output }}
+                  </div>
+                  <!-- tool call -->
+                  <div v-if="conversationItem.formatted.tool">
+                    {{ conversationItem.formatted.tool.name }}(
+                    {{ conversationItem.formatted.tool.arguments }})
+                  </div>
+                  <div
+                      v-if="
+                      !conversationItem.formatted.tool &&
+                      conversationItem.role === 'user'
+                    "
+                  >
+                    {{
+                      conversationItem.formatted.transcript ||
+                      (conversationItem.formatted.audio?.length
+                          ? '(awaiting transcript)'
+                          : conversationItem.formatted.text || '(item sent)')
+                    }}
+                  </div>
+                  <div
+                      v-if="
+                      !conversationItem.formatted.tool &&
+                      conversationItem.role === 'assistant'
+                    "
+                  >
+                    {{
+                      conversationItem.formatted.transcript ||
+                      conversationItem.formatted.text ||
+                      '(truncated)'
+                    }}
+                  </div>
+                  <audio
+                      v-if="conversationItem.formatted.file"
+                      :src="conversationItem.formatted.file.url"
+                      controls
+                  />
+                </div>
+              </div>
+            </template>
+          </div>
+        </div>
+        <div class="content-actions" style="position:absolute; top: 0; left: 0">
+          <el-button
+              :type="isConnected ? '' : 'primary'"
+              @click="connectConversation"
+          >
+            {{isConnected ? '断开连接' : '连接对话'}}
+          </el-button>
+
+          <el-button @mousedown="startRecording" @mouseup="stopRecording">开始讲话</el-button>
+        </div>
       </div>
-      <div class="voice-indicator right">
-        <canvas ref="canvasServerRef" width="600" height="200"></canvas>
-      </div>
-    </div>
-    <div class="call-controls">
-      <button class="call-button hangup" @click="hangUp">
-        <i class="iconfont icon-hung-up"></i>
-      </button>
-      <button class="call-button answer" @click="answer">
-        <i class="iconfont icon-call"></i>
-      </button>
+
     </div>
   </div>
 </template>
 
 <script setup>
-// Script 部分保持不变
-import {ref, onMounted, onUnmounted} from 'vue';
+import { ref, reactive, onMounted, onUnmounted, watch } from 'vue';
+import { RealtimeClient } from '@openai/realtime-api-beta';
+import { WavRecorder, WavStreamPlayer } from '@/lib/wavtools/index.js';
+import { instructions } from '@/utils/conversation_config.js';
+import { WavRenderer } from '@/utils/wav_renderer';
 
-const leftVoiceActive = ref(false);
-const rightVoiceActive = ref(false);
+// Constants
+const LOCAL_RELAY_SERVER_URL = process.env.REACT_APP_LOCAL_RELAY_SERVER_URL || '';
 
-const animateVoice = () => {
-  leftVoiceActive.value = Math.random() > 0.5;
-  rightVoiceActive.value = Math.random() > 0.5;
+// Reactive state
+const apiKey = ref(
+    LOCAL_RELAY_SERVER_URL
+        ? ''
+        : localStorage.getItem('tmp::voice_api_key') || prompt('OpenAI API Key') || ''
+);
+const wavRecorder = ref(new WavRecorder({ sampleRate: 24000 }));
+const wavStreamPlayer = ref(new WavStreamPlayer({ sampleRate: 24000 }));
+const client = ref(
+    new RealtimeClient({
+      url: "wss://api.geekai.pro/v1/realtime",
+      apiKey: "sk-Gc5cEzDzGQLIqxWA9d62089350F3454bB359C4A3Fa21B3E4",
+      dangerouslyAllowAPIKeyInBrowser: true,
+    })
+);
+
+const clientCanvasRef = ref(null);
+const serverCanvasRef = ref(null);
+const eventsScrollRef = ref(null);
+const startTime = ref(new Date().toISOString());
+
+const items = ref([]);
+const realtimeEvents = ref([]);
+const expandedEvents = reactive({});
+const isConnected = ref(false);
+const canPushToTalk = ref(true);
+const isRecording = ref(false);
+const memoryKv = ref({});
+const coords = ref({ lat: 37.775593, lng: -122.418137 });
+const marker = ref(null);
+
+// Methods
+const formatTime = (timestamp) => {
+  const t0 = new Date(startTime.value).valueOf();
+  const t1 = new Date(timestamp).valueOf();
+  const delta = t1 - t0;
+  const hs = Math.floor(delta / 10) % 100;
+  const s = Math.floor(delta / 1000) % 60;
+  const m = Math.floor(delta / 60_000) % 60;
+  const pad = (n) => {
+    let s = n + '';
+    while (s.length < 2) {
+      s = '0' + s;
+    }
+    return s;
+  };
+  return `${pad(m)}:${pad(s)}.${pad(hs)}`;
 };
 
-let voiceInterval;
-const canvasClientRef = ref(null);
-const canvasServerRef = ref(null);
+const connectConversation = async () => {
+  alert(123)
+  startTime.value = new Date().toISOString();
+  isConnected.value = true;
+  realtimeEvents.value = [];
+  items.value = client.value.conversation.getItems();
 
+  await wavRecorder.value.begin();
+  await wavStreamPlayer.value.connect();
+  await client.value.connect();
+  client.value.sendUserMessageContent([
+    {
+      type: 'input_text',
+      text: '你好，我是老阳!',
+    },
+  ]);
+
+  if (client.value.getTurnDetectionType() === 'server_vad') {
+    await wavRecorder.value.record((data) => client.value.appendInputAudio(data.mono));
+  }
+};
+
+const disconnectConversation = async () => {
+  isConnected.value = false;
+  realtimeEvents.value = [];
+  items.value = [];
+  memoryKv.value = {};
+  coords.value = { lat: 37.775593, lng: -122.418137 };
+  marker.value = null;
+
+  client.value.disconnect();
+  await wavRecorder.value.end();
+  await wavStreamPlayer.value.interrupt();
+};
+
+const deleteConversationItem = async (id) => {
+  client.value.deleteItem(id);
+};
+
+const startRecording = async () => {
+  isRecording.value = true;
+  const trackSampleOffset = await wavStreamPlayer.value.interrupt();
+  if (trackSampleOffset?.trackId) {
+    const { trackId, offset } = trackSampleOffset;
+    await client.value.cancelResponse(trackId, offset);
+  }
+  await wavRecorder.value.record((data) => client.value.appendInputAudio(data.mono));
+};
+
+const stopRecording = async () => {
+  isRecording.value = false;
+  await wavRecorder.value.pause();
+  client.value.createResponse();
+};
+
+const changeTurnEndType = async (value) => {
+  if (value === 'none' && wavRecorder.value.getStatus() === 'recording') {
+    await wavRecorder.value.pause();
+  }
+  client.value.updateSession({
+    turn_detection: value === 'none' ? null : { type: 'server_vad' },
+  });
+  if (value === 'server_vad' && client.value.isConnected()) {
+    await wavRecorder.value.record((data) => client.value.appendInputAudio(data.mono));
+  }
+  canPushToTalk.value = value === 'none';
+};
+
+const toggleEventDetails = (eventId) => {
+  if (expandedEvents[eventId]) {
+    delete expandedEvents[eventId];
+  } else {
+    expandedEvents[eventId] = true;
+  }
+};
+
+// Lifecycle hooks and watchers
 onMounted(() => {
-  voiceInterval = setInterval(animateVoice, 500);
+  if (apiKey.value !== '') {
+    localStorage.setItem('tmp::voice_api_key', apiKey.value);
+  }
 
+  // Set up render loops for the visualization canvas
+  let isLoaded = true;
+  const render = () => {
+    if (isLoaded) {
+      if (clientCanvasRef.value) {
+        const canvas = clientCanvasRef.value;
+        if (!canvas.width || !canvas.height) {
+          canvas.width = canvas.offsetWidth;
+          canvas.height = canvas.offsetHeight;
+        }
+        const ctx = canvas.getContext('2d');
+        if (ctx) {
+          ctx.clearRect(0, 0, canvas.width, canvas.height);
+          const result = wavRecorder.value.recording
+              ? wavRecorder.value.getFrequencies('voice')
+              : { values: new Float32Array([0]) };
+          WavRenderer.drawBars(canvas, ctx, result.values, '#0099ff', 10, 0, 8);
+        }
+      }
+      if (serverCanvasRef.value) {
+        const canvas = serverCanvasRef.value;
+        if (!canvas.width || !canvas.height) {
+          canvas.width = canvas.offsetWidth;
+          canvas.height = canvas.offsetHeight;
+        }
+        const ctx = canvas.getContext('2d');
+        if (ctx) {
+          ctx.clearRect(0, 0, canvas.width, canvas.height);
+          const result = wavStreamPlayer.value.analyser
+              ?  wavStreamPlayer.value.getFrequencies('voice')
+                  : { values: new Float32Array([0]) };
+          WavRenderer.drawBars(canvas, ctx, result.values, '#009900', 10, 0, 8);
+        }
+      }
+      requestAnimationFrame(render);
+    }
+  };
+  render();
 
+  // Set up client event listeners
+  client.value.on('realtime.event', (realtimeEvent) => {
+    realtimeEvents.value = realtimeEvents.value.slice();
+    const lastEvent = realtimeEvents.value[realtimeEvents.value.length - 1];
+    if (lastEvent?.event.type === realtimeEvent.event.type) {
+      lastEvent.count = (lastEvent.count || 0) + 1;
+      realtimeEvents.value.splice(-1, 1, lastEvent);
+    } else {
+      realtimeEvents.value.push(realtimeEvent);
+    }
+  });
 
+  client.value.on('error', (event) => console.error(event));
 
-  //setupAudioProcessing(canvasServerRef.value, '#2ecc71');
+  client.value.on('conversation.interrupted', async () => {
+    const trackSampleOffset = await wavStreamPlayer.value.interrupt();
+    if (trackSampleOffset?.trackId) {
+      const { trackId, offset } = trackSampleOffset;
+      await client.value.cancelResponse(trackId, offset);
+    }
+  });
 
+  client.value.on('conversation.updated', async ({ item, delta }) => {
+    items.value = client.value.conversation.getItems();
+    if (delta?.audio) {
+      wavStreamPlayer.value.add16BitPCM(delta.audio, item.id);
+    }
+    if (item.status === 'completed' && item.formatted.audio?.length) {
+      const wavFile = await WavRecorder.decode(
+          item.formatted.audio,
+          24000,
+          24000
+      );
+      item.formatted.file = wavFile;
+    }
+  });
+
+  // Set up client instructions and tools
+  client.value.updateSession({ instructions: instructions });
+  client.value.updateSession({ input_audio_transcription: { model: 'whisper-1' } });
+
+  client.value.addTool(
+      {
+        name: 'set_memory',
+        description: 'Saves important data about the user into memory.',
+        parameters: {
+          type: 'object',
+          properties: {
+            key: {
+              type: 'string',
+              description:
+                  'The key of the memory value. Always use lowercase and underscores, no other characters.',
+            },
+            value: {
+              type: 'string',
+              description: 'Value can be anything represented as a string',
+            },
+          },
+          required: ['key', 'value'],
+        },
+      },
+      async ({ key, value }) => {
+        memoryKv.value = { ...memoryKv.value, [key]: value };
+        return { ok: true };
+      }
+  );
+
+  client.value.addTool(
+      {
+        name: 'get_weather',
+        description:
+            'Retrieves the weather for a given lat, lng coordinate pair. Specify a label for the location.',
+        parameters: {
+          type: 'object',
+          properties: {
+            lat: {
+              type: 'number',
+              description: 'Latitude',
+            },
+            lng: {
+              type: 'number',
+              description: 'Longitude',
+            },
+            location: {
+              type: 'string',
+              description: 'Name of the location',
+            },
+          },
+          required: ['lat', 'lng', 'location'],
+        },
+      },
+      async ({ lat, lng, location }) => {
+        marker.value = { lat, lng, location };
+        coords.value = { lat, lng, location };
+        const result = await fetch(
+            `https://api.open-meteo.com/v1/forecast?latitude=${lat}&longitude=${lng}&current=temperature_2m,wind_speed_10m`
+        );
+        const json = await result.json();
+        const temperature = {
+          value: json.current.temperature_2m,
+          units: json.current_units.temperature_2m,
+        };
+        const wind_speed = {
+          value: json.current.wind_speed_10m,
+          units: json.current_units.wind_speed_10m,
+        };
+        marker.value = { lat, lng, location, temperature, wind_speed };
+        return json;
+      }
+  );
+
+  items.value = client.value.conversation.getItems();
 });
 
-const setupAudioProcessing = async (canvas, color) => {
-  try {
-    const audioContext = new (window.AudioContext || window.webkitAudioContext)();
-    const data = JSON.parse(localStorage.getItem("chat_data"))
-
-    // 将 Int16Array 转换为 Float32Array (Web Audio API 使用 Float32)
-    let float32Array = new Float32Array(data.length);
-    for (let i = 0; i < data.length; i++) {
-      float32Array[i] = data[i] / 32768; // Int16 转换为 Float32
-    }
-
-    // 创建 AudioBuffer
-    const audioBuffer = audioContext.createBuffer(1, float32Array.length, 24000); // 单声道
-    audioBuffer.getChannelData(0).set(float32Array); // 设置音频数据
-
-    // 创建 AudioBufferSourceNode 并播放音频
-    const source = audioContext.createBufferSource();
-    source.buffer = audioBuffer;
-    const analyser = audioContext.createAnalyser();
-    analyser.fftSize = 256;
-    const bufferLength = analyser.frequencyBinCount;
-    // 连接到输入源（模拟麦克风）
-    source.connect(analyser);
-    // 同时连接到扬声器播放语音
-    source.connect(audioContext.destination);
-    source.start(); // 播放
-    const dataArray = new Uint8Array(bufferLength);
-    const ctx = canvas.getContext('2d')
-
-    const draw = () => {
-      analyser.getByteFrequencyData(dataArray);
-
-      // 检查音量是否安静
-      const maxVolume = Math.max(...dataArray);
-      if (maxVolume < 100) {
-        // 如果音量很小，则停止绘制
-        ctx.clearRect(0, 0, canvas.width, canvas.height);
-        requestAnimationFrame(draw);
-        return;
-      }
-
-      ctx.clearRect(0, 0, canvas.width, canvas.height);
-
-      const barWidth = (canvas.width / bufferLength) * 2.5;
-      let x = 0;
-
-      for (let i = 0; i < bufferLength; i++) {
-        const barHeight = dataArray[i] / 2;
-
-        ctx.fillStyle = color; // 淡蓝色
-        ctx.fillRect(x, canvas.height - barHeight, barWidth, barHeight);
-
-        x += barWidth + 2;
-      }
-      requestAnimationFrame(draw);
-    }
-
-    draw();
-  } catch (err) {
-    console.error('获取麦克风权限失败:', err);
-  }
-}
-
-
-const speaker = ref(null)
-// 假设 PCM16 数据已经存储在一个 Int16Array 中
-function playPCM16(pcm16Array, sampleRate = 44100) {
-  // 创建 AudioContext
-  const audioContext = new (window.AudioContext || window.webkitAudioContext)();
-
-  // 将 Int16Array 转换为 Float32Array (Web Audio API 使用 Float32)
-  let float32Array = new Float32Array(pcm16Array.length);
-  for (let i = 0; i < pcm16Array.length; i++) {
-    float32Array[i] = pcm16Array[i] / 32768; // Int16 转换为 Float32
-  }
-
-  // 创建 AudioBuffer
-  const audioBuffer = audioContext.createBuffer(1, float32Array.length, sampleRate); // 单声道
-  audioBuffer.getChannelData(0).set(float32Array); // 设置音频数据
-
-  // 创建 AudioBufferSourceNode 并播放音频
-  const source = audioContext.createBufferSource();
-  source.buffer = audioBuffer;
-  source.connect(audioContext.destination); // 连接到扬声器
-  source.start(); // 播放
-  speaker.value = source
-
-}
-
 onUnmounted(() => {
-  clearInterval(voiceInterval);
+  client.value.reset();
 });
 
-const hangUp = () => {
-  console.log('Call hung up');
-};
-
-const answer = () => {
-  console.log('Call answered');
-  setupAudioProcessing(canvasServerRef.value, '#2ecc71');
-};
-
+// Watchers
+watch(realtimeEvents, () => {
+  if (eventsScrollRef.value) {
+    const eventsEl = eventsScrollRef.value;
+    eventsEl.scrollTop = eventsEl.scrollHeight;
+  }
+});
 
+watch(items, () => {
+  const conversationEls = document.querySelectorAll('[data-conversation-content]');
+  conversationEls.forEach((el) => {
+    el.scrollTop = el.scrollHeight;
+  });
+});
 </script>
 
-<style scoped lang="stylus">
-.video-call-container {
-  background: linear-gradient(to right, #2c3e50, #4a5568, #6b46c1);
-  height: 100vh;
-  display: flex;
-  flex-direction: column;
-  justify-content: space-between;
-  align-items: center;
-  padding: 0;
-
-  .wave-container {
-    padding 2rem
-    .wave-animation {
-      display: flex;
-      justify-content: center;
-      align-items: center;
-      gap: 10px;
-    }
-  }
-
-
-  .wave-ellipse {
-    width: 40px;
-    height: 40px;
-    background-color: white;
-    border-radius: 20px;
-    animation: wave 0.8s infinite ease-in-out;
-  }
-
-  .wave-ellipse:nth-child(odd) {
-    height: 60px;
-  }
-
-  .wave-ellipse:nth-child(even) {
-    height: 80px;
-  }
-
-  @keyframes wave {
-    0%, 100% {
-      transform: scaleY(0.8);
-    }
-    50% {
-      transform: scaleY(1.2);
-    }
-  }
-
-  .wave-ellipse:nth-child(2) {
-    animation-delay: 0.1s;
-  }
-
-  .wave-ellipse:nth-child(3) {
-    animation-delay: 0.2s;
-  }
-
-  .wave-ellipse:nth-child(4) {
-    animation-delay: 0.3s;
-  }
-
-  .wave-ellipse:nth-child(5) {
-    animation-delay: 0.4s;
-  }
-
-  /* 其余样式保持不变 */
-  .voice-indicators {
-    display: flex;
-    justify-content: space-between;
-    width: 100%;
-  }
-
-  .voice-indicator {
-    display: flex;
-    align-items: flex-end;
-  }
-
-  .bar {
-    width: 10px;
-    height: 20px;
-    background-color: #3498db;
-    margin: 0 2px;
-    transition: height 0.2s ease;
-  }
-
-  .voice-indicator.left .bar:nth-child(1) {
-    height: 15px;
-  }
-
-  .voice-indicator.left .bar:nth-child(2) {
-    height: 25px;
-  }
-
-  .voice-indicator.left .bar:nth-child(3) {
-    height: 20px;
-  }
-
-  .voice-indicator.right .bar:nth-child(1) {
-    height: 20px;
-  }
-
-  .voice-indicator.right .bar:nth-child(2) {
-    height: 10px;
-  }
-
-  .voice-indicator.right .bar:nth-child(3) {
-    height: 30px;
-  }
-
-  .call-controls {
-    display: flex;
-    justify-content: center;
-    gap: 2rem;
-    padding 2rem
-
-    .call-button {
-      width: 60px;
-      height: 60px;
-      border-radius: 50%;
-      border: none;
-      display: flex;
-      justify-content: center;
-      align-items: center;
-      font-size: 24px;
-      color: white;
-      cursor: pointer;
-
-      .iconfont {
-        font-size 24px
-      }
-    }
-    .hangup {
-      background-color: #e74c3c;
-    }
-
-    .answer {
-      background-color: #2ecc71;
-    }
-
-    .icon {
-      font-size: 28px;
-    }
-  }
-
-}
-
-canvas {
-  background-color: transparent;
-}
-
-
+<style scoped>
+/* You can add your component-specific styles here */
+/* If you're using SCSS, you might want to import your existing SCSS file */
+/* @import './ConsolePage.scss'; */
 </style>
\ No newline at end of file
diff --git a/web/src/views/Test2.vue b/web/src/views/Test2.vue
index ae198da9..447d0ca2 100644
--- a/web/src/views/Test2.vue
+++ b/web/src/views/Test2.vue
@@ -81,7 +81,7 @@ onMounted(() => {
 
             x += barWidth + 2;
           }
-        requestAnimationFrame(draw);
+        //requestAnimationFrame(draw);
       }
 
       draw();
diff --git a/web/src/views/Test3.vue b/web/src/views/Test3.vue
new file mode 100644
index 00000000..da2492a2
--- /dev/null
+++ b/web/src/views/Test3.vue
@@ -0,0 +1,184 @@
+<template>
+  <div class="audio-chat-page">
+    <el-button style="margin: 20px" type="primary" size="large" @click="connect">开始语音对话</el-button>
+
+    <el-dialog v-model="showDialog" title="语音通话" :fullscreen="true">
+      <el-container>
+        <calling v-if="!connected" :height="dialogHeight+'px'" />
+        <conversation v-else :height="dialogHeight+'px'" @hang-up="hangUp" />
+      </el-container>
+    </el-dialog>
+  </div>
+</template>
+
+<script setup>
+import {ref} from 'vue';
+import { RealtimeClient } from '@openai/realtime-api-beta';
+import Calling from "@/components/Calling.vue";
+import Conversation from "@/components/Conversation .vue";
+import {playPCM16} from "@/utils/wav_player";
+import {showMessageError} from "@/utils/dialog";
+
+const showDialog = ref(false);
+const connected = ref(false);
+const dialogHeight = ref(window.innerHeight - 75);
+
+const recognition = ref(null)
+if (!('webkitSpeechRecognition' in window)) {
+  alert("你的浏览器不支持语音识别，请使用最新版本的 Chrome 浏览器。");
+} else {
+  recognition.value = new webkitSpeechRecognition();
+  recognition.value.lang = 'zh-CN'; // 设置语言为简体中文
+  recognition.value.continuous = false; // 设置为单句识别
+  recognition.value.interimResults = false; // 不需要中间结果
+
+  recognition.value.onresult = function(event) {
+    const transcript = event.results[0][0].transcript;
+    try {
+      client.cancelResponse(chatId.value)
+      speaker.value.stop()
+    } catch (e) {
+      console.warn(e)
+    }
+    console.log(`你说的是: ${transcript}`)
+    console.log(client.isConnected())
+   if (client.isConnected()){
+     client.sendUserMessageContent([{ type: 'input_text', text: transcript }]);
+   }
+    //recognition.value.start()
+  };
+
+  recognition.value.onerror = function(event) {
+    showMessageError("识别失败:", event.error)
+  };
+  recognition.value.onend = function() {
+    console.log('语音识别结束，重新开始');
+    recognition.value.start(); // 在结束时重新开始
+  };
+
+  //recognition.value.start()
+}
+
+const client = new RealtimeClient({
+  url: "wss://api.geekai.pro/v1/realtime",
+  apiKey: "sk-Gc5cEzDzGQLIqxWA9d62089350F3454bB359C4A3Fa21B3E4",
+  dangerouslyAllowAPIKeyInBrowser: true,
+});
+
+// Can set parameters ahead of connecting, either separately or all at once
+client.updateSession({ instructions: 'You are a great, upbeat friend.' });
+client.updateSession({ voice: 'nova' });
+client.updateSession({
+  turn_detection: 'disabled', // or 'server_vad'
+  input_audio_transcription: { model: 'whisper-1' },
+});
+
+const chatId = ref("")
+const audioChunks = ref([])
+// Set up event handling
+client.on('conversation.updated', ({ item, delta }) => {
+  chatId.value = item.id
+  //console.info('conversation.updated', item, delta)
+  switch (item.type) {
+    case 'message':
+      // system, user, or assistant message (item.role)
+      localStorage.setItem("chat_data", JSON.stringify(Array.from(item.formatted.audio)))
+      console.log(item)
+      break;
+    case 'function_call':
+      // always a function call from the model
+      break;
+    case 'function_call_output':
+      // always a response from the user / application
+      break;
+  }
+  if (delta) {
+    // console.info(delta.audio)
+    if (delta.audio && delta.audio.length > 1) {
+      audioChunks.value.push(delta.audio)
+    }
+    if (audioChunks.value.length === 1) {
+      playAudio(0)
+    }
+
+    //localStorage.setItem("chat_data", JSON.stringify(Array.from(delta.audio)))
+    // Only one of the following will be populated for any given event
+    // delta.audio = Int16Array, audio added
+    // delta.transcript = string, transcript added
+    // delta.arguments = string, function arguments added
+  }
+});
+
+const speaker = ref(null)
+const playAudio = (index) => {
+  if (index === 0 && speaker.value) {
+    speaker.value.stop()
+  }
+  const data = audioChunks.value[index]
+  console.log(data)
+  if (index === audioChunks.value.length-1) {
+    audioChunks.value = []
+  }
+  speaker.value = playPCM16(data, 24000);
+  if (speaker.value !== null) {
+    speaker.value.onended = () => {
+      playAudio(index + 1)
+    }
+  }
+
+}
+
+client.on('conversation.interrupted', async () => {
+  console.log('聊天中断')
+});
+
+client.on('conversation.item.appended', ({ item }) => {
+  if (item.role === 'assistant') {
+    // playPCM16(item.formatted.audio, 24000);
+    // console.log(item)
+  }
+});
+
+const connect = () => {
+  showDialog.value = true
+  client.connect().then(res => {
+    if (res) {
+      console.log("连接成功!")
+      connected.value = true
+      // const data = JSON.parse(localStorage.getItem("chat_data"))
+      // playPCM16(data, 24000)
+      client.sendUserMessageContent([{ type: 'input_text', text: `你好，我是老阳。` }]);
+    }
+  }).catch(e => {
+    console.log(e)
+  })
+}
+
+const hangUp = () => {
+ try {
+   client.cancelResponse(chatId.value)
+   speaker.value.stop()
+ } catch (e) {
+   console.warn(e)
+  }
+  showDialog.value = false
+  connected.value = false
+}
+
+</script>
+
+<style scoped lang="stylus">
+.audio-chat-page {
+  display flex
+  flex-flow column
+  justify-content center
+  align-items center
+
+}
+
+canvas {
+  background-color: transparent;
+}
+
+
+</style>
\ No newline at end of file