integrated openai realtime console

web/src/utils/conversation_config.js

@@ -0,0 +1,16 @@
+export const instructions = `System settings:
+Tool use: enabled.
+
+Instructions:
+- You are an artificial intelligence agent responsible for helping test realtime voice capabilities
+- Please make sure to respond with a helpful voice via audio
+- Be kind, helpful, and curteous
+- It is okay to ask the user questions
+- Use tools and functions you have available liberally, it is part of the training apparatus
+- Be open to exploration and conversation
+- Remember: this is just for fun and testing!
+
+Personality:
+- Be upbeat and genuine
+- Try speaking quickly as if excited
+`;

web/src/utils/wav_player.js

@@ -0,0 +1,27 @@
+// 播放 PCM16 语音流
+export const playPCM16 = (pcm16Array, sampleRate = 44100) => {
+    try {
+        // 创建 AudioContext
+        const audioContext = new (window.AudioContext || window.webkitAudioContext)();
+
+        // 将 Int16Array 转换为 Float32Array (Web Audio API 使用 Float32)
+        let float32Array = new Float32Array(pcm16Array.length);
+        for (let i = 0; i < pcm16Array.length; i++) {
+            float32Array[i] = pcm16Array[i] / 32768; // Int16 转换为 Float32
+        }
+
+        // 创建 AudioBuffer
+        const audioBuffer = audioContext.createBuffer(1, float32Array.length, sampleRate); // 单声道
+        audioBuffer.getChannelData(0).set(float32Array); // 设置音频数据
+
+        // 创建 AudioBufferSourceNode 并播放音频
+        const source = audioContext.createBufferSource();
+        source.buffer = audioBuffer;
+        source.connect(audioContext.destination); // 连接到扬声器
+        source.start(); // 播放
+        return source
+    } catch (e) {
+        console.warn(e)
+        return null
+    }
+}

web/src/utils/wav_renderer.js

@@ -0,0 +1,81 @@
+const dataMap = new WeakMap();
+
+const normalizeArray = (data, m, downsamplePeaks = false, memoize = false) => {
+  let cache, mKey, dKey;
+  if (memoize) {
+    mKey = m.toString();
+    dKey = downsamplePeaks.toString();
+    cache = dataMap.has(data)? dataMap.get(data) : {};
+    dataMap.set(data, cache);
+    cache[mKey] = cache[mKey] || {};
+    if (cache[mKey][dKey]) {
+      return cache[mKey][dKey];
+    }
+  }
+  const n = data.length;
+  const result = new Array(m);
+  if (m <= n) {
+    // Downsampling
+    result.fill(0);
+    const count = new Array(m).fill(0);
+    for (let i = 0; i < n; i++) {
+      const index = Math.floor(i * (m / n));
+      if (downsamplePeaks) {
+        // take highest result in the set
+        result[index] = Math.max(result[index], Math.abs(data[i]));
+      } else {
+        result[index] += Math.abs(data[i]);
+      }
+      count[index]++;
+    }
+    if (!downsamplePeaks) {
+      for (let i = 0; i < result.length; i++) {
+        result[i] = result[i] / count[i];
+      }
+    }
+  } else {
+    for (let i = 0; i < m; i++) {
+      const index = (i * (n - 1)) / (m - 1);
+      const low = Math.floor(index);
+      const high = Math.ceil(index);
+      const t = index - low;
+      if (high >= n) {
+        result[i] = data[n - 1];
+      } else {
+        result[i] = data[low] * (1 - t) + data[high] * t;
+      }
+    }
+  }
+  if (memoize) {
+    cache[mKey][dKey] = result;
+  }
+  return result;
+};
+
+export const WavRenderer = {
+  drawBars: (canvas, ctx, data, color, pointCount = 0, barWidth = 0, barSpacing = 0, center = false) => {
+    pointCount = Math.floor(
+        Math.min(
+            pointCount,
+            (canvas.width - barSpacing) / (Math.max(barWidth, 1) + barSpacing)
+        )
+    );
+    if (!pointCount) {
+      pointCount = Math.floor(
+          (canvas.width - barSpacing) / (Math.max(barWidth, 1) + barSpacing)
+      );
+    }
+    if (!barWidth) {
+      barWidth = (canvas.width - barSpacing) / pointCount - barSpacing;
+    }
+    const points = normalizeArray(data, pointCount, true);
+    for (let i = 0; i < pointCount; i++) {
+      const amplitude = Math.abs(points[i]);
+      const height = Math.max(1, amplitude * canvas.height);
+      const x = barSpacing + i * (barWidth + barSpacing);
+      const y = center? (canvas.height - height) / 2 : canvas.height - height;
+      ctx.fillStyle = color;
+      ctx.fillRect(x, y, barWidth, height);
+    }
+  },
+};