feat: optimize animation

app/components/chat.module.scss

@@ -815,3 +815,34 @@
     }
   }
 }
+
+.chat-message-checkmark {
+  display: inline-block;
+  margin-right: 5px;
+  height: 12px;
+  width: 12px;
+  color: #13a10e;
+  fill: #13a10e;
+  user-select: none;
+  backface-visibility: hidden;
+  transform: translateZ(0px);
+}
+
+.chat-message-tools-status {
+  display: flex;
+  justify-content: center;
+  align-items: center;
+  font-size: 12px;
+  margin-top: 5px;
+  line-height: 1.5;
+}
+
+.chat-message-tools-name {
+  color: #aaa;
+}
+
+.chat-message-tools-details {
+  margin-left: 5px;
+  font-weight: bold;
+  color: #999;
+}

app/components/chat.tsx

@@ -2039,6 +2039,30 @@ function _Chat() {
                             </div>
                           )}
                         </div>
+                        {!isUser &&
+                          message.toolMessages &&
+                          message.toolMessages.map((tool, index) => (
+                            <div
+                              className={styles["chat-message-tools-status"]}
+                              key={index}
+                            >
+                              <div
+                                className={styles["chat-message-tools-name"]}
+                              >
+                                <CheckmarkIcon
+                                  className={styles["chat-message-checkmark"]}
+                                />
+                                {tool.toolName}:
+                                <code
+                                  className={
+                                    styles["chat-message-tools-details"]
+                                  }
+                                >
+                                  {tool.toolInput}
+                                </code>
+                              </div>
+                            </div>
+                          ))}
                         {message?.tools?.length == 0 && showTyping && (
                           <div className={styles["chat-message-status"]}>
                             {Locale.Chat.Typing}

app/components/openai-voice-visualizer/index.ts

@@ -0,0 +1 @@
+export * from "./openai-voice-visualizer";

app/components/openai-voice-visualizer/openai-voice-visualizer.module.scss

@@ -0,0 +1,9 @@
+.openai-voice-visualizer {
+  width: 100%;
+  height: 100%;
+
+  canvas {
+    width: 100%;
+    height: 100%;
+  }
+}

app/components/openai-voice-visualizer/openai-voice-visualizer.tsx

@@ -0,0 +1,372 @@
+import { useEffect, useRef, useCallback, useState } from "react";
+import styles from "./openai-voice-visualizer.module.scss";
+import { initWebGL } from "../../utils/webgl";
+import vertexShaderSource from "../../shaders/vertex.glsl";
+import fragmentShaderSource from "../../shaders/fragment.glsl";
+import { loadImage } from "canvas";
+
+const CANVAS_SIZE = 208;
+const DEFAULT_VIEWPORT_SIZE: [number, number] = [300, 300];
+const NOISE_TEXTURE_OPTIONS = {
+  format: "webp",
+  width: 512,
+  height: 512,
+  space: "srgb",
+  channels: 3,
+  depth: "uchar",
+  density: 72,
+  isProgressive: false,
+  paletteBitDepth: 8,
+  hasProfile: false,
+  hasAlpha: false,
+  src: "./noise-texture.webp",
+} as const;
+
+interface ColorTheme {
+  bloopColorMain: Float32Array;
+  bloopColorLow: Float32Array;
+  bloopColorMid: Float32Array;
+  bloopColorHigh: Float32Array;
+}
+
+export interface AudioData {
+  avgMag: Float32Array;
+  micLevel: number;
+  cumulativeAudio: Float32Array;
+}
+
+const hexToFloatArray = (hex: string): Float32Array => {
+  const hexWithoutHash = hex.replace("#", "");
+  const red = parseInt(hexWithoutHash.substring(0, 2), 16) / 255;
+  const green = parseInt(hexWithoutHash.substring(2, 4), 16) / 255;
+  const blue = parseInt(hexWithoutHash.substring(4, 6), 16) / 255;
+  return new Float32Array([red, green, blue]);
+};
+
+const colorThemes = {
+  BLUE: {
+    bloopColorMain: hexToFloatArray("#DCF7FF"),
+    bloopColorLow: hexToFloatArray("#0181FE"),
+    bloopColorMid: hexToFloatArray("#A4EFFF"),
+    bloopColorHigh: hexToFloatArray("#FFFDEF"),
+  },
+  DARK_BLUE: {
+    bloopColorMain: hexToFloatArray("#DAF5FF"),
+    bloopColorLow: hexToFloatArray("#0066CC"),
+    bloopColorMid: hexToFloatArray("#2EC6F5"),
+    bloopColorHigh: hexToFloatArray("#72EAF5"),
+  },
+  GREYSCALE: {
+    bloopColorMain: hexToFloatArray("#D7D7D7"),
+    bloopColorLow: hexToFloatArray("#303030"),
+    bloopColorMid: hexToFloatArray("#989898"),
+    bloopColorHigh: hexToFloatArray("#FFFFFF"),
+  },
+  WHITE: {
+    bloopColorMain: hexToFloatArray("#FFFFFF"),
+    bloopColorLow: hexToFloatArray("#FFFFFF"),
+    bloopColorMid: hexToFloatArray("#FFFFFF"),
+    bloopColorHigh: hexToFloatArray("#FFFFFF"),
+  },
+  BLACK: {
+    bloopColorMain: hexToFloatArray("#000000"),
+    bloopColorLow: hexToFloatArray("#000000"),
+    bloopColorMid: hexToFloatArray("#000000"),
+    bloopColorHigh: hexToFloatArray("#000000"),
+  },
+} as const;
+
+interface OpenAIVoiceVisualizerProps {
+  audioData?: AudioData;
+  isActive?: boolean;
+}
+
+export class NormalBlorpUniformsSetter {
+  static uniformBlockName = "BlorbUniformsObject";
+  private gl: WebGL2RenderingContext;
+  private uniformBuffer: WebGLBuffer;
+  private uniformNames: string[];
+  private uniformOffsets: { [key: string]: number };
+  private dataBuffer: ArrayBuffer;
+  private floatView: Float32Array;
+  private intView: Int32Array;
+
+  constructor(gl: WebGL2RenderingContext, program: WebGLProgram) {
+    this.gl = gl;
+    const uniformBlockIndex = gl.getUniformBlockIndex(
+      program,
+      NormalBlorpUniformsSetter.uniformBlockName,
+    );
+    const uniformBlockSize = gl.getActiveUniformBlockParameter(
+      program,
+      uniformBlockIndex,
+      gl.UNIFORM_BLOCK_DATA_SIZE,
+    );
+
+    this.uniformBuffer = gl.createBuffer()!;
+    gl.bindBuffer(gl.UNIFORM_BUFFER, this.uniformBuffer);
+    gl.bufferData(gl.UNIFORM_BUFFER, uniformBlockSize, gl.DYNAMIC_DRAW);
+
+    const bindingPoint = 0;
+    gl.bindBufferBase(gl.UNIFORM_BUFFER, bindingPoint, this.uniformBuffer);
+    gl.uniformBlockBinding(program, uniformBlockIndex, bindingPoint);
+
+    const uniformIndices = gl.getActiveUniformBlockParameter(
+      program,
+      uniformBlockIndex,
+      gl.UNIFORM_BLOCK_ACTIVE_UNIFORM_INDICES,
+    );
+
+    this.uniformNames = [];
+    this.uniformOffsets = {};
+    for (let i = 0; i < uniformIndices.length; i++) {
+      const uniformIndex = uniformIndices[i];
+      const uniformInfo = gl.getActiveUniform(program, uniformIndex);
+      if (!uniformInfo) {
+        throw new Error("No uniformInfo for index " + uniformIndex);
+      }
+      let uniformName = uniformInfo.name;
+      uniformName = uniformName.replace(/\[0\]$/, "");
+      const uniformOffset = gl.getActiveUniforms(
+        program,
+        [uniformIndex],
+        gl.UNIFORM_OFFSET,
+      )[0];
+      this.uniformNames.push(uniformName);
+      this.uniformOffsets[uniformName] = uniformOffset;
+    }
+
+    this.dataBuffer = new ArrayBuffer(uniformBlockSize);
+    this.floatView = new Float32Array(this.dataBuffer);
+    this.intView = new Int32Array(this.dataBuffer);
+  }
+
+  setVariablesAndRender(variables: {
+    [key: string]: number | boolean | number[];
+  }) {
+    for (const uniformName of this.uniformNames) {
+      const [, name] = uniformName.split(".");
+      const offset = this.uniformOffsets[uniformName] / 4;
+      const value = variables[name];
+
+      if (typeof value === "number") {
+        this.floatView[offset] = value;
+      } else if (typeof value === "boolean") {
+        this.intView[offset] = value ? 1 : 0;
+      } else if (Array.isArray(value)) {
+        this.floatView.set(value, offset);
+      }
+    }
+
+    this.gl.bindBuffer(this.gl.UNIFORM_BUFFER, this.uniformBuffer);
+    this.gl.bufferSubData(this.gl.UNIFORM_BUFFER, 0, this.dataBuffer);
+    this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 6);
+  }
+}
+
+export function OpenAIVoiceVisualizer({
+  audioData,
+  isActive,
+}: OpenAIVoiceVisualizerProps) {
+  const canvasRef = useRef<HTMLCanvasElement>(null);
+  const glRef = useRef<WebGL2RenderingContext | null>(null);
+  const programRef = useRef<WebGLProgram | null>(null);
+  const animationFrameRef = useRef<number>(0);
+  const uniformSetterRef = useRef<NormalBlorpUniformsSetter | null>(null);
+  const startTimeRef = useRef<number>(performance.now() / 1000);
+  const readyTimeRef = useRef<number>(performance.now() / 1000);
+
+  const variablesRef = useRef({
+    time: 0,
+    micLevel: 0,
+    stateListen: 0,
+    listenTimestamp: 0,
+    stateThink: 0.0,
+    thinkTimestamp: 0.0,
+    stateSpeak: 1,
+    speakTimestamp: 0,
+    readyTimestamp: 0,
+    stateHalt: 0.0,
+    haltTimestamp: 0.0,
+    touchDownTimestamp: 0.0,
+    touchUpTimestamp: 0.0,
+    stateFailedToConnect: 0.0,
+    failedToConnectTimestamp: 0.0,
+    avgMag: new Array(4).fill(0),
+    cumulativeAudio: new Array(4).fill(0),
+    isNewBloop: true,
+    isAdvancedBloop: true,
+    bloopColorMain: [0, 0, 0],
+    bloopColorLow: [0, 0, 0],
+    bloopColorMid: [0, 0, 0],
+    bloopColorHigh: [0, 0, 0],
+    isDarkMode: false,
+    screenScaleFactor: 1.0,
+    viewport: DEFAULT_VIEWPORT_SIZE,
+    silenceAmount: 0.0,
+    silenceTimestamp: 0.0,
+    fadeBloopWhileListening: false,
+  });
+
+  const audioDataRef = useRef<AudioData>({
+    avgMag: new Float32Array(4),
+    micLevel: 0,
+    cumulativeAudio: new Float32Array(4),
+  });
+
+  const handleAudioData = useCallback((data: AudioData) => {
+    audioDataRef.current = data;
+  }, []);
+
+  const [viewportSize] = useState<[number, number]>(DEFAULT_VIEWPORT_SIZE);
+  const [noiseTextureImage, setNoiseTextureImage] =
+    useState<HTMLImageElement | null>(null);
+  const getColorTheme = useCallback((isAdvanced: boolean): ColorTheme => {
+    return colorThemes.BLUE;
+  }, []);
+
+  const initializeWebGL = useCallback(() => {
+    if (!canvasRef.current) return;
+
+    canvasRef.current.width = CANVAS_SIZE;
+    canvasRef.current.height = CANVAS_SIZE;
+
+    const { gl, program } = initWebGL(
+      canvasRef.current,
+      vertexShaderSource,
+      fragmentShaderSource,
+    );
+
+    if (!gl || !program) {
+      console.error("WebGL 初始化失败");
+      return;
+    }
+
+    glRef.current = gl;
+    programRef.current = program;
+    uniformSetterRef.current = new NormalBlorpUniformsSetter(gl, program);
+
+    return { gl, program };
+  }, []);
+
+  const initializeNoiseTexture = useCallback(
+    (gl: WebGL2RenderingContext, program: WebGLProgram) => {
+      const noiseTexture = gl.createTexture();
+      if (!noiseTexture) {
+        console.error("创建噪声纹理失败");
+        return;
+      }
+
+      gl.bindTexture(gl.TEXTURE_2D, noiseTexture);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
+      gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
+
+      if (noiseTextureImage) {
+        gl.texImage2D(
+          gl.TEXTURE_2D,
+          0,
+          gl.RGBA,
+          gl.RGBA,
+          gl.UNSIGNED_BYTE,
+          noiseTextureImage,
+        );
+      }
+
+      const location = gl.getUniformLocation(program, "uTextureNoise");
+      gl.activeTexture(gl.TEXTURE0);
+      gl.bindTexture(gl.TEXTURE_2D, noiseTexture);
+      gl.uniform1i(location, 0);
+
+      return noiseTexture;
+    },
+    [noiseTextureImage],
+  );
+
+  const renderFrame = useCallback(() => {
+    if (!glRef.current || !uniformSetterRef.current) return;
+
+    if (!audioData) {
+      handleAudioData({
+        avgMag: new Float32Array(4),
+        micLevel: 0,
+        cumulativeAudio: new Float32Array(4),
+      });
+      // return;
+    } else {
+      handleAudioData(audioData);
+    }
+
+    const currentFrameTime = performance.now() / 1000;
+    const colorTheme = getColorTheme(true);
+
+    const variables = variablesRef.current;
+    variables.time = currentFrameTime;
+    variables.micLevel = audioDataRef.current.micLevel;
+    variables.speakTimestamp = readyTimeRef.current;
+    variables.readyTimestamp = startTimeRef.current;
+    variables.avgMag = Array.from(audioDataRef.current.avgMag);
+    variables.cumulativeAudio = Array.from(
+      audioDataRef.current.cumulativeAudio,
+    );
+    variables.bloopColorMain = Array.from(colorTheme.bloopColorMain);
+    variables.bloopColorLow = Array.from(colorTheme.bloopColorLow);
+    variables.bloopColorMid = Array.from(colorTheme.bloopColorMid);
+    variables.bloopColorHigh = Array.from(colorTheme.bloopColorHigh);
+    variables.screenScaleFactor = window.devicePixelRatio || 1.0;
+    variables.viewport = viewportSize;
+
+    uniformSetterRef.current.setVariablesAndRender(variables);
+    animationFrameRef.current = requestAnimationFrame(renderFrame);
+  }, [audioData, getColorTheme, handleAudioData, viewportSize]);
+
+  useEffect(() => {
+    const loadNoiseTexture = async () => {
+      try {
+        const image = await loadImage(NOISE_TEXTURE_OPTIONS.src);
+        setNoiseTextureImage(image as unknown as HTMLImageElement);
+      } catch (error) {
+        console.error("加载噪声纹理失败:", error);
+      }
+    };
+    loadNoiseTexture();
+  }, []);
+
+  useEffect(() => {
+    const canvas = canvasRef.current;
+    if (!canvas) return;
+
+    const webglContext = initializeWebGL();
+    if (!webglContext) return;
+
+    const { gl, program } = webglContext;
+    const noiseTexture = initializeNoiseTexture(gl, program);
+
+    renderFrame();
+
+    return () => {
+      if (animationFrameRef.current) {
+        cancelAnimationFrame(animationFrameRef.current);
+      }
+      if (gl) {
+        if (noiseTexture) {
+          gl.deleteTexture(noiseTexture);
+        }
+        gl.deleteProgram(program);
+      }
+    };
+  }, [
+    initializeWebGL,
+    initializeNoiseTexture,
+    renderFrame,
+    audioData,
+    isActive,
+  ]);
+
+  return (
+    <div className={styles["openai-voice-visualizer"]}>
+      <canvas ref={canvasRef} />
+    </div>
+  );
+}

app/components/realtime-chat/realtime-chat.module.scss

@@ -8,15 +8,15 @@
   height: 100%;
   padding: 20px;
   box-sizing: border-box;
-  .circle-mic {
-    width: 150px;
-    height: 150px;
-    border-radius: 50%;
-    background: linear-gradient(to bottom right, #a0d8ef, #f0f8ff);
-    display: flex;
-    justify-content: center;
-    align-items: center;
-  }
+  // .circle-mic {
+  //   width: 150px;
+  //   height: 150px;
+  //   border-radius: 50%;
+  //   background: linear-gradient(to bottom right, #a0d8ef, #f0f8ff);
+  //   display: flex;
+  //   justify-content: center;
+  //   align-items: center;
+  // }
   .icon-center {
     font-size: 24px;
   }

app/components/realtime-chat/realtime-chat.tsx

@@ -21,6 +21,10 @@ import {
 import { AudioHandler } from "@/app/lib/audio";
 import { uploadImage } from "@/app/utils/chat";
 import { VoicePrint } from "@/app/components/voice-print";
+import {
+  OpenAIVoiceVisualizer,
+  AudioData,
+} from "../openai-voice-visualizer/openai-voice-visualizer";
 
 interface RealtimeChatProps {
   onClose?: () => void;
@@ -43,6 +47,7 @@ export function RealtimeChat({
   const [modality, setModality] = useState("audio");
   const [useVAD, setUseVAD] = useState(true);
   const [frequencies, setFrequencies] = useState<Uint8Array | undefined>();
+  const [audioData, setAudioData] = useState<AudioData | undefined>();
 
   const clientRef = useRef<RTClient | null>(null);
   const audioHandlerRef = useRef<AudioHandler | null>(null);
@@ -292,6 +297,9 @@ export function RealtimeChat({
         if (audioHandlerRef.current) {
           const freqData = audioHandlerRef.current.getByteFrequencyData();
           setFrequencies(freqData);
+
+          const audioData = audioHandlerRef.current.getAudioData();
+          setAudioData(audioData);
         }
         animationFrameId = requestAnimationFrame(animationFrame);
       };
@@ -299,6 +307,7 @@ export function RealtimeChat({
       animationFrameId = requestAnimationFrame(animationFrame);
     } else {
       setFrequencies(undefined);
+      setAudioData(undefined);
     }
 
     return () => {
@@ -327,11 +336,11 @@ export function RealtimeChat({
   return (
     <div className={styles["realtime-chat"]}>
       <div
-        className={clsx(styles["circle-mic"], {
-          [styles["pulse"]]: isRecording,
-        })}
+      // className={clsx(styles["circle-mic"], {
+      //   [styles["pulse"]]: isRecording,
+      // })}
       >
-        <VoicePrint frequencies={frequencies} isActive={isRecording} />
+        <OpenAIVoiceVisualizer audioData={audioData} isActive={isRecording} />
       </div>
 
       <div className={styles["bottom-icons"]}>