mirror of
https://github.com/ChatGPTNextWeb/ChatGPT-Next-Web.git
synced 2025-11-13 12:43:42 +08:00
feat: optimize animation
This commit is contained in:
Hk-Gosuto
1
app/components/openai-voice-visualizer/index.ts
Normal file
1
app/components/openai-voice-visualizer/index.ts
Normal file
@@ -0,0 +1 @@
|
||||
export * from "./openai-voice-visualizer";
|
||||
@@ -0,0 +1,9 @@
|
||||
.openai-voice-visualizer {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
|
||||
canvas {
|
||||
width: 100%;
|
||||
height: 100%;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,372 @@
|
||||
import { useEffect, useRef, useCallback, useState } from "react";
|
||||
import styles from "./openai-voice-visualizer.module.scss";
|
||||
import { initWebGL } from "../../utils/webgl";
|
||||
import vertexShaderSource from "../../shaders/vertex.glsl";
|
||||
import fragmentShaderSource from "../../shaders/fragment.glsl";
|
||||
import { loadImage } from "canvas";
|
||||
|
||||
const CANVAS_SIZE = 208;
|
||||
const DEFAULT_VIEWPORT_SIZE: [number, number] = [300, 300];
|
||||
const NOISE_TEXTURE_OPTIONS = {
|
||||
format: "webp",
|
||||
width: 512,
|
||||
height: 512,
|
||||
space: "srgb",
|
||||
channels: 3,
|
||||
depth: "uchar",
|
||||
density: 72,
|
||||
isProgressive: false,
|
||||
paletteBitDepth: 8,
|
||||
hasProfile: false,
|
||||
hasAlpha: false,
|
||||
src: "./noise-texture.webp",
|
||||
} as const;
|
||||
|
||||
interface ColorTheme {
|
||||
bloopColorMain: Float32Array;
|
||||
bloopColorLow: Float32Array;
|
||||
bloopColorMid: Float32Array;
|
||||
bloopColorHigh: Float32Array;
|
||||
}
|
||||
|
||||
export interface AudioData {
|
||||
avgMag: Float32Array;
|
||||
micLevel: number;
|
||||
cumulativeAudio: Float32Array;
|
||||
}
|
||||
|
||||
const hexToFloatArray = (hex: string): Float32Array => {
|
||||
const hexWithoutHash = hex.replace("#", "");
|
||||
const red = parseInt(hexWithoutHash.substring(0, 2), 16) / 255;
|
||||
const green = parseInt(hexWithoutHash.substring(2, 4), 16) / 255;
|
||||
const blue = parseInt(hexWithoutHash.substring(4, 6), 16) / 255;
|
||||
return new Float32Array([red, green, blue]);
|
||||
};
|
||||
|
||||
const colorThemes = {
|
||||
BLUE: {
|
||||
bloopColorMain: hexToFloatArray("#DCF7FF"),
|
||||
bloopColorLow: hexToFloatArray("#0181FE"),
|
||||
bloopColorMid: hexToFloatArray("#A4EFFF"),
|
||||
bloopColorHigh: hexToFloatArray("#FFFDEF"),
|
||||
},
|
||||
DARK_BLUE: {
|
||||
bloopColorMain: hexToFloatArray("#DAF5FF"),
|
||||
bloopColorLow: hexToFloatArray("#0066CC"),
|
||||
bloopColorMid: hexToFloatArray("#2EC6F5"),
|
||||
bloopColorHigh: hexToFloatArray("#72EAF5"),
|
||||
},
|
||||
GREYSCALE: {
|
||||
bloopColorMain: hexToFloatArray("#D7D7D7"),
|
||||
bloopColorLow: hexToFloatArray("#303030"),
|
||||
bloopColorMid: hexToFloatArray("#989898"),
|
||||
bloopColorHigh: hexToFloatArray("#FFFFFF"),
|
||||
},
|
||||
WHITE: {
|
||||
bloopColorMain: hexToFloatArray("#FFFFFF"),
|
||||
bloopColorLow: hexToFloatArray("#FFFFFF"),
|
||||
bloopColorMid: hexToFloatArray("#FFFFFF"),
|
||||
bloopColorHigh: hexToFloatArray("#FFFFFF"),
|
||||
},
|
||||
BLACK: {
|
||||
bloopColorMain: hexToFloatArray("#000000"),
|
||||
bloopColorLow: hexToFloatArray("#000000"),
|
||||
bloopColorMid: hexToFloatArray("#000000"),
|
||||
bloopColorHigh: hexToFloatArray("#000000"),
|
||||
},
|
||||
} as const;
|
||||
|
||||
interface OpenAIVoiceVisualizerProps {
|
||||
audioData?: AudioData;
|
||||
isActive?: boolean;
|
||||
}
|
||||
|
||||
export class NormalBlorpUniformsSetter {
|
||||
static uniformBlockName = "BlorbUniformsObject";
|
||||
private gl: WebGL2RenderingContext;
|
||||
private uniformBuffer: WebGLBuffer;
|
||||
private uniformNames: string[];
|
||||
private uniformOffsets: { [key: string]: number };
|
||||
private dataBuffer: ArrayBuffer;
|
||||
private floatView: Float32Array;
|
||||
private intView: Int32Array;
|
||||
|
||||
constructor(gl: WebGL2RenderingContext, program: WebGLProgram) {
|
||||
this.gl = gl;
|
||||
const uniformBlockIndex = gl.getUniformBlockIndex(
|
||||
program,
|
||||
NormalBlorpUniformsSetter.uniformBlockName,
|
||||
);
|
||||
const uniformBlockSize = gl.getActiveUniformBlockParameter(
|
||||
program,
|
||||
uniformBlockIndex,
|
||||
gl.UNIFORM_BLOCK_DATA_SIZE,
|
||||
);
|
||||
|
||||
this.uniformBuffer = gl.createBuffer()!;
|
||||
gl.bindBuffer(gl.UNIFORM_BUFFER, this.uniformBuffer);
|
||||
gl.bufferData(gl.UNIFORM_BUFFER, uniformBlockSize, gl.DYNAMIC_DRAW);
|
||||
|
||||
const bindingPoint = 0;
|
||||
gl.bindBufferBase(gl.UNIFORM_BUFFER, bindingPoint, this.uniformBuffer);
|
||||
gl.uniformBlockBinding(program, uniformBlockIndex, bindingPoint);
|
||||
|
||||
const uniformIndices = gl.getActiveUniformBlockParameter(
|
||||
program,
|
||||
uniformBlockIndex,
|
||||
gl.UNIFORM_BLOCK_ACTIVE_UNIFORM_INDICES,
|
||||
);
|
||||
|
||||
this.uniformNames = [];
|
||||
this.uniformOffsets = {};
|
||||
for (let i = 0; i < uniformIndices.length; i++) {
|
||||
const uniformIndex = uniformIndices[i];
|
||||
const uniformInfo = gl.getActiveUniform(program, uniformIndex);
|
||||
if (!uniformInfo) {
|
||||
throw new Error("No uniformInfo for index " + uniformIndex);
|
||||
}
|
||||
let uniformName = uniformInfo.name;
|
||||
uniformName = uniformName.replace(/\[0\]$/, "");
|
||||
const uniformOffset = gl.getActiveUniforms(
|
||||
program,
|
||||
[uniformIndex],
|
||||
gl.UNIFORM_OFFSET,
|
||||
)[0];
|
||||
this.uniformNames.push(uniformName);
|
||||
this.uniformOffsets[uniformName] = uniformOffset;
|
||||
}
|
||||
|
||||
this.dataBuffer = new ArrayBuffer(uniformBlockSize);
|
||||
this.floatView = new Float32Array(this.dataBuffer);
|
||||
this.intView = new Int32Array(this.dataBuffer);
|
||||
}
|
||||
|
||||
setVariablesAndRender(variables: {
|
||||
[key: string]: number | boolean | number[];
|
||||
}) {
|
||||
for (const uniformName of this.uniformNames) {
|
||||
const [, name] = uniformName.split(".");
|
||||
const offset = this.uniformOffsets[uniformName] / 4;
|
||||
const value = variables[name];
|
||||
|
||||
if (typeof value === "number") {
|
||||
this.floatView[offset] = value;
|
||||
} else if (typeof value === "boolean") {
|
||||
this.intView[offset] = value ? 1 : 0;
|
||||
} else if (Array.isArray(value)) {
|
||||
this.floatView.set(value, offset);
|
||||
}
|
||||
}
|
||||
|
||||
this.gl.bindBuffer(this.gl.UNIFORM_BUFFER, this.uniformBuffer);
|
||||
this.gl.bufferSubData(this.gl.UNIFORM_BUFFER, 0, this.dataBuffer);
|
||||
this.gl.drawArrays(this.gl.TRIANGLE_STRIP, 0, 6);
|
||||
}
|
||||
}
|
||||
|
||||
export function OpenAIVoiceVisualizer({
|
||||
audioData,
|
||||
isActive,
|
||||
}: OpenAIVoiceVisualizerProps) {
|
||||
const canvasRef = useRef<HTMLCanvasElement>(null);
|
||||
const glRef = useRef<WebGL2RenderingContext | null>(null);
|
||||
const programRef = useRef<WebGLProgram | null>(null);
|
||||
const animationFrameRef = useRef<number>(0);
|
||||
const uniformSetterRef = useRef<NormalBlorpUniformsSetter | null>(null);
|
||||
const startTimeRef = useRef<number>(performance.now() / 1000);
|
||||
const readyTimeRef = useRef<number>(performance.now() / 1000);
|
||||
|
||||
const variablesRef = useRef({
|
||||
time: 0,
|
||||
micLevel: 0,
|
||||
stateListen: 0,
|
||||
listenTimestamp: 0,
|
||||
stateThink: 0.0,
|
||||
thinkTimestamp: 0.0,
|
||||
stateSpeak: 1,
|
||||
speakTimestamp: 0,
|
||||
readyTimestamp: 0,
|
||||
stateHalt: 0.0,
|
||||
haltTimestamp: 0.0,
|
||||
touchDownTimestamp: 0.0,
|
||||
touchUpTimestamp: 0.0,
|
||||
stateFailedToConnect: 0.0,
|
||||
failedToConnectTimestamp: 0.0,
|
||||
avgMag: new Array(4).fill(0),
|
||||
cumulativeAudio: new Array(4).fill(0),
|
||||
isNewBloop: true,
|
||||
isAdvancedBloop: true,
|
||||
bloopColorMain: [0, 0, 0],
|
||||
bloopColorLow: [0, 0, 0],
|
||||
bloopColorMid: [0, 0, 0],
|
||||
bloopColorHigh: [0, 0, 0],
|
||||
isDarkMode: false,
|
||||
screenScaleFactor: 1.0,
|
||||
viewport: DEFAULT_VIEWPORT_SIZE,
|
||||
silenceAmount: 0.0,
|
||||
silenceTimestamp: 0.0,
|
||||
fadeBloopWhileListening: false,
|
||||
});
|
||||
|
||||
const audioDataRef = useRef<AudioData>({
|
||||
avgMag: new Float32Array(4),
|
||||
micLevel: 0,
|
||||
cumulativeAudio: new Float32Array(4),
|
||||
});
|
||||
|
||||
const handleAudioData = useCallback((data: AudioData) => {
|
||||
audioDataRef.current = data;
|
||||
}, []);
|
||||
|
||||
const [viewportSize] = useState<[number, number]>(DEFAULT_VIEWPORT_SIZE);
|
||||
const [noiseTextureImage, setNoiseTextureImage] =
|
||||
useState<HTMLImageElement | null>(null);
|
||||
const getColorTheme = useCallback((isAdvanced: boolean): ColorTheme => {
|
||||
return colorThemes.BLUE;
|
||||
}, []);
|
||||
|
||||
const initializeWebGL = useCallback(() => {
|
||||
if (!canvasRef.current) return;
|
||||
|
||||
canvasRef.current.width = CANVAS_SIZE;
|
||||
canvasRef.current.height = CANVAS_SIZE;
|
||||
|
||||
const { gl, program } = initWebGL(
|
||||
canvasRef.current,
|
||||
vertexShaderSource,
|
||||
fragmentShaderSource,
|
||||
);
|
||||
|
||||
if (!gl || !program) {
|
||||
console.error("WebGL 初始化失败");
|
||||
return;
|
||||
}
|
||||
|
||||
glRef.current = gl;
|
||||
programRef.current = program;
|
||||
uniformSetterRef.current = new NormalBlorpUniformsSetter(gl, program);
|
||||
|
||||
return { gl, program };
|
||||
}, []);
|
||||
|
||||
const initializeNoiseTexture = useCallback(
|
||||
(gl: WebGL2RenderingContext, program: WebGLProgram) => {
|
||||
const noiseTexture = gl.createTexture();
|
||||
if (!noiseTexture) {
|
||||
console.error("创建噪声纹理失败");
|
||||
return;
|
||||
}
|
||||
|
||||
gl.bindTexture(gl.TEXTURE_2D, noiseTexture);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_S, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_WRAP_T, gl.CLAMP_TO_EDGE);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MIN_FILTER, gl.LINEAR);
|
||||
gl.texParameteri(gl.TEXTURE_2D, gl.TEXTURE_MAG_FILTER, gl.LINEAR);
|
||||
|
||||
if (noiseTextureImage) {
|
||||
gl.texImage2D(
|
||||
gl.TEXTURE_2D,
|
||||
0,
|
||||
gl.RGBA,
|
||||
gl.RGBA,
|
||||
gl.UNSIGNED_BYTE,
|
||||
noiseTextureImage,
|
||||
);
|
||||
}
|
||||
|
||||
const location = gl.getUniformLocation(program, "uTextureNoise");
|
||||
gl.activeTexture(gl.TEXTURE0);
|
||||
gl.bindTexture(gl.TEXTURE_2D, noiseTexture);
|
||||
gl.uniform1i(location, 0);
|
||||
|
||||
return noiseTexture;
|
||||
},
|
||||
[noiseTextureImage],
|
||||
);
|
||||
|
||||
const renderFrame = useCallback(() => {
|
||||
if (!glRef.current || !uniformSetterRef.current) return;
|
||||
|
||||
if (!audioData) {
|
||||
handleAudioData({
|
||||
avgMag: new Float32Array(4),
|
||||
micLevel: 0,
|
||||
cumulativeAudio: new Float32Array(4),
|
||||
});
|
||||
// return;
|
||||
} else {
|
||||
handleAudioData(audioData);
|
||||
}
|
||||
|
||||
const currentFrameTime = performance.now() / 1000;
|
||||
const colorTheme = getColorTheme(true);
|
||||
|
||||
const variables = variablesRef.current;
|
||||
variables.time = currentFrameTime;
|
||||
variables.micLevel = audioDataRef.current.micLevel;
|
||||
variables.speakTimestamp = readyTimeRef.current;
|
||||
variables.readyTimestamp = startTimeRef.current;
|
||||
variables.avgMag = Array.from(audioDataRef.current.avgMag);
|
||||
variables.cumulativeAudio = Array.from(
|
||||
audioDataRef.current.cumulativeAudio,
|
||||
);
|
||||
variables.bloopColorMain = Array.from(colorTheme.bloopColorMain);
|
||||
variables.bloopColorLow = Array.from(colorTheme.bloopColorLow);
|
||||
variables.bloopColorMid = Array.from(colorTheme.bloopColorMid);
|
||||
variables.bloopColorHigh = Array.from(colorTheme.bloopColorHigh);
|
||||
variables.screenScaleFactor = window.devicePixelRatio || 1.0;
|
||||
variables.viewport = viewportSize;
|
||||
|
||||
uniformSetterRef.current.setVariablesAndRender(variables);
|
||||
animationFrameRef.current = requestAnimationFrame(renderFrame);
|
||||
}, [audioData, getColorTheme, handleAudioData, viewportSize]);
|
||||
|
||||
useEffect(() => {
|
||||
const loadNoiseTexture = async () => {
|
||||
try {
|
||||
const image = await loadImage(NOISE_TEXTURE_OPTIONS.src);
|
||||
setNoiseTextureImage(image as unknown as HTMLImageElement);
|
||||
} catch (error) {
|
||||
console.error("加载噪声纹理失败:", error);
|
||||
}
|
||||
};
|
||||
loadNoiseTexture();
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const canvas = canvasRef.current;
|
||||
if (!canvas) return;
|
||||
|
||||
const webglContext = initializeWebGL();
|
||||
if (!webglContext) return;
|
||||
|
||||
const { gl, program } = webglContext;
|
||||
const noiseTexture = initializeNoiseTexture(gl, program);
|
||||
|
||||
renderFrame();
|
||||
|
||||
return () => {
|
||||
if (animationFrameRef.current) {
|
||||
cancelAnimationFrame(animationFrameRef.current);
|
||||
}
|
||||
if (gl) {
|
||||
if (noiseTexture) {
|
||||
gl.deleteTexture(noiseTexture);
|
||||
}
|
||||
gl.deleteProgram(program);
|
||||
}
|
||||
};
|
||||
}, [
|
||||
initializeWebGL,
|
||||
initializeNoiseTexture,
|
||||
renderFrame,
|
||||
audioData,
|
||||
isActive,
|
||||
]);
|
||||
|
||||
return (
|
||||
<div className={styles["openai-voice-visualizer"]}>
|
||||
<canvas ref={canvasRef} />
|
||||
</div>
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user