Merge remote-tracking branch 'up/main'

# Conflicts:
#	app/components/chat.tsx
#	app/store/chat.ts
This commit is contained in:
织梦人
2024-12-13 09:30:14 +08:00
54 changed files with 2293 additions and 791 deletions

View File

@@ -18,6 +18,8 @@ import {
trackSettingsPageGuideToCPaymentClick,
trackAuthorizationPageButtonToCPaymentClick,
} from "../utils/auth-settings-events";
import clsx from "clsx";
const storage = safeLocalStorage();
export function AuthPage() {
@@ -54,7 +56,7 @@ export function AuthPage() {
onClick={() => navigate(Path.Home)}
></IconButton>
</div>
<div className={`no-dark ${styles["auth-logo"]}`}>
<div className={clsx("no-dark", styles["auth-logo"])}>
<BotIcon />
</div>
@@ -163,7 +165,7 @@ function TopBanner() {
onMouseEnter={handleMouseEnter}
onMouseLeave={handleMouseLeave}
>
<div className={`${styles["top-banner-inner"]} no-dark`}>
<div className={clsx(styles["top-banner-inner"], "no-dark")}>
<Logo className={styles["top-banner-logo"]}></Logo>
<span>
{Locale.Auth.TopTips}

View File

@@ -2,6 +2,7 @@ import * as React from "react";
import styles from "./button.module.scss";
import { CSSProperties } from "react";
import clsx from "clsx";
export type ButtonType = "primary" | "danger" | null;
@@ -22,12 +23,16 @@ export function IconButton(props: {
}) {
return (
<button
className={
styles["icon-button"] +
` ${props.bordered && styles.border} ${props.shadow && styles.shadow} ${
props.className ?? ""
} clickable ${styles[props.type ?? ""]}`
}
className={clsx(
"clickable",
styles["icon-button"],
{
[styles.border]: props.bordered,
[styles.shadow]: props.shadow,
},
styles[props.type ?? ""],
props.className,
)}
onClick={props.onClick}
title={props.title}
disabled={props.disabled}
@@ -40,10 +45,9 @@ export function IconButton(props: {
{props.icon && (
<div
aria-label={props.text || props.title}
className={
styles["icon-button-icon"] +
` ${props.type === "primary" && "no-dark"}`
}
className={clsx(styles["icon-button-icon"], {
"no-dark": props.type === "primary",
})}
>
{props.icon}
</div>

View File

@@ -18,6 +18,7 @@ import { Mask } from "../store/mask";
import { useRef, useEffect } from "react";
import { showConfirm } from "./ui-lib";
import { useMobileScreen } from "../utils";
import clsx from "clsx";
export function ChatItem(props: {
onClick?: () => void;
@@ -45,11 +46,11 @@ export function ChatItem(props: {
<Draggable draggableId={`${props.id}`} index={props.index}>
{(provided) => (
<div
className={`${styles["chat-item"]} ${
props.selected &&
(currentPath === Path.Chat || currentPath === Path.Home) &&
styles["chat-item-selected"]
}`}
className={clsx(styles["chat-item"], {
[styles["chat-item-selected"]]:
props.selected &&
(currentPath === Path.Chat || currentPath === Path.Home),
})}
onClick={props.onClick}
ref={(ele) => {
draggableRef.current = ele;
@@ -63,7 +64,7 @@ export function ChatItem(props: {
>
{props.narrow ? (
<div className={styles["chat-item-narrow"]}>
<div className={styles["chat-item-avatar"] + " no-dark"}>
<div className={clsx(styles["chat-item-avatar"], "no-dark")}>
<MaskAvatar
avatar={props.mask.avatar}
model={props.mask.modelConfig.model}

View File

@@ -45,6 +45,14 @@
.chat-input-actions {
display: flex;
flex-wrap: wrap;
justify-content: space-between;
gap: 5px;
&-end {
display: flex;
margin-left: auto;
gap: 5px;
}
.chat-input-action {
display: inline-flex;
@@ -62,10 +70,6 @@
width: var(--icon-width);
overflow: hidden;
&:not(:last-child) {
margin-right: 5px;
}
.text {
white-space: nowrap;
padding-left: 5px;
@@ -231,10 +235,12 @@
animation: slide-in ease 0.3s;
$linear: linear-gradient(to right,
rgba(0, 0, 0, 0),
rgba(0, 0, 0, 1),
rgba(0, 0, 0, 0));
$linear: linear-gradient(
to right,
rgba(0, 0, 0, 0),
rgba(0, 0, 0, 1),
rgba(0, 0, 0, 0)
);
mask-image: $linear;
@mixin show {
@@ -373,7 +379,7 @@
}
}
.chat-message-user>.chat-message-container {
.chat-message-user > .chat-message-container {
align-items: flex-end;
}
@@ -443,6 +449,25 @@
transition: all ease 0.3s;
}
.chat-message-audio {
display: flex;
align-items: center;
justify-content: space-between;
border-radius: 10px;
background-color: rgba(0, 0, 0, 0.05);
border: var(--border-in-light);
position: relative;
transition: all ease 0.3s;
margin-top: 10px;
font-size: 14px;
user-select: text;
word-break: break-word;
box-sizing: border-box;
audio {
height: 30px; /* 调整高度 */
}
}
.chat-message-item-image {
width: 100%;
margin-top: 10px;
@@ -471,23 +496,27 @@
border: rgba($color: #888, $alpha: 0.2) 1px solid;
}
@media only screen and (max-width: 600px) {
$calc-image-width: calc(100vw/3*2/var(--image-count));
$calc-image-width: calc(100vw / 3 * 2 / var(--image-count));
.chat-message-item-image-multi {
width: $calc-image-width;
height: $calc-image-width;
}
.chat-message-item-image {
max-width: calc(100vw/3*2);
max-width: calc(100vw / 3 * 2);
}
}
@media screen and (min-width: 600px) {
$max-image-width: calc(calc(1200px - var(--sidebar-width))/3*2/var(--image-count));
$image-width: calc(calc(var(--window-width) - var(--sidebar-width))/3*2/var(--image-count));
$max-image-width: calc(
calc(1200px - var(--sidebar-width)) / 3 * 2 / var(--image-count)
);
$image-width: calc(
calc(var(--window-width) - var(--sidebar-width)) / 3 * 2 /
var(--image-count)
);
.chat-message-item-image-multi {
width: $image-width;
@@ -497,7 +526,7 @@
}
.chat-message-item-image {
max-width: calc(calc(1200px - var(--sidebar-width))/3*2);
max-width: calc(calc(1200px - var(--sidebar-width)) / 3 * 2);
}
}
@@ -515,7 +544,7 @@
z-index: 1;
}
.chat-message-user>.chat-message-container>.chat-message-item {
.chat-message-user > .chat-message-container > .chat-message-item {
background-color: var(--second);
&:hover {
@@ -626,7 +655,8 @@
min-height: 68px;
}
.chat-input:focus {}
.chat-input:focus {
}
.chat-input-send {
background-color: var(--primary);
@@ -693,4 +723,31 @@
.shortcut-key span {
font-size: 12px;
color: var(--black);
}
}
.chat-main {
display: flex;
height: 100%;
width: 100%;
position: relative;
overflow: hidden;
.chat-body-container {
height: 100%;
display: flex;
flex-direction: column;
flex: 1;
width: 100%;
}
.chat-side-panel {
position: absolute;
inset: 0;
background: var(--white);
overflow: hidden;
z-index: 10;
transform: translateX(100%);
transition: all ease 0.3s;
&-show {
transform: translateX(0);
}
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -37,7 +37,8 @@ export function Avatar(props: { model?: ModelType; avatar?: string }) {
return (
<div className="no-dark">
{props.model?.startsWith("gpt-4") ||
props.model?.startsWith("chatgpt-4o") ? (
props.model?.startsWith("chatgpt-4o") ||
props.model?.startsWith("o1") ? (
<BlackBotIcon className="user-avatar" />
) : (
<BotIcon className="user-avatar" />

View File

@@ -40,6 +40,7 @@ import { EXPORT_MESSAGE_CLASS_NAME } from "../constant";
import { getClientConfig } from "../config/client";
import { type ClientApi, getClientApi } from "../client/api";
import { getMessageTextContent } from "../utils";
import clsx from "clsx";
const Markdown = dynamic(async () => (await import("./markdown")).Markdown, {
loading: () => <LoadingIcon />,
@@ -118,9 +119,10 @@ function Steps<
return (
<div
key={i}
className={`${styles["step"]} ${
styles[i <= props.index ? "step-finished" : ""]
} ${i === props.index && styles["step-current"]} clickable`}
className={clsx("clickable", styles["step"], {
[styles["step-finished"]]: i <= props.index,
[styles["step-current"]]: i === props.index,
})}
onClick={() => {
props.onStepChange?.(i);
}}
@@ -525,11 +527,11 @@ export function ImagePreviewer(props: {
messages={props.messages}
/>
<div
className={`${styles["preview-body"]} ${styles["default-theme"]}`}
className={clsx(styles["preview-body"], styles["default-theme"])}
ref={previewRef}
>
<div className={styles["chat-info"]}>
<div className={styles["logo"] + " no-dark"}>
<div className={clsx(styles["logo"], "no-dark")}>
<NextImage
src={ChatGptIcon.src}
alt="logo"
@@ -570,7 +572,7 @@ export function ImagePreviewer(props: {
{props.messages.map((m, i) => {
return (
<div
className={styles["message"] + " " + styles["message-" + m.role]}
className={clsx(styles["message"], styles["message-" + m.role])}
key={i}
>
<div className={styles["avatar"]}>

View File

@@ -3,7 +3,6 @@
require("../polyfill");
import { useState, useEffect } from "react";
import styles from "./home.module.scss";
import BotIcon from "../icons/bot.svg";
@@ -29,10 +28,11 @@ import { AuthPage } from "./auth";
import { getClientConfig } from "../config/client";
import { type ClientApi, getClientApi } from "../client/api";
import { useAccessStore } from "../store";
import clsx from "clsx";
export function Loading(props: { noLogo?: boolean }) {
return (
<div className={styles["loading-content"] + " no-dark"}>
<div className={clsx("no-dark", styles["loading-content"])}>
{!props.noLogo && <BotIcon />}
<LoadingIcon />
</div>
@@ -179,7 +179,11 @@ function Screen() {
if (isSdNew) return <Sd />;
return (
<>
<SideBar className={isHome ? styles["sidebar-show"] : ""} />
<SideBar
className={clsx({
[styles["sidebar-show"]]: isHome,
})}
/>
<WindowContent>
<Routes>
<Route path={Path.Home} element={<Chat />} />
@@ -197,9 +201,10 @@ function Screen() {
return (
<div
className={`${styles.container} ${
shouldTightBorder ? styles["tight-container"] : styles.container
} ${getLang() === "ar" ? styles["rtl-screen"] : ""}`}
className={clsx(styles.container, {
[styles["tight-container"]]: shouldTightBorder,
[styles["rtl-screen"]]: getLang() === "ar",
})}
>
{renderContent()}
</div>

View File

@@ -1,5 +1,6 @@
import * as React from "react";
import styles from "./input-range.module.scss";
import clsx from "clsx";
interface InputRangeProps {
onChange: React.ChangeEventHandler<HTMLInputElement>;
@@ -23,7 +24,7 @@ export function InputRange({
aria,
}: InputRangeProps) {
return (
<div className={styles["input-range"] + ` ${className ?? ""}`}>
<div className={clsx(styles["input-range"], className)}>
{title || value}
<input
aria-label={aria}

View File

@@ -23,6 +23,7 @@ import { useChatStore } from "../store";
import { IconButton } from "./button";
import { useAppConfig } from "../store/config";
import clsx from "clsx";
export function Mermaid(props: { code: string }) {
const ref = useRef<HTMLDivElement>(null);
@@ -57,7 +58,7 @@ export function Mermaid(props: { code: string }) {
return (
<div
className="no-dark mermaid"
className={clsx("no-dark", "mermaid")}
style={{
cursor: "pointer",
overflow: "auto",
@@ -89,7 +90,11 @@ export function PreCode(props: { children: any }) {
const refText = ref.current.querySelector("code")?.innerText;
if (htmlDom) {
setHtmlCode((htmlDom as HTMLElement).innerText);
} else if (refText?.startsWith("<!DOCTYPE")) {
} else if (
refText?.startsWith("<!DOCTYPE") ||
refText?.startsWith("<svg") ||
refText?.startsWith("<?xml")
) {
setHtmlCode(refText);
}
}, 600);
@@ -193,7 +198,12 @@ function CustomCode(props: { children: any; className?: string }) {
const renderShowMoreButton = () => {
if (showToggle && enableCodeFold && collapsed) {
return (
<div className={`show-hide-button ${collapsed ? "collapsed" : "expanded"}`}>
<div
className={clsx("show-hide-button", {
collapsed,
expanded: !collapsed,
})}
>
<button onClick={toggleCollapsed}>{Locale.NewChat.More}</button>
</div>
);
@@ -203,7 +213,7 @@ function CustomCode(props: { children: any; className?: string }) {
return (
<>
<code
className={props?.className}
className={clsx(props?.className)}
ref={ref}
style={{
maxHeight: enableCodeFold && collapsed ? "400px" : "none",
@@ -238,6 +248,10 @@ function escapeBrackets(text: string) {
function tryWrapHtmlCode(text: string) {
// try add wrap html code (fixed: html codeblock include 2 newline)
// ignore embed codeblock
if (text.includes("```")) {
return text;
}
return text
.replace(
/([`]*?)(\w*?)([\n\r]*?)(<!DOCTYPE html>)/g,

View File

@@ -55,6 +55,7 @@ import {
OnDragEndResponder,
} from "@hello-pangea/dnd";
import { getMessageTextContent } from "../utils";
import clsx from "clsx";
// drag and drop helper function
function reorder<T>(list: T[], startIndex: number, endIndex: number): T[] {
@@ -588,7 +589,7 @@ export function MaskPage() {
</div>
<div className={styles["mask-title"]}>
<div className={styles["mask-name"]}>{m.name}</div>
<div className={styles["mask-info"] + " one-line"}>
<div className={clsx(styles["mask-info"], "one-line")}>
{`${Locale.Mask.Item.Info(m.context.length)} / ${
ALL_LANG_OPTIONS[m.lang]
} / ${m.modelConfig.model}`}

View File

@@ -8,6 +8,7 @@ import Locale from "../locales";
import styles from "./message-selector.module.scss";
import { getMessageTextContent } from "../utils";
import clsx from "clsx";
function useShiftRange() {
const [startIndex, setStartIndex] = useState<number>();
@@ -71,6 +72,7 @@ export function MessageSelector(props: {
defaultSelectAll?: boolean;
onSelected?: (messages: ChatMessage[]) => void;
}) {
const LATEST_COUNT = 4;
const chatStore = useChatStore();
const session = chatStore.currentSession();
const isValid = (m: ChatMessage) => m.content && !m.isError && !m.streaming;
@@ -141,15 +143,13 @@ export function MessageSelector(props: {
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [startIndex, endIndex]);
const LATEST_COUNT = 4;
return (
<div className={styles["message-selector"]}>
<div className={styles["message-filter"]}>
<input
type="text"
placeholder={Locale.Select.Search}
className={styles["filter-item"] + " " + styles["search-bar"]}
className={clsx(styles["filter-item"], styles["search-bar"])}
value={searchInput}
onInput={(e) => {
setSearchInput(e.currentTarget.value);
@@ -196,9 +196,9 @@ export function MessageSelector(props: {
return (
<div
className={`${styles["message"]} ${
props.selection.has(m.id!) && styles["message-selected"]
}`}
className={clsx(styles["message"], {
[styles["message-selected"]]: props.selection.has(m.id!),
})}
key={i}
onClick={() => {
props.updateSelection((selection) => {
@@ -221,7 +221,7 @@ export function MessageSelector(props: {
<div className={styles["date"]}>
{new Date(m.date).toLocaleString()}
</div>
<div className={`${styles["content"]} one-line`}>
<div className={clsx(styles["content"], "one-line")}>
{getMessageTextContent(m)}
</div>
</div>

View File

@@ -7,6 +7,7 @@ import { ListItem, Select } from "./ui-lib";
import { useAllModels } from "../utils/hooks";
import { groupBy } from "lodash-es";
import styles from "./model-config.module.scss";
import { getModelProvider } from "../utils/model";
export function ModelConfigList(props: {
modelConfig: ModelConfig;
@@ -28,7 +29,9 @@ export function ModelConfigList(props: {
value={value}
align="left"
onChange={(e) => {
const [model, providerName] = e.currentTarget.value.split("@");
const [model, providerName] = getModelProvider(
e.currentTarget.value,
);
props.updateConfig((config) => {
config.model = ModalConfigValidator.model(model);
config.providerName = providerName as ServiceProvider;
@@ -247,7 +250,9 @@ export function ModelConfigList(props: {
aria-label={Locale.Settings.CompressModel.Title}
value={compressModelValue}
onChange={(e) => {
const [model, providerName] = e.currentTarget.value.split("@");
const [model, providerName] = getModelProvider(
e.currentTarget.value,
);
props.updateConfig((config) => {
config.compressModel = ModalConfigValidator.model(model);
config.compressProviderName = providerName as ServiceProvider;

View File

@@ -16,6 +16,7 @@ import { MaskAvatar } from "./mask";
import { useCommand } from "../command";
import { showConfirm } from "./ui-lib";
import { BUILTIN_MASK_STORE } from "../masks";
import clsx from "clsx";
function MaskItem(props: { mask: Mask; onClick?: () => void }) {
return (
@@ -24,7 +25,9 @@ function MaskItem(props: { mask: Mask; onClick?: () => void }) {
avatar={props.mask.avatar}
model={props.mask.modelConfig.model}
/>
<div className={styles["mask-name"] + " one-line"}>{props.mask.name}</div>
<div className={clsx(styles["mask-name"], "one-line")}>
{props.mask.name}
</div>
</div>
);
}

View File

@@ -28,6 +28,7 @@ import {
import Locale from "../locales";
import { useNavigate } from "react-router-dom";
import { useState } from "react";
import clsx from "clsx";
export function PluginPage() {
const navigate = useNavigate();
@@ -199,7 +200,7 @@ export function PluginPage() {
<div className={styles["mask-name"]}>
{m.title}@<small>{m.version}</small>
</div>
<div className={styles["mask-info"] + " one-line"}>
<div className={clsx(styles["mask-info"], "one-line")}>
{Locale.Plugin.Item.Info(
FunctionToolService.add(m).length,
)}
@@ -335,7 +336,10 @@ export function PluginPage() {
<ListItem
subTitle={
<div
className={`markdown-body ${pluginStyles["plugin-content"]}`}
className={clsx(
"markdown-body",
pluginStyles["plugin-content"],
)}
dir="auto"
>
<pre>

View File

@@ -0,0 +1 @@
export * from "./realtime-chat";

View File

@@ -0,0 +1,74 @@
.realtime-chat {
width: 100%;
justify-content: center;
align-items: center;
position: relative;
display: flex;
flex-direction: column;
height: 100%;
padding: 20px;
box-sizing: border-box;
.circle-mic {
width: 150px;
height: 150px;
border-radius: 50%;
background: linear-gradient(to bottom right, #a0d8ef, #f0f8ff);
display: flex;
justify-content: center;
align-items: center;
}
.icon-center {
font-size: 24px;
}
.bottom-icons {
display: flex;
justify-content: space-between;
align-items: center;
width: 100%;
position: absolute;
bottom: 20px;
box-sizing: border-box;
padding: 0 20px;
}
.icon-left,
.icon-right {
width: 46px;
height: 46px;
font-size: 36px;
background: var(--second);
border-radius: 50%;
padding: 2px;
display: flex;
justify-content: center;
align-items: center;
cursor: pointer;
&:hover {
opacity: 0.8;
}
}
&.mobile {
display: none;
}
}
.pulse {
animation: pulse 1.5s infinite;
}
@keyframes pulse {
0% {
transform: scale(1);
opacity: 0.7;
}
50% {
transform: scale(1.1);
opacity: 1;
}
100% {
transform: scale(1);
opacity: 0.7;
}
}

View File

@@ -0,0 +1,359 @@
import VoiceIcon from "@/app/icons/voice.svg";
import VoiceOffIcon from "@/app/icons/voice-off.svg";
import PowerIcon from "@/app/icons/power.svg";
import styles from "./realtime-chat.module.scss";
import clsx from "clsx";
import { useState, useRef, useEffect } from "react";
import { useChatStore, createMessage, useAppConfig } from "@/app/store";
import { IconButton } from "@/app/components/button";
import {
Modality,
RTClient,
RTInputAudioItem,
RTResponse,
TurnDetection,
} from "rt-client";
import { AudioHandler } from "@/app/lib/audio";
import { uploadImage } from "@/app/utils/chat";
import { VoicePrint } from "@/app/components/voice-print";
interface RealtimeChatProps {
onClose?: () => void;
onStartVoice?: () => void;
onPausedVoice?: () => void;
}
export function RealtimeChat({
onClose,
onStartVoice,
onPausedVoice,
}: RealtimeChatProps) {
const chatStore = useChatStore();
const session = chatStore.currentSession();
const config = useAppConfig();
const [status, setStatus] = useState("");
const [isRecording, setIsRecording] = useState(false);
const [isConnected, setIsConnected] = useState(false);
const [isConnecting, setIsConnecting] = useState(false);
const [modality, setModality] = useState("audio");
const [useVAD, setUseVAD] = useState(true);
const [frequencies, setFrequencies] = useState<Uint8Array | undefined>();
const clientRef = useRef<RTClient | null>(null);
const audioHandlerRef = useRef<AudioHandler | null>(null);
const initRef = useRef(false);
const temperature = config.realtimeConfig.temperature;
const apiKey = config.realtimeConfig.apiKey;
const model = config.realtimeConfig.model;
const azure = config.realtimeConfig.provider === "Azure";
const azureEndpoint = config.realtimeConfig.azure.endpoint;
const azureDeployment = config.realtimeConfig.azure.deployment;
const voice = config.realtimeConfig.voice;
const handleConnect = async () => {
if (isConnecting) return;
if (!isConnected) {
try {
setIsConnecting(true);
clientRef.current = azure
? new RTClient(
new URL(azureEndpoint),
{ key: apiKey },
{ deployment: azureDeployment },
)
: new RTClient({ key: apiKey }, { model });
const modalities: Modality[] =
modality === "audio" ? ["text", "audio"] : ["text"];
const turnDetection: TurnDetection = useVAD
? { type: "server_vad" }
: null;
await clientRef.current.configure({
instructions: "",
voice,
input_audio_transcription: { model: "whisper-1" },
turn_detection: turnDetection,
tools: [],
temperature,
modalities,
});
startResponseListener();
setIsConnected(true);
// TODO
// try {
// const recentMessages = chatStore.getMessagesWithMemory();
// for (const message of recentMessages) {
// const { role, content } = message;
// if (typeof content === "string") {
// await clientRef.current.sendItem({
// type: "message",
// role: role as any,
// content: [
// {
// type: (role === "assistant" ? "text" : "input_text") as any,
// text: content as string,
// },
// ],
// });
// }
// }
// // await clientRef.current.generateResponse();
// } catch (error) {
// console.error("Set message failed:", error);
// }
} catch (error) {
console.error("Connection failed:", error);
setStatus("Connection failed");
} finally {
setIsConnecting(false);
}
} else {
await disconnect();
}
};
const disconnect = async () => {
if (clientRef.current) {
try {
await clientRef.current.close();
clientRef.current = null;
setIsConnected(false);
} catch (error) {
console.error("Disconnect failed:", error);
}
}
};
const startResponseListener = async () => {
if (!clientRef.current) return;
try {
for await (const serverEvent of clientRef.current.events()) {
if (serverEvent.type === "response") {
await handleResponse(serverEvent);
} else if (serverEvent.type === "input_audio") {
await handleInputAudio(serverEvent);
}
}
} catch (error) {
if (clientRef.current) {
console.error("Response iteration error:", error);
}
}
};
const handleResponse = async (response: RTResponse) => {
for await (const item of response) {
if (item.type === "message" && item.role === "assistant") {
const botMessage = createMessage({
role: item.role,
content: "",
});
// add bot message first
chatStore.updateTargetSession(session, (session) => {
session.messages = session.messages.concat([botMessage]);
});
let hasAudio = false;
for await (const content of item) {
if (content.type === "text") {
for await (const text of content.textChunks()) {
botMessage.content += text;
}
} else if (content.type === "audio") {
const textTask = async () => {
for await (const text of content.transcriptChunks()) {
botMessage.content += text;
}
};
const audioTask = async () => {
audioHandlerRef.current?.startStreamingPlayback();
for await (const audio of content.audioChunks()) {
hasAudio = true;
audioHandlerRef.current?.playChunk(audio);
}
};
await Promise.all([textTask(), audioTask()]);
}
// update message.content
chatStore.updateTargetSession(session, (session) => {
session.messages = session.messages.concat();
});
}
if (hasAudio) {
// upload audio get audio_url
const blob = audioHandlerRef.current?.savePlayFile();
uploadImage(blob!).then((audio_url) => {
botMessage.audio_url = audio_url;
// update text and audio_url
chatStore.updateTargetSession(session, (session) => {
session.messages = session.messages.concat();
});
});
}
}
}
};
const handleInputAudio = async (item: RTInputAudioItem) => {
await item.waitForCompletion();
if (item.transcription) {
const userMessage = createMessage({
role: "user",
content: item.transcription,
});
chatStore.updateTargetSession(session, (session) => {
session.messages = session.messages.concat([userMessage]);
});
// save input audio_url, and update session
const { audioStartMillis, audioEndMillis } = item;
// upload audio get audio_url
const blob = audioHandlerRef.current?.saveRecordFile(
audioStartMillis,
audioEndMillis,
);
uploadImage(blob!).then((audio_url) => {
userMessage.audio_url = audio_url;
chatStore.updateTargetSession(session, (session) => {
session.messages = session.messages.concat();
});
});
}
// stop streaming play after get input audio.
audioHandlerRef.current?.stopStreamingPlayback();
};
const toggleRecording = async () => {
if (!isRecording && clientRef.current) {
try {
if (!audioHandlerRef.current) {
audioHandlerRef.current = new AudioHandler();
await audioHandlerRef.current.initialize();
}
await audioHandlerRef.current.startRecording(async (chunk) => {
await clientRef.current?.sendAudio(chunk);
});
setIsRecording(true);
} catch (error) {
console.error("Failed to start recording:", error);
}
} else if (audioHandlerRef.current) {
try {
audioHandlerRef.current.stopRecording();
if (!useVAD) {
const inputAudio = await clientRef.current?.commitAudio();
await handleInputAudio(inputAudio!);
await clientRef.current?.generateResponse();
}
setIsRecording(false);
} catch (error) {
console.error("Failed to stop recording:", error);
}
}
};
useEffect(() => {
// 防止重复初始化
if (initRef.current) return;
initRef.current = true;
const initAudioHandler = async () => {
const handler = new AudioHandler();
await handler.initialize();
audioHandlerRef.current = handler;
await handleConnect();
await toggleRecording();
};
initAudioHandler().catch((error) => {
setStatus(error);
console.error(error);
});
return () => {
if (isRecording) {
toggleRecording();
}
audioHandlerRef.current?.close().catch(console.error);
disconnect();
};
}, []);
useEffect(() => {
let animationFrameId: number;
if (isConnected && isRecording) {
const animationFrame = () => {
if (audioHandlerRef.current) {
const freqData = audioHandlerRef.current.getByteFrequencyData();
setFrequencies(freqData);
}
animationFrameId = requestAnimationFrame(animationFrame);
};
animationFrameId = requestAnimationFrame(animationFrame);
} else {
setFrequencies(undefined);
}
return () => {
if (animationFrameId) {
cancelAnimationFrame(animationFrameId);
}
};
}, [isConnected, isRecording]);
// update session params
useEffect(() => {
clientRef.current?.configure({ voice });
}, [voice]);
useEffect(() => {
clientRef.current?.configure({ temperature });
}, [temperature]);
const handleClose = async () => {
onClose?.();
if (isRecording) {
await toggleRecording();
}
disconnect().catch(console.error);
};
return (
<div className={styles["realtime-chat"]}>
<div
className={clsx(styles["circle-mic"], {
[styles["pulse"]]: isRecording,
})}
>
<VoicePrint frequencies={frequencies} isActive={isRecording} />
</div>
<div className={styles["bottom-icons"]}>
<div>
<IconButton
icon={isRecording ? <VoiceIcon /> : <VoiceOffIcon />}
onClick={toggleRecording}
disabled={!isConnected}
shadow
bordered
/>
</div>
<div className={styles["icon-center"]}>{status}</div>
<div>
<IconButton
icon={<PowerIcon />}
onClick={handleClose}
shadow
bordered
/>
</div>
</div>
</div>
);
}

View File

@@ -0,0 +1,173 @@
import { RealtimeConfig } from "@/app/store";
import Locale from "@/app/locales";
import { ListItem, Select, PasswordInput } from "@/app/components/ui-lib";
import { InputRange } from "@/app/components/input-range";
import { Voice } from "rt-client";
import { ServiceProvider } from "@/app/constant";
const providers = [ServiceProvider.OpenAI, ServiceProvider.Azure];
const models = ["gpt-4o-realtime-preview-2024-10-01"];
const voice = ["alloy", "shimmer", "echo"];
export function RealtimeConfigList(props: {
realtimeConfig: RealtimeConfig;
updateConfig: (updater: (config: RealtimeConfig) => void) => void;
}) {
const azureConfigComponent = props.realtimeConfig.provider ===
ServiceProvider.Azure && (
<>
<ListItem
title={Locale.Settings.Realtime.Azure.Endpoint.Title}
subTitle={Locale.Settings.Realtime.Azure.Endpoint.SubTitle}
>
<input
value={props.realtimeConfig?.azure?.endpoint}
type="text"
placeholder={Locale.Settings.Realtime.Azure.Endpoint.Title}
onChange={(e) => {
props.updateConfig(
(config) => (config.azure.endpoint = e.currentTarget.value),
);
}}
/>
</ListItem>
<ListItem
title={Locale.Settings.Realtime.Azure.Deployment.Title}
subTitle={Locale.Settings.Realtime.Azure.Deployment.SubTitle}
>
<input
value={props.realtimeConfig?.azure?.deployment}
type="text"
placeholder={Locale.Settings.Realtime.Azure.Deployment.Title}
onChange={(e) => {
props.updateConfig(
(config) => (config.azure.deployment = e.currentTarget.value),
);
}}
/>
</ListItem>
</>
);
return (
<>
<ListItem
title={Locale.Settings.Realtime.Enable.Title}
subTitle={Locale.Settings.Realtime.Enable.SubTitle}
>
<input
type="checkbox"
checked={props.realtimeConfig.enable}
onChange={(e) =>
props.updateConfig(
(config) => (config.enable = e.currentTarget.checked),
)
}
></input>
</ListItem>
{props.realtimeConfig.enable && (
<>
<ListItem
title={Locale.Settings.Realtime.Provider.Title}
subTitle={Locale.Settings.Realtime.Provider.SubTitle}
>
<Select
aria-label={Locale.Settings.Realtime.Provider.Title}
value={props.realtimeConfig.provider}
onChange={(e) => {
props.updateConfig(
(config) =>
(config.provider = e.target.value as ServiceProvider),
);
}}
>
{providers.map((v, i) => (
<option value={v} key={i}>
{v}
</option>
))}
</Select>
</ListItem>
<ListItem
title={Locale.Settings.Realtime.Model.Title}
subTitle={Locale.Settings.Realtime.Model.SubTitle}
>
<Select
aria-label={Locale.Settings.Realtime.Model.Title}
value={props.realtimeConfig.model}
onChange={(e) => {
props.updateConfig((config) => (config.model = e.target.value));
}}
>
{models.map((v, i) => (
<option value={v} key={i}>
{v}
</option>
))}
</Select>
</ListItem>
<ListItem
title={Locale.Settings.Realtime.ApiKey.Title}
subTitle={Locale.Settings.Realtime.ApiKey.SubTitle}
>
<PasswordInput
aria={Locale.Settings.ShowPassword}
aria-label={Locale.Settings.Realtime.ApiKey.Title}
value={props.realtimeConfig.apiKey}
type="text"
placeholder={Locale.Settings.Realtime.ApiKey.Placeholder}
onChange={(e) => {
props.updateConfig(
(config) => (config.apiKey = e.currentTarget.value),
);
}}
/>
</ListItem>
{azureConfigComponent}
<ListItem
title={Locale.Settings.TTS.Voice.Title}
subTitle={Locale.Settings.TTS.Voice.SubTitle}
>
<Select
value={props.realtimeConfig.voice}
onChange={(e) => {
props.updateConfig(
(config) => (config.voice = e.currentTarget.value as Voice),
);
}}
>
{voice.map((v, i) => (
<option value={v} key={i}>
{v}
</option>
))}
</Select>
</ListItem>
<ListItem
title={Locale.Settings.Realtime.Temperature.Title}
subTitle={Locale.Settings.Realtime.Temperature.SubTitle}
>
<InputRange
aria={Locale.Settings.Temperature.Title}
value={props.realtimeConfig?.temperature?.toFixed(1)}
min="0.6"
max="1"
step="0.1"
onChange={(e) => {
props.updateConfig(
(config) =>
(config.temperature = e.currentTarget.valueAsNumber),
);
}}
></InputRange>
</ListItem>
</>
)}
</>
);
}

View File

@@ -4,6 +4,7 @@ import { Select } from "@/app/components/ui-lib";
import { IconButton } from "@/app/components/button";
import Locale from "@/app/locales";
import { useSdStore } from "@/app/store/sd";
import clsx from "clsx";
export const params = [
{
@@ -136,7 +137,7 @@ export function ControlParamItem(props: {
className?: string;
}) {
return (
<div className={styles["ctrl-param-item"] + ` ${props.className || ""}`}>
<div className={clsx(styles["ctrl-param-item"], props.className)}>
<div className={styles["ctrl-param-item-header"]}>
<div className={styles["ctrl-param-item-title"]}>
<div>

View File

@@ -36,6 +36,7 @@ import { removeImage } from "@/app/utils/chat";
import { SideBar } from "./sd-sidebar";
import { WindowContent } from "@/app/components/home";
import { params } from "./sd-panel";
import clsx from "clsx";
function getSdTaskStatus(item: any) {
let s: string;
@@ -104,7 +105,7 @@ export function Sd() {
return (
<>
<SideBar className={isSd ? homeStyles["sidebar-show"] : ""} />
<SideBar className={clsx({ [homeStyles["sidebar-show"]]: isSd })} />
<WindowContent>
<div className={chatStyles.chat} key={"1"}>
<div className="window-header" data-tauri-drag-region>
@@ -121,7 +122,10 @@ export function Sd() {
</div>
)}
<div
className={`window-header-title ${chatStyles["chat-body-title"]}`}
className={clsx(
"window-header-title",
chatStyles["chat-body-title"],
)}
>
<div className={`window-header-main-title`}>Stability AI</div>
<div className="window-header-sub-title">

View File

@@ -85,6 +85,7 @@ import { nanoid } from "nanoid";
import { useMaskStore } from "../store/mask";
import { ProviderType } from "../utils/cloud";
import { TTSConfigList } from "./tts-config";
import { RealtimeConfigList } from "./realtime-chat/realtime-config";
function EditPromptModal(props: { id: string; onClose: () => void }) {
const promptStore = usePromptStore();
@@ -1785,9 +1786,11 @@ export function Settings() {
<ListItem
title={Locale.Settings.Access.CustomModel.Title}
subTitle={Locale.Settings.Access.CustomModel.SubTitle}
vertical={true}
>
<input
aria-label={Locale.Settings.Access.CustomModel.Title}
style={{ width: "100%", maxWidth: "unset", textAlign: "left" }}
type="text"
value={config.customModels}
placeholder="model1,model2,model3"
@@ -1814,7 +1817,18 @@ export function Settings() {
{shouldShowPromptModal && (
<UserPromptModal onClose={() => setShowPromptModal(false)} />
)}
<List>
<RealtimeConfigList
realtimeConfig={config.realtimeConfig}
updateConfig={(updater) => {
const realtimeConfig = { ...config.realtimeConfig };
updater(realtimeConfig);
config.update(
(config) => (config.realtimeConfig = realtimeConfig),
);
}}
/>
</List>
<List>
<TTSConfigList
ttsConfig={config.ttsConfig}

View File

@@ -30,6 +30,7 @@ import { Link, useNavigate } from "react-router-dom";
import { isIOS, useMobileScreen } from "../utils";
import dynamic from "next/dynamic";
import { showConfirm, Selector } from "./ui-lib";
import clsx from "clsx";
const ChatList = dynamic(async () => (await import("./chat-list")).ChatList, {
loading: () => null,
@@ -141,9 +142,9 @@ export function SideBarContainer(props: {
const { children, className, onDragStart, shouldNarrow } = props;
return (
<div
className={`${styles.sidebar} ${className} ${
shouldNarrow && styles["narrow-sidebar"]
}`}
className={clsx(styles.sidebar, className, {
[styles["narrow-sidebar"]]: shouldNarrow,
})}
style={{
// #3016 disable transition on ios mobile screen
transition: isMobileScreen && isIOSMobile ? "none" : undefined,
@@ -171,9 +172,9 @@ export function SideBarHeader(props: {
return (
<Fragment>
<div
className={`${styles["sidebar-header"]} ${
shouldNarrow ? styles["sidebar-header-narrow"] : ""
}`}
className={clsx(styles["sidebar-header"], {
[styles["sidebar-header-narrow"]]: shouldNarrow,
})}
data-tauri-drag-region
>
<div className={styles["sidebar-title-container"]}>
@@ -182,7 +183,7 @@ export function SideBarHeader(props: {
</div>
<div className={styles["sidebar-sub-title"]}>{subTitle}</div>
</div>
<div className={styles["sidebar-logo"] + " no-dark"}>{logo}</div>
<div className={clsx(styles["sidebar-logo"], "no-dark")}>{logo}</div>
</div>
{children}
</Fragment>
@@ -286,7 +287,7 @@ export function SideBar(props: { className?: string }) {
<SideBarTail
primaryAction={
<>
<div className={styles["sidebar-action"] + " " + styles.mobile}>
<div className={clsx(styles["sidebar-action"], styles.mobile)}>
<IconButton
icon={<DeleteIcon />}
onClick={async () => {

View File

@@ -23,6 +23,7 @@ import React, {
useRef,
} from "react";
import { IconButton } from "./button";
import clsx from "clsx";
export function Popover(props: {
children: JSX.Element;
@@ -45,7 +46,7 @@ export function Popover(props: {
export function Card(props: { children: JSX.Element[]; className?: string }) {
return (
<div className={styles.card + " " + props.className}>{props.children}</div>
<div className={clsx(styles.card, props.className)}>{props.children}</div>
);
}
@@ -60,11 +61,13 @@ export function ListItem(props: {
}) {
return (
<div
className={
styles["list-item"] +
` ${props.vertical ? styles["vertical"] : ""} ` +
` ${props.className || ""}`
}
className={clsx(
styles["list-item"],
{
[styles["vertical"]]: props.vertical,
},
props.className,
)}
onClick={props.onClick}
>
<div className={styles["list-header"]}>
@@ -135,9 +138,9 @@ export function Modal(props: ModalProps) {
return (
<div
className={
styles["modal-container"] + ` ${isMax && styles["modal-container-max"]}`
}
className={clsx(styles["modal-container"], {
[styles["modal-container-max"]]: isMax,
})}
>
<div className={styles["modal-header"]}>
<div className={styles["modal-title"]}>{props.title}</div>
@@ -260,7 +263,7 @@ export function Input(props: InputProps) {
return (
<textarea
{...props}
className={`${styles["input"]} ${props.className}`}
className={clsx(styles["input"], props.className)}
></textarea>
);
}
@@ -301,9 +304,13 @@ export function Select(
const { className, children, align, ...otherProps } = props;
return (
<div
className={`${styles["select-with-icon"]} ${
align === "left" ? styles["left-align-option"] : ""
} ${className}`}
className={clsx(
styles["select-with-icon"],
{
[styles["left-align-option"]]: align === "left",
},
className,
)}
>
<select className={styles["select-with-icon-select"]} {...otherProps}>
{children}
@@ -509,9 +516,9 @@ export function Selector<T>(props: {
const selected = selectedValues.includes(item.value);
return (
<ListItem
className={`${styles["selector-item"]} ${
item.disable && styles["selector-item-disabled"]
}`}
className={clsx(styles["selector-item"], {
[styles["selector-item-disabled"]]: item.disable,
})}
key={i}
title={item.title}
subTitle={item.subTitle}

View File

@@ -0,0 +1 @@
export * from "./voice-print";

View File

@@ -0,0 +1,11 @@
.voice-print {
width: 100%;
height: 60px;
margin: 20px 0;
canvas {
width: 100%;
height: 100%;
filter: brightness(1.2); // 增加整体亮度
}
}

View File

@@ -0,0 +1,180 @@
import { useEffect, useRef, useCallback } from "react";
import styles from "./voice-print.module.scss";
interface VoicePrintProps {
frequencies?: Uint8Array;
isActive?: boolean;
}
export function VoicePrint({ frequencies, isActive }: VoicePrintProps) {
// Canvas引用用于获取绘图上下文
const canvasRef = useRef<HTMLCanvasElement>(null);
// 存储历史频率数据,用于平滑处理
const historyRef = useRef<number[][]>([]);
// 控制保留的历史数据帧数,影响平滑度
const historyLengthRef = useRef(10);
// 存储动画帧ID用于清理
const animationFrameRef = useRef<number>();
/**
* 更新频率历史数据
* 使用FIFO队列维护固定长度的历史记录
*/
const updateHistory = useCallback((freqArray: number[]) => {
historyRef.current.push(freqArray);
if (historyRef.current.length > historyLengthRef.current) {
historyRef.current.shift();
}
}, []);
useEffect(() => {
const canvas = canvasRef.current;
if (!canvas) return;
const ctx = canvas.getContext("2d");
if (!ctx) return;
/**
* 处理高DPI屏幕显示
* 根据设备像素比例调整canvas实际渲染分辨率
*/
const dpr = window.devicePixelRatio || 1;
canvas.width = canvas.offsetWidth * dpr;
canvas.height = canvas.offsetHeight * dpr;
ctx.scale(dpr, dpr);
/**
* 主要绘制函数
* 使用requestAnimationFrame实现平滑动画
* 包含以下步骤:
* 1. 清空画布
* 2. 更新历史数据
* 3. 计算波形点
* 4. 绘制上下对称的声纹
*/
const draw = () => {
// 清空画布
ctx.clearRect(0, 0, canvas.width, canvas.height);
if (!frequencies || !isActive) {
historyRef.current = [];
return;
}
const freqArray = Array.from(frequencies);
updateHistory(freqArray);
// 绘制声纹
const points: [number, number][] = [];
const centerY = canvas.height / 2;
const width = canvas.width;
const sliceWidth = width / (frequencies.length - 1);
// 绘制主波形
ctx.beginPath();
ctx.moveTo(0, centerY);
/**
* 声纹绘制算法:
* 1. 使用历史数据平均值实现平滑过渡
* 2. 通过正弦函数添加自然波动
* 3. 使用贝塞尔曲线连接点,使曲线更平滑
* 4. 绘制对称部分形成完整声纹
*/
for (let i = 0; i < frequencies.length; i++) {
const x = i * sliceWidth;
let avgFrequency = frequencies[i];
/**
* 波形平滑处理:
* 1. 收集历史数据中对应位置的频率值
* 2. 计算当前值与历史值的加权平均
* 3. 根据平均值计算实际显示高度
*/
if (historyRef.current.length > 0) {
const historicalValues = historyRef.current.map((h) => h[i] || 0);
avgFrequency =
(avgFrequency + historicalValues.reduce((a, b) => a + b, 0)) /
(historyRef.current.length + 1);
}
/**
* 波形变换:
* 1. 归一化频率值到0-1范围
* 2. 添加时间相关的正弦变换
* 3. 使用贝塞尔曲线平滑连接点
*/
const normalized = avgFrequency / 255.0;
const height = normalized * (canvas.height / 2);
const y = centerY + height * Math.sin(i * 0.2 + Date.now() * 0.002);
points.push([x, y]);
if (i === 0) {
ctx.moveTo(x, y);
} else {
// 使用贝塞尔曲线使波形更平滑
const prevPoint = points[i - 1];
const midX = (prevPoint[0] + x) / 2;
ctx.quadraticCurveTo(
prevPoint[0],
prevPoint[1],
midX,
(prevPoint[1] + y) / 2,
);
}
}
// 绘制对称的下半部分
for (let i = points.length - 1; i >= 0; i--) {
const [x, y] = points[i];
const symmetricY = centerY - (y - centerY);
if (i === points.length - 1) {
ctx.lineTo(x, symmetricY);
} else {
const nextPoint = points[i + 1];
const midX = (nextPoint[0] + x) / 2;
ctx.quadraticCurveTo(
nextPoint[0],
centerY - (nextPoint[1] - centerY),
midX,
centerY - ((nextPoint[1] + y) / 2 - centerY),
);
}
}
ctx.closePath();
/**
* 渐变效果:
* 从左到右应用三色渐变,带透明度
* 使用蓝色系配色提升视觉效果
*/
const gradient = ctx.createLinearGradient(0, 0, canvas.width, 0);
gradient.addColorStop(0, "rgba(100, 180, 255, 0.95)");
gradient.addColorStop(0.5, "rgba(140, 200, 255, 0.9)");
gradient.addColorStop(1, "rgba(180, 220, 255, 0.95)");
ctx.fillStyle = gradient;
ctx.fill();
animationFrameRef.current = requestAnimationFrame(draw);
};
// 启动动画循环
draw();
// 清理函数:在组件卸载时取消动画
return () => {
if (animationFrameRef.current) {
cancelAnimationFrame(animationFrameRef.current);
}
};
}, [frequencies, isActive, updateHistory]);
return (
<div className={styles["voice-print"]}>
<canvas ref={canvasRef} />
</div>
);
}