333 lines
9.1 KiB
TypeScript
333 lines
9.1 KiB
TypeScript
import { ChatEventType, RoleType } from "@coze/api";
|
|
import {
|
|
EventNames,
|
|
RealtimeAPIError,
|
|
RealtimeClient,
|
|
RealtimeError,
|
|
RealtimeUtils,
|
|
} from "@coze/realtime-api";
|
|
import {
|
|
createContext,
|
|
ReactNode,
|
|
useCallback,
|
|
useContext,
|
|
useEffect,
|
|
useRef,
|
|
useState,
|
|
} from "react";
|
|
import { useToast } from "@/hooks/use-toast";
|
|
|
|
type RoomInfo = {
|
|
appId: string;
|
|
roomId: string;
|
|
token: string;
|
|
uid: string;
|
|
};
|
|
|
|
export const RealtimeClientContext = createContext<{
|
|
client: RealtimeClient | null;
|
|
isConnecting: boolean;
|
|
isConnected: boolean;
|
|
audioEnabled: boolean;
|
|
isSupportVideo: boolean;
|
|
messageList: { content: string; role: RoleType }[];
|
|
isAiTalking: boolean;
|
|
roomInfo: RoomInfo | null;
|
|
initClient: () => void;
|
|
handleConnect: () => void;
|
|
handleInterrupt: () => void;
|
|
handleDisconnect: () => void;
|
|
toggleMicrophone: () => void;
|
|
sendUserMessageWithText: (message: string) => void;
|
|
setInitMessage: (message: string) => void;
|
|
}>({
|
|
client: null,
|
|
isConnecting: false,
|
|
isConnected: false,
|
|
audioEnabled: true,
|
|
isSupportVideo: false,
|
|
messageList: [],
|
|
isAiTalking: false,
|
|
roomInfo: null,
|
|
initClient: () => {},
|
|
handleConnect: () => {},
|
|
handleInterrupt: () => {},
|
|
handleDisconnect: () => {},
|
|
toggleMicrophone: () => {},
|
|
sendUserMessageWithText: () => {},
|
|
setInitMessage: () => {},
|
|
});
|
|
|
|
// 添加自定义hook
|
|
export const useRealtimeClient = () => {
|
|
const context = useContext(RealtimeClientContext);
|
|
|
|
if (context === undefined) {
|
|
throw new Error("useRealtimeClient必须在RealtimeClientProvider内部使用");
|
|
}
|
|
|
|
return { ...context };
|
|
};
|
|
|
|
export const RealtimeClientProvider = ({
|
|
children,
|
|
}: {
|
|
children: ReactNode;
|
|
}) => {
|
|
const token = import.meta.env.VITE_COZE_TOKEN;
|
|
const botId = import.meta.env.VITE_COZE_BOT_ID;
|
|
const voiceId = import.meta.env.VITE_COZE_VOICE_ID;
|
|
const connectorId = "1024";
|
|
|
|
const clientRef = useRef<RealtimeClient | null>(null);
|
|
// 实时语音回复消息列表
|
|
const [messageList, setMessageList] = useState<
|
|
{ content: string; role: RoleType }[]
|
|
>([]);
|
|
// 是否正在连接
|
|
const [isConnecting, setIsConnecting] = useState(false);
|
|
// 是否已连接
|
|
const [isConnected, setIsConnected] = useState(false);
|
|
// 是否开启麦克风
|
|
const [audioEnabled, setAudioEnabled] = useState(false);
|
|
// 是否支持视频
|
|
const [isSupportVideo] = useState(false);
|
|
// 是否正在说话
|
|
const [isAiTalking, setIsAiTalking] = useState(false);
|
|
|
|
const [initMessage, setInitMessage] = useState("");
|
|
const [isClientInitialized, setIsClientInitialized] = useState(false);
|
|
const [roomInfo, setRoomInfo] = useState<RoomInfo | null>(null);
|
|
|
|
const { toast } = useToast();
|
|
|
|
const initClient = async () => {
|
|
const permission = await RealtimeUtils.checkDevicePermission(false);
|
|
if (!permission.audio) {
|
|
toast({
|
|
title: "连接错误",
|
|
description: "需要麦克风访问权限",
|
|
});
|
|
throw new Error("需要麦克风访问权限");
|
|
}else{
|
|
const client = new RealtimeClient({
|
|
accessToken: token,
|
|
botId: botId,
|
|
voiceId: voiceId,
|
|
connectorId: connectorId,
|
|
allowPersonalAccessTokenInBrowser: true, // 可选：允许在浏览器中使用个人访问令牌
|
|
});
|
|
|
|
clientRef.current = client;
|
|
setupEventListeners(client);
|
|
setIsClientInitialized(true);
|
|
}
|
|
};
|
|
|
|
useEffect(() => {
|
|
if (clientRef.current) {
|
|
setupMessageEventListeners(clientRef.current);
|
|
if (initMessage) {
|
|
setupInitMessageEventListener(clientRef.current);
|
|
}
|
|
}
|
|
}, [initMessage, isClientInitialized]);
|
|
|
|
const handleConnect = async () => {
|
|
try {
|
|
if (!clientRef.current) {
|
|
await initClient();
|
|
}
|
|
|
|
await clientRef.current?.connect();
|
|
} catch (error) {
|
|
console.error(error);
|
|
if (error instanceof RealtimeAPIError) {
|
|
switch (error.code) {
|
|
case RealtimeError.CREATE_ROOM_ERROR:
|
|
console.error(`创建房间失败: ${error.message}`);
|
|
break;
|
|
case RealtimeError.CONNECTION_ERROR:
|
|
console.error(`加入房间失败: ${error.message}`);
|
|
break;
|
|
case RealtimeError.DEVICE_ACCESS_ERROR:
|
|
console.error(`获取设备失败: ${error.message}`);
|
|
break;
|
|
default:
|
|
console.error(`连接错误: ${error.message}`);
|
|
}
|
|
} else {
|
|
console.error("连接错误：" + error);
|
|
}
|
|
}
|
|
};
|
|
|
|
const handleInterrupt = () => {
|
|
try {
|
|
clientRef.current?.interrupt();
|
|
} catch (error) {
|
|
console.error("打断失败：" + error);
|
|
}
|
|
};
|
|
|
|
const handleDisconnect = () => {
|
|
try {
|
|
// 关闭客户的时候清除一些信息
|
|
setIsAiTalking(false);
|
|
setIsClientInitialized(false);
|
|
setMessageList([]);
|
|
|
|
clientRef.current?.disconnect();
|
|
clientRef.current?.clearEventHandlers();
|
|
clientRef.current = null;
|
|
setIsConnected(false);
|
|
} catch (error) {
|
|
console.error("断开失败：" + error);
|
|
}
|
|
};
|
|
|
|
const toggleMicrophone = async () => {
|
|
try {
|
|
await clientRef.current?.setAudioEnable(!audioEnabled);
|
|
setAudioEnabled(!audioEnabled);
|
|
} catch (error) {
|
|
console.error("切换麦克风状态失败：" + error);
|
|
}
|
|
};
|
|
|
|
const setupInitMessageEventListener = (client: RealtimeClient) => {
|
|
client.on(EventNames.ALL_SERVER, (eventName, _event: any) => {
|
|
if (eventName === "server.session.created") {
|
|
// 这里需要加个 server. 前缀
|
|
sendUserMessageWithText(initMessage);
|
|
}
|
|
});
|
|
};
|
|
|
|
const setupMessageEventListeners = (client: RealtimeClient) => {
|
|
let lastEvent: any;
|
|
client.on(EventNames.ALL, (_eventName, event: any) => {
|
|
// AI智能体设置
|
|
|
|
if (
|
|
event.event_type !== ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
|
event.event_type !== ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&
|
|
event.event_type !== "conversation.created"
|
|
) {
|
|
return;
|
|
}
|
|
const content = event.data.content;
|
|
setMessageList((prev) => {
|
|
// 如果上一个事件是增量更新，则附加到最后一条消息
|
|
if (
|
|
lastEvent?.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
|
(event.data.type === "answer" || event.data.type === "question")
|
|
) {
|
|
return [
|
|
...prev.slice(0, -1),
|
|
{
|
|
content: prev[prev.length - 1].content + content,
|
|
role: prev[prev.length - 1].role,
|
|
},
|
|
];
|
|
}
|
|
|
|
// 添加AI的欢迎语
|
|
if (initMessage === "" && event.event_type === "conversation.created") {
|
|
return [
|
|
...prev,
|
|
{ content: event.data.prologue, role: RoleType.Assistant },
|
|
];
|
|
}
|
|
|
|
// 否则添加新消息
|
|
if (
|
|
(content !== "" &&
|
|
event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA) ||
|
|
(event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&
|
|
(event.data.type === "answer" || event.data.type === "question") &&
|
|
event.data.role !== RoleType.Assistant)
|
|
) {
|
|
return [...prev, { content: content, role: event.data.role }];
|
|
}
|
|
return prev;
|
|
});
|
|
lastEvent = event;
|
|
});
|
|
};
|
|
|
|
// 设置事件监听器
|
|
const setupEventListeners = useCallback(
|
|
(client: RealtimeClient) => {
|
|
// 监听 AI 开始说话事件
|
|
client.on(EventNames.AUDIO_AGENT_SPEECH_STARTED, () => {
|
|
// console.log("AI开始说话");
|
|
setIsAiTalking(true);
|
|
setAudioEnabled(false);
|
|
});
|
|
|
|
// 监听 AI 结束说话事件
|
|
client.on(EventNames.AUDIO_AGENT_SPEECH_STOPPED, () => {
|
|
// console.log("AI结束说话");
|
|
setIsAiTalking(false);
|
|
setAudioEnabled(true);
|
|
});
|
|
|
|
// 监听连接客户端
|
|
client.on(EventNames.CONNECTING, () => {
|
|
setIsConnecting(true);
|
|
setIsConnected(false);
|
|
});
|
|
|
|
// 客户端连接成功
|
|
client.on(EventNames.CONNECTED, (_eventName: string, event: any) => {
|
|
setRoomInfo(event);
|
|
setIsConnecting(false);
|
|
setIsConnected(true);
|
|
});
|
|
},
|
|
[clientRef.current, initMessage]
|
|
);
|
|
|
|
// 发送信息
|
|
const sendUserMessageWithText = async (message: string) => {
|
|
try {
|
|
await clientRef.current?.sendMessage({
|
|
id: "",
|
|
event_type: "conversation.message.create",
|
|
data: {
|
|
role: "user",
|
|
content_type: "text",
|
|
content: message,
|
|
},
|
|
});
|
|
} catch (error) {
|
|
console.error("发送消息失败：" + error);
|
|
}
|
|
};
|
|
|
|
return (
|
|
<RealtimeClientContext.Provider
|
|
value={{
|
|
client: clientRef.current,
|
|
isConnecting,
|
|
isConnected,
|
|
audioEnabled,
|
|
isSupportVideo,
|
|
messageList,
|
|
isAiTalking,
|
|
roomInfo,
|
|
initClient,
|
|
handleConnect,
|
|
handleInterrupt,
|
|
handleDisconnect,
|
|
toggleMicrophone,
|
|
sendUserMessageWithText,
|
|
setInitMessage,
|
|
}}
|
|
>
|
|
{children}
|
|
</RealtimeClientContext.Provider>
|
|
);
|
|
};
|