import { ChatEventType, RoleType } from "@coze/api"; import { EventNames, RealtimeAPIError, RealtimeClient, RealtimeError, RealtimeUtils, } from "@coze/realtime-api"; import { createContext, ReactNode, useCallback, useContext, useRef, useState, } from "react"; import { useToast } from "@/hooks/use-toast"; type RoomInfo = { appId: string; roomId: string; token: string; uid: string; }; export const RealtimeClientContext = createContext<{ client: RealtimeClient | null; isConnecting: boolean; isConnected: boolean; audioEnabled: boolean; isSupportVideo: boolean; messageList: { content: string; role: RoleType }[]; isAiTalking: boolean; roomInfo: RoomInfo | null; initClient: (initMessage?:string) => void; handleConnect: (initMessage?:string) => Promise; handleInterrupt: () => void; handleDisconnect: () => void; toggleMicrophone: () => void; }>({ client: null, isConnecting: false, isConnected: false, audioEnabled: true, isSupportVideo: false, messageList: [], isAiTalking: false, roomInfo: null, initClient: () => {}, handleConnect: () => Promise.resolve(), handleInterrupt: () => {}, handleDisconnect: () => {}, toggleMicrophone: () => {}, }); // 添加自定义hook export const useRealtimeClient = () => { const context = useContext(RealtimeClientContext); if (context === undefined) { throw new Error("useRealtimeClient必须在RealtimeClientProvider内部使用"); } return { ...context }; }; export const RealtimeClientProvider = ({ children, }: { children: ReactNode; }) => { const token = import.meta.env.VITE_COZE_TOKEN; const botId = import.meta.env.VITE_COZE_BOT_ID; const voiceId = import.meta.env.VITE_COZE_VOICE_ID; const connectorId = "1024"; const clientRef = useRef(null); // 实时语音回复消息列表 const [messageList, setMessageList] = useState< { content: string; role: RoleType }[] >([]); // 是否正在连接 const [isConnecting, setIsConnecting] = useState(false); // 是否已连接 const [isConnected, setIsConnected] = useState(false); // 是否开启麦克风 const [audioEnabled, setAudioEnabled] = useState(true); // 是否支持视频 const [isSupportVideo] = useState(false); // 是否正在说话 const [isAiTalking, setIsAiTalking] = useState(false); const [roomInfo, setRoomInfo] = useState(null); const { toast } = useToast(); const initClient = async (_initMessage?:string) => { const permission = await RealtimeUtils.checkDevicePermission(false); const device = await RealtimeUtils.getAudioDevices(); if (!permission.audio) { toast({ title: "连接错误", description: "需要麦克风访问权限", }); throw new Error("需要麦克风访问权限"); } if (device.audioInputs.length === 0) { toast({ title: "连接错误", description: "没有麦克风设备", }); throw new Error("没有麦克风设备"); } const client = new RealtimeClient({ accessToken: token, botId: botId, voiceId: voiceId, connectorId: connectorId, allowPersonalAccessTokenInBrowser: true, // 可选:允许在浏览器中使用个人访问令牌 }); clientRef.current = client; setupEventListeners(client); setupMessageEventListeners(client,_initMessage ?? ''); setupInitMessageEventListener(client,_initMessage) }; const handleConnect = async (initMessage?:string) => { try { if (!clientRef.current) { await initClient(initMessage); } await clientRef.current?.connect(); await toggleMicrophone(); } catch (error) { console.error(error); if (error instanceof RealtimeAPIError) { switch (error.code) { case RealtimeError.CREATE_ROOM_ERROR: console.error(`创建房间失败: ${error.message}`); break; case RealtimeError.CONNECTION_ERROR: console.error(`加入房间失败: ${error.message}`); break; case RealtimeError.DEVICE_ACCESS_ERROR: console.error(`获取设备失败: ${error.message}`); break; default: console.error(`连接错误: ${error.message}`); } } else { console.error("连接错误:" + error); } } }; const handleInterrupt = () => { try { clientRef.current?.interrupt(); } catch (error) { console.error("打断失败:" + error); } }; const handleDisconnect = async() => { try { // 关闭客户的时候清除一些信息 setIsAiTalking(false); setMessageList([]); await clientRef.current?.setAudioEnable(false); setAudioEnabled(false); clientRef.current?.disconnect(); clientRef.current?.clearEventHandlers(); clientRef.current = null; setIsConnected(false); } catch (error) { console.error("断开失败:" + error); } }; const toggleMicrophone = async () => { try { await clientRef.current?.setAudioEnable(!audioEnabled); setAudioEnabled(!audioEnabled); } catch (error) { console.error("切换麦克风状态失败:" + error); } }; const setupInitMessageEventListener = useCallback((client: RealtimeClient,_initMessage?:string) => { client.on(EventNames.ALL_SERVER, async(eventName, _event: any) => { if (eventName === "server.session.created") { await client.sendMessage({ id:'', "event_type":"session.update", data:{ chat_config:{ allow_voice_interrupt:false } } }) } if(eventName === "server.bot.join" && _initMessage){ // 这里需要加个 server. 前缀 await clientRef.current?.sendMessage({ id: "", event_type: "conversation.message.create", data: { role: "user", content_type: "text", content: _initMessage, }, }); } }); },[clientRef.current]); const setupMessageEventListeners = (client: RealtimeClient,_initMessage:string) => { let lastEvent: any; client.on(EventNames.ALL, (_eventName, event: any) => { // AI智能体设置 if ( event.event_type !== ChatEventType.CONVERSATION_MESSAGE_DELTA && event.event_type !== ChatEventType.CONVERSATION_MESSAGE_COMPLETED && event.event_type !== "conversation.created" ) { return; } const content = event.data.content; setMessageList((prev) => { // 如果上一个事件是增量更新,则附加到最后一条消息 if ( lastEvent?.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA && (event.data.type === "answer" || event.data.type === "question") ) { return [ ...prev.slice(0, -1), { content: prev[prev.length - 1].content + content, role: prev[prev.length - 1].role, }, ]; } // 添加AI的欢迎语 if (_initMessage === "" && event.event_type === "conversation.created") { return [ ...prev, { content: event.data.prologue, role: RoleType.Assistant }, ]; } // 否则添加新消息 if ( (content !== "" && event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA) || (event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED && (event.data.type === "answer" || event.data.type === "question") && event.data.role !== RoleType.Assistant) ) { return [...prev, { content: content, role: event.data.role }]; } return prev; }); lastEvent = event; }); }; // 设置事件监听器 const setupEventListeners = useCallback( (client: RealtimeClient) => { // 监听 AI 开始说话事件 client.on(EventNames.AUDIO_AGENT_SPEECH_STARTED, async() => { // console.log("AI开始说话"); setIsAiTalking(true); await clientRef.current?.setAudioEnable(false); setAudioEnabled(false); }); // 监听 AI 结束说话事件 client.on(EventNames.AUDIO_AGENT_SPEECH_STOPPED, async() => { // console.log("AI结束说话"); setIsAiTalking(false); await clientRef.current?.setAudioEnable(true); setAudioEnabled(true); }); // 监听连接客户端 client.on(EventNames.CONNECTING, () => { setIsConnecting(true); setIsConnected(false); }); // 客户端连接成功 client.on(EventNames.CONNECTED, (_eventName: string, event: any) => { setRoomInfo(event); setIsConnecting(false); setIsConnected(true); }); }, [clientRef.current] ); // 发送信息 // const sendUserMessageWithText = async (message: string) => { // try { // await clientRef.current?.sendMessage({ // id: "", // event_type: "conversation.message.create", // data: { // role: "user", // content_type: "text", // content: message, // }, // }); // } catch (error) { // console.error("发送消息失败:" + error); // } // }; return ( {children} ); };