import { ChatEventType, RoleType } from "@coze/api"; import { EventNames, RealtimeAPIError, RealtimeClient, RealtimeUtils, } from "@coze/realtime-api"; import { createContext, ReactNode, useCallback, useContext, useRef, useState, } from "react"; import { useToast } from "@/hooks/use-toast"; import { FileParser, FileParseStatus } from "./FileParser"; type RoomInfo = { appId: string; roomId: string; token: string; uid: string; }; export type FileInfo = { type: string; url: string; tableName: string; provinceName: string; subjectClaim: string; }; export const RealtimeClientContext = createContext<{ client: RealtimeClient | null; isConnecting: boolean; isConnected: boolean; audioEnabled: boolean; isSupportVideo: boolean; messageList: { content: string; role: RoleType; event?: any; fileInfo?: FileInfo; fileParseStatus?: number; }[]; isAiTalking: boolean; roomInfo: RoomInfo | null; initClient: (opts: { initMessage?: string; fileInfo?: FileInfo }) => void; handleConnect: (opts: { initMessage?: string; fileInfo?: FileInfo }) => Promise; handleInterrupt: () => void; handleDisconnect: () => void; toggleMicrophone: () => void; }>(/* 默认值省略 */ null!); export const useRealtimeClient = () => { const ctx = useContext(RealtimeClientContext); if (!ctx) throw new Error("useRealtimeClient 必须在 RealtimeClientProvider 内部使用"); return ctx; }; export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => { const token = import.meta.env.VITE_COZE_TOKEN; const botId = import.meta.env.VITE_COZE_BOT_ID; const voiceId = import.meta.env.VITE_COZE_VOICE_ID; const connectorId = "1024"; const clientRef = useRef(null); const connectingLockRef = useRef(false); const [messageList, setMessageList] = useState<{ content: string; role: RoleType; event?: any; fileInfo?: FileInfo; fileParseStatus?: number; }[]>([]); const [isConnecting, setIsConnecting] = useState(false); const [isConnected, setIsConnected] = useState(false); const [audioEnabled, setAudioEnabled] = useState(true); const [isSupportVideo] = useState(false); const [isAiTalking, setIsAiTalking] = useState(false); const [roomInfo, setRoomInfo] = useState(null); // 引入状态机 const fileParseStatusRef = useRef(-1); const fileParserRef = useRef( new FileParser((newStatus:any) => { fileParseStatusRef.current = newStatus; // 根据不同状态更新 messageList if (newStatus === 0) { appendAssistantMessage("AI正在解析您的文档..."); } else if (newStatus === 1) { replaceLastAssistantMessage("AI正在调用插件"); } else if (newStatus === 2) { replaceLastAssistantMessage("文档解析完成"); } }) ); const { toast } = useToast(); /** Helpers */ const appendAssistantMessage = (content: string) => { setMessageList(prev => { return [ ...prev, { content, role: RoleType.Assistant } ] }); }; const replaceLastAssistantMessage = (content: string) => { setMessageList(prev => { return [ ...prev.slice(0, -1), { content, role: RoleType.Assistant } ] }); }; /** 初始化客户端并设置监听 */ const initClient = async ({ initMessage, fileInfo, }: { initMessage?: string; fileInfo?: FileInfo; }) => { const perm = await RealtimeUtils.checkDevicePermission(false); const device = await RealtimeUtils.getAudioDevices(); if (!perm.audio) { toast({ title: "连接错误", description: "需要麦克风访问权限" }); throw new Error("需要麦克风访问权限"); } if (device.audioInputs.length === 0) { toast({ title: "连接错误", description: "没有麦克风设备" }); throw new Error("没有麦克风设备"); } const client = new RealtimeClient({ accessToken: token, botId, voiceId, connectorId, allowPersonalAccessTokenInBrowser: true, suppressStationaryNoise: true, suppressNonStationaryNoise: true, debug: true, }); clientRef.current = client; setupEventListeners(client); setupMessageEventListeners(client, { initMessage, fileInfo }); setupInitMessageEventListener(client, { initMessage, fileInfo }); }; /** 连接房间 */ const handleConnect = async ({ initMessage, fileInfo, }: { initMessage?: string; fileInfo?: FileInfo; }) => { if (connectingLockRef.current) return; connectingLockRef.current = true; if (isConnected || isConnecting) { connectingLockRef.current = false; return; } try { if (!clientRef.current) { await initClient({ initMessage, fileInfo }); } await clientRef.current!.connect(); await clientRef.current!.setAudioEnable(false); setAudioEnabled(false); } catch (error: any) { if (error instanceof RealtimeAPIError) { console.error(`连接错误 (${error.code}): ${error.message}`); } else { console.error("连接错误:" + error); } } finally { connectingLockRef.current = false; } }; const handleInterrupt = () => { clientRef.current?.interrupt(); }; const handleDisconnect = async () => { setIsAiTalking(false); setMessageList([]); await clientRef.current?.setAudioEnable(false); setAudioEnabled(false); await clientRef.current?.disconnect(); clientRef.current?.clearEventHandlers(); clientRef.current = null; setIsConnected(false); }; const toggleMicrophone = async () => { await clientRef.current?.setAudioEnable(!audioEnabled); setAudioEnabled(!audioEnabled); }; /** 首条初始化消息(session.create & bot.join) */ const setupInitMessageEventListener = useCallback( ( client: RealtimeClient, { initMessage, fileInfo }: { initMessage?: string; fileInfo?: FileInfo } ) => { client.on(EventNames.ALL_SERVER, async (eventName, event: any) => { if (eventName === "server.session.created") { await client.sendMessage({ id: "", event_type: "session.update", data: { chat_config: { allow_voice_interrupt: false }, turn_detection: { silence_duration_ms: 2000 }, }, }); } if (eventName === "server.bot.join") { if (initMessage) { await clientRef.current!.sendMessage({ id: "", event_type: "conversation.message.create", data: { role: "user", content_type: "text", content: initMessage, }, }); } else if (fileInfo) { await clientRef.current!.sendMessage({ id: "", event_type: "conversation.message.create", data: { role: "user", content_type: "object_string", content: JSON.stringify([ { type: "text", text: "帮我解读这个文件,结合当下的专业行情以及对该专业未来的发展趋势,简介的给出志愿建议" }, { type: "image", file_url: fileInfo.url }, ]), }, }); } } }); }, [] ); /** 消息及文件解析监听 */ const setupMessageEventListeners = ( client: RealtimeClient, opts: { initMessage?: string; fileInfo?: FileInfo } ) => { client.on(EventNames.ALL, (_eventName, event: any) => { // 交给状态机处理解析流程 fileParserRef.current.handleEvent(_eventName, event); // 普通消息流处理 if ( event.event_type !== ChatEventType.CONVERSATION_MESSAGE_DELTA && event.event_type !== ChatEventType.CONVERSATION_MESSAGE_COMPLETED ) { // 处理conversation.created事件 if (event.event_type === "conversation.created" && !opts.initMessage && !opts.fileInfo) { setMessageList(prev => [ ...prev, { content: event.data.prologue, role: RoleType.Assistant, event } ]); } return; } // 如果是assistant的completed消息或verbose类型,直接返回 if ( (event.data.role === "assistant" && event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&event.data.type === "verbose" ) || event.data.type === "answer" && event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED ) { return; } // 如果没有fileInfo,过滤掉function_call和tool_response类型的消息 if (!opts.fileInfo && (event.data.type === "function_call" || event.data.type === "tool_response")) { return; } const content = event.data.content; setMessageList(prev => { // 合并增量 console.log("合并增量",prev); console.log("信息",content); // 处理工具回调结果 try { const parsedContent = JSON.parse(content); if (parsedContent.msg_type === "time_capsule_recall" || (parsedContent.name && parsedContent.arguments)) { // 如果没有fileInfo,不创建假信息 if (!opts.fileInfo) { return prev; } // 检查是否已存在工具回调消息 const existingToolMessageIndex = prev.findIndex(msg => msg.content === "正在处理您的请求..." && msg.fileInfo === opts.fileInfo ); if (existingToolMessageIndex !== -1) { // 更新已存在的消息的fileParseStatus const newStatus = parsedContent.msg_type === "time_capsule_recall" ? 0 : 2; return [ ...prev.slice(0, existingToolMessageIndex), { ...prev[existingToolMessageIndex], fileParseStatus: newStatus }, ...prev.slice(existingToolMessageIndex + 1) ]; } else { // 创建新的工具回调消息 return [ ...prev, { content: "正在处理您的请求...", role: event.data.role, event, fileInfo: opts.fileInfo, fileParseStatus: parsedContent.msg_type === "time_capsule_recall" ? 0 : 2 }, ]; } } } catch (e) { // 如果不是JSON格式,继续正常处理 } if ( prev.length > 0 && prev[prev.length - 1].event?.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA && event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA && prev[prev.length - 1].event.data.answer_id === event.data.answer_id ) { return [ ...prev.slice(0, -1), { content: prev[prev.length - 1].content + content, role: prev[prev.length - 1].role, event, }, ]; } // 新消息追加 return [ ...prev, { content, role: event.data.role, event }, ]; }); }); }; /** 基本连接状态 & 语音事件监听 */ const setupEventListeners = useCallback( (client: RealtimeClient) => { client.on(EventNames.AUDIO_AGENT_SPEECH_STARTED, async () => { setIsAiTalking(true); await clientRef.current?.setAudioEnable(false); setAudioEnabled(false); }); client.on(EventNames.AUDIO_AGENT_SPEECH_STOPPED, async () => { setIsAiTalking(false); await clientRef.current?.setAudioEnable(true); setAudioEnabled(true); }); client.on(EventNames.CONNECTING, () => { setIsConnecting(true); setIsConnected(false); }); client.on(EventNames.CONNECTED, (_name, evt) => { setRoomInfo(evt as RoomInfo); setIsConnecting(false); setIsConnected(true); }); client.on(EventNames.ALL_SERVER, (name, evt) => { // 其它全局服务端事件可在此处理 }); }, [] ); return ( {children} ); };