From 207f28c72841c378de2cb13b0fccaf3e9766df15 Mon Sep 17 00:00:00 2001 From: xjs Date: Wed, 21 May 2025 11:33:36 +0800 Subject: [PATCH] =?UTF-8?q?fix:=20=E4=BF=AE=E5=A4=8D=E6=8F=92=E4=BB=B6?= =?UTF-8?q?=E8=B0=83=E7=94=A8=E7=9A=84=E5=A4=84=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Provider/RealtimeClientProvider.tsx | 246 ++++++++---------- 1 file changed, 106 insertions(+), 140 deletions(-) diff --git a/src/components/Provider/RealtimeClientProvider.tsx b/src/components/Provider/RealtimeClientProvider.tsx index 3308224..0a6a8f4 100644 --- a/src/components/Provider/RealtimeClientProvider.tsx +++ b/src/components/Provider/RealtimeClientProvider.tsx @@ -14,7 +14,6 @@ import { useState, } from "react"; import { useToast } from "@/hooks/use-toast"; -import { FileParser, FileParseStatus } from "./fileParser"; type RoomInfo = { appId: string; @@ -37,8 +36,8 @@ export const RealtimeClientContext = createContext<{ isConnected: boolean; audioEnabled: boolean; isSupportVideo: boolean; - messageList: { - content: string; + messageList: { + content: string; role: RoleType; event?: any; fileInfo?: FileInfo; @@ -48,7 +47,10 @@ export const RealtimeClientContext = createContext<{ roomInfo: RoomInfo | null; initClient: (opts: { initMessage?: string; fileInfo?: FileInfo }) => void; - handleConnect: (opts: { initMessage?: string; fileInfo?: FileInfo }) => Promise; + handleConnect: (opts: { + initMessage?: string; + fileInfo?: FileInfo; + }) => Promise; handleInterrupt: () => void; handleDisconnect: () => void; toggleMicrophone: () => void; @@ -56,11 +58,16 @@ export const RealtimeClientContext = createContext<{ export const useRealtimeClient = () => { const ctx = useContext(RealtimeClientContext); - if (!ctx) throw new Error("useRealtimeClient 必须在 RealtimeClientProvider 内部使用"); + if (!ctx) + throw new Error("useRealtimeClient 必须在 RealtimeClientProvider 内部使用"); return ctx; }; -export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => { +export const RealtimeClientProvider = ({ + children, +}: { + children: ReactNode; +}) => { const token = import.meta.env.VITE_COZE_TOKEN; const botId = import.meta.env.VITE_COZE_BOT_ID; const voiceId = import.meta.env.VITE_COZE_VOICE_ID; @@ -69,13 +76,15 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => const clientRef = useRef(null); const connectingLockRef = useRef(false); - const [messageList, setMessageList] = useState<{ - content: string; - role: RoleType; - event?: any; - fileInfo?: FileInfo; - fileParseStatus?: number; - }[]>([]); + const [messageList, setMessageList] = useState< + { + content: string; + role: RoleType; + event?: any; + fileInfo?: FileInfo; + fileParseStatus?: number; + }[] + >([]); const [isConnecting, setIsConnecting] = useState(false); const [isConnected, setIsConnected] = useState(false); const [audioEnabled, setAudioEnabled] = useState(true); @@ -83,42 +92,8 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => const [isAiTalking, setIsAiTalking] = useState(false); const [roomInfo, setRoomInfo] = useState(null); - // 引入状态机 - const fileParseStatusRef = useRef(-1); - const fileParserRef = useRef( - new FileParser((newStatus:any) => { - fileParseStatusRef.current = newStatus; - // 根据不同状态更新 messageList - if (newStatus === 0) { - appendAssistantMessage("AI正在解析您的文档..."); - } else if (newStatus === 1) { - replaceLastAssistantMessage("AI正在调用插件"); - } else if (newStatus === 2) { - replaceLastAssistantMessage("文档解析完成"); - } - }) - ); - const { toast } = useToast(); - /** Helpers */ - const appendAssistantMessage = (content: string) => { - setMessageList(prev => { - return [ - ...prev, - { content, role: RoleType.Assistant } - ] - }); - }; - const replaceLastAssistantMessage = (content: string) => { - setMessageList(prev => { - return [ - ...prev.slice(0, -1), - { content, role: RoleType.Assistant } - ] - }); - }; - /** 初始化客户端并设置监听 */ const initClient = async ({ initMessage, @@ -147,7 +122,7 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => allowPersonalAccessTokenInBrowser: true, suppressStationaryNoise: true, suppressNonStationaryNoise: true, - debug: true, + debug: false, }); clientRef.current = client; @@ -227,7 +202,7 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => }, }); } - if (eventName === "server.bot.join") { + if (eventName === "server.bot.join") { if (initMessage) { await clientRef.current!.sendMessage({ id: "", @@ -246,7 +221,10 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => role: "user", content_type: "object_string", content: JSON.stringify([ - { type: "text", text: "帮我解读这个文件,结合当下的专业行情以及对该专业未来的发展趋势,简介的给出志愿建议" }, + { + type: "text", + text: "帮我解读这个文件,结合当下的专业行情以及对该专业未来的发展趋势,简介的给出志愿建议", + }, { type: "image", file_url: fileInfo.url }, ]), }, @@ -265,7 +243,6 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => ) => { client.on(EventNames.ALL, (_eventName, event: any) => { // 交给状态机处理解析流程 - fileParserRef.current.handleEvent(_eventName, event); // 普通消息流处理 if ( @@ -273,14 +250,18 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => event.event_type !== ChatEventType.CONVERSATION_MESSAGE_COMPLETED ) { // 处理conversation.created事件 - if (event.event_type === "conversation.created" && !opts.initMessage && !opts.fileInfo) { - setMessageList(prev => [ + if ( + event.event_type === "conversation.created" && + !opts.initMessage && + !opts.fileInfo + ) { + setMessageList((prev) => [ ...prev, { content: event.data.prologue, role: RoleType.Assistant, - event - } + event, + }, ]); } return; @@ -288,72 +269,62 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => // 如果是assistant的completed消息或verbose类型，直接返回 if ( - (event.data.role === "assistant" && event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&event.data.type === "verbose" ) || - event.data.type === "answer" && event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED + (event.data.role === "assistant" && + event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED && + event.data.type === "verbose") || + (event.data.type === "answer" && + event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED) ) { return; } - // 如果没有fileInfo，过滤掉function_call和tool_response类型的消息 - if (!opts.fileInfo && (event.data.type === "function_call" || event.data.type === "tool_response")) { - return; - } - const content = event.data.content; - setMessageList(prev => { - // 合并增量 - // 处理工具回调结果 - try { - const parsedContent = JSON.parse(content); - if (parsedContent.msg_type === "time_capsule_recall" || - (parsedContent.name && parsedContent.arguments)) { - // 如果没有fileInfo，不创建假信息 - if (!opts.fileInfo) { - return prev; - } - - // 检查是否已存在工具回调消息 - const existingToolMessageIndex = prev.findIndex(msg => - msg.content === "正在处理您的请求..." && - msg.fileInfo === opts.fileInfo - ); + setMessageList((prev) => { + // 如果是工具调用相关的消息，不添加到消息列表 + if ( + event.data.type === "function_call" || + event.data.type === "tool_response" + ) { + const jsonContent = JSON.parse(event.data.content); + const lastMessage = prev[prev.length - 1]; - if (existingToolMessageIndex !== -1) { - // 更新已存在的消息的fileParseStatus - const newStatus = parsedContent.msg_type === "time_capsule_recall" ? 0 : 2; - return [ - ...prev.slice(0, existingToolMessageIndex), - { - ...prev[existingToolMessageIndex], - fileParseStatus: newStatus - }, - ...prev.slice(existingToolMessageIndex + 1) - ]; - } else { - // 创建新的工具回调消息 - return [ - ...prev, - { - content: "正在处理您的请求...", - role: event.data.role, - event, - fileInfo: opts.fileInfo, - fileParseStatus: parsedContent.msg_type === "time_capsule_recall" ? 0 : 2 - }, - ]; - } + if (jsonContent.name === "doc_reader-PDF_reader") { + return [ + ...prev, + { + content: "", + role: RoleType.Assistant, + fileInfo: opts.fileInfo, + fileParseStatus: 1, + event, + }, + ]; + } + else if ( + lastMessage.event.type === "function_call" && + event.data.type === "tool_response" + ) { + return [ + ...prev.slice(0, prev.length - 2), + { + content: "", + role: RoleType.Assistant, + fileInfo: opts.fileInfo, + fileParseStatus: 2, + event, + }, + ]; + }else{ + return [...prev] } - } catch (e) { - // 如果不是JSON格式，继续正常处理 } - + if ( prev.length > 0 && prev[prev.length - 1].event?.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA && event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA && - prev[prev.length - 1].event.data.answer_id === - event.data.answer_id + prev[prev.length - 1].event.data.answer_id === event.data.answer_id ) { return [ ...prev.slice(0, -1), @@ -363,44 +334,39 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => event, }, ]; + } else { + // 新消息追加 + return [...prev, { content, role: event.data.role, event }]; } - // 新消息追加 - return [ - ...prev, - { content, role: event.data.role, event }, - ]; }); }); }; /** 基本连接状态 & 语音事件监听 */ - const setupEventListeners = useCallback( - (client: RealtimeClient) => { - client.on(EventNames.AUDIO_AGENT_SPEECH_STARTED, async () => { - setIsAiTalking(true); - await clientRef.current?.setAudioEnable(false); - setAudioEnabled(false); - }); - client.on(EventNames.AUDIO_AGENT_SPEECH_STOPPED, async () => { - setIsAiTalking(false); - await clientRef.current?.setAudioEnable(true); - setAudioEnabled(true); - }); - client.on(EventNames.CONNECTING, () => { - setIsConnecting(true); - setIsConnected(false); - }); - client.on(EventNames.CONNECTED, (_name, evt) => { - setRoomInfo(evt as RoomInfo); - setIsConnecting(false); - setIsConnected(true); - }); - client.on(EventNames.ALL_SERVER, (_name, _evt) => { - // 其它全局服务端事件可在此处理 - }); - }, - [] - ); + const setupEventListeners = useCallback((client: RealtimeClient) => { + client.on(EventNames.AUDIO_AGENT_SPEECH_STARTED, async () => { + setIsAiTalking(true); + await clientRef.current?.setAudioEnable(false); + setAudioEnabled(false); + }); + client.on(EventNames.AUDIO_AGENT_SPEECH_STOPPED, async () => { + setIsAiTalking(false); + await clientRef.current?.setAudioEnable(true); + setAudioEnabled(true); + }); + client.on(EventNames.CONNECTING, () => { + setIsConnecting(true); + setIsConnected(false); + }); + client.on(EventNames.CONNECTED, (_name, evt) => { + setRoomInfo(evt as RoomInfo); + setIsConnecting(false); + setIsConnected(true); + }); + client.on(EventNames.ALL_SERVER, (_name, _evt) => { + // 其它全局服务端事件可在此处理 + }); + }, []); return (