fix: 修复插件调用的处理
parent
e1acc4973a
commit
207f28c728
|
|
@ -14,7 +14,6 @@ import {
|
|||
useState,
|
||||
} from "react";
|
||||
import { useToast } from "@/hooks/use-toast";
|
||||
import { FileParser, FileParseStatus } from "./fileParser";
|
||||
|
||||
type RoomInfo = {
|
||||
appId: string;
|
||||
|
|
@ -48,7 +47,10 @@ export const RealtimeClientContext = createContext<{
|
|||
roomInfo: RoomInfo | null;
|
||||
|
||||
initClient: (opts: { initMessage?: string; fileInfo?: FileInfo }) => void;
|
||||
handleConnect: (opts: { initMessage?: string; fileInfo?: FileInfo }) => Promise<void>;
|
||||
handleConnect: (opts: {
|
||||
initMessage?: string;
|
||||
fileInfo?: FileInfo;
|
||||
}) => Promise<void>;
|
||||
handleInterrupt: () => void;
|
||||
handleDisconnect: () => void;
|
||||
toggleMicrophone: () => void;
|
||||
|
|
@ -56,11 +58,16 @@ export const RealtimeClientContext = createContext<{
|
|||
|
||||
export const useRealtimeClient = () => {
|
||||
const ctx = useContext(RealtimeClientContext);
|
||||
if (!ctx) throw new Error("useRealtimeClient 必须在 RealtimeClientProvider 内部使用");
|
||||
if (!ctx)
|
||||
throw new Error("useRealtimeClient 必须在 RealtimeClientProvider 内部使用");
|
||||
return ctx;
|
||||
};
|
||||
|
||||
export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => {
|
||||
export const RealtimeClientProvider = ({
|
||||
children,
|
||||
}: {
|
||||
children: ReactNode;
|
||||
}) => {
|
||||
const token = import.meta.env.VITE_COZE_TOKEN;
|
||||
const botId = import.meta.env.VITE_COZE_BOT_ID;
|
||||
const voiceId = import.meta.env.VITE_COZE_VOICE_ID;
|
||||
|
|
@ -69,13 +76,15 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
const clientRef = useRef<RealtimeClient | null>(null);
|
||||
const connectingLockRef = useRef(false);
|
||||
|
||||
const [messageList, setMessageList] = useState<{
|
||||
const [messageList, setMessageList] = useState<
|
||||
{
|
||||
content: string;
|
||||
role: RoleType;
|
||||
event?: any;
|
||||
fileInfo?: FileInfo;
|
||||
fileParseStatus?: number;
|
||||
}[]>([]);
|
||||
}[]
|
||||
>([]);
|
||||
const [isConnecting, setIsConnecting] = useState(false);
|
||||
const [isConnected, setIsConnected] = useState(false);
|
||||
const [audioEnabled, setAudioEnabled] = useState(true);
|
||||
|
|
@ -83,42 +92,8 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
const [isAiTalking, setIsAiTalking] = useState(false);
|
||||
const [roomInfo, setRoomInfo] = useState<RoomInfo | null>(null);
|
||||
|
||||
// 引入状态机
|
||||
const fileParseStatusRef = useRef<FileParseStatus>(-1);
|
||||
const fileParserRef = useRef(
|
||||
new FileParser((newStatus:any) => {
|
||||
fileParseStatusRef.current = newStatus;
|
||||
// 根据不同状态更新 messageList
|
||||
if (newStatus === 0) {
|
||||
appendAssistantMessage("AI正在解析您的文档...");
|
||||
} else if (newStatus === 1) {
|
||||
replaceLastAssistantMessage("AI正在调用插件");
|
||||
} else if (newStatus === 2) {
|
||||
replaceLastAssistantMessage("文档解析完成");
|
||||
}
|
||||
})
|
||||
);
|
||||
|
||||
const { toast } = useToast();
|
||||
|
||||
/** Helpers */
|
||||
const appendAssistantMessage = (content: string) => {
|
||||
setMessageList(prev => {
|
||||
return [
|
||||
...prev,
|
||||
{ content, role: RoleType.Assistant }
|
||||
]
|
||||
});
|
||||
};
|
||||
const replaceLastAssistantMessage = (content: string) => {
|
||||
setMessageList(prev => {
|
||||
return [
|
||||
...prev.slice(0, -1),
|
||||
{ content, role: RoleType.Assistant }
|
||||
]
|
||||
});
|
||||
};
|
||||
|
||||
/** 初始化客户端并设置监听 */
|
||||
const initClient = async ({
|
||||
initMessage,
|
||||
|
|
@ -147,7 +122,7 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
allowPersonalAccessTokenInBrowser: true,
|
||||
suppressStationaryNoise: true,
|
||||
suppressNonStationaryNoise: true,
|
||||
debug: true,
|
||||
debug: false,
|
||||
});
|
||||
clientRef.current = client;
|
||||
|
||||
|
|
@ -246,7 +221,10 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
role: "user",
|
||||
content_type: "object_string",
|
||||
content: JSON.stringify([
|
||||
{ type: "text", text: "帮我解读这个文件,结合当下的专业行情以及对该专业未来的发展趋势,简介的给出志愿建议" },
|
||||
{
|
||||
type: "text",
|
||||
text: "帮我解读这个文件,结合当下的专业行情以及对该专业未来的发展趋势,简介的给出志愿建议",
|
||||
},
|
||||
{ type: "image", file_url: fileInfo.url },
|
||||
]),
|
||||
},
|
||||
|
|
@ -265,7 +243,6 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
) => {
|
||||
client.on(EventNames.ALL, (_eventName, event: any) => {
|
||||
// 交给状态机处理解析流程
|
||||
fileParserRef.current.handleEvent(_eventName, event);
|
||||
|
||||
// 普通消息流处理
|
||||
if (
|
||||
|
|
@ -273,14 +250,18 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
event.event_type !== ChatEventType.CONVERSATION_MESSAGE_COMPLETED
|
||||
) {
|
||||
// 处理conversation.created事件
|
||||
if (event.event_type === "conversation.created" && !opts.initMessage && !opts.fileInfo) {
|
||||
setMessageList(prev => [
|
||||
if (
|
||||
event.event_type === "conversation.created" &&
|
||||
!opts.initMessage &&
|
||||
!opts.fileInfo
|
||||
) {
|
||||
setMessageList((prev) => [
|
||||
...prev,
|
||||
{
|
||||
content: event.data.prologue,
|
||||
role: RoleType.Assistant,
|
||||
event
|
||||
}
|
||||
event,
|
||||
},
|
||||
]);
|
||||
}
|
||||
return;
|
||||
|
|
@ -288,63 +269,54 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
|
||||
// 如果是assistant的completed消息或verbose类型，直接返回
|
||||
if (
|
||||
(event.data.role === "assistant" && event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&event.data.type === "verbose" ) ||
|
||||
event.data.type === "answer" && event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED
|
||||
(event.data.role === "assistant" &&
|
||||
event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&
|
||||
event.data.type === "verbose") ||
|
||||
(event.data.type === "answer" &&
|
||||
event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED)
|
||||
) {
|
||||
return;
|
||||
}
|
||||
|
||||
// 如果没有fileInfo，过滤掉function_call和tool_response类型的消息
|
||||
if (!opts.fileInfo && (event.data.type === "function_call" || event.data.type === "tool_response")) {
|
||||
return;
|
||||
}
|
||||
|
||||
const content = event.data.content;
|
||||
setMessageList(prev => {
|
||||
// 合并增量
|
||||
// 处理工具回调结果
|
||||
try {
|
||||
const parsedContent = JSON.parse(content);
|
||||
if (parsedContent.msg_type === "time_capsule_recall" ||
|
||||
(parsedContent.name && parsedContent.arguments)) {
|
||||
// 如果没有fileInfo，不创建假信息
|
||||
if (!opts.fileInfo) {
|
||||
return prev;
|
||||
}
|
||||
setMessageList((prev) => {
|
||||
// 如果是工具调用相关的消息，不添加到消息列表
|
||||
if (
|
||||
event.data.type === "function_call" ||
|
||||
event.data.type === "tool_response"
|
||||
) {
|
||||
const jsonContent = JSON.parse(event.data.content);
|
||||
const lastMessage = prev[prev.length - 1];
|
||||
|
||||
// 检查是否已存在工具回调消息
|
||||
const existingToolMessageIndex = prev.findIndex(msg =>
|
||||
msg.content === "正在处理您的请求..." &&
|
||||
msg.fileInfo === opts.fileInfo
|
||||
);
|
||||
|
||||
if (existingToolMessageIndex !== -1) {
|
||||
// 更新已存在的消息的fileParseStatus
|
||||
const newStatus = parsedContent.msg_type === "time_capsule_recall" ? 0 : 2;
|
||||
return [
|
||||
...prev.slice(0, existingToolMessageIndex),
|
||||
{
|
||||
...prev[existingToolMessageIndex],
|
||||
fileParseStatus: newStatus
|
||||
},
|
||||
...prev.slice(existingToolMessageIndex + 1)
|
||||
];
|
||||
} else {
|
||||
// 创建新的工具回调消息
|
||||
if (jsonContent.name === "doc_reader-PDF_reader") {
|
||||
return [
|
||||
...prev,
|
||||
{
|
||||
content: "正在处理您的请求...",
|
||||
role: event.data.role,
|
||||
event,
|
||||
content: "",
|
||||
role: RoleType.Assistant,
|
||||
fileInfo: opts.fileInfo,
|
||||
fileParseStatus: parsedContent.msg_type === "time_capsule_recall" ? 0 : 2
|
||||
fileParseStatus: 1,
|
||||
event,
|
||||
},
|
||||
];
|
||||
}
|
||||
else if (
|
||||
lastMessage.event.type === "function_call" &&
|
||||
event.data.type === "tool_response"
|
||||
) {
|
||||
return [
|
||||
...prev.slice(0, prev.length - 2),
|
||||
{
|
||||
content: "",
|
||||
role: RoleType.Assistant,
|
||||
fileInfo: opts.fileInfo,
|
||||
fileParseStatus: 2,
|
||||
event,
|
||||
},
|
||||
];
|
||||
}else{
|
||||
return [...prev]
|
||||
}
|
||||
} catch (e) {
|
||||
// 如果不是JSON格式，继续正常处理
|
||||
}
|
||||
|
||||
if (
|
||||
|
|
@ -352,8 +324,7 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
prev[prev.length - 1].event?.event_type ===
|
||||
ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
||||
event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
||||
prev[prev.length - 1].event.data.answer_id ===
|
||||
event.data.answer_id
|
||||
prev[prev.length - 1].event.data.answer_id === event.data.answer_id
|
||||
) {
|
||||
return [
|
||||
...prev.slice(0, -1),
|
||||
|
|
@ -363,19 +334,16 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
event,
|
||||
},
|
||||
];
|
||||
}
|
||||
} else {
|
||||
// 新消息追加
|
||||
return [
|
||||
...prev,
|
||||
{ content, role: event.data.role, event },
|
||||
];
|
||||
return [...prev, { content, role: event.data.role, event }];
|
||||
}
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
/** 基本连接状态 & 语音事件监听 */
|
||||
const setupEventListeners = useCallback(
|
||||
(client: RealtimeClient) => {
|
||||
const setupEventListeners = useCallback((client: RealtimeClient) => {
|
||||
client.on(EventNames.AUDIO_AGENT_SPEECH_STARTED, async () => {
|
||||
setIsAiTalking(true);
|
||||
await clientRef.current?.setAudioEnable(false);
|
||||
|
|
@ -398,9 +366,7 @@ export const RealtimeClientProvider = ({ children }: { children: ReactNode }) =>
|
|||
client.on(EventNames.ALL_SERVER, (_name, _evt) => {
|
||||
// 其它全局服务端事件可在此处理
|
||||
});
|
||||
},
|
||||
[]
|
||||
);
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<RealtimeClientContext.Provider
|
||||
|
|
|
|||
Loading…
Reference in New Issue