fix: coze api流程调整
parent
5549450523
commit
f6bf09e2f7
|
|
@ -72,19 +72,20 @@ export default function Antechamber() {
|
||||||
setIsLoading(false);
|
setIsLoading(false);
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="flex flex-col items-center h-full overflow-y-auto relative">
|
<div className="flex flex-col items-center h-full overflow-y-auto relative">
|
||||||
<AntechamberHeader />
|
<AntechamberHeader />
|
||||||
<AntechamberScore />
|
<AntechamberScore />
|
||||||
<AntechamberWishList />
|
<AntechamberWishList handleLoading={setIsLoading}/>
|
||||||
<AntechamberFile handleLoading={setIsLoading} />
|
<AntechamberFile handleLoading={setIsLoading} />
|
||||||
<AntechamberReport handleLoading={setIsLoading} />
|
<AntechamberReport handleLoading={setIsLoading} />
|
||||||
<InvokeButton disable={disable} onClick={() => toRoom({})} />
|
<InvokeButton disable={disable} onClick={() => toRoom({})} />
|
||||||
{
|
{
|
||||||
isLoading ? <div className="w-[108px] h-[108px] absolute top-[50%] left-[50%] translate-x-[-50%] translate-y-[-50%] bg-black/60 rounded-[20px] flex flex-col items-center justify-center">
|
isLoading ? <div className="absolute w-full h-full bg-red"><div className="w-[108px] h-[108px] absolute top-[50%] left-[50%] translate-x-[-50%] translate-y-[-50%] bg-black/60 rounded-[20px] flex flex-col items-center justify-center">
|
||||||
<img src="/icons/loading.gif" alt="loading" className="w-[68px] h-[68px]" />
|
<img src="/icons/loading.gif" alt="loading" className="w-[68px] h-[68px]" />
|
||||||
<span className="text-[14px] text-[#fff]">加载中</span>
|
<span className="text-[14px] text-[#fff]">加载中</span>
|
||||||
</div> : <></>
|
</div></div> : <></>
|
||||||
}
|
}
|
||||||
</div>
|
</div>
|
||||||
);
|
);
|
||||||
|
|
|
||||||
|
|
@ -20,7 +20,8 @@ export default function AntechamberFile({handleLoading}:Props) {
|
||||||
const { setHasHandledReport,hasHandledReport } = useContext(ReportContext);
|
const { setHasHandledReport,hasHandledReport } = useContext(ReportContext);
|
||||||
const { handleConnect } = useContext(RealtimeClientContext);
|
const { handleConnect } = useContext(RealtimeClientContext);
|
||||||
|
|
||||||
const useFileFetch = async () => {
|
const useFileFetch = async () => {
|
||||||
|
handleLoading(true)
|
||||||
const result = await fetchFile({
|
const result = await fetchFile({
|
||||||
params: { id: fileId, location: locationCode },
|
params: { id: fileId, location: locationCode },
|
||||||
options: {
|
options: {
|
||||||
|
|
@ -28,13 +29,14 @@ export default function AntechamberFile({handleLoading}:Props) {
|
||||||
headers: { Authorization: `Bearer ${token}` },
|
headers: { Authorization: `Bearer ${token}` },
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
if (result.message) {
|
if (result.message) {
|
||||||
toast({
|
toast({
|
||||||
title: result.message,
|
title: result.message,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
let resp = result.result as FileInfo;
|
let resp = result.result as FileInfo;
|
||||||
handleLoading(true)
|
|
||||||
handleConnect({
|
handleConnect({
|
||||||
fileInfo: {type: resp.type,url: resp.url,tableName: resp.tableName,provinceName: resp.provinceName,subjectClaim: resp.subjectClaim},
|
fileInfo: {type: resp.type,url: resp.url,tableName: resp.tableName,provinceName: resp.provinceName,subjectClaim: resp.subjectClaim},
|
||||||
});
|
});
|
||||||
|
|
@ -43,6 +45,7 @@ export default function AntechamberFile({handleLoading}:Props) {
|
||||||
|
|
||||||
|
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
|
|
||||||
if (fileId && locationCode && !hasHandledReport) {
|
if (fileId && locationCode && !hasHandledReport) {
|
||||||
useFileFetch();
|
useFileFetch();
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -19,8 +19,11 @@ import { useAbortController } from "@/hooks/useAbortController";
|
||||||
import { useSearchParams } from "react-router-dom";
|
import { useSearchParams } from "react-router-dom";
|
||||||
|
|
||||||
import { toast } from "@/hooks/use-toast";
|
import { toast } from "@/hooks/use-toast";
|
||||||
|
type Props = {
|
||||||
|
handleLoading:(val:boolean) => void
|
||||||
|
}
|
||||||
|
|
||||||
export default function AntechamberWishList() {
|
export default function AntechamberWishList({handleLoading}:Props) {
|
||||||
const { wishList } = useWishList();
|
const { wishList } = useWishList();
|
||||||
const { handleConnect } = useContext(RealtimeClientContext);
|
const { handleConnect } = useContext(RealtimeClientContext);
|
||||||
const { getSignal } = useAbortController();
|
const { getSignal } = useAbortController();
|
||||||
|
|
@ -28,10 +31,7 @@ export default function AntechamberWishList() {
|
||||||
const token = searchParams.get("token") || "";
|
const token = searchParams.get("token") || "";
|
||||||
|
|
||||||
const handleNavigate = async (item: any) => {
|
const handleNavigate = async (item: any) => {
|
||||||
const loadingId = toast({
|
handleLoading(true)
|
||||||
title:'文件生成中...',
|
|
||||||
duration: 10000,
|
|
||||||
})
|
|
||||||
const result = await fetchFile({
|
const result = await fetchFile({
|
||||||
params: { id: item.vId, location: item.personlocationCode },
|
params: { id: item.vId, location: item.personlocationCode },
|
||||||
options: {
|
options: {
|
||||||
|
|
@ -39,7 +39,7 @@ export default function AntechamberWishList() {
|
||||||
headers: { Authorization: `Bearer ${token}` },
|
headers: { Authorization: `Bearer ${token}` },
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
loadingId.dismiss();
|
|
||||||
if (result.message) {
|
if (result.message) {
|
||||||
toast({
|
toast({
|
||||||
title: result.message,
|
title: result.message,
|
||||||
|
|
|
||||||
|
|
@ -3,7 +3,6 @@ import {
|
||||||
EventNames,
|
EventNames,
|
||||||
RealtimeAPIError,
|
RealtimeAPIError,
|
||||||
RealtimeClient,
|
RealtimeClient,
|
||||||
RealtimeError,
|
|
||||||
RealtimeUtils,
|
RealtimeUtils,
|
||||||
} from "@coze/realtime-api";
|
} from "@coze/realtime-api";
|
||||||
import {
|
import {
|
||||||
|
|
@ -15,6 +14,7 @@ import {
|
||||||
useState,
|
useState,
|
||||||
} from "react";
|
} from "react";
|
||||||
import { useToast } from "@/hooks/use-toast";
|
import { useToast } from "@/hooks/use-toast";
|
||||||
|
import { FileParser, FileParseStatus } from "./FileParser";
|
||||||
|
|
||||||
type RoomInfo = {
|
type RoomInfo = {
|
||||||
appId: string;
|
appId: string;
|
||||||
|
|
@ -23,7 +23,13 @@ type RoomInfo = {
|
||||||
uid: string;
|
uid: string;
|
||||||
};
|
};
|
||||||
|
|
||||||
export type FileInfo = {type:string,url:string,tableName:string,provinceName:string,subjectClaim:string}
|
export type FileInfo = {
|
||||||
|
type: string;
|
||||||
|
url: string;
|
||||||
|
tableName: string;
|
||||||
|
provinceName: string;
|
||||||
|
subjectClaim: string;
|
||||||
|
};
|
||||||
|
|
||||||
export const RealtimeClientContext = createContext<{
|
export const RealtimeClientContext = createContext<{
|
||||||
client: RealtimeClient | null;
|
client: RealtimeClient | null;
|
||||||
|
|
@ -31,91 +37,89 @@ export const RealtimeClientContext = createContext<{
|
||||||
isConnected: boolean;
|
isConnected: boolean;
|
||||||
audioEnabled: boolean;
|
audioEnabled: boolean;
|
||||||
isSupportVideo: boolean;
|
isSupportVideo: boolean;
|
||||||
messageList: { content: string; role: RoleType }[];
|
messageList: {
|
||||||
|
content: string;
|
||||||
|
role: RoleType;
|
||||||
|
event?: any;
|
||||||
|
fileInfo?: FileInfo;
|
||||||
|
fileParseStatus?: number;
|
||||||
|
}[];
|
||||||
isAiTalking: boolean;
|
isAiTalking: boolean;
|
||||||
roomInfo: RoomInfo | null;
|
roomInfo: RoomInfo | null;
|
||||||
|
|
||||||
initClient: ({
|
initClient: (opts: { initMessage?: string; fileInfo?: FileInfo }) => void;
|
||||||
initMessage,
|
handleConnect: (opts: { initMessage?: string; fileInfo?: FileInfo }) => Promise<void>;
|
||||||
fileInfo,
|
|
||||||
}: {
|
|
||||||
initMessage?: string;
|
|
||||||
fileInfo?: FileInfo;
|
|
||||||
}) => void;
|
|
||||||
handleConnect: ({
|
|
||||||
initMessage,
|
|
||||||
fileInfo,
|
|
||||||
}: {
|
|
||||||
initMessage?: string;
|
|
||||||
fileInfo?: FileInfo;
|
|
||||||
}) => Promise<void>;
|
|
||||||
handleInterrupt: () => void;
|
handleInterrupt: () => void;
|
||||||
handleDisconnect: () => void;
|
handleDisconnect: () => void;
|
||||||
toggleMicrophone: () => void;
|
toggleMicrophone: () => void;
|
||||||
}>({
|
}>(/* 默认值省略 */ null!);
|
||||||
client: null,
|
|
||||||
isConnecting: false,
|
|
||||||
isConnected: false,
|
|
||||||
audioEnabled: true,
|
|
||||||
isSupportVideo: false,
|
|
||||||
messageList: [],
|
|
||||||
isAiTalking: false,
|
|
||||||
roomInfo: null,
|
|
||||||
|
|
||||||
initClient: () => {},
|
|
||||||
handleConnect: () => Promise.resolve(),
|
|
||||||
handleInterrupt: () => {},
|
|
||||||
handleDisconnect: () => {},
|
|
||||||
toggleMicrophone: () => {},
|
|
||||||
});
|
|
||||||
|
|
||||||
// 添加自定义hook
|
|
||||||
export const useRealtimeClient = () => {
|
export const useRealtimeClient = () => {
|
||||||
const context = useContext(RealtimeClientContext);
|
const ctx = useContext(RealtimeClientContext);
|
||||||
|
if (!ctx) throw new Error("useRealtimeClient 必须在 RealtimeClientProvider 内部使用");
|
||||||
if (context === undefined) {
|
return ctx;
|
||||||
throw new Error("useRealtimeClient必须在RealtimeClientProvider内部使用");
|
|
||||||
}
|
|
||||||
|
|
||||||
return { ...context };
|
|
||||||
};
|
};
|
||||||
|
|
||||||
export const RealtimeClientProvider = ({
|
export const RealtimeClientProvider = ({ children }: { children: ReactNode }) => {
|
||||||
children,
|
|
||||||
}: {
|
|
||||||
children: ReactNode;
|
|
||||||
}) => {
|
|
||||||
const token = import.meta.env.VITE_COZE_TOKEN;
|
const token = import.meta.env.VITE_COZE_TOKEN;
|
||||||
const botId = import.meta.env.VITE_COZE_BOT_ID;
|
const botId = import.meta.env.VITE_COZE_BOT_ID;
|
||||||
const voiceId = import.meta.env.VITE_COZE_VOICE_ID;
|
const voiceId = import.meta.env.VITE_COZE_VOICE_ID;
|
||||||
const connectorId = "1024";
|
const connectorId = "1024";
|
||||||
|
|
||||||
const clientRef = useRef<RealtimeClient | null>(null);
|
const clientRef = useRef<RealtimeClient | null>(null);
|
||||||
// 添加连接锁
|
const connectingLockRef = useRef(false);
|
||||||
const connectingLockRef = useRef<boolean>(false);
|
|
||||||
// 实时语音回复消息列表
|
|
||||||
const [messageList, setMessageList] = useState<
|
|
||||||
{ content: string; role: RoleType; event?: any }[]
|
|
||||||
>([]);
|
|
||||||
// 是否正在连接
|
|
||||||
const [isConnecting, setIsConnecting] = useState(false);
|
|
||||||
// 是否已连接
|
|
||||||
const [isConnected, setIsConnected] = useState(false);
|
|
||||||
// 是否开启麦克风
|
|
||||||
const [audioEnabled, setAudioEnabled] = useState(true);
|
|
||||||
// 是否支持视频
|
|
||||||
const [isSupportVideo] = useState(false);
|
|
||||||
// 是否正在说话
|
|
||||||
const [isAiTalking, setIsAiTalking] = useState(false);
|
|
||||||
|
|
||||||
|
const [messageList, setMessageList] = useState<{
|
||||||
|
content: string;
|
||||||
|
role: RoleType;
|
||||||
|
event?: any;
|
||||||
|
fileInfo?: FileInfo;
|
||||||
|
fileParseStatus?: number;
|
||||||
|
}[]>([]);
|
||||||
|
const [isConnecting, setIsConnecting] = useState(false);
|
||||||
|
const [isConnected, setIsConnected] = useState(false);
|
||||||
|
const [audioEnabled, setAudioEnabled] = useState(true);
|
||||||
|
const [isSupportVideo] = useState(false);
|
||||||
|
const [isAiTalking, setIsAiTalking] = useState(false);
|
||||||
const [roomInfo, setRoomInfo] = useState<RoomInfo | null>(null);
|
const [roomInfo, setRoomInfo] = useState<RoomInfo | null>(null);
|
||||||
|
|
||||||
// 记录文件解析是否完成 分为 没有解析 -1,未解析 0,解析中 1,解析完成 2
|
// 引入状态机
|
||||||
|
const fileParseStatusRef = useRef<FileParseStatus>(-1);
|
||||||
const fileParseStatusRef = useRef(-1);
|
const fileParserRef = useRef(
|
||||||
|
new FileParser((newStatus:any) => {
|
||||||
|
fileParseStatusRef.current = newStatus;
|
||||||
|
// 根据不同状态更新 messageList
|
||||||
|
if (newStatus === 0) {
|
||||||
|
appendAssistantMessage("AI正在解析您的文档...");
|
||||||
|
} else if (newStatus === 1) {
|
||||||
|
replaceLastAssistantMessage("AI正在调用插件");
|
||||||
|
} else if (newStatus === 2) {
|
||||||
|
replaceLastAssistantMessage("文档解析完成");
|
||||||
|
}
|
||||||
|
})
|
||||||
|
);
|
||||||
|
|
||||||
const { toast } = useToast();
|
const { toast } = useToast();
|
||||||
|
|
||||||
|
/** Helpers */
|
||||||
|
const appendAssistantMessage = (content: string) => {
|
||||||
|
setMessageList(prev => {
|
||||||
|
return [
|
||||||
|
...prev,
|
||||||
|
{ content, role: RoleType.Assistant }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
};
|
||||||
|
const replaceLastAssistantMessage = (content: string) => {
|
||||||
|
setMessageList(prev => {
|
||||||
|
return [
|
||||||
|
...prev.slice(0, -1),
|
||||||
|
{ content, role: RoleType.Assistant }
|
||||||
|
]
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
/** 初始化客户端并设置监听 */
|
||||||
const initClient = async ({
|
const initClient = async ({
|
||||||
initMessage,
|
initMessage,
|
||||||
fileInfo,
|
fileInfo,
|
||||||
|
|
@ -123,36 +127,28 @@ export const RealtimeClientProvider = ({
|
||||||
initMessage?: string;
|
initMessage?: string;
|
||||||
fileInfo?: FileInfo;
|
fileInfo?: FileInfo;
|
||||||
}) => {
|
}) => {
|
||||||
const permission = await RealtimeUtils.checkDevicePermission(false);
|
const perm = await RealtimeUtils.checkDevicePermission(false);
|
||||||
const device = await RealtimeUtils.getAudioDevices();
|
const device = await RealtimeUtils.getAudioDevices();
|
||||||
|
|
||||||
if (!permission.audio) {
|
if (!perm.audio) {
|
||||||
toast({
|
toast({ title: "连接错误", description: "需要麦克风访问权限" });
|
||||||
title: "连接错误",
|
|
||||||
description: "需要麦克风访问权限",
|
|
||||||
});
|
|
||||||
throw new Error("需要麦克风访问权限");
|
throw new Error("需要麦克风访问权限");
|
||||||
}
|
}
|
||||||
|
|
||||||
if (device.audioInputs.length === 0) {
|
if (device.audioInputs.length === 0) {
|
||||||
toast({
|
toast({ title: "连接错误", description: "没有麦克风设备" });
|
||||||
title: "连接错误",
|
|
||||||
description: "没有麦克风设备",
|
|
||||||
});
|
|
||||||
throw new Error("没有麦克风设备");
|
throw new Error("没有麦克风设备");
|
||||||
}
|
}
|
||||||
|
|
||||||
const client = new RealtimeClient({
|
const client = new RealtimeClient({
|
||||||
accessToken: token,
|
accessToken: token,
|
||||||
botId: botId,
|
botId,
|
||||||
voiceId: voiceId,
|
voiceId,
|
||||||
connectorId: connectorId,
|
connectorId,
|
||||||
allowPersonalAccessTokenInBrowser: true, // 可选:允许在浏览器中使用个人访问令牌
|
allowPersonalAccessTokenInBrowser: true,
|
||||||
suppressStationaryNoise: true,
|
suppressStationaryNoise: true,
|
||||||
suppressNonStationaryNoise: true,
|
suppressNonStationaryNoise: true,
|
||||||
debug: false,
|
debug: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
clientRef.current = client;
|
clientRef.current = client;
|
||||||
|
|
||||||
setupEventListeners(client);
|
setupEventListeners(client);
|
||||||
|
|
@ -160,6 +156,7 @@ export const RealtimeClientProvider = ({
|
||||||
setupInitMessageEventListener(client, { initMessage, fileInfo });
|
setupInitMessageEventListener(client, { initMessage, fileInfo });
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** 连接房间 */
|
||||||
const handleConnect = async ({
|
const handleConnect = async ({
|
||||||
initMessage,
|
initMessage,
|
||||||
fileInfo,
|
fileInfo,
|
||||||
|
|
@ -167,313 +164,245 @@ export const RealtimeClientProvider = ({
|
||||||
initMessage?: string;
|
initMessage?: string;
|
||||||
fileInfo?: FileInfo;
|
fileInfo?: FileInfo;
|
||||||
}) => {
|
}) => {
|
||||||
|
if (connectingLockRef.current) return;
|
||||||
|
connectingLockRef.current = true;
|
||||||
|
|
||||||
|
if (isConnected || isConnecting) {
|
||||||
|
connectingLockRef.current = false;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// 使用连接锁确保原子性
|
|
||||||
if (connectingLockRef.current) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
connectingLockRef.current = true;
|
|
||||||
|
|
||||||
// 如果已经连接或正在连接中,直接返回
|
|
||||||
if (isConnected || isConnecting) {
|
|
||||||
connectingLockRef.current = false;
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!clientRef.current) {
|
if (!clientRef.current) {
|
||||||
await initClient({ initMessage, fileInfo });
|
await initClient({ initMessage, fileInfo });
|
||||||
}
|
}
|
||||||
await clientRef.current?.connect();
|
await clientRef.current!.connect();
|
||||||
await clientRef.current?.setAudioEnable(false);
|
await clientRef.current!.setAudioEnable(false);
|
||||||
setAudioEnabled(false);
|
setAudioEnabled(false);
|
||||||
|
} catch (error: any) {
|
||||||
} catch (error) {
|
|
||||||
// console.error(error);
|
|
||||||
if (error instanceof RealtimeAPIError) {
|
if (error instanceof RealtimeAPIError) {
|
||||||
switch (error.code) {
|
console.error(`连接错误 (${error.code}): ${error.message}`);
|
||||||
case RealtimeError.CREATE_ROOM_ERROR:
|
|
||||||
console.error(`创建房间失败: ${error.message}`);
|
|
||||||
break;
|
|
||||||
case RealtimeError.CONNECTION_ERROR:
|
|
||||||
console.error(`加入房间失败: ${error.message}`);
|
|
||||||
break;
|
|
||||||
case RealtimeError.DEVICE_ACCESS_ERROR:
|
|
||||||
console.error(`获取设备失败: ${error.message}`);
|
|
||||||
break;
|
|
||||||
default:
|
|
||||||
console.error(`连接错误: ${error.message}`);
|
|
||||||
}
|
|
||||||
} else {
|
} else {
|
||||||
console.error("连接错误:" + error);
|
console.error("连接错误:" + error);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
// 确保在任何情况下都释放连接锁
|
|
||||||
connectingLockRef.current = false;
|
connectingLockRef.current = false;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleInterrupt = () => {
|
const handleInterrupt = () => {
|
||||||
try {
|
clientRef.current?.interrupt();
|
||||||
clientRef.current?.interrupt();
|
|
||||||
} catch (error) {
|
|
||||||
console.error("打断失败:" + error);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const handleDisconnect = async () => {
|
const handleDisconnect = async () => {
|
||||||
try {
|
setIsAiTalking(false);
|
||||||
// 关闭客户的时候清除一些信息
|
setMessageList([]);
|
||||||
setIsAiTalking(false);
|
await clientRef.current?.setAudioEnable(false);
|
||||||
setMessageList([]);
|
setAudioEnabled(false);
|
||||||
await clientRef.current?.setAudioEnable(false);
|
await clientRef.current?.disconnect();
|
||||||
setAudioEnabled(false);
|
clientRef.current?.clearEventHandlers();
|
||||||
|
clientRef.current = null;
|
||||||
await clientRef.current?.disconnect();
|
setIsConnected(false);
|
||||||
clientRef.current?.clearEventHandlers();
|
|
||||||
clientRef.current = null;
|
|
||||||
setIsConnected(false);
|
|
||||||
} catch (error) {
|
|
||||||
console.error("断开失败:" + error);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
const toggleMicrophone = async () => {
|
const toggleMicrophone = async () => {
|
||||||
try {
|
await clientRef.current?.setAudioEnable(!audioEnabled);
|
||||||
await clientRef.current?.setAudioEnable(!audioEnabled);
|
setAudioEnabled(!audioEnabled);
|
||||||
setAudioEnabled(!audioEnabled);
|
|
||||||
} catch (error) {
|
|
||||||
console.error("切换麦克风状态失败:" + error);
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
|
|
||||||
|
/** 首条初始化消息(session.create & bot.join) */
|
||||||
const setupInitMessageEventListener = useCallback(
|
const setupInitMessageEventListener = useCallback(
|
||||||
(
|
(
|
||||||
client: RealtimeClient,
|
client: RealtimeClient,
|
||||||
{ initMessage, fileInfo }: { initMessage?: string; fileInfo?: FileInfo }
|
{ initMessage, fileInfo }: { initMessage?: string; fileInfo?: FileInfo }
|
||||||
) => {
|
) => {
|
||||||
client.on(EventNames.ALL_SERVER, async (eventName, _event: any) => {
|
client.on(EventNames.ALL_SERVER, async (eventName, event: any) => {
|
||||||
if (eventName === "server.session.created") {
|
if (eventName === "server.session.created") {
|
||||||
await client.sendMessage({
|
await client.sendMessage({
|
||||||
id: "",
|
id: "",
|
||||||
event_type: "session.update",
|
event_type: "session.update",
|
||||||
data: {
|
data: {
|
||||||
chat_config: {
|
chat_config: { allow_voice_interrupt: false },
|
||||||
allow_voice_interrupt: false,
|
turn_detection: { silence_duration_ms: 2000 },
|
||||||
},
|
|
||||||
turn_detection: {
|
|
||||||
silence_duration_ms: 2000,
|
|
||||||
},
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (eventName === "server.bot.join" && initMessage) {
|
if (eventName === "server.bot.join") {
|
||||||
// 这里需要加个 server. 前缀
|
if (initMessage) {
|
||||||
await clientRef.current?.sendMessage({
|
await clientRef.current!.sendMessage({
|
||||||
id: "",
|
id: "",
|
||||||
event_type: "conversation.message.create",
|
event_type: "conversation.message.create",
|
||||||
data: {
|
data: {
|
||||||
role: "user",
|
role: "user",
|
||||||
content_type: "text",
|
content_type: "text",
|
||||||
content: initMessage,
|
content: initMessage,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
} else if (eventName === "server.bot.join" && fileInfo) {
|
} else if (fileInfo) {
|
||||||
fileParseStatusRef.current = 0;
|
await clientRef.current!.sendMessage({
|
||||||
|
id: "",
|
||||||
await clientRef.current?.sendMessage({
|
event_type: "conversation.message.create",
|
||||||
id: "",
|
data: {
|
||||||
event_type: "conversation.message.create",
|
role: "user",
|
||||||
data: {
|
content_type: "object_string",
|
||||||
role: "user",
|
content: JSON.stringify([
|
||||||
content_type: "object_string",
|
{ type: "text", text: "帮我解读这个文件,结合当下的专业行情以及对该专业未来的发展趋势,简介的给出志愿建议" },
|
||||||
content: JSON.stringify([
|
{ type: "image", file_url: fileInfo.url },
|
||||||
{
|
]),
|
||||||
type: "text",
|
},
|
||||||
text: "帮我解读这个文件,结合当下的专业行情以及对该专业未来的发展趋势,简介的给出志愿建议",
|
});
|
||||||
},
|
}
|
||||||
{ type: "image", file_url: fileInfo.url },
|
|
||||||
]),
|
|
||||||
},
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
},
|
},
|
||||||
[clientRef.current]
|
[]
|
||||||
);
|
);
|
||||||
|
|
||||||
|
/** 消息及文件解析监听 */
|
||||||
const setupMessageEventListeners = (
|
const setupMessageEventListeners = (
|
||||||
client: RealtimeClient,
|
client: RealtimeClient,
|
||||||
{ initMessage, fileInfo }: { initMessage?: string; fileInfo?: FileInfo }
|
opts: { initMessage?: string; fileInfo?: FileInfo }
|
||||||
) => {
|
) => {
|
||||||
client.on(EventNames.ALL, (_eventName, event: any) => {
|
client.on(EventNames.ALL, (_eventName, event: any) => {
|
||||||
// AI智能体设置
|
// 交给状态机处理解析流程
|
||||||
|
fileParserRef.current.handleEvent(_eventName, event);
|
||||||
if(_eventName === 'server.error'){
|
|
||||||
// 长期不活动,服务端终结了
|
|
||||||
handleDisconnect();
|
|
||||||
}
|
|
||||||
|
|
||||||
|
|
||||||
|
// 普通消息流处理
|
||||||
if (
|
if (
|
||||||
event.event_type !== ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
event.event_type !== ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
||||||
event.event_type !== ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&
|
event.event_type !== ChatEventType.CONVERSATION_MESSAGE_COMPLETED
|
||||||
event.event_type !== "conversation.created" &&
|
) {
|
||||||
event.event_type !== "conversation.message.create"
|
// 处理conversation.created事件
|
||||||
|
if (event.event_type === "conversation.created" && !opts.initMessage && !opts.fileInfo) {
|
||||||
|
setMessageList(prev => [
|
||||||
|
...prev,
|
||||||
|
{
|
||||||
|
content: event.data.prologue,
|
||||||
|
role: RoleType.Assistant,
|
||||||
|
event
|
||||||
|
}
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 如果是assistant的completed消息或verbose类型,直接返回
|
||||||
|
if (
|
||||||
|
(event.data.role === "assistant" && event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&event.data.type === "verbose" ) ||
|
||||||
|
event.data.type === "answer" && event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED
|
||||||
) {
|
) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const content = event.data.content;
|
// 如果没有fileInfo,过滤掉function_call和tool_response类型的消息
|
||||||
if (
|
if (!opts.fileInfo && (event.data.type === "function_call" || event.data.type === "tool_response")) {
|
||||||
fileParseStatusRef.current === 0 &&
|
return;
|
||||||
event.data.type === "function_call" &&
|
|
||||||
JSON.parse(content).name === "doc_reader-PDF_reader"
|
|
||||||
) {
|
|
||||||
fileParseStatusRef.current = 1;
|
|
||||||
} else if (
|
|
||||||
event.data.type === "tool_response" &&
|
|
||||||
fileParseStatusRef.current === 1
|
|
||||||
) {
|
|
||||||
fileParseStatusRef.current = 2;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
setMessageList((prev) => {
|
const content = event.data.content;
|
||||||
// 如果上一个事件是增量更新,则附加到最后一条消息
|
setMessageList(prev => {
|
||||||
|
// 合并增量
|
||||||
|
console.log("合并增量",prev);
|
||||||
|
console.log("信息",content);
|
||||||
|
|
||||||
|
// 处理工具回调结果
|
||||||
|
try {
|
||||||
|
const parsedContent = JSON.parse(content);
|
||||||
|
if (parsedContent.msg_type === "time_capsule_recall" ||
|
||||||
|
(parsedContent.name && parsedContent.arguments)) {
|
||||||
|
// 如果没有fileInfo,不创建假信息
|
||||||
|
if (!opts.fileInfo) {
|
||||||
|
return prev;
|
||||||
|
}
|
||||||
|
|
||||||
|
// 检查是否已存在工具回调消息
|
||||||
|
const existingToolMessageIndex = prev.findIndex(msg =>
|
||||||
|
msg.content === "正在处理您的请求..." &&
|
||||||
|
msg.fileInfo === opts.fileInfo
|
||||||
|
);
|
||||||
|
|
||||||
|
if (existingToolMessageIndex !== -1) {
|
||||||
|
// 更新已存在的消息的fileParseStatus
|
||||||
|
const newStatus = parsedContent.msg_type === "time_capsule_recall" ? 0 : 2;
|
||||||
|
return [
|
||||||
|
...prev.slice(0, existingToolMessageIndex),
|
||||||
|
{
|
||||||
|
...prev[existingToolMessageIndex],
|
||||||
|
fileParseStatus: newStatus
|
||||||
|
},
|
||||||
|
...prev.slice(existingToolMessageIndex + 1)
|
||||||
|
];
|
||||||
|
} else {
|
||||||
|
// 创建新的工具回调消息
|
||||||
|
return [
|
||||||
|
...prev,
|
||||||
|
{
|
||||||
|
content: "正在处理您的请求...",
|
||||||
|
role: event.data.role,
|
||||||
|
event,
|
||||||
|
fileInfo: opts.fileInfo,
|
||||||
|
fileParseStatus: parsedContent.msg_type === "time_capsule_recall" ? 0 : 2
|
||||||
|
},
|
||||||
|
];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e) {
|
||||||
|
// 如果不是JSON格式,继续正常处理
|
||||||
|
}
|
||||||
|
|
||||||
if (
|
if (
|
||||||
prev.length > 0 &&
|
prev.length > 0 &&
|
||||||
prev[prev.length - 1].event?.event_type ===
|
prev[prev.length - 1].event?.event_type ===
|
||||||
ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
||||||
event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
||||||
prev[prev.length - 1].event.data.type === event.data.type &&
|
prev[prev.length - 1].event.data.answer_id ===
|
||||||
prev[prev.length - 1].event.data.answer_id === event.data.answer_id
|
event.data.answer_id
|
||||||
) {
|
) {
|
||||||
return [
|
return [
|
||||||
...prev.slice(0, -1),
|
...prev.slice(0, -1),
|
||||||
{
|
{
|
||||||
content: prev[prev.length - 1].content + content,
|
content: prev[prev.length - 1].content + content,
|
||||||
role: prev[prev.length - 1].role,
|
role: prev[prev.length - 1].role,
|
||||||
event: event,
|
event,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
// 新消息追加
|
||||||
// 添加AI的欢迎语
|
return [
|
||||||
if (
|
...prev,
|
||||||
typeof initMessage === "undefined" &&
|
{ content, role: event.data.role, event },
|
||||||
typeof fileInfo === "undefined" &&
|
];
|
||||||
event.event_type === "conversation.created"
|
|
||||||
) {
|
|
||||||
return [
|
|
||||||
...prev,
|
|
||||||
{
|
|
||||||
content: event.data.prologue,
|
|
||||||
role: RoleType.Assistant,
|
|
||||||
event: event,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
// 否则添加新消息
|
|
||||||
if (
|
|
||||||
(content !== "" &&
|
|
||||||
event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA) ||
|
|
||||||
(event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&
|
|
||||||
(event.data.type === "answer" || event.data.type === "question") &&
|
|
||||||
event.data.role !== RoleType.Assistant)
|
|
||||||
) {
|
|
||||||
// lastEvent = event;
|
|
||||||
if(event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA && fileParseStatusRef.current === 2){
|
|
||||||
fileParseStatusRef.current = -1;
|
|
||||||
}
|
|
||||||
return [
|
|
||||||
...prev,
|
|
||||||
{ content: content, role: event.data.role, event: event },
|
|
||||||
];
|
|
||||||
}
|
|
||||||
|
|
||||||
// 添加一个文件解析的信息
|
|
||||||
if (
|
|
||||||
fileParseStatusRef.current === 0 &&
|
|
||||||
event.event_type === "conversation.message.completed"
|
|
||||||
) {
|
|
||||||
return [
|
|
||||||
...prev,
|
|
||||||
{
|
|
||||||
content: "AI正在解析您的文档...",
|
|
||||||
role: RoleType.Assistant,
|
|
||||||
event: event,
|
|
||||||
fileParseStatus: fileParseStatusRef.current,
|
|
||||||
fileInfo: fileInfo,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
} else if (
|
|
||||||
fileParseStatusRef.current === 1 &&
|
|
||||||
event.event_type === "conversation.message.completed"
|
|
||||||
) {
|
|
||||||
return [
|
|
||||||
...prev.slice(0, -1),
|
|
||||||
{
|
|
||||||
content: "AI正在调用插件",
|
|
||||||
role: prev[prev.length - 1].role,
|
|
||||||
event: event,
|
|
||||||
fileParseStatus: fileParseStatusRef.current,
|
|
||||||
fileInfo: fileInfo,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
} else if (
|
|
||||||
fileParseStatusRef.current === 2 &&
|
|
||||||
event.event_type === "conversation.message.completed"
|
|
||||||
) {
|
|
||||||
return [
|
|
||||||
...prev.slice(0, -1),
|
|
||||||
{
|
|
||||||
content: "文档解析完成",
|
|
||||||
role: RoleType.Assistant,
|
|
||||||
event: event,
|
|
||||||
fileParseStatus: fileParseStatusRef.current,
|
|
||||||
fileInfo: fileInfo,
|
|
||||||
},
|
|
||||||
];
|
|
||||||
}
|
|
||||||
return prev;
|
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
// 设置事件监听器
|
/** 基本连接状态 & 语音事件监听 */
|
||||||
const setupEventListeners = useCallback(
|
const setupEventListeners = useCallback(
|
||||||
(client: RealtimeClient) => {
|
(client: RealtimeClient) => {
|
||||||
// 监听 AI 开始说话事件
|
|
||||||
client.on(EventNames.AUDIO_AGENT_SPEECH_STARTED, async () => {
|
client.on(EventNames.AUDIO_AGENT_SPEECH_STARTED, async () => {
|
||||||
// console.log("AI开始说话");
|
|
||||||
setIsAiTalking(true);
|
setIsAiTalking(true);
|
||||||
await clientRef.current?.setAudioEnable(false);
|
await clientRef.current?.setAudioEnable(false);
|
||||||
setAudioEnabled(false);
|
setAudioEnabled(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
// 监听 AI 结束说话事件
|
|
||||||
client.on(EventNames.AUDIO_AGENT_SPEECH_STOPPED, async () => {
|
client.on(EventNames.AUDIO_AGENT_SPEECH_STOPPED, async () => {
|
||||||
// console.log("AI结束说话");
|
|
||||||
setIsAiTalking(false);
|
setIsAiTalking(false);
|
||||||
await clientRef.current?.setAudioEnable(true);
|
await clientRef.current?.setAudioEnable(true);
|
||||||
setAudioEnabled(true);
|
setAudioEnabled(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
// 监听连接客户端
|
|
||||||
client.on(EventNames.CONNECTING, () => {
|
client.on(EventNames.CONNECTING, () => {
|
||||||
setIsConnecting(true);
|
setIsConnecting(true);
|
||||||
setIsConnected(false);
|
setIsConnected(false);
|
||||||
});
|
});
|
||||||
|
client.on(EventNames.CONNECTED, (_name, evt) => {
|
||||||
// 客户端连接成功
|
setRoomInfo(evt as RoomInfo);
|
||||||
client.on(EventNames.CONNECTED, (_eventName: string, event: any) => {
|
|
||||||
setRoomInfo(event);
|
|
||||||
setIsConnecting(false);
|
setIsConnecting(false);
|
||||||
setIsConnected(true);
|
setIsConnected(true);
|
||||||
});
|
});
|
||||||
|
client.on(EventNames.ALL_SERVER, (name, evt) => {
|
||||||
|
// 其它全局服务端事件可在此处理
|
||||||
|
});
|
||||||
},
|
},
|
||||||
[clientRef.current]
|
[]
|
||||||
);
|
);
|
||||||
|
|
||||||
return (
|
return (
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,93 @@
|
||||||
|
import { ChatEventType } from "@coze/api";
|
||||||
|
|
||||||
|
/** 文件解析状态枚举 */
|
||||||
|
export type FileParseStatus = -1 | 0 | 1 | 2;
|
||||||
|
|
||||||
|
/** 状态接口 */
|
||||||
|
export interface IFileParseState {
|
||||||
|
status: FileParseStatus;
|
||||||
|
handleEvent(eventType: string, event: any): void;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 文件解析器:状态模式 + 回调(观察者)
|
||||||
|
*/
|
||||||
|
export class FileParser {
|
||||||
|
private state: IFileParseState|null=null;
|
||||||
|
|
||||||
|
constructor(
|
||||||
|
/** 状态变更时回调,更新 UI 或者其他逻辑 */
|
||||||
|
private onStatusChange: (newStatus: FileParseStatus) => void
|
||||||
|
) {
|
||||||
|
this.transitionTo(new UninitializedState(this));
|
||||||
|
}
|
||||||
|
|
||||||
|
get status() {
|
||||||
|
return this.state?.status;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 外部事件统一入口,交由当前状态去处理 */
|
||||||
|
handleEvent(eventType: string, event: any) {
|
||||||
|
this.state?.handleEvent(eventType, event);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 切换状态并触发回调 */
|
||||||
|
transitionTo(state: IFileParseState) {
|
||||||
|
this.state = state;
|
||||||
|
this.onStatusChange(state.status);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 初始状态:-1,等待 bot.join 且带 fileInfo */
|
||||||
|
class UninitializedState implements IFileParseState {
|
||||||
|
status: FileParseStatus = -1;
|
||||||
|
constructor(private ctx: FileParser) {}
|
||||||
|
|
||||||
|
handleEvent(eventType: string, event: any) {
|
||||||
|
if (eventType === "server.bot.join" && event.fileInfo) {
|
||||||
|
this.ctx.transitionTo(new WaitingState(this.ctx));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 等待发送解析请求:0 */
|
||||||
|
class WaitingState implements IFileParseState {
|
||||||
|
status: FileParseStatus = 0;
|
||||||
|
constructor(private ctx: FileParser) {}
|
||||||
|
|
||||||
|
handleEvent(eventType: string, event: any) {
|
||||||
|
if (
|
||||||
|
event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
||||||
|
event.data.type === "function_call" &&
|
||||||
|
JSON.parse(event.data.content).name === "doc_reader-PDF_reader"
|
||||||
|
) {
|
||||||
|
this.ctx.transitionTo(new ParsingState(this.ctx));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 正在调用插件解析:1 */
|
||||||
|
class ParsingState implements IFileParseState {
|
||||||
|
status: FileParseStatus = 1;
|
||||||
|
constructor(private ctx: FileParser) {}
|
||||||
|
|
||||||
|
handleEvent(eventType: string, event: any) {
|
||||||
|
if (
|
||||||
|
event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&
|
||||||
|
event.data.type === "tool_response"
|
||||||
|
) {
|
||||||
|
this.ctx.transitionTo(new CompletedState(this.ctx));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 解析完成:2 */
|
||||||
|
class CompletedState implements IFileParseState {
|
||||||
|
status: FileParseStatus = 2;
|
||||||
|
constructor(private ctx: FileParser) {}
|
||||||
|
|
||||||
|
handleEvent(eventType: string, event: any) {
|
||||||
|
// 如需重置或其他后续逻辑,可在此处理
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
@ -68,18 +68,18 @@ export default function RoomConversation() {
|
||||||
<div className="flex flex-col items-start ml-[10px]">
|
<div className="flex flex-col items-start ml-[10px]">
|
||||||
<div className="flex items-center">
|
<div className="flex items-center">
|
||||||
<span className="text-[15px] text-[#303030] mr-[8px] leading-[1]">
|
<span className="text-[15px] text-[#303030] mr-[8px] leading-[1]">
|
||||||
{message.fileInfo.tableName}
|
{message?.fileInfo?.tableName}
|
||||||
</span>
|
</span>
|
||||||
<div className="bg-[#F4F6FA] rounded-[4px] w-[48px] h-[16px] px-[4px] py-[2px] text-[10px]">
|
<div className="bg-[#F4F6FA] rounded-[4px] w-[48px] h-[16px] px-[4px] py-[2px] text-[10px]">
|
||||||
{message.fileInfo.type}
|
{message?.fileInfo?.type}
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div className="text-[12px] text-[#303030] mt-[6px] flex items-center">
|
<div className="text-[12px] text-[#303030] mt-[6px] flex items-center">
|
||||||
<span className="mr-[10px]">
|
<span className="mr-[10px]">
|
||||||
{message.fileInfo.provinceName}·{message.fileInfo.score}
|
{message?.fileInfo?.provinceName}·{message?.fileInfo?.score}
|
||||||
</span>
|
</span>
|
||||||
<span>
|
<span>
|
||||||
{message.fileInfo.subjectClaim.split(",").join("/")}
|
{message?.fileInfo?.subjectClaim?.split(",").join("/")}
|
||||||
</span>
|
</span>
|
||||||
{
|
{
|
||||||
message.fileParseStatus < 2 && (
|
message.fileParseStatus < 2 && (
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue