357 lines
9.7 KiB
TypeScript
357 lines
9.7 KiB
TypeScript
import { ChatEventType, RoleType } from "@coze/api";
|
|
import {
|
|
EventNames,
|
|
RealtimeAPIError,
|
|
RealtimeClient,
|
|
RealtimeError,
|
|
RealtimeUtils,
|
|
} from "@coze/realtime-api";
|
|
import {
|
|
createContext,
|
|
ReactNode,
|
|
useCallback,
|
|
useContext,
|
|
useRef,
|
|
useState,
|
|
} from "react";
|
|
import { useToast } from "@/hooks/use-toast";
|
|
|
|
type RoomInfo = {
|
|
appId: string;
|
|
roomId: string;
|
|
token: string;
|
|
uid: string;
|
|
};
|
|
|
|
export const RealtimeClientContext = createContext<{
|
|
client: RealtimeClient | null;
|
|
isConnecting: boolean;
|
|
isConnected: boolean;
|
|
audioEnabled: boolean;
|
|
isSupportVideo: boolean;
|
|
messageList: { content: string; role: RoleType }[];
|
|
isAiTalking: boolean;
|
|
roomInfo: RoomInfo | null;
|
|
initClient: (initMessage?:string) => void;
|
|
handleConnect: (initMessage?:string) => Promise<void>;
|
|
handleInterrupt: () => void;
|
|
handleDisconnect: () => void;
|
|
toggleMicrophone: () => void;
|
|
|
|
}>({
|
|
client: null,
|
|
isConnecting: false,
|
|
isConnected: false,
|
|
audioEnabled: true,
|
|
isSupportVideo: false,
|
|
messageList: [],
|
|
isAiTalking: false,
|
|
roomInfo: null,
|
|
initClient: () => {},
|
|
handleConnect: () => Promise.resolve(),
|
|
handleInterrupt: () => {},
|
|
handleDisconnect: () => {},
|
|
toggleMicrophone: () => {},
|
|
|
|
});
|
|
|
|
// 添加自定义hook
|
|
export const useRealtimeClient = () => {
|
|
const context = useContext(RealtimeClientContext);
|
|
|
|
if (context === undefined) {
|
|
throw new Error("useRealtimeClient必须在RealtimeClientProvider内部使用");
|
|
}
|
|
|
|
return { ...context };
|
|
};
|
|
|
|
export const RealtimeClientProvider = ({
|
|
children,
|
|
}: {
|
|
children: ReactNode;
|
|
}) => {
|
|
const token = import.meta.env.VITE_COZE_TOKEN;
|
|
const botId = import.meta.env.VITE_COZE_BOT_ID;
|
|
const voiceId = import.meta.env.VITE_COZE_VOICE_ID;
|
|
const connectorId = "1024";
|
|
|
|
const clientRef = useRef<RealtimeClient | null>(null);
|
|
// 实时语音回复消息列表
|
|
const [messageList, setMessageList] = useState<
|
|
{ content: string; role: RoleType }[]
|
|
>([]);
|
|
// 是否正在连接
|
|
const [isConnecting, setIsConnecting] = useState(false);
|
|
// 是否已连接
|
|
const [isConnected, setIsConnected] = useState(false);
|
|
// 是否开启麦克风
|
|
const [audioEnabled, setAudioEnabled] = useState(true);
|
|
// 是否支持视频
|
|
const [isSupportVideo] = useState(false);
|
|
// 是否正在说话
|
|
const [isAiTalking, setIsAiTalking] = useState(false);
|
|
|
|
|
|
|
|
const [roomInfo, setRoomInfo] = useState<RoomInfo | null>(null);
|
|
|
|
const { toast } = useToast();
|
|
|
|
const initClient = async (_initMessage?:string) => {
|
|
const permission = await RealtimeUtils.checkDevicePermission(false);
|
|
const device = await RealtimeUtils.getAudioDevices();
|
|
|
|
if (!permission.audio) {
|
|
toast({
|
|
title: "连接错误",
|
|
description: "需要麦克风访问权限",
|
|
});
|
|
throw new Error("需要麦克风访问权限");
|
|
}
|
|
|
|
if (device.audioInputs.length === 0) {
|
|
toast({
|
|
title: "连接错误",
|
|
description: "没有麦克风设备",
|
|
});
|
|
throw new Error("没有麦克风设备");
|
|
}
|
|
|
|
const client = new RealtimeClient({
|
|
accessToken: token,
|
|
botId: botId,
|
|
voiceId: voiceId,
|
|
connectorId: connectorId,
|
|
allowPersonalAccessTokenInBrowser: true, // 可选：允许在浏览器中使用个人访问令牌
|
|
});
|
|
|
|
clientRef.current = client;
|
|
|
|
|
|
setupEventListeners(client);
|
|
setupMessageEventListeners(client,_initMessage ?? '');
|
|
setupInitMessageEventListener(client,_initMessage)
|
|
};
|
|
|
|
|
|
const handleConnect = async (initMessage?:string) => {
|
|
try {
|
|
if (!clientRef.current) {
|
|
await initClient(initMessage);
|
|
}
|
|
await clientRef.current?.connect();
|
|
await toggleMicrophone();
|
|
} catch (error) {
|
|
console.error(error);
|
|
if (error instanceof RealtimeAPIError) {
|
|
switch (error.code) {
|
|
case RealtimeError.CREATE_ROOM_ERROR:
|
|
console.error(`创建房间失败: ${error.message}`);
|
|
break;
|
|
case RealtimeError.CONNECTION_ERROR:
|
|
console.error(`加入房间失败: ${error.message}`);
|
|
break;
|
|
case RealtimeError.DEVICE_ACCESS_ERROR:
|
|
console.error(`获取设备失败: ${error.message}`);
|
|
break;
|
|
default:
|
|
console.error(`连接错误: ${error.message}`);
|
|
}
|
|
} else {
|
|
console.error("连接错误：" + error);
|
|
}
|
|
}
|
|
};
|
|
|
|
const handleInterrupt = () => {
|
|
try {
|
|
clientRef.current?.interrupt();
|
|
} catch (error) {
|
|
console.error("打断失败：" + error);
|
|
}
|
|
};
|
|
|
|
const handleDisconnect = async() => {
|
|
try {
|
|
// 关闭客户的时候清除一些信息
|
|
setIsAiTalking(false);
|
|
setMessageList([]);
|
|
await clientRef.current?.setAudioEnable(false);
|
|
setAudioEnabled(false);
|
|
|
|
clientRef.current?.disconnect();
|
|
clientRef.current?.clearEventHandlers();
|
|
clientRef.current = null;
|
|
setIsConnected(false);
|
|
} catch (error) {
|
|
console.error("断开失败：" + error);
|
|
}
|
|
};
|
|
|
|
const toggleMicrophone = async () => {
|
|
try {
|
|
await clientRef.current?.setAudioEnable(!audioEnabled);
|
|
setAudioEnabled(!audioEnabled);
|
|
} catch (error) {
|
|
console.error("切换麦克风状态失败：" + error);
|
|
}
|
|
};
|
|
|
|
|
|
const setupInitMessageEventListener = useCallback((client: RealtimeClient,_initMessage?:string) => {
|
|
client.on(EventNames.ALL_SERVER, async(eventName, _event: any) => {
|
|
if (eventName === "server.session.created") {
|
|
await client.sendMessage({
|
|
id:'',
|
|
"event_type":"session.update",
|
|
data:{
|
|
chat_config:{
|
|
allow_voice_interrupt:false
|
|
}
|
|
}
|
|
})
|
|
}
|
|
if(eventName === "server.bot.join" && _initMessage){
|
|
// 这里需要加个 server. 前缀
|
|
await clientRef.current?.sendMessage({
|
|
id: "",
|
|
event_type: "conversation.message.create",
|
|
data: {
|
|
role: "user",
|
|
content_type: "text",
|
|
content: _initMessage,
|
|
},
|
|
});
|
|
}
|
|
});
|
|
},[clientRef.current]);
|
|
|
|
const setupMessageEventListeners = (client: RealtimeClient,_initMessage:string) => {
|
|
let lastEvent: any;
|
|
client.on(EventNames.ALL, (_eventName, event: any) => {
|
|
// AI智能体设置
|
|
|
|
if (
|
|
event.event_type !== ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
|
event.event_type !== ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&
|
|
event.event_type !== "conversation.created"
|
|
) {
|
|
return;
|
|
}
|
|
const content = event.data.content;
|
|
setMessageList((prev) => {
|
|
// 如果上一个事件是增量更新，则附加到最后一条消息
|
|
if (
|
|
lastEvent?.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA &&
|
|
(event.data.type === "answer" || event.data.type === "question")
|
|
) {
|
|
return [
|
|
...prev.slice(0, -1),
|
|
{
|
|
content: prev[prev.length - 1].content + content,
|
|
role: prev[prev.length - 1].role,
|
|
},
|
|
];
|
|
}
|
|
|
|
// 添加AI的欢迎语
|
|
if (_initMessage === "" && event.event_type === "conversation.created") {
|
|
return [
|
|
...prev,
|
|
{ content: event.data.prologue, role: RoleType.Assistant },
|
|
];
|
|
}
|
|
|
|
// 否则添加新消息
|
|
if (
|
|
(content !== "" &&
|
|
event.event_type === ChatEventType.CONVERSATION_MESSAGE_DELTA) ||
|
|
(event.event_type === ChatEventType.CONVERSATION_MESSAGE_COMPLETED &&
|
|
(event.data.type === "answer" || event.data.type === "question") &&
|
|
event.data.role !== RoleType.Assistant)
|
|
) {
|
|
return [...prev, { content: content, role: event.data.role }];
|
|
}
|
|
return prev;
|
|
});
|
|
lastEvent = event;
|
|
});
|
|
};
|
|
|
|
// 设置事件监听器
|
|
const setupEventListeners = useCallback(
|
|
(client: RealtimeClient) => {
|
|
// 监听 AI 开始说话事件
|
|
client.on(EventNames.AUDIO_AGENT_SPEECH_STARTED, async() => {
|
|
// console.log("AI开始说话");
|
|
setIsAiTalking(true);
|
|
await clientRef.current?.setAudioEnable(false);
|
|
setAudioEnabled(false);
|
|
});
|
|
|
|
// 监听 AI 结束说话事件
|
|
client.on(EventNames.AUDIO_AGENT_SPEECH_STOPPED, async() => {
|
|
// console.log("AI结束说话");
|
|
setIsAiTalking(false);
|
|
await clientRef.current?.setAudioEnable(true);
|
|
setAudioEnabled(true);
|
|
});
|
|
|
|
// 监听连接客户端
|
|
client.on(EventNames.CONNECTING, () => {
|
|
setIsConnecting(true);
|
|
setIsConnected(false);
|
|
});
|
|
|
|
// 客户端连接成功
|
|
client.on(EventNames.CONNECTED, (_eventName: string, event: any) => {
|
|
setRoomInfo(event);
|
|
setIsConnecting(false);
|
|
setIsConnected(true);
|
|
});
|
|
},
|
|
[clientRef.current]
|
|
);
|
|
|
|
// 发送信息
|
|
// const sendUserMessageWithText = async (message: string) => {
|
|
// try {
|
|
// await clientRef.current?.sendMessage({
|
|
// id: "",
|
|
// event_type: "conversation.message.create",
|
|
// data: {
|
|
// role: "user",
|
|
// content_type: "text",
|
|
// content: message,
|
|
// },
|
|
// });
|
|
// } catch (error) {
|
|
// console.error("发送消息失败：" + error);
|
|
// }
|
|
// };
|
|
|
|
return (
|
|
<RealtimeClientContext.Provider
|
|
value={{
|
|
client: clientRef.current,
|
|
isConnecting,
|
|
isConnected,
|
|
audioEnabled,
|
|
isSupportVideo,
|
|
messageList,
|
|
isAiTalking,
|
|
roomInfo,
|
|
initClient,
|
|
handleConnect,
|
|
handleInterrupt,
|
|
handleDisconnect,
|
|
toggleMicrophone,
|
|
|
|
}}
|
|
>
|
|
{children}
|
|
</RealtimeClientContext.Provider>
|
|
);
|
|
};
|