/** * Copyright 2025 Beijing Volcano Engine Technology Co., Ltd. All Rights Reserved. * SPDX-license-identifier: BSD-3-Clause */ import VERTC, { MirrorType, StreamIndex, IRTCEngine, RoomProfileType, onUserJoinedEvent, onUserLeaveEvent, MediaType, LocalStreamStats, RemoteStreamStats, StreamRemoveReason, LocalAudioPropertiesInfo, RemoteAudioPropertiesInfo, AudioProfileType, DeviceInfo, AutoPlayFailedEvent, PlayerEvent, NetworkQuality, VideoRenderMode, } from '@volcengine/rtc'; import RTCAIAnsExtension from '@volcengine/rtc/extension-ainr'; import openAPIs from '@/app/api'; import aigcConfig from '@/config'; import Utils from '@/utils/utils'; import { COMMAND, INTERRUPT_PRIORITY } from '@/utils/handler'; export interface IEventListener { handleError: (e: { errorCode: any }) => void; handleUserJoin: (e: onUserJoinedEvent) => void; handleUserLeave: (e: onUserLeaveEvent) => void; handleUserPublishStream: (e: { userId: string; mediaType: MediaType }) => void; handleUserUnpublishStream: (e: { userId: string; mediaType: MediaType; reason: StreamRemoveReason; }) => void; handleRemoteStreamStats: (e: RemoteStreamStats) => void; handleLocalStreamStats: (e: LocalStreamStats) => void; handleLocalAudioPropertiesReport: (e: LocalAudioPropertiesInfo[]) => void; handleRemoteAudioPropertiesReport: (e: RemoteAudioPropertiesInfo[]) => void; handleAudioDeviceStateChanged: (e: DeviceInfo) => void; handleUserMessageReceived: (e: { userId: string; message: any }) => void; handleAutoPlayFail: (e: AutoPlayFailedEvent) => void; handlePlayerEvent: (e: PlayerEvent) => void; handleUserStartAudioCapture: (e: { userId: string }) => void; handleUserStopAudioCapture: (e: { userId: string }) => void; handleRoomBinaryMessageReceived: (e: { userId: string; message: ArrayBuffer }) => void; handleNetworkQuality: ( uplinkNetworkQuality: NetworkQuality, downlinkNetworkQuality: NetworkQuality ) => void; } interface EngineOptions { appId: string; uid: string; roomId: string; } export interface BasicBody { room_id: string; user_id: string; login_token: string | null; } export const AIAnsExtension = new RTCAIAnsExtension(); /** * @brief RTC Core Client * @notes Refer to official website documentation to get more information about the API. */ export class RTCClient { engine!: IRTCEngine; config!: EngineOptions; basicInfo!: BasicBody; private _audioCaptureDevice?: string; private _videoCaptureDevice?: string; audioBotEnabled = false; audioBotStartTime = 0; createEngine = async (props: EngineOptions) => { this.config = props; this.basicInfo = { room_id: props.roomId, user_id: props.uid, login_token: aigcConfig.BaseConfig.Token, }; this.engine = VERTC.createEngine(this.config.appId); try { await this.engine.registerExtension(AIAnsExtension); AIAnsExtension.enable(); } catch (error) { console.error((error as any).message); } }; addEventListeners = ({ handleError, handleUserJoin, handleUserLeave, handleUserPublishStream, handleUserUnpublishStream, handleRemoteStreamStats, handleLocalStreamStats, handleLocalAudioPropertiesReport, handleRemoteAudioPropertiesReport, handleAudioDeviceStateChanged, handleUserMessageReceived, handleAutoPlayFail, handlePlayerEvent, handleUserStartAudioCapture, handleUserStopAudioCapture, handleRoomBinaryMessageReceived, handleNetworkQuality, }: IEventListener) => { this.engine.on(VERTC.events.onError, handleError); this.engine.on(VERTC.events.onUserJoined, handleUserJoin); this.engine.on(VERTC.events.onUserLeave, handleUserLeave); this.engine.on(VERTC.events.onUserPublishStream, handleUserPublishStream); this.engine.on(VERTC.events.onUserUnpublishStream, handleUserUnpublishStream); this.engine.on(VERTC.events.onRemoteStreamStats, handleRemoteStreamStats); this.engine.on(VERTC.events.onLocalStreamStats, handleLocalStreamStats); this.engine.on(VERTC.events.onAudioDeviceStateChanged, handleAudioDeviceStateChanged); this.engine.on(VERTC.events.onLocalAudioPropertiesReport, handleLocalAudioPropertiesReport); this.engine.on(VERTC.events.onRemoteAudioPropertiesReport, handleRemoteAudioPropertiesReport); this.engine.on(VERTC.events.onUserMessageReceived, handleUserMessageReceived); this.engine.on(VERTC.events.onAutoplayFailed, handleAutoPlayFail); this.engine.on(VERTC.events.onPlayerEvent, handlePlayerEvent); this.engine.on(VERTC.events.onUserStartAudioCapture, handleUserStartAudioCapture); this.engine.on(VERTC.events.onUserStopAudioCapture, handleUserStopAudioCapture); this.engine.on(VERTC.events.onRoomBinaryMessageReceived, handleRoomBinaryMessageReceived); this.engine.on(VERTC.events.onNetworkQuality, handleNetworkQuality); }; joinRoom = (token: string | null, username: string): Promise => { this.engine.enableAudioPropertiesReport({ interval: 1000 }); return this.engine.joinRoom( token, `${this.config.roomId!}`, { userId: this.config.uid!, extraInfo: JSON.stringify({ call_scene: 'RTC-AIGC', user_name: username, user_id: this.config.uid, }), }, { isAutoPublish: true, isAutoSubscribeAudio: true, roomProfileType: RoomProfileType.chat, } ); }; leaveRoom = () => { this.stopAudioBot(); this.audioBotEnabled = false; this.engine.leaveRoom(); VERTC.destroyEngine(this.engine); this._audioCaptureDevice = undefined; }; checkPermission(): Promise<{ video: boolean; audio: boolean; }> { return VERTC.enableDevices({ video: false, audio: true, }); } /** * @brief get the devices * @returns */ async getDevices(props?: { video?: boolean; audio?: boolean }): Promise<{ audioInputs: MediaDeviceInfo[]; audioOutputs: MediaDeviceInfo[]; videoInputs: MediaDeviceInfo[]; }> { const { video, audio = true } = props || {}; let audioInputs: MediaDeviceInfo[] = []; let audioOutputs: MediaDeviceInfo[] = []; let videoInputs: MediaDeviceInfo[] = []; if (audio) { const inputs = await VERTC.enumerateAudioCaptureDevices(); const outputs = await VERTC.enumerateAudioPlaybackDevices(); audioInputs = inputs.filter((i) => i.deviceId && i.kind === 'audioinput'); audioOutputs = outputs.filter((i) => i.deviceId && i.kind === 'audiooutput'); this._audioCaptureDevice = audioInputs.filter((i) => i.deviceId)?.[0]?.deviceId; } if (video) { videoInputs = await VERTC.enumerateVideoCaptureDevices(); videoInputs = videoInputs.filter((i) => i.deviceId && i.kind === 'videoinput'); this._videoCaptureDevice = videoInputs?.[0]?.deviceId; } return { audioInputs, audioOutputs, videoInputs, }; } startVideoCapture = async (camera?: string) => { await this.engine.startVideoCapture(camera || this._videoCaptureDevice); }; stopVideoCapture = async () => { this.engine.setLocalVideoMirrorType(MirrorType.MIRROR_TYPE_RENDER); await this.engine.stopVideoCapture(); }; startAudioCapture = async (mic?: string) => { await this.engine.startAudioCapture(mic || this._audioCaptureDevice); }; stopAudioCapture = async () => { await this.engine.stopAudioCapture(); }; publishStream = (mediaType: MediaType) => { this.engine.publishStream(mediaType); }; unpublishStream = (mediaType: MediaType) => { this.engine.unpublishStream(mediaType); }; /** * @brief 设置业务标识参数 * @param businessId */ setBusinessId = (businessId: string) => { this.engine.setBusinessId(businessId); }; setAudioVolume = (volume: number) => { this.engine.setCaptureVolume(StreamIndex.STREAM_INDEX_MAIN, volume); this.engine.setCaptureVolume(StreamIndex.STREAM_INDEX_SCREEN, volume); }; /** * @brief 设置音质档位 */ setAudioProfile = (profile: AudioProfileType) => { this.engine.setAudioProfile(profile); }; /** * @brief 切换设备 */ switchDevice = (deviceType: MediaType, deviceId: string) => { if (deviceType === MediaType.AUDIO) { this._audioCaptureDevice = deviceId; this.engine.setAudioCaptureDevice(deviceId); } if (deviceType === MediaType.VIDEO) { this._videoCaptureDevice = deviceId; this.engine.setVideoCaptureDevice(deviceId); } if (deviceType === MediaType.AUDIO_AND_VIDEO) { this._audioCaptureDevice = deviceId; this._videoCaptureDevice = deviceId; this.engine.setVideoCaptureDevice(deviceId); this.engine.setAudioCaptureDevice(deviceId); } }; setLocalVideoMirrorType = (type: MirrorType) => { return this.engine.setLocalVideoMirrorType(type); }; setLocalVideoPlayer = (userId: string, renderDom?: string | HTMLElement) => { return this.engine.setLocalVideoPlayer(StreamIndex.STREAM_INDEX_MAIN, { renderDom, userId, renderMode: VideoRenderMode.RENDER_MODE_HIDDEN, }); }; /** * @brief 启用 AIGC */ startAudioBot = async () => { const roomId = this.basicInfo.room_id; const userId = this.basicInfo.user_id; if (this.audioBotEnabled) { await this.stopAudioBot(); } const agentConfig = aigcConfig.aigcConfig.AgentConfig; const options = { AppId: aigcConfig.BaseConfig.AppId, BusinessId: aigcConfig.BaseConfig.BusinessId, RoomId: roomId, TaskId: userId, AgentConfig: { ...agentConfig, TargetUserId: [userId], }, Config: aigcConfig.aigcConfig.Config, }; await openAPIs.StartVoiceChat(options); this.audioBotEnabled = true; this.audioBotStartTime = Date.now(); Utils.setSessionInfo({ audioBotEnabled: 'enable' }); }; /** * @brief 关闭 AIGC */ stopAudioBot = async () => { const roomId = this.basicInfo.room_id; const userId = this.basicInfo.user_id; if (this.audioBotEnabled || sessionStorage.getItem('audioBotEnabled')) { await openAPIs.StopVoiceChat({ AppId: aigcConfig.BaseConfig.AppId, BusinessId: aigcConfig.BaseConfig.BusinessId, RoomId: roomId, TaskId: userId, }); this.audioBotStartTime = 0; sessionStorage.removeItem('audioBotEnabled'); } this.audioBotEnabled = false; }; /** * @brief 命令 AIGC */ commandAudioBot = ( command: COMMAND, interruptMode = INTERRUPT_PRIORITY.NONE, message = '' ) => { if (this.audioBotEnabled) { this.engine.sendUserBinaryMessage( aigcConfig.BotName, Utils.string2tlv( JSON.stringify({ Command: command, InterruptMode: interruptMode, Message: message, }), 'ctrl' ) ); return; } console.warn('Interrupt failed, bot not enabled.'); }; /** * @brief 更新 AIGC 配置 */ updateAudioBot = async () => { if (this.audioBotEnabled) { await this.stopAudioBot(); await this.startAudioBot(); } else { await this.startAudioBot(); } }; /** * @brief 获取当前 AI 是否启用 */ getAudioBotEnabled = () => { return this.audioBotEnabled; }; } export default new RTCClient();