390 lines
11 KiB
TypeScript
390 lines
11 KiB
TypeScript
/**
|
|
* Copyright 2025 Beijing Volcano Engine Technology Co., Ltd. All Rights Reserved.
|
|
* SPDX-license-identifier: BSD-3-Clause
|
|
*/
|
|
|
|
|
|
import VERTC, {
|
|
MirrorType,
|
|
StreamIndex,
|
|
IRTCEngine,
|
|
RoomProfileType,
|
|
onUserJoinedEvent,
|
|
onUserLeaveEvent,
|
|
MediaType,
|
|
LocalStreamStats,
|
|
RemoteStreamStats,
|
|
StreamRemoveReason,
|
|
LocalAudioPropertiesInfo,
|
|
RemoteAudioPropertiesInfo,
|
|
AudioProfileType,
|
|
DeviceInfo,
|
|
AutoPlayFailedEvent,
|
|
PlayerEvent,
|
|
NetworkQuality,
|
|
VideoRenderMode,
|
|
} from '@volcengine/rtc';
|
|
import RTCAIAnsExtension from '@volcengine/rtc/extension-ainr';
|
|
import openAPIs from '@/app/api';
|
|
import aigcConfig from '@/config';
|
|
import Utils from '@/utils/utils';
|
|
import { COMMAND, INTERRUPT_PRIORITY } from '@/utils/handler';
|
|
|
|
export interface IEventListener {
|
|
handleError: (e: { errorCode: any }) => void;
|
|
handleUserJoin: (e: onUserJoinedEvent) => void;
|
|
handleUserLeave: (e: onUserLeaveEvent) => void;
|
|
handleUserPublishStream: (e: { userId: string; mediaType: MediaType }) => void;
|
|
handleUserUnpublishStream: (e: {
|
|
userId: string;
|
|
mediaType: MediaType;
|
|
reason: StreamRemoveReason;
|
|
}) => void;
|
|
handleRemoteStreamStats: (e: RemoteStreamStats) => void;
|
|
handleLocalStreamStats: (e: LocalStreamStats) => void;
|
|
handleLocalAudioPropertiesReport: (e: LocalAudioPropertiesInfo[]) => void;
|
|
handleRemoteAudioPropertiesReport: (e: RemoteAudioPropertiesInfo[]) => void;
|
|
handleAudioDeviceStateChanged: (e: DeviceInfo) => void;
|
|
handleUserMessageReceived: (e: { userId: string; message: any }) => void;
|
|
handleAutoPlayFail: (e: AutoPlayFailedEvent) => void;
|
|
handlePlayerEvent: (e: PlayerEvent) => void;
|
|
handleUserStartAudioCapture: (e: { userId: string }) => void;
|
|
handleUserStopAudioCapture: (e: { userId: string }) => void;
|
|
handleRoomBinaryMessageReceived: (e: { userId: string; message: ArrayBuffer }) => void;
|
|
handleNetworkQuality: (
|
|
uplinkNetworkQuality: NetworkQuality,
|
|
downlinkNetworkQuality: NetworkQuality
|
|
) => void;
|
|
}
|
|
|
|
interface EngineOptions {
|
|
appId: string;
|
|
uid: string;
|
|
roomId: string;
|
|
}
|
|
|
|
export interface BasicBody {
|
|
room_id: string;
|
|
user_id: string;
|
|
login_token: string | null;
|
|
}
|
|
|
|
export const AIAnsExtension = new RTCAIAnsExtension();
|
|
|
|
/**
|
|
* @brief RTC Core Client
|
|
* @notes Refer to official website documentation to get more information about the API.
|
|
*/
|
|
export class RTCClient {
|
|
engine!: IRTCEngine;
|
|
|
|
config!: EngineOptions;
|
|
|
|
basicInfo!: BasicBody;
|
|
|
|
private _audioCaptureDevice?: string;
|
|
|
|
private _videoCaptureDevice?: string;
|
|
|
|
audioBotEnabled = false;
|
|
|
|
audioBotStartTime = 0;
|
|
|
|
createEngine = async (props: EngineOptions) => {
|
|
this.config = props;
|
|
this.basicInfo = {
|
|
room_id: props.roomId,
|
|
user_id: props.uid,
|
|
login_token: aigcConfig.BaseConfig.Token,
|
|
};
|
|
|
|
this.engine = VERTC.createEngine(this.config.appId);
|
|
try {
|
|
await this.engine.registerExtension(AIAnsExtension);
|
|
AIAnsExtension.enable();
|
|
} catch (error) {
|
|
console.error((error as any).message);
|
|
}
|
|
};
|
|
|
|
addEventListeners = ({
|
|
handleError,
|
|
handleUserJoin,
|
|
handleUserLeave,
|
|
handleUserPublishStream,
|
|
handleUserUnpublishStream,
|
|
handleRemoteStreamStats,
|
|
handleLocalStreamStats,
|
|
handleLocalAudioPropertiesReport,
|
|
handleRemoteAudioPropertiesReport,
|
|
handleAudioDeviceStateChanged,
|
|
handleUserMessageReceived,
|
|
handleAutoPlayFail,
|
|
handlePlayerEvent,
|
|
handleUserStartAudioCapture,
|
|
handleUserStopAudioCapture,
|
|
handleRoomBinaryMessageReceived,
|
|
handleNetworkQuality,
|
|
}: IEventListener) => {
|
|
this.engine.on(VERTC.events.onError, handleError);
|
|
this.engine.on(VERTC.events.onUserJoined, handleUserJoin);
|
|
this.engine.on(VERTC.events.onUserLeave, handleUserLeave);
|
|
this.engine.on(VERTC.events.onUserPublishStream, handleUserPublishStream);
|
|
this.engine.on(VERTC.events.onUserUnpublishStream, handleUserUnpublishStream);
|
|
this.engine.on(VERTC.events.onRemoteStreamStats, handleRemoteStreamStats);
|
|
this.engine.on(VERTC.events.onLocalStreamStats, handleLocalStreamStats);
|
|
this.engine.on(VERTC.events.onAudioDeviceStateChanged, handleAudioDeviceStateChanged);
|
|
this.engine.on(VERTC.events.onLocalAudioPropertiesReport, handleLocalAudioPropertiesReport);
|
|
this.engine.on(VERTC.events.onRemoteAudioPropertiesReport, handleRemoteAudioPropertiesReport);
|
|
this.engine.on(VERTC.events.onUserMessageReceived, handleUserMessageReceived);
|
|
this.engine.on(VERTC.events.onAutoplayFailed, handleAutoPlayFail);
|
|
this.engine.on(VERTC.events.onPlayerEvent, handlePlayerEvent);
|
|
this.engine.on(VERTC.events.onUserStartAudioCapture, handleUserStartAudioCapture);
|
|
this.engine.on(VERTC.events.onUserStopAudioCapture, handleUserStopAudioCapture);
|
|
this.engine.on(VERTC.events.onRoomBinaryMessageReceived, handleRoomBinaryMessageReceived);
|
|
this.engine.on(VERTC.events.onNetworkQuality, handleNetworkQuality);
|
|
};
|
|
|
|
joinRoom = (token: string | null, username: string): Promise<void> => {
|
|
this.engine.enableAudioPropertiesReport({ interval: 1000 });
|
|
return this.engine.joinRoom(
|
|
token,
|
|
`${this.config.roomId!}`,
|
|
{
|
|
userId: this.config.uid!,
|
|
extraInfo: JSON.stringify({
|
|
call_scene: 'RTC-AIGC',
|
|
user_name: username,
|
|
user_id: this.config.uid,
|
|
}),
|
|
},
|
|
{
|
|
isAutoPublish: true,
|
|
isAutoSubscribeAudio: true,
|
|
roomProfileType: RoomProfileType.chat,
|
|
}
|
|
);
|
|
};
|
|
|
|
leaveRoom = () => {
|
|
this.stopAudioBot();
|
|
this.audioBotEnabled = false;
|
|
this.engine.leaveRoom();
|
|
VERTC.destroyEngine(this.engine);
|
|
this._audioCaptureDevice = undefined;
|
|
};
|
|
|
|
checkPermission(): Promise<{
|
|
video: boolean;
|
|
audio: boolean;
|
|
}> {
|
|
return VERTC.enableDevices({
|
|
video: false,
|
|
audio: true,
|
|
});
|
|
}
|
|
|
|
/**
|
|
* @brief get the devices
|
|
* @returns
|
|
*/
|
|
async getDevices(props?: { video?: boolean; audio?: boolean }): Promise<{
|
|
audioInputs: MediaDeviceInfo[];
|
|
audioOutputs: MediaDeviceInfo[];
|
|
videoInputs: MediaDeviceInfo[];
|
|
}> {
|
|
const { video, audio = true } = props || {};
|
|
let audioInputs: MediaDeviceInfo[] = [];
|
|
let audioOutputs: MediaDeviceInfo[] = [];
|
|
let videoInputs: MediaDeviceInfo[] = [];
|
|
if (audio) {
|
|
const inputs = await VERTC.enumerateAudioCaptureDevices();
|
|
const outputs = await VERTC.enumerateAudioPlaybackDevices();
|
|
audioInputs = inputs.filter((i) => i.deviceId && i.kind === 'audioinput');
|
|
audioOutputs = outputs.filter((i) => i.deviceId && i.kind === 'audiooutput');
|
|
this._audioCaptureDevice = audioInputs.filter((i) => i.deviceId)?.[0]?.deviceId;
|
|
}
|
|
if (video) {
|
|
videoInputs = await VERTC.enumerateVideoCaptureDevices();
|
|
videoInputs = videoInputs.filter((i) => i.deviceId && i.kind === 'videoinput');
|
|
this._videoCaptureDevice = videoInputs?.[0]?.deviceId;
|
|
}
|
|
|
|
return {
|
|
audioInputs,
|
|
audioOutputs,
|
|
videoInputs,
|
|
};
|
|
}
|
|
|
|
startVideoCapture = async (camera?: string) => {
|
|
await this.engine.startVideoCapture(camera || this._videoCaptureDevice);
|
|
};
|
|
|
|
stopVideoCapture = async () => {
|
|
this.engine.setLocalVideoMirrorType(MirrorType.MIRROR_TYPE_RENDER);
|
|
await this.engine.stopVideoCapture();
|
|
};
|
|
|
|
startAudioCapture = async (mic?: string) => {
|
|
await this.engine.startAudioCapture(mic || this._audioCaptureDevice);
|
|
};
|
|
|
|
stopAudioCapture = async () => {
|
|
await this.engine.stopAudioCapture();
|
|
};
|
|
|
|
publishStream = (mediaType: MediaType) => {
|
|
this.engine.publishStream(mediaType);
|
|
};
|
|
|
|
unpublishStream = (mediaType: MediaType) => {
|
|
this.engine.unpublishStream(mediaType);
|
|
};
|
|
|
|
/**
|
|
* @brief 设置业务标识参数
|
|
* @param businessId
|
|
*/
|
|
setBusinessId = (businessId: string) => {
|
|
this.engine.setBusinessId(businessId);
|
|
};
|
|
|
|
setAudioVolume = (volume: number) => {
|
|
this.engine.setCaptureVolume(StreamIndex.STREAM_INDEX_MAIN, volume);
|
|
this.engine.setCaptureVolume(StreamIndex.STREAM_INDEX_SCREEN, volume);
|
|
};
|
|
|
|
/**
|
|
* @brief 设置音质档位
|
|
*/
|
|
setAudioProfile = (profile: AudioProfileType) => {
|
|
this.engine.setAudioProfile(profile);
|
|
};
|
|
|
|
/**
|
|
* @brief 切换设备
|
|
*/
|
|
switchDevice = (deviceType: MediaType, deviceId: string) => {
|
|
if (deviceType === MediaType.AUDIO) {
|
|
this._audioCaptureDevice = deviceId;
|
|
this.engine.setAudioCaptureDevice(deviceId);
|
|
}
|
|
if (deviceType === MediaType.VIDEO) {
|
|
this._videoCaptureDevice = deviceId;
|
|
this.engine.setVideoCaptureDevice(deviceId);
|
|
}
|
|
if (deviceType === MediaType.AUDIO_AND_VIDEO) {
|
|
this._audioCaptureDevice = deviceId;
|
|
this._videoCaptureDevice = deviceId;
|
|
this.engine.setVideoCaptureDevice(deviceId);
|
|
this.engine.setAudioCaptureDevice(deviceId);
|
|
}
|
|
};
|
|
|
|
setLocalVideoMirrorType = (type: MirrorType) => {
|
|
return this.engine.setLocalVideoMirrorType(type);
|
|
};
|
|
|
|
setLocalVideoPlayer = (userId: string, renderDom?: string | HTMLElement) => {
|
|
return this.engine.setLocalVideoPlayer(StreamIndex.STREAM_INDEX_MAIN, {
|
|
renderDom,
|
|
userId,
|
|
renderMode: VideoRenderMode.RENDER_MODE_HIDDEN,
|
|
});
|
|
};
|
|
|
|
/**
|
|
* @brief 启用 AIGC
|
|
*/
|
|
startAudioBot = async () => {
|
|
const roomId = this.basicInfo.room_id;
|
|
const userId = this.basicInfo.user_id;
|
|
if (this.audioBotEnabled) {
|
|
await this.stopAudioBot();
|
|
}
|
|
const agentConfig = aigcConfig.aigcConfig.AgentConfig;
|
|
|
|
const options = {
|
|
AppId: aigcConfig.BaseConfig.AppId,
|
|
BusinessId: aigcConfig.BaseConfig.BusinessId,
|
|
RoomId: roomId,
|
|
TaskId: userId,
|
|
AgentConfig: {
|
|
...agentConfig,
|
|
TargetUserId: [userId],
|
|
},
|
|
Config: aigcConfig.aigcConfig.Config,
|
|
};
|
|
await openAPIs.StartVoiceChat(options);
|
|
this.audioBotEnabled = true;
|
|
this.audioBotStartTime = Date.now();
|
|
Utils.setSessionInfo({ audioBotEnabled: 'enable' });
|
|
};
|
|
|
|
/**
|
|
* @brief 关闭 AIGC
|
|
*/
|
|
stopAudioBot = async () => {
|
|
const roomId = this.basicInfo.room_id;
|
|
const userId = this.basicInfo.user_id;
|
|
if (this.audioBotEnabled || sessionStorage.getItem('audioBotEnabled')) {
|
|
await openAPIs.StopVoiceChat({
|
|
AppId: aigcConfig.BaseConfig.AppId,
|
|
BusinessId: aigcConfig.BaseConfig.BusinessId,
|
|
RoomId: roomId,
|
|
TaskId: userId,
|
|
});
|
|
this.audioBotStartTime = 0;
|
|
sessionStorage.removeItem('audioBotEnabled');
|
|
}
|
|
this.audioBotEnabled = false;
|
|
};
|
|
|
|
/**
|
|
* @brief 命令 AIGC
|
|
*/
|
|
commandAudioBot = (
|
|
command: COMMAND,
|
|
interruptMode = INTERRUPT_PRIORITY.NONE,
|
|
message = ''
|
|
) => {
|
|
if (this.audioBotEnabled) {
|
|
this.engine.sendUserBinaryMessage(
|
|
aigcConfig.BotName,
|
|
Utils.string2tlv(
|
|
JSON.stringify({
|
|
Command: command,
|
|
InterruptMode: interruptMode,
|
|
Message: message,
|
|
}),
|
|
'ctrl'
|
|
)
|
|
);
|
|
return;
|
|
}
|
|
console.warn('Interrupt failed, bot not enabled.');
|
|
};
|
|
|
|
/**
|
|
* @brief 更新 AIGC 配置
|
|
*/
|
|
updateAudioBot = async () => {
|
|
if (this.audioBotEnabled) {
|
|
await this.stopAudioBot();
|
|
await this.startAudioBot();
|
|
} else {
|
|
await this.startAudioBot();
|
|
}
|
|
};
|
|
|
|
/**
|
|
* @brief 获取当前 AI 是否启用
|
|
*/
|
|
getAudioBotEnabled = () => {
|
|
return this.audioBotEnabled;
|
|
};
|
|
}
|
|
|
|
export default new RTCClient();
|