From 033127de9488b8fba0b17abb4b4e620a32b19bf2 Mon Sep 17 00:00:00 2001 From: xjs Date: Tue, 1 Apr 2025 11:38:53 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E5=8F=B7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- package.json | 2 +- public/favicon.png | Bin 3251 -> 823 bytes src/config/index.ts | 4 +- src/lib/RtcClient.ts | 88 ++++++++++++++---- src/lib/listenerHooks.ts | 38 +++----- .../MainPage/MainArea/Room/Conversation.tsx | 8 +- .../MainPage/MainArea/Room/index.module.less | 2 +- yarn.lock | 8 +- 8 files changed, 96 insertions(+), 54 deletions(-) diff --git a/package.json b/package.json index 85c1c3b..91c5c48 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "dependencies": { "@arco-design/web-react": "^2.65.0", "@reduxjs/toolkit": "^1.8.3", - "@volcengine/rtc": "4.58.9", + "@volcengine/rtc": "4.66.1", "autolinker": "^4.0.0", "i18next": "^21.8.16", "react": "^18.2.0", diff --git a/public/favicon.png b/public/favicon.png index 87017b545d29aa1c731a4af31f0941a80ddf4085..50492875fa2ed5831242cf37a6c396a31068e650 100644 GIT binary patch literal 823 zcmV-71IYY|P)nMu{(&?gMGqyLi-n4`J$Mmp6Y<{mIxHJ=_98`r$yV6hrR))fi$>(dFABg?!O#w=?00YWy zLSY=rcmiIg*!Yy22-*osBUHECVG*egkae)+t#Rgb0wU_N1Cq;h3?u=-l1Hs;!E9uv!8fg zdS46``pmHy+um+&iz$tX#p|z^WtgMHD2dD`4fBNK(3)f29Np?9NOgnM_ff4MqR6fO z!yGe7*4=$9?;ap?;k>X-p{|uJrL5f$w*oW!4rrW2;&kU(&{J6V;pG@@?(f1s*b^1) z1C7Jh6NkXBIx;&VgYQ3qFFz#7?Ws)@sRbM9n=%hybX_o^s`N2Ozx6pCMu{RjXCYWI zA1mdQGV`Q#V3n?lLy*2Z?f*5whsVu7xW{p?A)wqoRzKgE&T^zZeWVK<>Jmo=PGmd` zuoMPc!Xn?U-u6~c+APU{6NQxBm~$pYHF_{p8w5l{65!!`-jbnZcMe)V-jU4VsDbL{ z)v0xtXfaU%Gl_`UPd|D!HVntXa9ap<3%psmY2LUMidA5f>mV{D`e&Iz_6&@nEkfFe zWEJl5aqrVER=;`_q~*rdZ;Msoo=w^$EY@*`{Q(y4hv`sgykY1zMfIwiAKLqk{=OCd^tc{Zhbleazv$Z>)2 z0|q!2%bLwrh!)643KXLR@@oz70Tah&gQ@l8Bz9;B5EP>Y09TI`_F9Po>m-2+h+5-g zfdarI1wv+|;CzEn&43~dQMxDqu>d3pgdm6&hWx)8ZfFRApee@-Ia}fcLaAs0j>B2E zgdHAMiV+fk9}oz7#TbFj)RYhbc>!f01q8v?EEd>q-744xfGU>-N>M@_Z~%FovvXtw zjJ4AQ`#2n7HP~!0Rf!cE!;g#rJ{W++ff>-&qZB|2Yhy6ov6dj1uBU9pX73yx283N) zu0rzv)l(=7XkhC#aqIDMrOPYpW3UA9^`^L9v9;3yCby(7DFgq&Owu1AVa+o*9S?!P zdU#mp;jGT18DIcZOt*a6qL*XRHe%d13alzpRxr<|H8K~q3+6EuGiDuQnpsQ6jbmG= zOF&KU#gxIzU&?Dq0X2Yt%4iq-R!dq2YQPUDizSn#aB&q=Ic-EAHSgsBx60T4Xyz{H z*Gz7IKMwRwJGokUb2=rz)zf$euO`(}c^2Q0&BxE6{si`4HTMTyh8dh`FV!kY# zOj<#Wob~&L>OzX@{7LhB_w?Hsyg{6}2*J`ez z%uGgRqWdQz{_3HR`WucMwq+$PEB%t#PNmn~D zxo4V3v@{S&hcQ|XC_7QPi&Jm9|05|XY4?cYM>(*^Bic#MZfc?bRsWlNZ+P-2E`;Tt zh>z!NXq#AUXm5X2N&65G5Tc#pFfWn-au`X}NMrc@+u-M+L>aG#Ms30p`Vn z)KM>2BYjso&kVr_^hCbA*>jju8VHTb`@=%ubt?tmd<|gGG)V^g z{PQF#Xjdg#A_{K`jYav&b~$IOvhTM*4#&;IJ4ndcV|HJB3wIVK6h3ra>8wP-IugIu zeQuK_)aw2PML$!^zcoYfv3)Fq+Z7MBg!8dRhKsN#&lzq z(7N^ngC$XZxtG28NWC#m5AIdJY4lY{`bOiFpa@h4BG>kZM`%G*c4&6&gcUurndks zURUoq9XO^(m;Q`^c;+w>IVSYC+O6C;`NNeZ188(J>zXM`YpMU+Rn2mf(y7kn+f zCeBn1MTvUXt29*eEg`w$`1$N7u$CWJ;F8ZT{n_=XOav{#)AJG`SQbL?$4rk5e`z&i zT+6?Ge7dC^p@MJMmI>ZncU9S?vJ|)+A#d~RqIe&VKhDSsGf_X8(9H&qnL%)Mk9HXK zcTiWKEL6mpccWeBxpfMG*pn-43?eUqZ+D~TCpmb|cC`Z*m=C3UPM0vTrLl&$j<)>R zL7`ULAJ9A*Aq#-3M?dUUgF3nTrTZHvrfc?oF-2~Q4aXlM#%c{aos%HBq_rP}stO^m zhx;2(PuFNiD_LV8T(!OIlO3qadhQ5X_e?&-WhBo86$69dZOl9=(@I z7Dq4YK8`D5e3Ht}k;b3!i@5j@!qS{dHk;$dW0Bw?8LYN9E+E}FpOb;_&Ipa*xr2nH zlcki7=QEVD298Isuw@Z`=QC3M<@u_8Y!c7MeWV)O9<-knl;x8EtMeStinKbb>Pt&b zNt(3s;qijY=r&2nJm_pc)|dACT*pR>GZCpC80enE%zVuvhjBBaJYV5yn=Hg7^W&j% zeCFuAaMQ9&hLKb=4!PDlwV2U_oc)9~&7mJL7sXx`>3GvVGgtF$_C`}f)&+a1+FlTehZjM|~A2UofzmQE~xZ(%Q75hL&i-g)wT z_WCvFi=#808&0HD;;k%<10E#M)q)^)K6KlQL#{_=+kn+3sE6V$>Qpv^J16mJBR^xm zDRi3&SsrGQ$B@}FB;ZXCp}|n59X!{OV-gmT>Z2R$a_(sO?YN)6NE8m1W>C?{ePl*M z<4EAVwpTT99V=;E9*JTr5nKK?WZ(Vx8Lxdk^}GSRK}@+^S8$ak6lV`Bd)mPL9_Z;x zfq+%VRlJp}ML(YJabYWz6t@qyjXOmSGxn2q3e_yfPPz_PPRk;g3dJ#@XtLEu zTi9}YQz{$=7zMv&U{OET?T;m+K0B zSmF&RKA>~%z?USrWT_}#PHoC|nW$VM$axqug+sm>D^!pDMGvc3T$rxZI)HUR|B^*MHaGpS(L_OZ_+b8vCjN whBq{RtnY@(_}{GN6cww`f$IMmj4eyN)vu%Hl26Bhe_jyL-o>um*6-4P0aoK$Hvj+t diff --git a/src/config/index.ts b/src/config/index.ts index a5227f4..fb78632 100644 --- a/src/config/index.ts +++ b/src/config/index.ts @@ -7,8 +7,8 @@ import { ConfigFactory } from './config'; export * from './common'; -export const AIGC_PROXY_HOST = 'http://192.168.31.106:3001'; -export const DEMO_VERSION = '1.4.0'; +export const AIGC_PROXY_HOST = 'https://aigc.ycymedu.com'; +export const DEMO_VERSION = '1.5.0'; export const Config = ConfigFactory; export default new ConfigFactory(); diff --git a/src/lib/RtcClient.ts b/src/lib/RtcClient.ts index e0f400c..3492f2b 100644 --- a/src/lib/RtcClient.ts +++ b/src/lib/RtcClient.ts @@ -3,7 +3,6 @@ * SPDX-license-identifier: BSD-3-Clause */ - import VERTC, { MirrorType, StreamIndex, @@ -23,8 +22,10 @@ import VERTC, { PlayerEvent, NetworkQuality, VideoRenderMode, + ScreenEncoderConfig, } from '@volcengine/rtc'; import RTCAIAnsExtension from '@volcengine/rtc/extension-ainr'; +import { Message } from '@arco-design/web-react'; import openAPIs from '@/app/api'; import aigcConfig from '@/config'; import Utils from '@/utils/utils'; @@ -34,6 +35,7 @@ export interface IEventListener { handleError: (e: { errorCode: any }) => void; handleUserJoin: (e: onUserJoinedEvent) => void; handleUserLeave: (e: onUserLeaveEvent) => void; + handleTrackEnded: (e: { kind: string; isScreen: boolean }) => void; handleUserPublishStream: (e: { userId: string; mediaType: MediaType }) => void; handleUserUnpublishStream: (e: { userId: string; @@ -45,7 +47,6 @@ export interface IEventListener { handleLocalAudioPropertiesReport: (e: LocalAudioPropertiesInfo[]) => void; handleRemoteAudioPropertiesReport: (e: RemoteAudioPropertiesInfo[]) => void; handleAudioDeviceStateChanged: (e: DeviceInfo) => void; - handleUserMessageReceived: (e: { userId: string; message: any }) => void; handleAutoPlayFail: (e: AutoPlayFailedEvent) => void; handlePlayerEvent: (e: PlayerEvent) => void; handleUserStartAudioCapture: (e: { userId: string }) => void; @@ -69,7 +70,7 @@ export interface BasicBody { room_id: string; user_id: string; login_token: string | null; - init_msg:string|null; + init_msg:string|null } export const AIAnsExtension = new RTCAIAnsExtension(); @@ -107,7 +108,9 @@ export class RTCClient { await this.engine.registerExtension(AIAnsExtension); AIAnsExtension.enable(); } catch (error) { - console.error((error as any).message); + console.warn( + `当前环境不支持 AI 降噪, 此错误可忽略, 不影响实际使用, e: ${(error as any).message}` + ); } }; @@ -115,6 +118,7 @@ export class RTCClient { handleError, handleUserJoin, handleUserLeave, + handleTrackEnded, handleUserPublishStream, handleUserUnpublishStream, handleRemoteStreamStats, @@ -122,7 +126,6 @@ export class RTCClient { handleLocalAudioPropertiesReport, handleRemoteAudioPropertiesReport, handleAudioDeviceStateChanged, - handleUserMessageReceived, handleAutoPlayFail, handlePlayerEvent, handleUserStartAudioCapture, @@ -133,6 +136,7 @@ export class RTCClient { this.engine.on(VERTC.events.onError, handleError); this.engine.on(VERTC.events.onUserJoined, handleUserJoin); this.engine.on(VERTC.events.onUserLeave, handleUserLeave); + this.engine.on(VERTC.events.onTrackEnded, handleTrackEnded); this.engine.on(VERTC.events.onUserPublishStream, handleUserPublishStream); this.engine.on(VERTC.events.onUserUnpublishStream, handleUserUnpublishStream); this.engine.on(VERTC.events.onRemoteStreamStats, handleRemoteStreamStats); @@ -140,7 +144,6 @@ export class RTCClient { this.engine.on(VERTC.events.onAudioDeviceStateChanged, handleAudioDeviceStateChanged); this.engine.on(VERTC.events.onLocalAudioPropertiesReport, handleLocalAudioPropertiesReport); this.engine.on(VERTC.events.onRemoteAudioPropertiesReport, handleRemoteAudioPropertiesReport); - this.engine.on(VERTC.events.onUserMessageReceived, handleUserMessageReceived); this.engine.on(VERTC.events.onAutoplayFailed, handleAutoPlayFail); this.engine.on(VERTC.events.onPlayerEvent, handlePlayerEvent); this.engine.on(VERTC.events.onUserStartAudioCapture, handleUserStartAudioCapture); @@ -197,21 +200,42 @@ export class RTCClient { audioOutputs: MediaDeviceInfo[]; videoInputs: MediaDeviceInfo[]; }> { - const { video, audio = true } = props || {}; + const { video = false, audio = true } = props || {}; let audioInputs: MediaDeviceInfo[] = []; let audioOutputs: MediaDeviceInfo[] = []; let videoInputs: MediaDeviceInfo[] = []; + const { video: hasVideoPermission, audio: hasAudioPermission } = await VERTC.enableDevices({ + video, + audio, + }); if (audio) { const inputs = await VERTC.enumerateAudioCaptureDevices(); const outputs = await VERTC.enumerateAudioPlaybackDevices(); audioInputs = inputs.filter((i) => i.deviceId && i.kind === 'audioinput'); audioOutputs = outputs.filter((i) => i.deviceId && i.kind === 'audiooutput'); this._audioCaptureDevice = audioInputs.filter((i) => i.deviceId)?.[0]?.deviceId; + if (hasAudioPermission) { + if (!audioInputs?.length) { + Message.error('无麦克风设备, 请先确认设备情况。'); + } + if (!audioOutputs?.length) { + // Message.error('无扬声器设备, 请先确认设备情况。'); + } + } else { + Message.error('暂无麦克风设备权限, 请先确认设备权限授予情况。'); + } } if (video) { videoInputs = await VERTC.enumerateVideoCaptureDevices(); videoInputs = videoInputs.filter((i) => i.deviceId && i.kind === 'videoinput'); this._videoCaptureDevice = videoInputs?.[0]?.deviceId; + if (hasVideoPermission) { + if (!videoInputs?.length) { + Message.error('无摄像头设备, 请先确认设备情况。'); + } + } else { + Message.error('暂无摄像头设备权限, 请先确认设备权限授予情况。'); + } } return { @@ -230,7 +254,17 @@ export class RTCClient { await this.engine.stopVideoCapture(); }; - startAudioCapture = async (mic?: string) => { + startScreenCapture = async (enableAudio = false) => { + await this.engine.startScreenCapture({ + enableAudio, + }); + }; + + stopScreenCapture = async () => { + await this.engine.stopScreenCapture(); + }; + + startAudioCapture = async (mic?: string) => { await this.engine.startAudioCapture(mic || this._audioCaptureDevice); }; @@ -246,6 +280,18 @@ export class RTCClient { this.engine.unpublishStream(mediaType); }; + publishScreenStream = async (mediaType: MediaType) => { + await this.engine.publishScreen(mediaType); + }; + + unpublishScreenStream = async (mediaType: MediaType) => { + await this.engine.unpublishScreen(mediaType); + }; + + setScreenEncoderConfig = async (description: ScreenEncoderConfig) => { + await this.engine.setScreenEncoderConfig(description); + }; + /** * @brief 设置业务标识参数 * @param businessId @@ -290,12 +336,19 @@ export class RTCClient { return this.engine.setLocalVideoMirrorType(type); }; - setLocalVideoPlayer = (userId: string, renderDom?: string | HTMLElement) => { - return this.engine.setLocalVideoPlayer(StreamIndex.STREAM_INDEX_MAIN, { - renderDom, - userId, - renderMode: VideoRenderMode.RENDER_MODE_HIDDEN, - }); + setLocalVideoPlayer = ( + userId: string, + renderDom?: string | HTMLElement, + isScreenShare = false + ) => { + return this.engine.setLocalVideoPlayer( + isScreenShare ? StreamIndex.STREAM_INDEX_SCREEN : StreamIndex.STREAM_INDEX_MAIN, + { + renderDom, + userId, + renderMode: VideoRenderMode.RENDER_MODE_FILL, + } + ); }; /** @@ -311,7 +364,6 @@ export class RTCClient { if(this.basicInfo.init_msg){ agentConfig.WelcomeMessage = "" } - const options = { AppId: aigcConfig.BaseConfig.AppId, BusinessId: aigcConfig.BaseConfig.BusinessId, @@ -351,11 +403,7 @@ export class RTCClient { /** * @brief 命令 AIGC */ - commandAudioBot = ( - command: COMMAND, - interruptMode = INTERRUPT_PRIORITY.NONE, - message = '' - ) => { + commandAudioBot = (command: COMMAND, interruptMode = INTERRUPT_PRIORITY.NONE, message = '') => { if (this.audioBotEnabled) { this.engine.sendUserBinaryMessage( aigcConfig.BotName, diff --git a/src/lib/listenerHooks.ts b/src/lib/listenerHooks.ts index ff2249c..af94c00 100644 --- a/src/lib/listenerHooks.ts +++ b/src/lib/listenerHooks.ts @@ -30,14 +30,11 @@ import { addAutoPlayFail, removeAutoPlayFail, updateAITalkState, - setHistoryMsg, - setCurrentMsg, updateNetworkQuality, } from '@/store/slices/room'; import RtcClient, { IEventListener } from './RtcClient'; import { setMicrophoneList, updateSelectedDevice } from '@/store/slices/device'; -import Utils from '@/utils/utils'; import { useMessageHandler } from '@/utils/handler'; const useRtcListeners = (): IEventListener => { @@ -45,12 +42,19 @@ const useRtcListeners = (): IEventListener => { const { parser } = useMessageHandler(); const playStatus = useRef<{ [key: string]: { audio: boolean; video: boolean } }>({}); - const debounceSetHistoryMsg = Utils.debounce((text: string, user: string) => { - const isAudioEnable = RtcClient.getAudioBotEnabled(); - if (isAudioEnable) { - dispatch(setHistoryMsg({ text, user })); + const handleTrackEnded = async (event: { kind: string; isScreen: boolean }) => { + const { kind, isScreen } = event; + /** 浏览器自带的屏幕共享关闭触发方式，通过 onTrackEnd 事件去关闭 */ + if (isScreen && kind === 'video') { + await RtcClient.stopScreenCapture(); + await RtcClient.unpublishScreenStream(MediaType.VIDEO); + dispatch( + updateLocalUser({ + publishScreen: false, + }) + ); } - }, 600); + }; const handleUserJoin = (e: onUserJoinedEvent) => { const extraInfo = JSON.parse(e.userInfo.extraInfo || '{}'); @@ -167,22 +171,6 @@ const useRtcListeners = (): IEventListener => { } }; - const handleUserMessageReceived = (e: { userId: string; message: any }) => { - /** debounce 记录用户输入文字 */ - if (e.message) { - const msgObj = JSON.parse(e.message || '{}'); - if (msgObj.text) { - const { text: msg, definite, user_id: user } = msgObj; - if ((window as any)._debug_mode) { - dispatch(setHistoryMsg({ msg, user })); - } else { - debounceSetHistoryMsg(msg, user); - } - dispatch(setCurrentMsg({ msg, definite, user })); - } - } - }; - const handleAutoPlayFail = (event: AutoPlayFailedEvent) => { const { userId, kind } = event; let playUser = playStatus.current?.[userId] || {}; @@ -264,6 +252,7 @@ const useRtcListeners = (): IEventListener => { handleError, handleUserJoin, handleUserLeave, + handleTrackEnded, handleUserPublishStream, handleUserUnpublishStream, handleRemoteStreamStats, @@ -271,7 +260,6 @@ const useRtcListeners = (): IEventListener => { handleLocalAudioPropertiesReport, handleRemoteAudioPropertiesReport, handleAudioDeviceStateChanged, - handleUserMessageReceived, handleAutoPlayFail, handlePlayerEvent, handleUserStartAudioCapture, diff --git a/src/pages/MainPage/MainArea/Room/Conversation.tsx b/src/pages/MainPage/MainArea/Room/Conversation.tsx index 0cbb013..aad131b 100644 --- a/src/pages/MainPage/MainArea/Room/Conversation.tsx +++ b/src/pages/MainPage/MainArea/Room/Conversation.tsx @@ -14,6 +14,12 @@ import { COMMAND, INTERRUPT_PRIORITY } from '@/utils/handler'; import RtcClient from '@/lib/RtcClient'; import { setCurrentMsg, setHistoryMsg } from '@/store/slices/room'; +interface Message { + value: string; + user: string; + isInterrupted?: boolean; +} + const lines: (string | React.ReactNode)[] = []; function Conversation(props: React.HTMLAttributes) { @@ -83,7 +89,7 @@ function Conversation(props: React.HTMLAttributes) { ) : ( '' )} - {msgHistory?.map(({ value, user, isInterrupted }, index) => { + {msgHistory?.map(({ value, user, isInterrupted }: Message, index: number) => { const isUserMsg = user === userId; const isRobotMsg = user === Config.BotName; if (!isUserMsg && !isRobotMsg) { diff --git a/src/pages/MainPage/MainArea/Room/index.module.less b/src/pages/MainPage/MainArea/Room/index.module.less index b7ce503..e233a3c 100644 --- a/src/pages/MainPage/MainArea/Room/index.module.less +++ b/src/pages/MainPage/MainArea/Room/index.module.less @@ -140,7 +140,7 @@ justify-content: center; gap: 16px; background-color: #fff; - padding: 8px 35px; + padding: 8px 15px; .talkWrapper { --h: 16px; diff --git a/yarn.lock b/yarn.lock index 7d8dc66..2f1e555 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2210,10 +2210,10 @@ "@typescript-eslint/types" "5.31.0" eslint-visitor-keys "^3.3.0" -"@volcengine/rtc@4.58.9": - version "4.58.9" - resolved "https://registry.yarnpkg.com/@volcengine/rtc/-/rtc-4.58.9.tgz#841ebaddd5d4963c71abd33037bd76d1d490d928" - integrity sha512-nnXnNW9pVo8ynBSxVe0ikNIdxWfoSx5oOnwK7EoMCXdc2bJgHATpz/B+Kv2F1k4GjYAbo7ZcOm/g3cchvHgH5Q== +"@volcengine/rtc@4.66.1": + version "4.66.1" + resolved "https://registry.yarnpkg.com/@volcengine/rtc/-/rtc-4.66.1.tgz#1934c269b31216f43718ae46b169c59ac5e474f2" + integrity sha512-APznH6eosmKJC1HYJJ8s6G3Mq3OSgw6ivv6uCiayM5QNMBj+GW6zxf+MVsk5rm6r4R92TLwQErWonJ8yzGO4xA== dependencies: eventemitter3 "^4.0.7"