chore: update comment & readme

master
quemingyi.wudong 2025-01-16 14:10:55 +08:00
parent cdb24f5b75
commit 540ecf261e
27 changed files with 353 additions and 85 deletions

View File

@ -5,7 +5,6 @@
- 用户只需调用基于标准的 OpenAPI 接口即可配置所需的 ASR、LLM、TTS 类型和参数。火山引擎云端计算服务负责边缘用户接入、云端资源调度、音视频流压缩、文本与语音转换处理以及数据订阅传输等环节。简化开发流程让开发者更专注在对大模型核心能力的训练及调试从而快速推进AIGC产品应用创新。
- 同时火山引擎 RTC拥有成熟的音频 3A 处理、视频处理等技术以及大规模音视频聊天能力,可支持 AIGC 产品更便捷的支持多模态交互、多人互动等场景能力,保持交互的自然性和高效性。
# 快速开始
## 【必看】环境准备
- Node 版本: 16.0+
1. 需要准备两个 Terminal分别启动服务端、前端页面。
@ -15,40 +14,43 @@ RoomId、UserId 以及申请的 AppID、BusinessID(如有)、Token、ASR AppID
4. 您需要在 [火山方舟-在线推理](https://console.volcengine.com/ark/region:ark+cn-beijing/endpoint?config=%7B%7D) 中创建接入点, 并将模型对应的接入点 ID 填入 `src/config/common.ts` 文件中的 `ARK_V3_MODEL_ID`, 否则无法正常启动智能体。
5. 如果您已经自行完成了服务端的逻辑,可以不依赖 Demo 中的 Server直接修改前端代码文件 `src/config/index.ts` 中的 `AIGC_PROXY_HOST` 请求域名和接口,并在 `src/app/api.ts` 中修改接口的参数配置 `APIS_CONFIG`
## 服务端
## 快速开始
### 服务端
进到项目根目录
### 安装依赖
#### 安装依赖
```shell
cd Server
yarn
```
### 运行项目
#### 运行项目
```shell
node app.js
```
## 前端页面
### 前端页面
进到项目根目录
### 安装依赖
#### 安装依赖
```shell
yarn
```
### 运行项目
#### 运行项目
```shell
yarn dev
```
## 常见问题
### 常见问题
| 问题 | 解决方案 |
| :-- | :-- |
| `Server/app.js` 中的 `sessionToken` 是什么,该怎么填,为什么要填 | `sessionToken` 是火山引擎子账号发起 OpenAPI 请求时所必须携带的临时 Token获取方式可参考 [此文章末尾](https://www.volcengine.com/docs/6348/1315561)。 |
| 什么是主账号,什么是子账号 | 可以参考[官方概念](https://www.volcengine.com/docs/6257/64963?hyperlink_open_type=lark.open_in_browser) 。|
| 启动智能体之后, 说话无反馈 | <li>可能是因为参数传递的有问题, 例如参数大小写、类型等问题,请再次确认下这类型问题是否存在。</li><li>另一方面,可能是因为控制台中相关权限没有正常授予,请参考[流程](https://www.volcengine.com/docs/6348/1315561)再次确认下是否完成相关操作。</li><li>相关资源可能未开通或者用量不足,请再次确认。</li> |
| 不清楚什么是主账号,什么是子账号 | 可以参考[官方概念](https://www.volcengine.com/docs/6257/64963?hyperlink_open_type=lark.open_in_browser) 。|
| **启动智能体之后, 对话无反馈** | <li>参数传递可能有问题, 例如参数大小写、类型等问题,请再次确认下这类型问题是否存在。</li><li>另一方面,可能是因为控制台中相关权限没有正常授予,请参考[流程](https://www.volcengine.com/docs/6348/1315561)再次确认下是否完成相关操作。</li><li>相关资源可能未开通或者用量不足,请再次确认。</li><li>请检查本地的网络/带宽情况</li> |
| **浏览器报了 `Uncaught (in promise) r: token_error` 错误** | 请检查您填在项目中的 RTC Token 是否合法,检测用于生成 Token 的 UserId、RoomId 是否与项目中填写的一致。 |
| 什么是 RTC | **R**eal **T**ime **C**ommunication, RTC 的概念可参考[官网文档](https://www.volcengine.com/docs/6348/66812)。 |
| 为什么我的麦克风正常、摄像头也正常,但是设备没有正常工作? | 可能是设备权限未授予,详情可参考 [Web 排查设备权限获取失败问题](https://www.volcengine.com/docs/6348/1356355)。 |
如果有上述以外的问题,欢迎联系我们反馈
如果有上述以外的问题,也可以参考[问题反馈收集](https://bytedance.larkoffice.com/docx/FM51drJNFoSFcAxciXYcZkpmnBl),或者联系我们帮忙排查处理
## 相关文档
### 相关文档
- [场景介绍](https://www.volcengine.com/docs/6348/1310537)
- [Demo 体验](https://www.volcengine.com/docs/6348/1310559)
- [场景搭建方案](https://www.volcengine.com/docs/6348/1310560)

View File

@ -26,7 +26,7 @@
"test": "craco test",
"eject": "react-scripts eject",
"prettier": "prettier --write '**/*.{js,jsx,tsx,ts,less,md,json}'",
"eslint": "eslint src/ --fix --quiet --ext .js,.jsx,.ts,.tsx",
"eslint": "eslint src/ --fix --cache --quiet --ext .js,.jsx,.ts,.tsx",
"stylelint": "stylelint 'src/**/*.less' --fix",
"pre-commit": "npm run eslint && npm run stylelint",
"echo": "node message.js"

View File

@ -28,13 +28,18 @@ type ApiConfig = typeof APIS_CONFIG;
type TupleToUnion<T extends readonly unknown[]> = T[number];
type ApiNames = Pick<TupleToUnion<ApiConfig>, 'action'>['action'];
type RequestFn = <T extends keyof RequestResponse>(params?: RequestParams[T]) => RequestResponse[T];
type PromiseRequestFn = <T extends keyof RequestResponse>(params?: RequestParams[T]) => Promise<RequestResponse[T]>;
type PromiseRequestFn = <T extends keyof RequestResponse>(
params?: RequestParams[T]
) => Promise<RequestResponse[T]>;
type Apis = Record<ApiNames, RequestFn | PromiseRequestFn>;
const APIS = APIS_CONFIG.reduce((store, cur) => {
const { action, apiBasicParams, method = 'get' } = cur;
store[action] = async (params) => {
const queryData = method === 'get' ? await requestGetMethod(apiBasicParams)(params) : await requestPostMethod(apiBasicParams)(params);
const queryData =
method === 'get'
? await requestGetMethod(apiBasicParams)(params)
: await requestPostMethod(apiBasicParams)(params);
const res = await queryData?.json();
return resultHandler(res);
};

View File

@ -33,7 +33,11 @@ export const requestGetMethod = (apiBasicParams: string, headers = {}) => {
* @param isJson
* @param headers
*/
export const requestPostMethod = (apiBasicParams: string, isJson: boolean = true, headers: Headers = {}) => {
export const requestPostMethod = (
apiBasicParams: string,
isJson: boolean = true,
headers: Headers = {}
) => {
return async <T>(params: T) => {
const res = await fetch(`${AIGC_PROXY_HOST}${apiBasicParams}`, {
method: 'post',
@ -57,5 +61,7 @@ export const resultHandler = (res: any) => {
return Result;
}
Message.error(`[${ResponseMetadata?.Action}]Failed(Reason: ${ResponseMetadata?.Error?.Code})`);
throw new Error(`[${ResponseMetadata?.Action}]Failed(${JSON.stringify(ResponseMetadata, null, 2)})`);
throw new Error(
`[${ResponseMetadata?.Action}]Failed(${JSON.stringify(ResponseMetadata, null, 2)})`
);
};

View File

@ -100,5 +100,9 @@ export interface RequestResponse {
}
export type DeepPartial<T> = {
[P in keyof T]?: T[P] extends Array<infer U> ? Array<DeepPartial<U>> : T[P] extends object ? DeepPartial<T[P]> : T[P];
[P in keyof T]?: T[P] extends Array<infer U>
? Array<DeepPartial<U>>
: T[P] extends object
? DeepPartial<T[P]>
: T[P];
};

View File

@ -8,7 +8,19 @@ import { useEffect, useState } from 'react';
import { useDispatch, useSelector } from 'react-redux';
import { IconSwap } from '@arco-design/web-react/icon';
import CheckIcon from '../CheckIcon';
import Config, { Icon, Name, SCENE, Prompt, Welcome, Voice, Model, AI_MODEL, ModelSourceType, VOICE_INFO_MAP, VOICE_TYPE } from '@/config';
import Config, {
Icon,
Name,
SCENE,
Prompt,
Welcome,
Voice,
Model,
AI_MODEL,
ModelSourceType,
VOICE_INFO_MAP,
VOICE_TYPE,
} from '@/config';
import TitleCard from '../TitleCard';
import CheckBoxSelector from '@/components/CheckBoxSelector';
import RtcClient from '@/lib/RtcClient';
@ -156,7 +168,9 @@ function AISettings() {
<span className={styles['special-text']}> AI </span>
</div>
<div className={styles['sub-title']}></div>
<div className={styles['sub-title']}>
</div>
<div className={utils.isMobile() ? styles['scenes-mobile'] : styles.scenes}>
{SCENES.map((key) => (
<CheckIcon
@ -282,11 +296,12 @@ function AISettings() {
<TitleCard title="官方模型">
<CheckBoxSelector
label="模型选择"
data={Object.keys(AI_MODEL).map((type) => ({
key: AI_MODEL[type as keyof typeof AI_MODEL],
label: type.replaceAll('_', ' '),
icon: DoubaoModelSVG,
}))}
data={Object.keys(AI_MODEL)
.map((type) => ({
key: AI_MODEL[type as keyof typeof AI_MODEL],
label: type.replaceAll('_', ' '),
icon: DoubaoModelSVG,
}))}
moreIcon={ModelChangeSVG}
moreText="更换模型"
placeHolder="请选择你需要的模型"

View File

@ -29,13 +29,20 @@ function AvatarCard(props: IAvatarCardProps) {
<div className={`${style.card} ${className}`} {...rest}>
<div className={style.corner} />
<div className={style.avatar}>
<img id="avatar-card" src={avatar || DouBaoAvatar} className={style['doubao-gif']} alt="Avatar" />
<img
id="avatar-card"
src={avatar || DouBaoAvatar}
className={style['doubao-gif']}
alt="Avatar"
/>
</div>
<div className={style.body} />
<div className={style['text-wrapper']}>
<div className={style['user-info']}>
<div className={style.title}>{Name[scene]}</div>
<div className={style.description}> {ReversedVoiceType[TTSConfig?.VoiceType || '']}</div>
<div className={style.description}>
{ReversedVoiceType[TTSConfig?.VoiceType || '']}
</div>
<div className={style.description}> {LLMConfig.ModelName}</div>
<AISettings />
</div>

View File

@ -22,7 +22,11 @@ function BubbleMsg(props: IBubbleMsgProps) {
return (
<div style={style} className={`${styles.bubbleWrapper} ${className}`}>
<img className={`${styles.bubbleLogo} ${styles[`bubble-direction-${direction}`]}`} src={Bubble} alt="Logo" />
<img
className={`${styles.bubbleLogo} ${styles[`bubble-direction-${direction}`]}`}
src={Bubble}
alt="Logo"
/>
<div className={styles.bubbleText}>{text}</div>
</div>
);

View File

@ -18,7 +18,8 @@ interface IProps {
function ButtonRadio(props: IProps) {
const { value, onChange, options } = props;
const selected = useMemo(() => options.find((item) => item.key === value), [value]) || options?.[0];
const selected =
useMemo(() => options.find((item) => item.key === value), [value]) || options?.[0];
const handleClick = (key: string) => {
onChange?.(key);
};
@ -26,7 +27,12 @@ function ButtonRadio(props: IProps) {
return (
<div className={styles.wrapper}>
{options.map(({ label, key }) => (
<Button type="text" key={key} className={`${styles.item} ${key === selected.key ? styles.selected : ''}`} onClick={() => handleClick(key)}>
<Button
type="text"
key={key}
className={`${styles.item} ${key === selected.key ? styles.selected : ''}`}
onClick={() => handleClick(key)}
>
<span className={key === selected.key ? styles['selected-text'] : ''}>{label}</span>
</Button>
))}

View File

@ -20,7 +20,16 @@ interface IProps {
}
function CheckBox(props: IProps) {
const { noStyle, className = '', icon = '', checked, label, description, suffix, onClick } = props;
const {
noStyle,
className = '',
icon = '',
checked,
label,
description,
suffix,
onClick,
} = props;
if (noStyle) {
return (
@ -35,7 +44,10 @@ function CheckBox(props: IProps) {
}
return (
<div className={`${className} ${styles.wrapper} ${checked ? styles.active : ''}`} onClick={onClick}>
<div
className={`${className} ${styles.wrapper} ${checked ? styles.active : ''}`}
onClick={onClick}
>
{icon ? <img className={styles.icon} src={icon} alt="icon" /> : ''}
<div className={styles.content}>
<div className={styles.label}>{label}</div>

View File

@ -42,7 +42,13 @@ function CheckBoxSelector(props: IProps) {
<>
<div className={styles.wrapper}>
{selectedOne ? (
<CheckBox className={styles.box} icon={selectedOne?.icon} label={selectedOne?.label || ''} description={selectedOne?.description} noStyle />
<CheckBox
className={styles.box}
icon={selectedOne?.icon}
label={selectedOne?.label || ''}
description={selectedOne?.description}
noStyle
/>
) : (
<div className={styles.placeholder}>{placeHolder}</div>
)}
@ -78,7 +84,15 @@ function CheckBoxSelector(props: IProps) {
>
<div className={styles.modalInner}>
{data.map((item) => (
<CheckBox className={styles.box} key={item.key} icon={item.icon} label={item.label} description={item.description} checked={item.key === selected} onClick={() => setSelected(item.key)} />
<CheckBox
className={styles.box}
key={item.key}
icon={item.icon}
label={item.label}
description={item.description}
checked={item.key === selected}
onClick={() => setSelected(item.key)}
/>
))}
</div>
</Drawer>

View File

@ -54,7 +54,15 @@ function DrawerRowItem(props: IDrawerRowItemProps) {
<IconRight className={styles.rightOutlined} />
</div>
</div>
<Drawer closable title={drawer?.title || ''} width={drawer?.width || 400} className={styles.drawer} visible={open} onCancel={handleClose} footer={null}>
<Drawer
closable
title={drawer?.title || ''}
width={drawer?.width || 400}
className={styles.drawer}
visible={open}
onCancel={handleClose}
footer={null}
>
<div className={styles.children}>{drawer?.children}</div>
</Drawer>
</>

View File

@ -74,12 +74,22 @@ function Header(props: HeaderProps) {
{children}
{utils.isMobile() ? null : (
<div className={styles['header-right']}>
<div className={styles['header-right-text']} onClick={() => window.open('https://www.volcengine.com/product/veRTC/ConversationalAI', '_blank')}>
<div
className={styles['header-right-text']}
onClick={() =>
window.open('https://www.volcengine.com/product/veRTC/ConversationalAI', '_blank')
}
>
</div>
<div
className={styles['header-right-text']}
onClick={() => window.open('https://www.volcengine.com/contact/product?t=%E5%AF%B9%E8%AF%9D%E5%BC%8Fai&source=%E4%BA%A7%E5%93%81%E5%92%A8%E8%AF%A2', '_blank')}
onClick={() =>
window.open(
'https://www.volcengine.com/contact/product?t=%E5%AF%B9%E8%AF%9D%E5%BC%8Fai&source=%E4%BA%A7%E5%93%81%E5%92%A8%E8%AF%A2',
'_blank'
)
}
>
</div>

View File

@ -34,7 +34,8 @@ function NetworkIndicator() {
const networkQuality = room.networkQuality;
const delay = room.localUser.audioStats?.rtt;
const audioLossRateUpper = room.localUser.audioStats?.audioLossRate || 0;
const audioLossRateLower = room.remoteUsers.find((user) => user.userId === Config.BotName)?.audioStats?.audioLossRate || 0;
const audioLossRateLower =
room.remoteUsers.find((user) => user.userId === Config.BotName)?.audioStats?.audioLossRate || 0;
const indicators = useMemo(() => {
switch (networkQuality) {
@ -75,11 +76,15 @@ function NetworkIndicator() {
<div className={style.loss}>
<div>
<IconArrowUp style={{ color: 'rgba(22, 100, 255, 1)' }} />
<span>{`${audioLossRateUpper}` ? (audioLossRateUpper * 100)?.toFixed(0) : '- '}%</span>
<span>
{`${audioLossRateUpper}` ? (audioLossRateUpper * 100)?.toFixed(0) : '- '}%
</span>
</div>
<div>
<IconArrowDown />
<span>{`${audioLossRateLower}` ? (audioLossRateLower * 100)?.toFixed(0) : '- '}%</span>
<span>
{`${audioLossRateLower}` ? (audioLossRateLower * 100)?.toFixed(0) : '- '}%
</span>
</div>
</div>
</div>

View File

@ -26,7 +26,9 @@ export enum CustomParamsType {
/**
* @brief AI
* @default
* @notes ,
* @notes ,
* ID https://console.volcengine.com/speech/service/8 中开通获取。
* "音色详情" , "Voice_type"
*/
export enum VOICE_TYPE {
'通用女声' = 'BV001_streaming',
@ -102,18 +104,14 @@ export const AI_MODE_MAP: Partial<Record<AI_MODEL, AI_MODEL_MODE>> = {
/**
* @brief ID
* @note ID https://console.volcengine.com/ark/region:ark+cn-beijing/endpoint 参看/创建
* ID ID, "接入点名称" , "ep-2024xxxxxx-xxx" ID
*/
export const ARK_V3_MODEL_ID: Partial<Record<AI_MODEL, string>> = {
/**
* @note ID https://console.volcengine.com/ark/region:ark+cn-beijing/endpoint 参看/创建
*/
[AI_MODEL.DOUBAO_LITE_4K]: '************** 此处填充方舟上的模型 ID *************',
[AI_MODEL.DOUBAO_PRO_4K]: '************** 此处填充方舟上的模型 ID *************',
[AI_MODEL.DOUBAO_PRO_32K]: '************** 此处填充方舟上的模型 ID *************',
[AI_MODEL.DOUBAO_PRO_128K]: '************** 此处填充方舟上的模型 ID *************',
/**
* @note , 使
*/
[AI_MODEL.VISION]: '************** 此处填充方舟上的模型 ID *************',
// ... 可根据所开通的模型进行扩充
};
@ -155,6 +153,9 @@ export const Name = {
[SCENE.CUSTOM]: '自定义',
};
/**
* @brief
*/
export const Welcome = {
[SCENE.INTELLIGENT_ASSISTANT]: '你好我是你的AI小助手有什么可以帮你的吗',
[SCENE.VIRTUAL_GIRL_FRIEND]: '你来啦,我好想你呀~今天有没有想我呢?',
@ -186,15 +187,38 @@ export const Voice = {
};
export const Questions = {
[SCENE.INTELLIGENT_ASSISTANT]: ['最近有什么好看的电影推荐吗?', '上海有什么好玩的地方吗?', '能给我讲一个故事吗?'],
[SCENE.VIRTUAL_GIRL_FRIEND]: ['我今天有点累。', '我们等会儿去看电影吧!', '明天我生日,你准备送给我什么礼物呢?'],
[SCENE.TRANSLATE]: ['道可道,非常道;名可名,非常名。', 'Stay hungry, stay foolish.', '天生我材必有用,千金散尽还复来。'],
[SCENE.CHILDREN_ENCYCLOPEDIA]: ['天上有多少颗星星?', '太阳为什么总是从东边升起?', '苹果的英语怎么说?'],
[SCENE.CUSTOMER_SERVICE]: ['我上次来你们店里吃饭,等了三十分钟菜才上来。', '你们店里卫生间有点脏。', '你们空调开得太冷了。'],
[SCENE.INTELLIGENT_ASSISTANT]: [
'最近有什么好看的电影推荐吗?',
'上海有什么好玩的地方吗?',
'能给我讲一个故事吗?',
],
[SCENE.VIRTUAL_GIRL_FRIEND]: [
'我今天有点累。',
'我们等会儿去看电影吧!',
'明天我生日,你准备送给我什么礼物呢?',
],
[SCENE.TRANSLATE]: [
'道可道,非常道;名可名,非常名。',
'Stay hungry, stay foolish.',
'天生我材必有用,千金散尽还复来。',
],
[SCENE.CHILDREN_ENCYCLOPEDIA]: [
'天上有多少颗星星?',
'太阳为什么总是从东边升起?',
'苹果的英语怎么说?',
],
[SCENE.CUSTOMER_SERVICE]: [
'我上次来你们店里吃饭,等了三十分钟菜才上来。',
'你们店里卫生间有点脏。',
'你们空调开得太冷了。',
],
[SCENE.TEACHING_ASSISTANT]: ['这个单词是什么意思?', '这道题该怎么做?', '我的表情是什么样的?'],
[SCENE.CUSTOM]: ['你能帮我解决什么问题?', '今天北京天气怎么样?', '你喜欢哪位流行歌手?'],
};
/**
* @brief System , Prompt
*/
export const Prompt = {
[SCENE.INTELLIGENT_ASSISTANT]: `##人设

View File

@ -3,40 +3,94 @@
* SPDX-license-identifier: BSD-3-Clause
*/
import { TTS_CLUSTER, ARK_V3_MODEL_ID, ModelSourceType, SCENE, Prompt, Welcome, Model, Voice, LLM_BOT_ID, AI_MODEL, AI_MODE_MAP, AI_MODEL_MODE } from '.';
import {
TTS_CLUSTER,
ARK_V3_MODEL_ID,
ModelSourceType,
SCENE,
Prompt,
Welcome,
Model,
Voice,
LLM_BOT_ID,
AI_MODEL,
AI_MODE_MAP,
AI_MODEL_MODE,
} from '.';
export const CONVERSATION_SIGNATURE = 'conversation';
/**
* @brief RTC & AIGC
* @notes : https://api.volcengine.com/api-explorer?action=StartVoiceChat&groupName=%E6%99%BA%E8%83%BD%E4%BD%93&serviceCode=rtc&version=2024-12-01
* @brief RTC & AIGC
* @notes
* https://www.volcengine.com/docs/6348/1404673
*/
export class ConfigFactory {
BaseConfig = {
AppId: 'Your AppId',
/**
* @brief ,
* @note , RTC AppId https://console.volcengine.com/rtc/listRTC 中获取。
*/
AppId: 'Your RTC AppId',
/**
* @brief ,
*/
BusinessId: undefined,
/**
* @brief , ID,
*/
RoomId: 'Your Room Id',
/**
* @brief , AI ID,
*/
UserId: 'Your User Id',
/**
* @brief , RTC Token, AppIdRoomIdUserId, https://console.volcengine.com/rtc/listRTC 列表中
* AppId "操作" "临时Token" , RTC
* @note Token , RoomId / UserId RoomId / UserId
* 使 https://www.volcengine.com/docs/6348/70121 通过代码生成 Token。
*/
Token: 'Your Token',
/**
* @brief , TTS() AppId, https://console.volcengine.com/speech/app 中获取, 若无可先创建应用。
* @note , "语音合成" , App
*/
TTSAppId: 'Your TTS AppId',
/**
* @brief , ASR() AppId, https://console.volcengine.com/speech/app 中获取, 若无可先创建应用。
* @note , "流式语音识别" , App
*/
ASRAppId: 'Your ASR AppId',
};
Model: AI_MODEL = Model[SCENE.INTELLIGENT_ASSISTANT];
/**
* @note , ID,
* ID VOICE_TYPE
* , ,
*/
VoiceType = Voice[SCENE.INTELLIGENT_ASSISTANT];
/**
* @note System , , Prompt
*/
Prompt = Prompt[SCENE.INTELLIGENT_ASSISTANT];
/**
* @note
*/
WelcomeSpeech = Welcome[SCENE.INTELLIGENT_ASSISTANT];
ModeSourceType = ModelSourceType.Available;
/**
* @note , 使,
*/
Url? = '';
/**
* @note , 使,
*/
APIKey? = '';
/**

View File

@ -3,6 +3,7 @@
* SPDX-license-identifier: BSD-3-Clause
*/
import VERTC, {
MirrorType,
StreamIndex,
@ -33,7 +34,11 @@ export interface IEventListener {
handleUserJoin: (e: onUserJoinedEvent) => void;
handleUserLeave: (e: onUserLeaveEvent) => void;
handleUserPublishStream: (e: { userId: string; mediaType: MediaType }) => void;
handleUserUnpublishStream: (e: { userId: string; mediaType: MediaType; reason: StreamRemoveReason }) => void;
handleUserUnpublishStream: (e: {
userId: string;
mediaType: MediaType;
reason: StreamRemoveReason;
}) => void;
handleRemoteStreamStats: (e: RemoteStreamStats) => void;
handleLocalStreamStats: (e: LocalStreamStats) => void;
handleLocalAudioPropertiesReport: (e: LocalAudioPropertiesInfo[]) => void;
@ -45,7 +50,10 @@ export interface IEventListener {
handleUserStartAudioCapture: (e: { userId: string }) => void;
handleUserStopAudioCapture: (e: { userId: string }) => void;
handleRoomBinaryMessageReceived: (e: { userId: string; message: ArrayBuffer }) => void;
handleNetworkQuality: (uplinkNetworkQuality: NetworkQuality, downlinkNetworkQuality: NetworkQuality) => void;
handleNetworkQuality: (
uplinkNetworkQuality: NetworkQuality,
downlinkNetworkQuality: NetworkQuality
) => void;
}
interface EngineOptions {

View File

@ -86,7 +86,11 @@ const useRtcListeners = (): IEventListener => {
dispatch(updateRemoteUser(payload));
};
const handleUserUnpublishStream = (e: { userId: string; mediaType: MediaType; reason: StreamRemoveReason }) => {
const handleUserUnpublishStream = (e: {
userId: string;
mediaType: MediaType;
reason: StreamRemoveReason;
}) => {
const { userId, mediaType } = e;
const payload: IUser = { userId };
@ -119,7 +123,9 @@ const useRtcListeners = (): IEventListener => {
};
const handleLocalAudioPropertiesReport = (e: LocalAudioPropertiesInfo[]) => {
const localAudioInfo = e.find((audioInfo) => audioInfo.streamIndex === StreamIndex.STREAM_INDEX_MAIN);
const localAudioInfo = e.find(
(audioInfo) => audioInfo.streamIndex === StreamIndex.STREAM_INDEX_MAIN
);
if (localAudioInfo) {
dispatch(
updateLocalUser({
@ -236,10 +242,15 @@ const useRtcListeners = (): IEventListener => {
dispatch(updateAITalkState({ isAITalking: false }));
};
const handleNetworkQuality = (uplinkNetworkQuality: NetworkQuality, downlinkNetworkQuality: NetworkQuality) => {
const handleNetworkQuality = (
uplinkNetworkQuality: NetworkQuality,
downlinkNetworkQuality: NetworkQuality
) => {
dispatch(
updateNetworkQuality({
networkQuality: Math.floor((uplinkNetworkQuality + downlinkNetworkQuality) / 2) as NetworkQuality,
networkQuality: Math.floor(
(uplinkNetworkQuality + downlinkNetworkQuality) / 2
) as NetworkQuality,
})
);
};

View File

@ -8,12 +8,23 @@ import { useSelector, useDispatch } from 'react-redux';
import { MediaType } from '@volcengine/rtc';
import Utils from '@/utils/utils';
import RtcClient from '@/lib/RtcClient';
import { clearCurrentMsg, clearHistoryMsg, localJoinRoom, localLeaveRoom, updateAIGCState, updateLocalUser } from '@/store/slices/room';
import {
clearCurrentMsg,
clearHistoryMsg,
localJoinRoom,
localLeaveRoom,
updateAIGCState,
updateLocalUser,
} from '@/store/slices/room';
import useRtcListeners from '@/lib/listenerHooks';
import { RootState } from '@/store';
import { updateMediaInputs, updateSelectedDevice, setDevicePermissions } from '@/store/slices/device';
import {
updateMediaInputs,
updateSelectedDevice,
setDevicePermissions,
} from '@/store/slices/device';
import logger from '@/utils/logger';
import aigcConfig, { AI_MODEL } from '@/config';
@ -45,7 +56,10 @@ export const useGetDevicePermission = () => {
return permission;
};
export const useJoin = (): [boolean, (formValues: FormProps, fromRefresh: boolean) => Promise<void | boolean>] => {
export const useJoin = (): [
boolean,
(formValues: FormProps, fromRefresh: boolean) => Promise<void | boolean>
] => {
const devicePermissions = useSelector((state: RootState) => state.device.devicePermissions);
const room = useSelector((state: RootState) => state.room);
@ -198,7 +212,9 @@ export const useDeviceState = () => {
const switchMic = (publish = true) => {
if (publish) {
!isAudioPublished ? RtcClient.publishStream(MediaType.AUDIO) : RtcClient.unpublishStream(MediaType.AUDIO);
!isAudioPublished
? RtcClient.publishStream(MediaType.AUDIO)
: RtcClient.unpublishStream(MediaType.AUDIO);
}
queryDevices(MediaType.AUDIO);
!isAudioPublished ? RtcClient.startAudioCapture() : RtcClient.stopAudioCapture();
@ -211,7 +227,9 @@ export const useDeviceState = () => {
const switchCamera = (publish = true) => {
if (publish) {
!isVideoPublished ? RtcClient.publishStream(MediaType.VIDEO) : RtcClient.unpublishStream(MediaType.VIDEO);
!isVideoPublished
? RtcClient.publishStream(MediaType.VIDEO)
: RtcClient.unpublishStream(MediaType.VIDEO);
}
queryDevices(MediaType.VIDEO);
!localUser.publishVideo ? RtcClient.startVideoCapture() : RtcClient.stopVideoCapture();

View File

@ -19,7 +19,11 @@ function InvokeButton(props: IInvokeButtonProps) {
<div className={`${style.wrapper} ${loading ? '' : style.cursor} ${className}`} {...rest}>
<div className={style.btn}>
<img src={CallButtonSVG} alt="call" />
{loading ? <Loading className={style.icon} /> : <img src={PhoneSVG} className={style.icon} alt="phone" />}
{loading ? (
<Loading className={style.icon} />
) : (
<img src={PhoneSVG} className={style.icon} alt="phone" />
)}
</div>
<div className={style.text}>{loading ? '连接中' : '通话'}</div>
</div>

View File

@ -26,7 +26,9 @@ function CameraArea(props: React.HTMLAttributes<HTMLDivElement>) {
const handleOperateCamera = () => {
!localUser.publishVideo ? RtcClient.startVideoCapture() : RtcClient.stopVideoCapture();
!localUser.publishVideo ? RtcClient.publishStream(MediaType.VIDEO) : RtcClient.unpublishStream(MediaType.VIDEO);
!localUser.publishVideo
? RtcClient.publishStream(MediaType.VIDEO)
: RtcClient.unpublishStream(MediaType.VIDEO);
dispatch(
updateLocalUser({
@ -49,7 +51,11 @@ function CameraArea(props: React.HTMLAttributes<HTMLDivElement>) {
<div id={LocalVideoID} className={styles['camera-player']} />
) : (
<div className={styles['camera-placeholder']}>
<img src={CameraCloseNoteSVG} alt="close" className={styles['camera-placeholder-close-note']} />
<img
src={CameraCloseNoteSVG}
alt="close"
className={styles['camera-placeholder-close-note']}
/>
<div>
<span onClick={handleOperateCamera} className={styles['camera-open-btn']}>

View File

@ -46,11 +46,18 @@ function Conversation(props: React.HTMLAttributes<HTMLDivElement>) {
return '';
}
return (
<div className={`${styles.sentence} ${isUserMsg ? styles.user : styles.robot}`} key={`msg-${index}`}>
<div
className={`${styles.sentence} ${isUserMsg ? styles.user : styles.robot}`}
key={`msg-${index}`}
>
<div className={styles.content}>
{value}
<div className={styles['loading-wrapper']}>
{isAIReady && (isUserTalking || isAITalking) && index === msgHistory.length - 1 ? <Loading gap={3} className={styles.loading} dotClassName={styles.dot} /> : ''}
{isAIReady && (isUserTalking || isAITalking) && index === msgHistory.length - 1 ? (
<Loading gap={3} className={styles.loading} dotClassName={styles.dot} />
) : (
''
)}
</div>
</div>
{!isUserMsg && isInterrupted ? <Tag className={styles.interruptTag}></Tag> : ''}

View File

@ -32,9 +32,25 @@ function ToolBar(props: React.HTMLAttributes<HTMLDivElement>) {
};
return (
<div className={`${className} ${style.btns} ${utils.isMobile() ? style.column : ''}`} {...rest}>
{utils.isMobile() ? <img src={SettingSVG} onClick={handleSetting} className={style.setting} alt="setting" /> : null}
<img src={isAudioPublished ? MicOpenSVG : MicCloseSVG} onClick={() => switchMic(true)} className={style.btn} alt="mic" />
{model === AI_MODEL.VISION ? <img src={isVideoPublished ? CameraOpenSVG : CameraCloseSVG} onClick={() => switchCamera(true)} className={style.btn} alt="camera" /> : ''}
{utils.isMobile() ? (
<img src={SettingSVG} onClick={handleSetting} className={style.setting} alt="setting" />
) : null}
<img
src={isAudioPublished ? MicOpenSVG : MicCloseSVG}
onClick={() => switchMic(true)}
className={style.btn}
alt="mic"
/>
{model === AI_MODEL.VISION ? (
<img
src={isVideoPublished ? CameraOpenSVG : CameraCloseSVG}
onClick={() => switchCamera(true)}
className={style.btn}
alt="camera"
/>
) : (
''
)}
<img src={LeaveRoomSVG} onClick={leaveRoom} className={style.btn} alt="leave" />
{utils.isMobile() ? (
<Drawer

View File

@ -31,11 +31,15 @@ function DeviceDrawerButton(props: IDeviceDrawerButtonProps) {
const switcher = type === MediaType.AUDIO ? device.switchMic : device.switchCamera;
const devicePermissions = useSelector((state: RootState) => state.device.devicePermissions);
const devices = useSelector((state: RootState) => state.device);
const selectedDevice = type === MediaType.AUDIO ? devices.selectedMicrophone : devices.selectedCamera;
const selectedDevice =
type === MediaType.AUDIO ? devices.selectedMicrophone : devices.selectedCamera;
const permission = devicePermissions?.[type === MediaType.AUDIO ? 'audio' : 'video'];
const dispatch = useDispatch();
const deviceList = useMemo(() => (type === MediaType.AUDIO ? devices.audioInputs : devices.videoInputs), [devices]);
const deviceList = useMemo(
() => (type === MediaType.AUDIO ? devices.audioInputs : devices.videoInputs),
[devices]
);
const handleDeviceChange = (value: string) => {
RtcClient.switchDevice(type, value);
@ -66,7 +70,12 @@ function DeviceDrawerButton(props: IDeviceDrawerButtonProps) {
<div className={styles.wrapper}>
<div className={styles.label}>{DEVICE_NAME[type]}</div>
<div className={styles.value}>
<Switch checked={isEnable} size="small" onChange={(enable) => switcher(enable)} disabled={!permission} />
<Switch
checked={isEnable}
size="small"
onChange={(enable) => switcher(enable)}
disabled={!permission}
/>
<Select style={{ width: 250 }} value={selectedDevice} onChange={handleDeviceChange}>
{deviceList.map((device) => (
<Select.Option key={device.deviceId} value={device.deviceId}>

View File

@ -68,6 +68,7 @@ export const DeviceSlice = createSlice({
},
},
});
export const { updateMediaInputs, updateSelectedDevice, setMicrophoneList, setDevicePermissions } = DeviceSlice.actions;
export const { updateMediaInputs, updateSelectedDevice, setMicrophoneList, setDevicePermissions } =
DeviceSlice.actions;
export default DeviceSlice.reducer;

View File

@ -4,7 +4,12 @@
*/
import { createSlice } from '@reduxjs/toolkit';
import { AudioPropertiesInfo, LocalAudioStats, NetworkQuality, RemoteAudioStats } from '@volcengine/rtc';
import {
AudioPropertiesInfo,
LocalAudioStats,
NetworkQuality,
RemoteAudioStats,
} from '@volcengine/rtc';
import config, { SCENE } from '@/config';
import utils from '@/utils/utils';

View File

@ -91,7 +91,10 @@ class Utils {
let hasLogin = true;
if (!_roomId || !_uid) {
hasLogin = false;
} else if (!/^[0-9a-zA-Z_\-@.]{1,128}$/.test(_roomId) || !/^[0-9a-zA-Z_\-@.]{1,128}$/.test(_uid)) {
} else if (
!/^[0-9a-zA-Z_\-@.]{1,128}$/.test(_roomId) ||
!/^[0-9a-zA-Z_\-@.]{1,128}$/.test(_uid)
) {
hasLogin = false;
}
return hasLogin;
@ -135,7 +138,10 @@ class Utils {
if (arr.length) {
const last = arr.at(-1)!;
const { user, value, isInterrupted } = last;
if ((added.user === RtcClient.basicInfo.user_id && last.user === added.user) || (user === added.user && added.value.startsWith(value) && value.trim())) {
if (
(added.user === RtcClient.basicInfo.user_id && last.user === added.user) ||
(user === added.user && added.value.startsWith(value) && value.trim())
) {
arr.pop();
added.isInterrupted = isInterrupted;
}
@ -177,7 +183,7 @@ class Utils {
* @note TLV
* | magic number | length(big-endian) | value |
* @param {ArrayBufferLike} tlvBuffer
* @returns
* @returns
*/
tlv2String(tlvBuffer: ArrayBufferLike) {
const typeBuffer = new Uint8Array(tlvBuffer, 0, 4);
@ -189,7 +195,8 @@ class Utils {
type += String.fromCharCode(typeBuffer[i]);
}
const length = (lengthBuffer[0] << 24) | (lengthBuffer[1] << 16) | (lengthBuffer[2] << 8) | lengthBuffer[3];
const length =
(lengthBuffer[0] << 24) | (lengthBuffer[1] << 16) | (lengthBuffer[2] << 8) | lengthBuffer[3];
const value = new TextDecoder().decode(valueBuffer.subarray(0, length));