- {isAITalking ? (
-
);
}
diff --git a/src/pages/MainPage/Menu/index.module.less b/src/pages/MainPage/Menu/index.module.less
index 27901a8..e33628f 100644
--- a/src/pages/MainPage/Menu/index.module.less
+++ b/src/pages/MainPage/Menu/index.module.less
@@ -112,6 +112,7 @@
font-weight: 400;
line-height: 20px;
color: rgba(66, 70, 78, 1);
+ cursor: pointer;
}
}
diff --git a/src/pages/MainPage/Menu/index.tsx b/src/pages/MainPage/Menu/index.tsx
index 8edeb02..a60a6ad 100644
--- a/src/pages/MainPage/Menu/index.tsx
+++ b/src/pages/MainPage/Menu/index.tsx
@@ -5,21 +5,45 @@
import VERTC from '@volcengine/rtc';
import { Tooltip, Typography } from '@arco-design/web-react';
-import { useSelector } from 'react-redux';
+import { useDispatch, useSelector } from 'react-redux';
import { useVisionMode } from '@/lib/useCommon';
import { RootState } from '@/store';
+import RtcClient from '@/lib/RtcClient';
import Operation from './components/Operation';
import { Questions } from '@/config';
+import { COMMAND, INTERRUPT_PRIORITY } from '@/utils/handler';
import CameraArea from '../MainArea/Room/CameraArea';
+import { setCurrentMsg, setHistoryMsg } from '@/store/slices/room';
import utils from '@/utils/utils';
import styles from './index.module.less';
function Menu() {
+ const dispatch = useDispatch();
const room = useSelector((state: RootState) => state.room);
const scene = room.scene;
const isJoined = room?.isJoined;
const isVisionMode = useVisionMode();
+ const handleQuestion = (question: string) => {
+ RtcClient.commandAudioBot(COMMAND.EXTERNAL_TEXT_TO_LLM, INTERRUPT_PRIORITY.HIGH, question);
+ dispatch(
+ setHistoryMsg({
+ text: question,
+ user: RtcClient.basicInfo.user_id,
+ paragraph: true,
+ definite: true,
+ })
+ );
+ dispatch(
+ setCurrentMsg({
+ text: question,
+ user: RtcClient.basicInfo.user_id,
+ paragraph: true,
+ definite: true,
+ })
+ );
+ };
+
return (
- 
- 点击打断
-
+ {isAudioPublished ? (
+ isAITalking ? (
+
+
+ 点击打断
+
+ ) : (
+ 正在听...
+ )
) : (
- 正在听...
+ 你已关闭麦克风
)}
-
{isJoined && utils.isMobile() && isVisionMode ? (
@@ -51,9 +75,9 @@ function Menu() {
{isJoined ? (
-
你可以问各类问题，比如
+ 点击下述问题进行提问:
{Questions[scene].map((question) => (
-
+
handleQuestion(question)} className={styles.line} key={question}>
{question}
))}
diff --git a/src/store/slices/room.ts b/src/store/slices/room.ts
index a047f83..0812fe7 100644
--- a/src/store/slices/room.ts
+++ b/src/store/slices/room.ts
@@ -231,15 +231,6 @@ export const roomSlice = createSlice({
state.isUserTalking = userTalking;
}
}
- /** 如果当前说话人是用户, 并且上一条记录是 AI 的话, 并且不成语句, 则是打断 */
- if (userTalking) {
- const lastMsg = state.msgHistory[state.msgHistory.length - 1];
- const isAI = lastMsg.user === config.BotName;
- if (!lastMsg.paragraph && isAI) {
- lastMsg.isInterrupted = true;
- state.msgHistory[state.msgHistory.length - 1] = lastMsg;
- }
- }
utils.addMsgWithoutDuplicate(state.msgHistory, {
user: payload.user,
value: payload.text,
diff --git a/src/utils/handler.ts b/src/utils/handler.ts
index beb53a1..79791c7 100644
--- a/src/utils/handler.ts
+++ b/src/utils/handler.ts
@@ -5,7 +5,12 @@
import { useDispatch } from 'react-redux';
import logger from './logger';
-import { setCurrentMsg, setHistoryMsg } from '@/store/slices/room';
+import {
+ setCurrentMsg,
+ setHistoryMsg,
+ setInterruptMsg,
+ updateAITalkState,
+} from '@/store/slices/room';
import RtcClient from '@/lib/RtcClient';
import Utils from '@/utils/utils';
@@ -26,6 +31,45 @@ export enum AGENT_BRIEF {
FINISHED,
}
+/**
+ * @brief 指令类型
+ */
+export enum COMMAND {
+ /**
+ * @brief 打断指令
+ */
+ INTERRUPT = 'interrupt',
+ /**
+ * @brief 发送外部文本驱动 TTS
+ */
+ EXTERNAL_TEXT_TO_SPEECH = 'ExternalTextToSpeech',
+ /**
+ * @brief 发送外部文本驱动 LLM
+ */
+ EXTERNAL_TEXT_TO_LLM = 'ExternalTextToLLM',
+}
+/**
+ * @brief 打断的类型
+ */
+export enum INTERRUPT_PRIORITY {
+ /**
+ * @brief 占位
+ */
+ NONE,
+ /**
+ * @brief 高优先级。传入信息直接打断交互，进行处理。
+ */
+ HIGH,
+ /**
+ * @brief 中优先级。等待当前交互结束后，进行处理。
+ */
+ MEDIUM,
+ /**
+ * @brief 低优先级。如当前正在发生交互，直接丢弃 Message 传入的信息。
+ */
+ LOW,
+}
+
export const MessageTypeCode = {
[MESSAGE_TYPE.SUBTITLE]: 1,
[MESSAGE_TYPE.FUNCTION_CALL]: 2,
@@ -44,6 +88,17 @@ export const useMessageHandler = () => {
const { Stage } = parsed || {};
const { Code, Description } = Stage || {};
logger.debug(Code, Description);
+ switch (Code) {
+ case AGENT_BRIEF.FINISHED:
+ dispatch(updateAITalkState({ isAITalking: false }));
+ break;
+ case AGENT_BRIEF.INTERRUPTED:
+ dispatch(updateAITalkState({ isAITalking: false }));
+ dispatch(setInterruptMsg());
+ break;
+ default:
+ break;
+ }
},
/**
* @brief 字幕
@@ -85,8 +140,9 @@ export const useMessageHandler = () => {
JSON.stringify({
ToolCallID: parsed?.tool_calls?.[0]?.id,
Content: map[name.toLocaleLowerCase().replaceAll('_', '')],
- })
- )
+ }),
+ 'func',
+ ),
);
},
};
diff --git a/src/utils/utils.ts b/src/utils/utils.ts
index 051124c..240dc93 100644
--- a/src/utils/utils.ts
+++ b/src/utils/utils.ts
@@ -152,8 +152,7 @@ class Utils {
/**
* @brief 将字符串包装成 TLV
*/
- string2tlv(str: string) {
- const type = 'func';
+ string2tlv(str: string, type: string) {
const typeBuffer = new Uint8Array(4);
for (let i = 0; i < type.length; i++) {