jiwanjun
/
ai-speech-build
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix/指令优化，可操作多步指令

This commit is contained in:
季万俊 2025-09-01 14:04:50 +08:00
parent 4b32c459b1
commit dd6a0f6155
4 changed files with 907 additions and 1997 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2350
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

280
src/config/index.js
View File

@ -4,87 +4,217 @@
* @Description:
*/
const config = {
"page": "Ecommerce Home",
"commands": [
page: "Ecommerce Home",
commands: [
{
"command": "open_login_111",
"description": "打开登录弹窗或页面",
"action": "click",
"selector": "#login-button",
"params": []
command: "add_project",
description: "新建",
action: "click",
selector: "#ai-speech-add-project",
params: [],
},
{
"command": "open_login_222_333",
"description": "打开登录弹窗或页面",
"action": "click",
"selector": "#login-button",
"params": []
command: "switch_jsc",
description: "驾驶舱",
action: "click",
selector: "#ai_speech_jsc",
params: [],
isIframe: true,
},
{
"command": "input_username",
"description": "在用户名输入框中填写用户名",
"action": "input",
"selector": "#username-input",
"params": [
command: "switch_zngk",
description: "智能管控",
action: "click",
selector: "#ai_speech_zngk",
params: [],
isIframe: true,
},
{
command: "switch_zndd",
description: "智能调度",
action: "click",
selector: "#ai_speech_zndd",
params: [],
isIframe: true,
},
{
command: "switch_znyw",
description: "智能运维 运维",
action: "click",
selector: "#ai_speech_znyw",
params: [],
isIframe: true,
children: [
{
"name": "username",
"type": "string",
"description": "要输入的用户名"
}
]
},
{
"command": "input_password",
"description": "在密码输入框中填写密码",
"action": "input",
"selector": "#password-input",
"params": [
command: "ai_speech_rygl",
description: "人员管理 人员",
action: "click",
selector: "#ai_speech_rygl",
params: [],
isIframe: true,
},
{
"name": "password",
"type": "string",
"description": "要输入的密码"
}
]
},
{
"command": "submit_login",
"description": "提交登录表单",
"action": "click",
"selector": "#submit-login",
"params": []
},
{
"command": "navigate_to_product",
"description": "导航到指定产品页面",
"action": "navigate",
"selector": "",
"params": [
command: "ai_speech_sbgl",
description: "设备管理",
action: "click",
selector: "#ai_speech_sbgl",
params: [],
isIframe: true,
children: [
// 用于测试搜索设备管理列表
{
command: "search_project",
description: "设备名称 设备搜索 搜索设备",
action: "input",
selector: "#search_project",
params: [
{
name: "projectname",
type: "string",
description: "要输入的设备名称",
},
],
isIframe: true,
},
]
},
{
"name": "product_url",
"type": "string",
"description": "产品页面的URL"
}
]
},
{
"command": "add_to_cart",
"description": "将当前产品添加到购物车",
"action": "click",
"selector": ".add-to-cart-btn",
"params": []
},
{
"command": "search_product",
"description": "在搜索框中输入关键词并搜索",
"action": "input_and_submit",
"selector": "#search-input",
"params": [
command: "ai_speech_gjgl",
description: "告警管理",
action: "click",
selector: "#ai_speech_gjgl",
params: [],
isIframe: true,
},
{
"name": "keyword",
"type": "string",
"description": "搜索关键词"
}
]
}
]
};
command: "ai_speech_zsk",
description: "知识库",
action: "click",
selector: "#ai_speech_zsk",
params: [],
isIframe: true,
},
{
command: "ai_speech_xcgl",
description: "巡查管理",
action: "click",
selector: "#ai_speech_xcgl",
params: [],
isIframe: true,
},
],
},
{
command: "switch_nyszzx",
description: "能源数据中心",
action: "click",
selector: "#ai_speech_nyszzx",
params: [],
isIframe: true,
children: [
{
command: "ai_speech_nhfx",
description: "能耗分析",
action: "click",
selector: "#ai_speech_nhfx",
params: [],
isIframe: true,
},
{
command: "ai_speech_nxrl",
description: "能耗日历",
action: "click",
selector: "#ai_speech_nxrl",
params: [],
isIframe: true,
},
{
command: "ai_speech_dbjc",
description: "电表监测",
action: "click",
selector: "#ai_speech_dbjc",
params: [],
isIframe: true,
},
{
command: "ai_speech_ynjc",
description: "用能监测",
action: "click",
selector: "#ai_speech_ynjc",
params: [],
isIframe: true,
},
{
command: "ai_speech_sbjc",
description: "设备监测",
action: "click",
selector: "#ai_speech_sbjc",
params: [],
isIframe: true,
},
{
command: "ai_speech_zljffx",
description: "制冷机房分析",
action: "click",
selector: "#ai_speech_zljffx",
params: [],
isIframe: true,
},
{
command: "ai_speech_ktjffx",
description: "空调机房分析",
action: "click",
selector: "#ai_speech_ktjffx",
params: [],
isIframe: true,
},
],
},
{
command: "switch_spjkzx",
description: "碳排放分析",
action: "click",
selector: "#ai_speech_spjkzx",
params: [],
isIframe: true,
},
{
command: "ai_speech_xcgl",
description: "巡查管理",
action: "click",
selector: "#ai_speech_xcgl",
params: [],
isIframe: true,
},
{
command: "input_project_name",
description: "搜索项目",
action: "input",
selector: "#ai-speech-project-name",
params: [
{
name: "projectname",
type: "string",
description: "输入项目名称",
},
],
},
{
command: "close_build_project",
description: "取消",
action: "click",
selector: "#ai-speech-close_buildproject",
params: [],
},
{
command: "build_project",
description: "创建",
action: "click",
selector: "#ai-speech-buildproject",
params: [],
},
],
};
export default config;

266
src/view/components/speechControl.vue
View File

@ -9,21 +9,36 @@
<div class="voice-control-container">
<div class="status-text">点击开始语音识别</div>
<button class="voice-btn" id="voice-btn" @click="toggleListening">
<svg style="
<svg
style="
position: absolute;
left: 50%;
top: 50%;
transform: translate3d(-50%, -50%, 0);
" t="1756171270753" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg"
p-id="14442" width="42" height="70">
"
t="1756171270753"
class="icon"
viewBox="0 0 1024 1024"
version="1.1"
xmlns="http://www.w3.org/2000/svg"
p-id="14442"
width="42"
height="70"
>
<path
d="M704 192v368c0 52.8-21.6 100.8-56.4 135.6S564.8 752 512 752c-105.6 0-192-86.4-192-192V192C320 86.4 406.4 0 512 0s192 86.4 192 192z"
p-id="14443" data-spm-anchor-id="a313x.search_index.0.i2.72cc3a81bxN4ca" class="selected" fill="#ffffff">
</path>
p-id="14443"
data-spm-anchor-id="a313x.search_index.0.i2.72cc3a81bxN4ca"
class="selected"
fill="#ffffff"
></path>
<path
d="M816 496v144c0 2.8-0.4 5.6-1.1 8.4-18.5 68.2-58.9 126.1-112.3 166.9-43.5 33.2-95.6 55.2-151.6 62.2-4 0.5-7 3.9-7 7.9V944c0 8.8 7.2 16 16 16h80c35.3 0 64 28.7 64 64H320c0-35.3 28.7-64 64-64h80c8.8 0 16-7.2 16-16v-58.5c0-4-3-7.4-7-7.9-124.8-15.7-230.3-105.5-263.9-229.2-0.7-2.7-1.1-5.6-1.1-8.4V496.7c0-17.4 13.7-32.2 31.1-32.7 18.1-0.5 32.9 14 32.9 32v129.8c0 6.9 1.1 13.8 3.3 20.3C309.3 746.9 404.6 816 512 816s202.7-69.1 236.7-169.9c2.2-6.5 3.3-13.4 3.3-20.3V496.7c0-17.4 13.7-32.2 31.1-32.7 18.1-0.5 32.9 14 32.9 32z"
p-id="14444" data-spm-anchor-id="a313x.search_index.0.i3.72cc3a81bxN4ca" class="selected" fill="#ffffff">
</path>
p-id="14444"
data-spm-anchor-id="a313x.search_index.0.i3.72cc3a81bxN4ca"
class="selected"
fill="#ffffff"
></path>
</svg>
<div class="pulse-ring"></div>
</button>
@ -32,9 +47,10 @@
<div class="command-action" v-for="item in action">
{{
"执行操作:" +
`${item.command} ${item.params && item.params.keyword
? "-" + item.params.keyword
: ""
`${item.command} ${
item.params && item.params.keyword
? "-" + item.params.keyword
: ""
}`
}}
</div>
@ -44,157 +60,17 @@
</template>
<script setup>
import { ref, onMounted, watch, nextTick } from "vue";
import config from "./../../config/index";
const config = {
page: "Ecommerce Home",
commands: [
{
command: "add_project",
description: "新建",
action: "click",
selector: "#ai-speech-add-project",
params: [],
},
{
command: "switch_jsc",
description: "驾驶舱",
action: "click",
selector: "#ai_speech_jsc",
params: [],
isIframe: true,
},
{
command: "switch_zngk",
description: "智能管控",
action: "click",
selector: "#ai_speech_zngk",
params: [],
isIframe: true,
},
{
command: "switch_zndd",
description: "智能调度",
action: "click",
selector: "#ai_speech_zndd",
params: [],
isIframe: true,
},
{
command: "switch_znyw",
description: "智能运维",
action: "click",
selector: "#ai_speech_znyw",
params: [],
isIframe: true,
},
{
command: "switch_nyszzx",
description: "能源数据中心",
action: "click",
selector: "#ai_speech_nyszzx",
params: [],
isIframe: true,
},
{
command: "switch_spjkzx",
description: "碳排放分析",
action: "click",
selector: "#ai_speech_spjkzx",
params: [],
isIframe: true,
},
{
command: "ai_speech_rygl",
description: "人员管理",
action: "click",
selector: "#ai_speech_rygl",
params: [],
isIframe: true,
},
{
command: "ai_speech_sbgl",
description: "设备管理",
action: "click",
selector: "#ai_speech_sbgl",
params: [],
isIframe: true,
},
{
command: "ai_speech_gjgl",
description: "告警管理",
action: "click",
selector: "#ai_speech_gjgl",
params: [],
isIframe: true,
},
{
command: "ai_speech_zsk",
description: "知识库",
action: "click",
selector: "#ai_speech_zsk",
params: [],
isIframe: true,
},
{
command: "ai_speech_xcgl",
description: "巡查管理",
action: "click",
selector: "#ai_speech_xcgl",
params: [],
isIframe: true,
},
console.log(config,"-====config");
// 用于测试搜索设备管理列表
{
command: "search_project",
description: "搜索",
action: "input",
selector: "#search_project",
params: [
{
name: "projectname",
type: "string",
description: "要输入的项目名称",
},
],
isIframe: true,
},
{
command: "input_project_name",
description: "名称",
action: "input",
selector: "#ai-speech-project-name",
params: [
{
name: "projectname",
type: "string",
description: "输入项目名称",
},
],
},
{
command: "close_build_project",
description: "取消",
action: "click",
selector: "#ai-speech-close_buildproject",
params: [],
},
{
command: "build_project",
description: "创建",
action: "click",
selector: "#ai-speech-buildproject",
params: [],
},
],
};
const listenStatus = ref(false);
const props = defineProps({
config: {
type: Object,
default: () => { },
default: () => {},
},
});
@ -244,7 +120,6 @@ class VoiceControl {
};
this.recognition.onresult = async (event) => {
// 避免同时处理多个结果
if (this.isProcessing) return;
@ -253,7 +128,6 @@ class VoiceControl {
const result = event.results[lastResultIndex];
let transcript = result[0].transcript;
this.showTranscript(transcript);
// 显示中间结果（灰色）和最终结果（黑色）
@ -312,15 +186,15 @@ class VoiceControl {
// 快速重启识别，减少等待时间
console.log("====> 走到这 ", listenStatus.value);
// TODO 模拟指令
setTimeout(async () => {
const sequence = await this.queryDeepSeek("搜索项目风冷热泵");
this.executeSequence(sequence);
if (sequence && sequence.sequence && sequence.sequence.length > 0) {
this.callback(sequence.sequence);
} else {
this.callback([]);
}
}, 100);
// setTimeout(async () => {
// const sequence = await this.queryDeepSeek("搜索项目风冷热泵");
// this.executeSequence(sequence);
// if (sequence && sequence.sequence && sequence.sequence.length > 0) {
// this.callback(sequence.sequence);
// } else {
// this.callback([]);
// }
// }, 100);
// END TODO
if (listenStatus.value) {
@ -367,10 +241,11 @@ class VoiceControl {
1. 只使用配置文件中定义的command
2.按照符合逻辑的执行顺序对筛选出的指令进行排序（例如：登录需遵循 "打开登录→输入用户名→输入密码→提交登录" 的顺序）
3.仅保留指令的 command 字段，形成有序数组
4. 如果用户指令中包含参数值（如用户名、密码、关键词），请正确提取并填充到params中
5.若请求涉及多个独立操作，需按操作逻辑拆分排序（如 "先登录再搜索商品" 需包含两部分完整指令链）
6.严格禁止添加指令集中不存在的指令，无关指令需排除
7.若无可匹配指令，返回空数组
4.若是command的children符合指令，则父指令也输出。比如: 输入人员管理，人员管理处于智能运维的children子菜单，则输出两条指令为智能运维→人员管理。
5. 如果用户指令中包含参数值（如用户名、密码、关键词），请正确提取并填充到params中
6.若请求涉及多个独立操作，需按操作逻辑拆分排序（如 "先登录再搜索商品" 需包含两部分完整指令链）
7.严格禁止添加指令集中不存在的指令，无关指令需排除
8.若无可匹配指令，返回空数组
现在请生成针对"${userQuery}"的JSON指令序列：
`;
@ -423,7 +298,11 @@ class VoiceControl {
let result = JSON.parse(content);
console.log(result, "=====> deepseek 返回", data);
// 验证响应结构
if (!result.sequence || !Array.isArray(result.sequence) || result.sequence.length === 0) {
if (
!result.sequence ||
!Array.isArray(result.sequence) ||
result.sequence.length === 0
) {
console.log(result, "DeepSeek返回了无效的指令序列格式");
throw new Error("DeepSeek返回了无效的指令序列格式");
}
@ -447,6 +326,8 @@ class VoiceControl {
async executeSequence(sequence) {
for (const [index, instruction] of sequence.sequence.entries()) {
console.log(instruction, "-----instruction");
try {
await this.executeInstruction(instruction);
@ -479,7 +360,7 @@ class VoiceControl {
try {
targetFrame.contentWindow.postMessage(message, "*");
console.log("=====> 发送到iframe");
console.log(message, "=====> 发送到iframe");
} catch (error) {
console.log(error, "=====> 发送到iframe");
}
@ -487,17 +368,52 @@ class VoiceControl {
}
async executeInstruction(instruction) {
const commandConfig = this.config.commands.find(
(c) => c.command === instruction.command
// 递归查找指令（支持搜索children中的嵌套指令）
function findCommandRecursively(commands, targetCommand) {
// 遍历当前层级的指令
for (const cmd of commands) {
// 1. 先检查当前指令是否匹配
if (cmd.command === targetCommand) {
return cmd;
}
// 2. 如果当前指令有children，递归查找子指令
if (
cmd.children &&
Array.isArray(cmd.children) &&
cmd.children.length > 0
) {
const foundInChildren = findCommandRecursively(
cmd.children,
targetCommand
);
// 子层级找到匹配指令，直接返回
if (foundInChildren) {
return foundInChildren;
}
}
}
// 所有层级都未找到
return null;
}
// 使用示例（替换原代码）
const commandConfig = findCommandRecursively(
this.config.commands,
instruction.command
);
console.log(commandConfig, "----commandConfig");
console.log(instruction, commandConfig, "----commandConfig");
if (!commandConfig) {
throw new Error(`未知指令: ${instruction.command}`);
}
if (commandConfig && commandConfig.isIframe) {
if (commandConfig.action == "input") {
const inputParam = commandConfig.params[0];
commandConfig.value = instruction.params[inputParam.name];
}
this.sendCommand(commandConfig);
return;
}
@ -576,7 +492,8 @@ class VoiceControl {
document.querySelector(".status-text").textContent = "正在聆听...";
} else {
voiceBtn.classList.remove("listening");
document.querySelector(".status-text").textContent = "点击开始语音识别2-1";
document.querySelector(".status-text").textContent =
"点击开始语音识别2-1";
}
}
}
@ -627,7 +544,6 @@ const toggleListening = () => {
}
alert("语音识别操作失败，请稍后再试");
}
};
onMounted(() => {

8
src/view/index.vue
View File

@ -91,14 +91,14 @@
<el-tab-pane label="云托管项目" name="cloud"></el-tab-pane>
</el-tabs>
<div class="search-box">
<el-input id="search_project" v-model="searchValue" style="width: 260px" class="responsive-input" placeholder="搜索项目"
<el-input id="ai-speech-project-name" v-model="searchValue" style="width: 260px" class="responsive-input" placeholder="搜索项目"
prefix-icon="Search" />
</div>
</section>
<!-- 项目列表 -->
<section class="project-list">
<div class="project-card" v-for="item in projects">
<div class="project-card" v-for="item in projects" @click="goDetail(item)">
<div class="project-item-info">
<div class="card-title">{{ item.name }}</div>
<div class="card-actions">
@ -211,6 +211,10 @@ const addProject = () => {
console.log("正在点击新建按钮 ======> addProject ");
}
const goDetail = () => {
router.push('/LargeScreen');
}
</script>
<style lang="less" scoped>