jiwanjun
/
ai-speech-build
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'main' of http://172.16.1.12/jiwanjun/ai-speech-build

This commit is contained in:
liangbin 2025-09-04 08:35:36 +08:00
parent 7064e47250 24945d62bc
commit 72e0ff660e
5 changed files with 941 additions and 2020 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2350
package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

265
src/config/index.js
View File

@ -4,87 +4,202 @@
* @Description:
*/
const config = {
"page": "Ecommerce Home",
"commands": [
page: "Ecommerce Home",
commands: [
{
"command": "open_login_111",
"description": "打开登录弹窗或页面",
"action": "click",
"selector": "#login-button",
"params": []
command: "add_project",
description: "新建",
action: "click",
selector: "#ai-speech-add-project",
params: [],
},
{
"command": "open_login_222_333",
"description": "打开登录弹窗或页面",
"action": "click",
"selector": "#login-button",
"params": []
command: "switch_jsc",
description: "驾驶舱",
action: "click",
selector: "#ai_speech_jsc",
params: [],
isIframe: true,
},
{
"command": "input_username",
"description": "在用户名输入框中填写用户名",
"action": "input",
"selector": "#username-input",
"params": [
command: "switch_zngk",
description: "智能管控",
action: "click",
selector: "#ai_speech_zngk",
params: [],
isIframe: true,
},
{
command: "switch_zndd",
description: "智能调度",
action: "click",
selector: "#ai_speech_zndd",
params: [],
isIframe: true,
},
{
command: "switch_znyw",
description: "智能运维 运维",
action: "click",
selector: "#ai_speech_znyw",
params: [],
isIframe: true,
children: [
{
"name": "username",
"type": "string",
"description": "要输入的用户名"
}
]
},
{
"command": "input_password",
"description": "在密码输入框中填写密码",
"action": "input",
"selector": "#password-input",
"params": [
command: "ai_speech_rygl",
description: "人员管理 人员",
action: "click",
selector: "#ai_speech_rygl",
params: [],
isIframe: true,
},
{
"name": "password",
"type": "string",
"description": "要输入的密码"
}
]
},
{
"command": "submit_login",
"description": "提交登录表单",
"action": "click",
"selector": "#submit-login",
"params": []
},
{
"command": "navigate_to_product",
"description": "导航到指定产品页面",
"action": "navigate",
"selector": "",
"params": [
command: "ai_speech_sbgl",
description: "设备管理",
action: "click",
selector: "#ai_speech_sbgl",
params: [],
isIframe: true,
children: [
{
command: "search_project",
description: "设备名称 设备搜索 搜索设备",
action: "input",
selector: "#search_project",
params: [
{
name: "projectname",
type: "string",
description: "要输入的设备名称",
},
],
isIframe: true,
},
]
},
{
"name": "product_url",
"type": "string",
"description": "产品页面的URL"
}
]
},
{
"command": "add_to_cart",
"description": "将当前产品添加到购物车",
"action": "click",
"selector": ".add-to-cart-btn",
"params": []
},
{
"command": "search_product",
"description": "在搜索框中输入关键词并搜索",
"action": "input_and_submit",
"selector": "#search-input",
"params": [
command: "ai_speech_gjgl",
description: "告警管理",
action: "click",
selector: "#ai_speech_gjgl",
params: [],
isIframe: true,
},
{
"name": "keyword",
"type": "string",
"description": "搜索关键词"
}
]
}
]
};
command: "ai_speech_zsk",
description: "知识库",
action: "click",
selector: "#ai_speech_zsk",
params: [],
isIframe: true,
},
{
command: "ai_speech_xcgl",
description: "巡查管理",
action: "click",
selector: "#ai_speech_xcgl",
params: [],
isIframe: true,
},
],
},
{
command: "switch_nyszzx",
description: "能源数据中心",
action: "click",
selector: "#ai_speech_nyszzx",
params: [],
isIframe: true,
children: [
{
command: "ai_speech_nhfx",
description: "能耗分析",
action: "click",
selector: "#ai_speech_nhfx",
params: [],
isIframe: true,
},
{
command: "ai_speech_nxrl",
description: "能耗日历",
action: "click",
selector: "#ai_speech_nxrl",
params: [],
isIframe: true,
},
{
command: "ai_speech_dbjc",
description: "电表监测",
action: "click",
selector: "#ai_speech_dbjc",
params: [],
isIframe: true,
},
{
command: "ai_speech_ynjc",
description: "用能监测",
action: "click",
selector: "#ai_speech_ynjc",
params: [],
isIframe: true,
},
{
command: "ai_speech_sbjc",
description: "设备监测",
action: "click",
selector: "#ai_speech_sbjc",
params: [],
isIframe: true,
},
{
command: "ai_speech_zljffx",
description: "制冷机房分析",
action: "click",
selector: "#ai_speech_zljffx",
params: [],
isIframe: true,
},
{
command: "ai_speech_ktjffx",
description: "空调机房分析",
action: "click",
selector: "#ai_speech_ktjffx",
params: [],
isIframe: true,
},
],
},
{
command: "switch_spjkzx",
description: "碳排放分析",
action: "click",
selector: "#ai_speech_spjkzx",
params: [],
isIframe: true,
},
{
command: "ai_speech_xcgl",
description: "巡查管理",
action: "click",
selector: "#ai_speech_xcgl",
params: [],
isIframe: true,
},
{
command: "close_build_project",
description: "取消",
action: "click",
selector: "#ai-speech-close_buildproject",
params: [],
},
{
command: "build_project",
description: "创建",
action: "click",
selector: "#ai-speech-buildproject",
params: [],
},
],
};
export default config;

2
src/view/components/projectModal.vue
View File

@ -293,7 +293,7 @@
<el-input v-model="projectName" id="ai-speech-project-name" placeholder="请输入名称" />
</div>
<div class="FootBtn">
<el-button class="close-btn" @click="handleClose">取消</el-button>
<el-button id="ai-speech-close_buildproject" class="close-btn" @click="handleClose">取消</el-button>
<el-button id="ai-speech-buildproject" type="primary" @click="submitFun">创建项目</el-button>
</div>
</div>

336
src/view/components/speechControl.vue
View File

@ -9,21 +9,36 @@
<div class="voice-control-container">
<div class="status-text">点击开始语音识别</div>
<button class="voice-btn" id="voice-btn" @click="toggleListening">
<svg style="
<svg
style="
position: absolute;
left: 50%;
top: 50%;
transform: translate3d(-50%, -50%, 0);
" t="1756171270753" class="icon" viewBox="0 0 1024 1024" version="1.1" xmlns="http://www.w3.org/2000/svg"
p-id="14442" width="42" height="70">
"
t="1756171270753"
class="icon"
viewBox="0 0 1024 1024"
version="1.1"
xmlns="http://www.w3.org/2000/svg"
p-id="14442"
width="42"
height="70"
>
<path
d="M704 192v368c0 52.8-21.6 100.8-56.4 135.6S564.8 752 512 752c-105.6 0-192-86.4-192-192V192C320 86.4 406.4 0 512 0s192 86.4 192 192z"
p-id="14443" data-spm-anchor-id="a313x.search_index.0.i2.72cc3a81bxN4ca" class="selected" fill="#ffffff">
</path>
p-id="14443"
data-spm-anchor-id="a313x.search_index.0.i2.72cc3a81bxN4ca"
class="selected"
fill="#ffffff"
></path>
<path
d="M816 496v144c0 2.8-0.4 5.6-1.1 8.4-18.5 68.2-58.9 126.1-112.3 166.9-43.5 33.2-95.6 55.2-151.6 62.2-4 0.5-7 3.9-7 7.9V944c0 8.8 7.2 16 16 16h80c35.3 0 64 28.7 64 64H320c0-35.3 28.7-64 64-64h80c8.8 0 16-7.2 16-16v-58.5c0-4-3-7.4-7-7.9-124.8-15.7-230.3-105.5-263.9-229.2-0.7-2.7-1.1-5.6-1.1-8.4V496.7c0-17.4 13.7-32.2 31.1-32.7 18.1-0.5 32.9 14 32.9 32v129.8c0 6.9 1.1 13.8 3.3 20.3C309.3 746.9 404.6 816 512 816s202.7-69.1 236.7-169.9c2.2-6.5 3.3-13.4 3.3-20.3V496.7c0-17.4 13.7-32.2 31.1-32.7 18.1-0.5 32.9 14 32.9 32z"
p-id="14444" data-spm-anchor-id="a313x.search_index.0.i3.72cc3a81bxN4ca" class="selected" fill="#ffffff">
</path>
p-id="14444"
data-spm-anchor-id="a313x.search_index.0.i3.72cc3a81bxN4ca"
class="selected"
fill="#ffffff"
></path>
</svg>
<div class="pulse-ring"></div>
</button>
@ -32,9 +47,10 @@
<div class="command-action" v-for="item in action">
{{
"执行操作:" +
`${item.command} ${item.params && item.params.keyword
? "-" + item.params.keyword
: ""
`${item.command} ${
item.params && item.params.keyword
? "-" + item.params.keyword
: ""
}`
}}
</div>
@ -44,157 +60,14 @@
</template>
<script setup>
import { ref, onMounted, watch, nextTick } from "vue";
const config = {
page: "Ecommerce Home",
commands: [
{
command: "add_project",
description: "新建",
action: "click",
selector: "#ai-speech-add-project",
params: [],
},
{
command: "switch_jsc",
description: "驾驶舱",
action: "click",
selector: "#ai_speech_jsc",
params: [],
isIframe: true,
},
{
command: "switch_zngk",
description: "智能管控",
action: "click",
selector: "#ai_speech_zngk",
params: [],
isIframe: true,
},
{
command: "switch_zndd",
description: "智能调度",
action: "click",
selector: "#ai_speech_zndd",
params: [],
isIframe: true,
},
{
command: "switch_znyw",
description: "智能运维",
action: "click",
selector: "#ai_speech_znyw",
params: [],
isIframe: true,
},
{
command: "switch_nyszzx",
description: "能源数据中心",
action: "click",
selector: "#ai_speech_nyszzx",
params: [],
isIframe: true,
},
{
command: "switch_spjkzx",
description: "碳排放分析",
action: "click",
selector: "#ai_speech_spjkzx",
params: [],
isIframe: true,
},
{
command: "ai_speech_rygl",
description: "人员管理",
action: "click",
selector: "#ai_speech_rygl",
params: [],
isIframe: true,
},
{
command: "ai_speech_sbgl",
description: "设备管理",
action: "click",
selector: "#ai_speech_sbgl",
params: [],
isIframe: true,
},
{
command: "ai_speech_gjgl",
description: "告警管理",
action: "click",
selector: "#ai_speech_gjgl",
params: [],
isIframe: true,
},
{
command: "ai_speech_zsk",
description: "知识库",
action: "click",
selector: "#ai_speech_zsk",
params: [],
isIframe: true,
},
{
command: "ai_speech_xcgl",
description: "巡查管理",
action: "click",
selector: "#ai_speech_xcgl",
params: [],
isIframe: true,
},
// 用于测试搜索设备管理列表
{
command: "search_project",
description: "搜索",
action: "input",
selector: "#search_project",
params: [
{
name: "projectname",
type: "string",
description: "要输入的项目名称",
},
],
isIframe: true,
},
{
command: "input_project_name",
description: "名称",
action: "input",
selector: "#ai-speech-project-name",
params: [
{
name: "projectname",
type: "string",
description: "输入项目名称",
},
],
},
{
command: "close_build_project",
description: "取消",
action: "click",
selector: "#ai-speech-close_buildproject",
params: [],
},
{
command: "build_project",
description: "创建",
action: "click",
selector: "#ai-speech-buildproject",
params: [],
},
],
};
import config from "./../../config/index";
const listenStatus = ref(false);
const props = defineProps({
config: {
type: Object,
default: () => { },
default: () => {},
},
});
@ -233,7 +106,7 @@ class VoiceControl {
// 音频处理参数（重要）
this.recognition.energy_threshold = 500; // 能量阈值
this.recognition.pause_threshold = 0.3; // 停顿时间阈值（秒）
this.recognition.pause_threshold = 0.5; // 停顿时间阈值（秒）
this.recognition.phrase_threshold = 0.2; // 短语识别阈值
this.lastFinalTranscript = ""; // 用于跟踪最终结果
@ -244,7 +117,6 @@ class VoiceControl {
};
this.recognition.onresult = async (event) => {
// 避免同时处理多个结果
if (this.isProcessing) return;
@ -253,7 +125,6 @@ class VoiceControl {
const result = event.results[lastResultIndex];
let transcript = result[0].transcript;
this.showTranscript(transcript);
// 显示中间结果（灰色）和最终结果（黑色）
@ -312,15 +183,15 @@ class VoiceControl {
// 快速重启识别，减少等待时间
console.log("====> 走到这 ", listenStatus.value);
// TODO 模拟指令
setTimeout(async () => {
const sequence = await this.queryDeepSeek("搜索项目风冷热泵");
this.executeSequence(sequence);
if (sequence && sequence.sequence && sequence.sequence.length > 0) {
this.callback(sequence.sequence);
} else {
this.callback([]);
}
}, 100);
// setTimeout(async () => {
// const sequence = await this.queryDeepSeek("搜索项目风冷热泵");
// this.executeSequence(sequence);
// if (sequence && sequence.sequence && sequence.sequence.length > 0) {
// this.callback(sequence.sequence);
// } else {
// this.callback([]);
// }
// }, 100);
// END TODO
if (listenStatus.value) {
@ -353,27 +224,61 @@ class VoiceControl {
}
const prompt = `
你是一个网页控制助手。请根据以下配置文件，分析用户自然语言请求，从可用指令集中筛选出与请求意图匹配的指令，将用户的自然语言指令解析成一个可执行的指令序列。
配置文件：
${JSON.stringify(this.config, null, 2)}
用户指令："${userQuery}"
请严格按照以下JSON格式输出，只包含一个名为"sequence"的数组：
{ "sequence": [ { "command": "command_name", "params": { "param_name": "value" } }, ... ] }
要求：
1. 只使用配置文件中定义的command
2.按照符合逻辑的执行顺序对筛选出的指令进行排序（例如：登录需遵循 "打开登录→输入用户名→输入密码→提交登录" 的顺序）
3.仅保留指令的 command 字段，形成有序数组
4. 如果用户指令中包含参数值（如用户名、密码、关键词），请正确提取并填充到params中
5.若请求涉及多个独立操作，需按操作逻辑拆分排序（如 "先登录再搜索商品" 需包含两部分完整指令链）
6.严格禁止添加指令集中不存在的指令，无关指令需排除
7.若无可匹配指令，返回空数组
现在请生成针对"${userQuery}"的JSON指令序列：
`;
你是一个网页控制助手，负责将用户的自然语言指令转换为可执行的指令序列。请基于提供的配置信息，分析用户请求的意图，从可用指令集中精准匹配相应指令。
# 配置信息
${JSON.stringify(this.config, null, 2)}
# 用户指令
"${userQuery}"
# 输出要求
请严格按照以下JSON格式输出，只包含名为"sequence"的数组：
{ "sequence": [ { "command": "command_name", "params": { "param_name": "value" } } ] }
# 处理规则
1. **指令匹配**：仅使用配置文件中明确定义的指令，禁止添加任何额外指令
2. **执行顺序**：按照逻辑顺序排列指令（如："打开页面→输入信息→提交操作"）
3. **层级处理**：若子指令被匹配，必须包含其父指令以形成完整路径（如：智能运维→人员管理）
4. **参数提取**：从用户指令中准确提取参数值并填充到对应字段
5. **多操作处理**：复合指令需拆分为多个完整操作序列并按逻辑排序
6. **语音容错**：对可能的语音识别错误（特别是专有名词）进行智能矫正
7. **场景判断**：根据关键词自动判断场景（设备相关→搜索设备，人员相关→搜索人员等）
8. **默认处理**：无匹配指令时返回空数组
# 特别注意
- 针对语音识别可能出现的同音词错误，使用上下文智能矫正（如："疯了"→"风冷"）
- 参数值提取时保持原始语义准确性
- 确保输出为纯JSON格式，无额外解释内容
请生成针对"${userQuery}"的指令序列：
`;
// const prompt = `
// 你是一个网页控制助手。请根据以下配置文件，分析用户自然语言请求，从可用指令集中筛选出与请求意图匹配的指令，将用户的自然语言指令解析成一个可执行的指令序列。
// 配置文件：
// ${JSON.stringify(this.config, null, 2)}
// 用户指令："${userQuery}"
// 请严格按照以下JSON格式输出，只包含一个名为"sequence"的数组：
// { "sequence": [ { "command": "command_name", "params": { "param_name": "value" } }, ... ] }
// 要求：
// 1. 只使用配置文件中定义的command
// 2.按照符合逻辑的执行顺序对筛选出的指令进行排序（例如：登录需遵循 "打开登录→输入用户名→输入密码→提交登录" 的顺序）
// 3.仅保留指令的 command 字段，形成有序数组
// 4.若是command的children符合指令，则父指令也输出。比如: 输入人员管理，人员管理处于智能运维的children子菜单，则输出两条指令为智能运维→人员管理。
// 5. 如果用户指令中包含参数值（如用户名、密码、关键词），请正确提取并填充到params中
// 6.若请求涉及多个独立操作，需按操作逻辑拆分排序（如 "先登录再搜索商品" 需包含两部分完整指令链）
// 7.严格禁止添加指令集中不存在的指令，无关指令需排除
// 8.若无可匹配指令，返回空数组
// 9.因为语音识别的问题，有的关键词可能是专有名词的原因没有识别正确，比如语音输入风冷，结果被识别成疯了，封了等。可能是设备或者专有名词，根据同音尽量矫正。
// 10.对于带有关键词的根据关键词智能判断场景，比如关键词是设备相关则是搜索设备，关键词是人物相关则是搜索人员等。
// 现在请生成针对"${userQuery}"的JSON指令序列：
// `;
const response = await fetch(
"https://api.deepseek.com/v1/chat/completions",
@ -423,7 +328,11 @@ class VoiceControl {
let result = JSON.parse(content);
console.log(result, "=====> deepseek 返回", data);
// 验证响应结构
if (!result.sequence || !Array.isArray(result.sequence) || result.sequence.length === 0) {
if (
!result.sequence ||
!Array.isArray(result.sequence) ||
result.sequence.length === 0
) {
console.log(result, "DeepSeek返回了无效的指令序列格式");
throw new Error("DeepSeek返回了无效的指令序列格式");
}
@ -479,7 +388,7 @@ class VoiceControl {
try {
targetFrame.contentWindow.postMessage(message, "*");
console.log("=====> 发送到iframe");
console.log(message, "=====> 发送到iframe");
} catch (error) {
console.log(error, "=====> 发送到iframe");
}
@ -487,25 +396,58 @@ class VoiceControl {
}
async executeInstruction(instruction) {
const commandConfig = this.config.commands.find(
(c) => c.command === instruction.command
// 递归查找指令（支持搜索children中的嵌套指令）
function findCommandRecursively(commands, targetCommand) {
// 遍历当前层级的指令
for (const cmd of commands) {
// 1. 先检查当前指令是否匹配
if (cmd.command === targetCommand) {
return cmd;
}
// 2. 如果当前指令有children，递归查找子指令
if (
cmd.children &&
Array.isArray(cmd.children) &&
cmd.children.length > 0
) {
const foundInChildren = findCommandRecursively(
cmd.children,
targetCommand
);
// 子层级找到匹配指令，直接返回
if (foundInChildren) {
return foundInChildren;
}
}
}
// 所有层级都未找到
return null;
}
// 使用示例（替换原代码）
const commandConfig = findCommandRecursively(
this.config.commands,
instruction.command
);
console.log(commandConfig, "----commandConfig");
console.log(commandConfig, "===> commandConfig");
if (!commandConfig) {
throw new Error(`未知指令: ${instruction.command}`);
}
if (commandConfig && commandConfig.isIframe) {
if (commandConfig.action == "input") {
const inputParam = commandConfig.params[0];
commandConfig.value = instruction.params[inputParam.name];
}
this.sendCommand(commandConfig);
return;
}
const element = document.querySelector(commandConfig.selector);
console.log(element, "====> 控制元素");
if (!element) {
throw new Error(`找不到元素: ${commandConfig.selector}`);
}
@ -576,7 +518,8 @@ class VoiceControl {
document.querySelector(".status-text").textContent = "正在聆听...";
} else {
voiceBtn.classList.remove("listening");
document.querySelector(".status-text").textContent = "点击开始语音识别2-1";
document.querySelector(".status-text").textContent =
"点击开始语音识别2-1";
}
}
}
@ -627,7 +570,6 @@ const toggleListening = () => {
}
alert("语音识别操作失败，请稍后再试");
}
};
onMounted(() => {

8
src/view/index.vue
View File

@ -91,14 +91,14 @@
<el-tab-pane label="云托管项目" name="cloud"></el-tab-pane>
</el-tabs>
<div class="search-box">
<el-input id="search_project" v-model="searchValue" style="width: 260px" class="responsive-input" placeholder="搜索项目"
<el-input id="ai-speech-project-name" v-model="searchValue" style="width: 260px" class="responsive-input" placeholder="搜索项目"
prefix-icon="Search" />
</div>
</section>
<!-- 项目列表 -->
<section class="project-list">
<div class="project-card" v-for="item in projects">
<div class="project-card" v-for="item in projects" @click="goDetail(item)">
<div class="project-item-info">
<div class="card-title">{{ item.name }}</div>
<div class="card-actions">
@ -211,6 +211,10 @@ const addProject = () => {
console.log("正在点击新建按钮 ======> addProject ");
}
const goDetail = () => {
router.push('/LargeScreen');
}
</script>
<style lang="less" scoped>