using System; using System.Collections; using System.Collections.Generic; using UnityEngine; using System.Text; using Newtonsoft.Json.Linq; using System.Net.WebSockets; using System.Threading; using System.Threading.Tasks; using System.Linq; using Newtonsoft.Json; using UnityEngine.Windows.Speech; using System.IO; using Crosstales.RTVoice; using System.Security.Cryptography; public class IFlytekManager : MonoBehaviour { public static IFlytekManager instance; public IFlytekManager() { instance = this; } ClientWebSocket ws; CancellationToken cancellation; /// /// 语音转文字 /// const string wsAudioToTextUrl = "wss://iat-api.xfyun.cn/v2/iat"; /// /// 文字转语音 /// const string wsTextToAudioUrl = "wss://tts-api.xfyun.cn/v2/tts"; //陈向学账号 //const string appId = "bcc904a6"; //const string apiKey = "edc3cb67c289b953747e31dae43b68e4"; //const string apiSecret = "NmY0ODE3YjdkNGFmN2VmYTYyYmQ4MDkz"; //夏云账号 const string appId = "241a1784"; const string apiKey = "86e2e8409788caacb8705ed07e31baa1"; const string apiSecret = "MGNmZWE0ZWFhYzhkN2VkNzFjZDI4ZWEx"; const int StatusFirstFrame = 0; const int StatusContinueFrame = 1; const int StatusLastFrame = 2; /// /// chat /// const string wsChatUrl = "ws://spark-api.xf-yun.com/v1.1/chat"; const string chat_appid = "a83ddb30"; const string chat_key = "df2f1b36593607691d184c2282977959"; const string chat_secret = "ZDUwMjczZjM2YTI2YTVhZDU2MjljZDI4"; /// /// 用户唯一id /// string clientId = System.Guid.NewGuid().ToString("N"); private void Awake() { //instance = this; } private void OnDestroy() { StopAllCoroutines(); if (ws != null && ws.State == WebSocketState.Open) { ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "", cancellation); ws = null; } } /// /// 语音转文字 /// /// /// public void AudioToText(byte[] AudioData, Action callback) { StartCoroutine(callApiAudioToText(AudioData, callback)); } /// /// 短文字转语音 /// /// /// /* public void TextToAudio(string text, Action callback, int speed = 1, string tone = "xiaoyan") { StartCoroutine(callApiTextToAudio(text,callback,speed,tone)); } */ //zhangheyi public void TextToAudio(string text) { StartCoroutine(callApiTextToAudio(text)); } /// /// 星火大模型对话(不联系上文) /// /// 问题 /// public void Chat(string question, Action callback) { JArray text = new JArray(); JObject jb = new JObject(); jb.Add("role", "user"); jb.Add("content", question); text.Add(jb); StartCoroutine(CallapiForChat(text, callback)); } /// /// 星火大模型对话(联系上文) /// /// 依次历史对话,不要多 /// 问题 /// public void Chat(List historyChat, string question, Action callback) { JArray text = new JArray(); for (int i = 0; i < historyChat.Count; i++) { JObject jb = new JObject(); jb.Add("role", "user"); jb.Add("content", historyChat[i].ask); text.Add(jb); JObject jb2 = new JObject(); jb2.Add("role", "assistant"); jb2.Add("content", historyChat[i].answer); text.Add(jb2); } //问题 JObject jb111 = new JObject(); jb111.Add("role", "user"); jb111.Add("content", question); text.Add(jb111); StartCoroutine(CallapiForChat(text, callback)); } //用网的功能 /// /// 调用语音听写 /// /// IEnumerator callApiAudioToText(byte[] AudioData, Action callback) { Debug.Log("开始调用API,音频长度:"+ AudioData.Length); MyDebugger.Log("开始转文字"); string url = GetAuthUrl(wsAudioToTextUrl, apiKey, apiSecret); using (ws = new ClientWebSocket()) { //连接 Task tConnect = ws.ConnectAsync(new Uri(url), cancellation); yield return new WaitUntil(() => tConnect.IsCompleted); if (ws.State == WebSocketState.Open) { //连接成功,开始发送数据 int frameSize = 8000; //每一帧音频的大小,建议每 40ms 发送 8000 int intervel = 40; //毫秒数 int status = 0; // 音频的状态 byte[] buffer /*= new byte[frameSize]*/; string str = ""; //开启异步接收 Task recive = Task.Run(() => ReciveText(ws)); //发送 for (int i = 0; i < AudioData.Length; i += frameSize) { buffer = SubArray(AudioData, i, frameSize); if (buffer == null) { status = StatusLastFrame; //文件读完,改变status 为 2 } switch (status) { case StatusFirstFrame: // 第一帧音频status = 0 //请求数据均为json字符串 JObject frame = new JObject(); JObject business = new JObject(); //第一帧必须发送 JObject common = new JObject(); //第一帧必须发送 JObject data = new JObject(); //每一帧都要发送 //公共参数 //填充common common.Add("app_id", appId); //业务参数 //填充business business.Add("language", "zh_cn"); business.Add("domain", "iat"); business.Add("accent", "mandarin"); //business.Add("nunum", 0); business.Add("vad_eos", 9000); business.Add("ptt", 1);//标点符号 //business.Add("rlang", "zh-hk"); // zh-cn :简体中文(默认值)zh-hk :繁体香港(若未授权不生效) //business.Add("vinfo", 1); //business.Add("dwa", "wpgs");//动态修正(若未授权不生效) //business.Add("nbest", 5);// 句子多候选(若未授权不生效) //business.Add("wbest", 3);// 词级多候选(若未授权不生效) //业务数据流参数 //填充data data.Add("status", StatusFirstFrame); data.Add("format", "audio/L16;rate=16000"); data.Add("audio", Convert.ToBase64String(buffer)); data.Add("encoding", "raw"); //填充frame frame.Add("common", common); frame.Add("business", business); frame.Add("data", data); var frameData = Encoding.UTF8.GetBytes(frame.ToString()); //发送数据 ws.SendAsync(new ArraySegment(frameData), WebSocketMessageType.Text, true, cancellation); status = StatusContinueFrame; // 发送完第一帧改变status 为 1 break; case StatusContinueFrame: //中间帧status = 1 JObject frame1 = new JObject(); JObject data1 = new JObject(); //每一帧都要发送 //填充data data1.Add("status", StatusContinueFrame); data1.Add("format", "audio/L16;rate=16000"); data1.Add("audio", Convert.ToBase64String(buffer)); data1.Add("encoding", "raw"); //填充frame frame1.Add("data", data1); var frameData1 = Encoding.UTF8.GetBytes(frame1.ToString()); ws.SendAsync(new ArraySegment(frameData1), WebSocketMessageType.Text, true, cancellation); break; case StatusLastFrame: // 最后一帧音频status = 2 ,标志音频发送结束 break; } //Task.Delay(intervel).Wait(); //模拟音频采样延时 //Debug.Log("发送片段"); yield return new WaitForSeconds(intervel * 0.001f);//模拟音频采样延时 } #region 结束 JObject frame2 = new JObject(); JObject data2 = new JObject(); //每一帧都要发送 //填充data data2.Add("status", StatusLastFrame); //填充frame frame2.Add("data", data2); var frameData2 = System.Text.Encoding.UTF8.GetBytes(frame2.ToString()); ws.SendAsync(new ArraySegment(frameData2), WebSocketMessageType.Text, true, cancellation); Debug.Log("发送最后一个片段"); #endregion Debug.Log("进入等待接收转换文字中..."); yield return new WaitUntil(() => recive.IsCompleted); MyDebugger.Log("文字转换完成"); callback(recive.Result); Debug.Log("结束调用API,文字转换完成"); } else { Debug.LogError("连接失败"); callback(null); } } } //zhangheyi IEnumerator callApiTextToAudio(string msg) { /* Debug.Log("开始调用API"); MyDebugger.Log("开始转语音"); string url = GetAuthUrl(wsTextToAudioUrl, apiKey, apiSecret); using (ws = new ClientWebSocket()) { //连接 Task tConnect=ws.ConnectAsync(new Uri(url), cancellation); yield return new WaitUntil(() => tConnect.IsCompleted); Debug.Log("连接状态:" + ws.State.ToString()); if (ws.State == WebSocketState.Open) { Task re = Task.Run(() => ReciveAudio(ws)); //连接成功,开始发送数据 //请求数据均为json字符串 JObject frame = new JObject(); JObject business = new JObject(); JObject common = new JObject(); JObject data = new JObject(); common.Add("app_id", appId); business.Add("aue", "lame"); business.Add("sfl", 1); //business.Add("speed", 3); business.Add("auf", "audio/L16;rate=16000"); business.Add("vcn", tone); //business.Add("bgs", 0);//是否有背景音 business.Add("tte", "utf8"); data.Add("status", 2); string str = Convert.ToBase64String(Encoding.UTF8.GetBytes(msg)); Debug.Log(str); data.Add("text", str); //填充frame frame.Add("common", common); frame.Add("business", business); frame.Add("data", data); string da = frame.ToString(); var frameData = Encoding.UTF8.GetBytes(da); //发送数据 ws.SendAsync(new ArraySegment(frameData), WebSocketMessageType.Text, true, cancellation); yield return new WaitUntil(() => re.IsCompleted); Debug.Log("完成"); MyDebugger.Log("语音转换完成"); //if (re.Result.Length > 0) //{ // File.WriteAllBytes(@"D:\chenxiangxue\test.mp3", re.Result); //} //转audio AudioClip clip = Mp3Tansform.LoadMp3Audio(re.Result, "demo.mp3"); callback(clip); } else { Debug.LogError("连接失败"); callback(null); } } */ yield return Speaker.Instance.Speak(msg, null, Speaker.Instance.Voices[1]); } IEnumerator CallapiForChat(JArray text, Action callback) { Debug.Log("开始调用API"); MyDebugger.Log("开始聊天"); string url = GetAuthUrl(wsChatUrl, chat_key, chat_secret); using (ws = new ClientWebSocket()) { //连接 ws.ConnectAsync(new Uri(url), cancellation).Wait(2000); if (ws.State == WebSocketState.Open) { Task re = Task.Run(() => ReciceChat(ws)); //连接成功,开始发送数据 //请求数据均为json字符串 JObject data = new JObject(); JObject header = new JObject(); JObject parameter = new JObject(); JObject payload = new JObject(); data.Add("header", header); data.Add("parameter", parameter); data.Add("payload", payload); header.Add("app_id", chat_appid); header.Add("uid", clientId); JObject chat = new JObject(); parameter.Add("chat", chat); chat.Add("domain", "general");//领域 chat.Add("max_tokens", 2048);//最大tokebs长度 chat.Add("auditing", "default");//审核 JObject message = new JObject(); payload.Add("message", message); message.Add("text", text); string da = data.ToString(); var frameData = Encoding.UTF8.GetBytes(da); //发送数据 ws.SendAsync(new ArraySegment(frameData), WebSocketMessageType.Text, true, cancellation); yield return new WaitUntil(() => re.IsCompleted); Debug.Log("完成"); callback(re.Result); } else { Debug.LogError("连接失败"); callback(null); } } } string ReciveText(ClientWebSocket ws) { string msgRecive = ""; //全部消息容器 List bs = new List(); //缓冲区 var buffer = new byte[1024 * 4]; while (ws.State == WebSocketState.Open) { Debug.Log("等待消息接收"); var task = ws.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); task.Wait(); WebSocketReceiveResult result = task.Result; //文本消息 if (result.CloseStatus.HasValue) { Debug.Log("接收关闭"); break; } if (result.MessageType == WebSocketMessageType.Text) { bs.AddRange(buffer.Take(result.Count)); //消息是否已接收完全 if (result.EndOfMessage) { //发送过来的消息 string userMsg = Encoding.UTF8.GetString(bs.ToArray(), 0, bs.Count); Debug.Log(userMsg); var resultObj = GetTextResultData(userMsg); string msg = resultObj.data.result.GetResultText(); //清空消息容器 bs = new List(); Debug.Log("接收片段:"+msg); msgRecive = msgRecive + msg; if (resultObj.data.result.ls) { //关闭 Debug.Log("接收是最后一块数据"); break; } } } } ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "", cancellation); Debug.Log("退出接收aaa"); return msgRecive; } //接收二进制语音 byte[] ReciveAudio(ClientWebSocket ws) { List msgRecive = new List(); //全部消息容器 List bs = new List(); //缓冲区 var buffer = new byte[1024 * 4]; while (ws.State == WebSocketState.Open) { //Debug.Log("等待消息接收"); var task = ws.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); task.Wait(); WebSocketReceiveResult result = task.Result; //文本消息 if (result.CloseStatus.HasValue) { Debug.Log("接收关闭"); break; } if (result.MessageType == WebSocketMessageType.Text) { bs.AddRange(buffer.Take(result.Count)); //消息是否已接收完全 if (result.EndOfMessage) { //发送过来的消息 string userMsg = Encoding.UTF8.GetString(bs.ToArray(), 0, bs.Count); bs = new List(); var resultObj = GetAudioResultData(userMsg); if (resultObj.code == 0) { string msg = resultObj.data.result.GetResultAudio(); //清空消息容器 Debug.Log(msg.Length); msgRecive.AddRange(Convert.FromBase64String(msg)); byte[] base64clip = Convert.FromBase64String(msg); } else { //关闭 Debug.Log("音频错误结束"); break; } if (resultObj.data.status == 2) { //关闭 Debug.Log("音频结束"); break; } } } } ws.CloseAsync(WebSocketCloseStatus.NormalClosure, "", cancellation); Debug.Log("退出接收"); return msgRecive.ToArray(); } //用网的功能 string ReciceChat(ClientWebSocket ws) { string msgRecive = ""; //全部消息容器 List bs = new List(); //缓冲区 var buffer = new byte[1024 * 4]; while (ws.State == WebSocketState.Open) { //Debug.Log("等待消息接收"); var task = ws.ReceiveAsync(new ArraySegment(buffer), CancellationToken.None); task.Wait(); WebSocketReceiveResult result = task.Result; //文本消息 if (result.CloseStatus.HasValue) { Debug.Log("接收关闭"); break; } if (result.MessageType == WebSocketMessageType.Text) { bs.AddRange(buffer.Take(result.Count)); //消息是否已接收完全 if (result.EndOfMessage) { //发送过来的消息 string userMsg = Encoding.UTF8.GetString(bs.ToArray(), 0, bs.Count); bs = new List(); JObject jb = JObject.Parse(userMsg); int code = jb["header"]["code"].ToObject(); string sid = jb["header"]["sid"].ToString(); int status = jb["header"]["status"].ToObject(); string message = jb["header"]["message"].ToString(); if (code == 0) { //正常 JToken[] ja = jb["payload"]["choices"]["text"].ToArray(); foreach (var item in ja) { msgRecive = msgRecive + item["content"].ToString(); //Debug.Log(item["content"].ToString()); } if (status == 2) { //完成 Debug.Log("最后一条"); break; } } else { //出错 Debug.LogError(message); break; } } } } Debug.Log("退出接收"); return msgRecive; } //用网的功能 /// /// 获取语音听写url /// /// private string GetAuthUrl(string whichUrl, string key, string secret) { //当前时间戳,RFC1123格式 string date = DateTime.UtcNow.ToString("r"); //string date = "Thu, 15 Jun 2023 02:32:41 GMT"; string host = whichUrl.Split("://")[1].Split('/')[0]; string localPath = whichUrl.Split(host)[1]; string str = "host: " + host + "\n" + "date: " + date + "\n" + "GET " + localPath + " HTTP/1.1"; //hmac-sha256计算签名 string sha = HMACsha256(secret, str); //授权(api_key,授权算法,头部,签名) //authorization格式:api_key="$api_key",algorithm="hmac-sha256",headers="host date request-line",signature="$signature" string authorization = string.Format("api_key=\"{0}\", algorithm=\"{1}\", headers=\"{2}\", signature=\"{3}\"", key, "hmac-sha256", "host date request-line", sha); //System.Web.HttpUtility.UrlEncode //鉴权参数(host,data,authorization): //authorization使用base64编码的签名相关信息(签名基于hmac-sha256计算) string path1 = "authorization" + "=" + Convert.ToBase64String(Encoding.UTF8.GetBytes(authorization)); date = date.Replace(" ", "%20").Replace(":", "%3A").Replace(",", "%2C"); string path2 = "date" + "=" + date; string path3 = "host" + "=" + host; string newurl = whichUrl + "?" + path1 + "&" + path2 + "&" + path3; return newurl; } private string HMACsha256(string apiSecretIsKey, string buider) { byte[] bytes = Encoding.UTF8.GetBytes(apiSecretIsKey); System.Security.Cryptography.HMACSHA256 hMACSHA256 = new System.Security.Cryptography.HMACSHA256(bytes); byte[] date = Encoding.UTF8.GetBytes(buider); date = hMACSHA256.ComputeHash(date); hMACSHA256.Clear(); return Convert.ToBase64String(date); } /// /// 从此实例检索子数组 /// /// 要检索的数组 /// 起始索引号 /// 检索最大长度 /// 与此实例中在 startIndex 处开头、长度为 length 的子数组等效的一个数组 public static byte[] SubArray(byte[] source, int startIndex, int length) { if (startIndex < 0 || startIndex > source.Length || length < 0) { return null; } byte[] Destination; if (startIndex + length <= source.Length) { Destination = new byte[length]; Array.Copy(source, startIndex, Destination, 0, length); } else { Destination = new byte[(source.Length - startIndex)]; Array.Copy(source, startIndex, Destination, 0, source.Length - startIndex); } return Destination; } /// /// 解析文本 /// /// /// private ResponseData GetTextResultData(string ReceviceStr) { ResponseData temp = new ResponseData(); ReaponseDataInfo dataInfo = new ReaponseDataInfo(); TextInfo resultInfo = new TextInfo(); List tempwsS; List tempcwS; Ws tempWs; Cw temocw; var jsonObj = (JObject)JsonConvert.DeserializeObject(ReceviceStr); temp.code = jsonObj["code"].ToObject(); temp.message = jsonObj["message"].ToObject(); temp.sid = jsonObj["sid"].ToObject(); if(temp.code!=0) { Debug.Log("返回码异常:" + temp.code); Debug.Log(temp.message); } var data = jsonObj["data"]/*.ToObject()*/; dataInfo.status = data["status"].ToObject(); Debug.Log("顺序:" + dataInfo.status); var result = data["result"]/*.ToObject()*/; resultInfo.bg = result["bg"].ToObject(); resultInfo.ed = result["ed"].ToObject(); //resultInfo.pgs = result["pgs"].ToObject(); //resultInfo.rg = result["rg"].ToObject(); resultInfo.sn = result["sn"].ToObject(); ; resultInfo.ls = result["ls"].ToObject(); ; var wss = result["ws"]; tempwsS = new List(); JArray wsArray = wss.ToObject(); for (int i = 0; i < wsArray.Count; i++) { tempWs = new Ws(); tempWs.bg = wsArray[i]["bg"].ToObject(); //tempWs.ed = wsArray[i]["ed"].ToObject(); var cws = wsArray[i]["cw"]; tempcwS = new List(); JArray cwArray = cws.ToObject(); for (int j = 0; j < cwArray.Count; j++) { temocw = new Cw(); temocw.sc = cwArray[j]["sc"].ToObject(); temocw.w = cwArray[j]["w"].ToObject(); tempcwS.Add(temocw); } tempWs.cw = tempcwS.ToArray(); tempwsS.Add(tempWs); } resultInfo.ws = tempwsS.ToArray(); dataInfo.result = resultInfo; temp.data = dataInfo; return temp; } private ResponseData GetAudioResultData(string ReceviceStr) { ResponseData temp = new ResponseData(); ReaponseDataInfo dataInfo = new ReaponseDataInfo(); AudioInfo resultInfo = new AudioInfo(); var jsonObj = (JObject)JsonConvert.DeserializeObject(ReceviceStr); temp.code = jsonObj["code"].ToObject(); temp.message = jsonObj["message"].ToObject(); temp.sid = jsonObj["sid"].ToObject(); if (temp.code != 0) { Debug.LogError("code:" + temp.code + ",message:" + temp.message); return temp; } temp.data = dataInfo; var data = jsonObj["data"]/*.ToObject()*/; dataInfo.status = data["status"].ToObject(); dataInfo.result = resultInfo; resultInfo.audio = data["audio"].ToString(); resultInfo.ced = data["ced"].ToString(); return temp; } } /// /// 返回数据 /// /// public class ResponseData { public int code; public string message; public string sid; public ReaponseDataInfo data; } public class ReaponseDataInfo { public int status; public T result; } /// /// 语音听写数据 /// public class TextInfo { public int bg; public int ed; public string pgs; public int[] rg; public int sn; public bool ls; public Ws[] ws; public string GetResultText() { StringBuilder strB = new StringBuilder(); for (int i = 0; i < ws.Length; i++) { strB.Append(ws[i].cw[0].w); } return strB.ToString(); } } public class Ws { public Cw[] cw; public int bg; public int ed; } public class Cw { public int sc; public string w; } /// /// 语音合成数据 /// public class AudioInfo { public string audio; public string ced; public string GetResultAudio() { return audio; } } /// /// 问答对话组 /// public class ChatData { public string ask; public string answer; }