esp32s3聊天机器人（三）

先放上最新的硬件图

添加了按钮、600毫安锂电池和充电板

关于 sherpa-onnx 语音生成的打断

按说明实现了一下，但是偶尔还是有问题，毕竟不是直接立刻打断生成

        private int OnAudioData(nint samples, int n){//Console.WriteLine("OnAudioData n:" + n);if (stopped){sendQueue.Clear();sendQueue = new(10240000 * 2);Console.WriteLine("停止生成");stopped = false;return 0;}float[] floatData = new float[n];Marshal.Copy(samples, floatData, 0, n);short[] shortData = new short[n];for (int i = 0; i < n; i++){shortData[i] = Math.Clamp((short)(floatData[i] * 32768f), short.MinValue, short.MaxValue);}HandleFloatData(shortData);return n;}

在生成过程中返回0就是停止生成，但是这个OnAudioData是个回调，需要等到有音频返回的时候才能停止…… 不知道为啥没有别的停止方法

关于语音增强

去除了之前的RNNoise.Net，改用OfflineSpeechDenoiser

           OfflineSpeechDenoiserGtcrnModelConfig osdgmc = new OfflineSpeechDenoiserGtcrnModelConfig();osdgmc.Model = Environment.CurrentDirectory + "/gtcrn_simple.onnx";OfflineSpeechDenoiserModelConfig osdmc = new OfflineSpeechDenoiserModelConfig();osdmc.NumThreads = numThreads;osdmc.Provider = "cpu";osdmc.Debug = 0;osdmc.Gtcrn = osdgmc;OfflineSpeechDenoiserConfig osdc = new OfflineSpeechDenoiserConfig();osdc.Model = osdmc;offlineSpeechDenoiser = new OfflineSpeechDenoiser(osdc);

        DenoisedAudio denoisedAudio;/// <summary>/// 识别语音数据/// </summary>short[] int16Array;float[] floatArray;private void Recognize(byte[] bytes){int16Array = new short[bytes.Length / 2];Buffer.BlockCopy(bytes, 0, int16Array, 0, bytes.Length);floatArray = new float[int16Array.Length];for (int i = 0; i < int16Array.Length; i++){floatArray[i] = int16Array[i] / 32768.0f;}// 语音增强denoisedAudio = offlineSpeechDenoiser.Run(floatArray, sampleRate);floatArray = denoisedAudio.Samples; keyword.Recognize(floatArray);offlineStream = recognizer.CreateStream();offlineStream.AcceptWaveform(sampleRate, floatArray);recognizer.Decode(offlineStream);string result = offlineStream.Result.Text;offlineStream.Dispose();Console.WriteLine("识别结果:" + result);if (!string.IsNullOrWhiteSpace(result)){result = offlinePunctuation.AddPunct(result.ToLower());BaseMsg textMsg = new BaseMsg(1, result);client.Send(JsonConvert.SerializeObject(textMsg));if (llm != null){llm.RequestAsync(result);}}}

关于多用户

最初的构想是服务端可以接入多个客户端，实际用发现，我电脑性能太差，而且ASR和TTS的初始化挺费时间的，直接改成了单用户

using Fleck;
using Newtonsoft.Json;namespace server
{public class Server{WebSocketServer webSocketServer = null;Asr asr = null;Tts tts = null;Llm llm = null;public Server(){asr = new Asr();llm = new Llm();tts = new Tts();llm.tts = tts;asr.llm = llm;Console.WriteLine("tts llm asr ok");webSocketServer = new WebSocketServer("ws://192.168.0.164:9999");//webSocketServer.Certificate =//    new System.Security.Cryptography.X509Certificates.X509Certificate2(//        Environment.CurrentDirectory + "/usherpa.xuefei.net.cn.pfx", "xb5ceehg");//webSocketServer.EnabledSslProtocols = System.Security.Authentication.SslProtocols.Tls12; //开启监听webSocketServer.Start(OnStart);}private void OnStart(IWebSocketConnection connection){connection.OnOpen += () => OnOpen(connection);connection.OnBinary = bytes => OnBinary(connection, bytes);connection.OnMessage = msg => OnMessage(connection, msg);connection.OnClose += () => OnClose(connection);}private void OnOpen(IWebSocketConnection connection){ tts.UpdateClient(connection);asr.UpdateClient(connection);Console.WriteLine("上线了");}private void OnBinary(IWebSocketConnection connection, byte[] bytes){if (asr != null){asr.Receive(bytes);}}private void OnMessage(IWebSocketConnection connection, string msg){BaseMsg baseMsg = null;try{baseMsg = JsonConvert.DeserializeObject<BaseMsg>(msg);}catch (Exception e){Console.WriteLine(e);}if (baseMsg != null){// 收到code 0时，心跳消息if (baseMsg.code == 0){}// 收到code 1时，开始录音if (baseMsg.code == 1){if (tts != null){tts.Interrupt();}}// 收到code 2时，开始识别if (baseMsg.code == 2){if (asr != null){asr.EndReceive();}}}}private void OnClose(IWebSocketConnection connection){tts.UpdateClient(null);asr.UpdateClient(null);Console.WriteLine("下线了");}~Server(){if (asr != null){asr.Stop();}if (tts != null){tts.Stop();}webSocketServer.Dispose();}}
}

工程地址，持续修改中
https://github.com/xue-fei/homeai
https://gitee.com/awnuxcvbn/homeai