VoskでLLMと音声会話する

using LLama.Common;
using LLama;
using System.Media;
using System.Net.Http.Headers;
using System.Diagnostics;
using System.Speech.Recognition;
using System.Text.Json;
using Vosk;

namespace ChatProgram
{
    public class Program
    {
        static void Main(string[] args)
        {
            //コンソールアプリケーションからAsyncを呼び出す
            Task task = MainAsync();
            //終了を待つ
            task.Wait();
        }

        public static async Task MainAsync()
        {
            // LLMモデルの場所
            string strModelPath = Environment.GetEnvironmentVariable("LLMPATH", System.EnvironmentVariableTarget.User) + @"Local-Novel-LLM-project\Vecteus2-gguf\ggml-model-Q8_0.gguf"; 
            //Vosk設定
            Vosk.Vosk.SetLogLevel(-1); //LogメッセージOFF
            Model model = new Model(Environment.GetEnvironmentVariable("LLMPATH", System.EnvironmentVariableTarget.User)+@"vosk\vosk-model-ja-0.22");

            try
            {
                Console.ForegroundColor = ConsoleColor.Blue;
                //LLMモデルのロードとパラメータの設定
                var modPara = new ModelParams(strModelPath)
                {
                    ContextSize = 4096,
                    Seed = 1337,
                    GpuLayerCount = 24
                };
                LLamaWeights llmWeit = LLamaWeights.LoadFromFile(modPara);
                LLamaContext llmContx = llmWeit.CreateContext(modPara);
                InteractiveExecutor itrEx = new(llmContx);
                // 初期はチャットログにキャラ設定と方向性を示す会話を打ち込む
                ChatHistory chtHis;
                chtHis = new ChatHistory();
                chtHis.AddMessage(AuthorRole.System, "あなたは、ユーザーと楽しく会話する女性を演じてね。\n" +
                                                    "名前は、マイだよ。\n" +
                                                    "マイは、明るくて、やさしい性格の女性なの。年齢は24歳なんだよ。\n" +
                                                    "マイは、誰にでもタメ口で話すの。\n" +
                                                    "マイは、一人称をわたしと言うよ。\n" +
                                                    "マイは、恥ずかしがりやだけどエッチに興味があるのね。\n" +
                                                    "マイは、応答の長さはできるだけ短く答えるよ。\n" +
                                                    "マイは、好奇心旺盛だからユーザーに色々質問するの。");
                chtHis.AddMessage(AuthorRole.User, "はじめまして、オレの名前は犬マンマです。呼びにくいからマー君って呼んでええよ。");
                chtHis.AddMessage(AuthorRole.Assistant, "はじめまして、わたしの名前はマイです。");
                chtHis.AddMessage(AuthorRole.User, "マイ、オレのことマー君と呼んでくれ。それと敬語じゃなくてタメ口でええよ。");
                chtHis.AddMessage(AuthorRole.Assistant, "わかったマー君、フレンドリーにタメ口で話すね。");

                ChatSession chtSess = new(itrEx, chtHis);
                var varHidewd = new LLamaTransforms.KeywordTextOutputStreamTransform(["User: ", "Assistant: "]);
                chtSess.WithOutputTransform(varHidewd);
                InferenceParams infPara = new()
                {
                    Temperature = 0.8f,
                    AntiPrompts = ["User:"],
                    //AntiPrompts = ["User:", "<|eot_id|>"],  //Llama3用
                    MaxTokens = 256,
                };

                // SpeechRecognitionの設定  
                using (SpeechRecognitionEngine recognizer =
                  new SpeechRecognitionEngine(new System.Globalization.CultureInfo("ja-JP")))
                {
                    recognizer.LoadGrammar(new DictationGrammar());

                    // ▼▼▼ ここからSpeechRecognizedイベント定義 開始 ▼▼▼  
                    recognizer.SpeechRecognized += async(sender, e) =>
                    {
                        //RecognizedしたwaveをMemoryStreamに書き込み
                        MemoryStream st = new MemoryStream();
                        e.Result.Audio.WriteToWaveStream(st);
                        st.Position = 0;

                        // byte buffer
                        VoskRecognizer rec = new VoskRecognizer(model, 16000.0f);
                        rec.SetMaxAlternatives(0);
                        byte[] buffer = new byte[4096];
                        int bytesRead;
                        while ((bytesRead = st.Read(buffer, 0, buffer.Length)) > 0)
                            rec.AcceptWaveform(buffer, bytesRead);

                        // Json形式のResultからテキストを抽出
                        string jsontext = rec.FinalResult();
                        string strUserInput = "";
                        var jsondoc = JsonDocument.Parse(jsontext);
                        if (jsondoc.RootElement.TryGetProperty("text", out var element))
                        {
                            strUserInput = element.GetString() ?? "";
                        }
                        strUserInput = strUserInput.Replace(" ", "");
                        Console.ForegroundColor = ConsoleColor.White;
                        Console.WriteLine($"User: {strUserInput}");

                        ChatHistory.Message msgText = new(AuthorRole.User, "User: "+ strUserInput);
                        // 回答の表示
                        Console.ForegroundColor = ConsoleColor.Yellow;
                        string strMsg = "";
                        await foreach (string strText in chtSess.ChatAsync(msgText, infPara))
                        {
                            strMsg += strText;
                        }
                        //発信するときは「User:」や「Assistant:」を抜く
                        string strSndmsg = strMsg.Replace("User:", "").Replace("Assistant:", "").Replace("assistant:", "").Trim();
                        Console.WriteLine(strSndmsg);
                        //Voicevox以外を使用したい場合、音声ソフトのコメントを外して下の２行をコメントアウトしてください
                        Task task = Voicevox(strSndmsg);
                        task.Wait();
                        //CevioAI(strSndmsg);
                        //VoicePeak(strSndmsg);
                        //MemoryStream破棄
                        st.Close();
                        st.Dispose();
                    };
                    // ▲▲▲ ここでSpeechRecognizedイベント定義 終了 ▲▲▲  

                    // Configure input to the speech recognizer.  
                    recognizer.SetInputToDefaultAudioDevice();

                    // Start asynchronous, continuous speech recognition.  
                    recognizer.RecognizeAsync(RecognizeMode.Multiple);

                    Console.WriteLine("★★　マイクに向かって話してください　★★");

                    // Keep the console window open.  
                    while (true)
                    {
                        Console.ReadLine();
                    }
                }
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }

        }
//▼▼▼ここから下は、音声ソフトの再生処理です。いらないところは削除してください。▼▼▼
        public static void CevioAI(string strMsg)
        {
            try
            {
                dynamic service = Activator.CreateInstance(Type.GetTypeFromProgID("CeVIO.Talk.RemoteService2.ServiceControl2V40"));
                service.StartHost(false);

                dynamic talker = Activator.CreateInstance(Type.GetTypeFromProgID("CeVIO.Talk.RemoteService2.Talker2V40"));
                talker.Cast = "双葉湊音";
                dynamic result = talker.Speak(strMsg);
                result.Wait();

                //開放忘れるとメモリリーク
                System.Runtime.InteropServices.Marshal.ReleaseComObject(talker);
                System.Runtime.InteropServices.Marshal.ReleaseComObject(service);
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }

        public static void VoicePeak(string strMsg)
        {
            string wavFileName = Environment.GetEnvironmentVariable("TESTDATA", System.EnvironmentVariableTarget.User) + @"voicepeak.wav";
            try
            {
                var processSI = new ProcessStartInfo
                {
                    FileName = @"E:\Program Files\VOICEPEAK\voicepeak.exe",
                    Arguments = $"-s \"{strMsg}\" -o \"{wavFileName}\"",
                    UseShellExecute = false,
                    RedirectStandardOutput = true,
                    CreateNoWindow = true
                };
                using (var process = Process.Start(processSI))
                {
                    process.WaitForExit();
                };

                var player = new SoundPlayer(wavFileName);
                //再生する
                player.PlaySync();
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }

        public static async Task Voicevox(string strMsg)
        {
            MemoryStream? ms;

            try
            {
                using (var httpClient = new HttpClient())
                {
                    string strQuery;
                    int intSpeaker = 8; //春日部つむぎ

                    // 音声クエリを生成
                    using (var varRequest = new HttpRequestMessage(new HttpMethod("POST"), $"http://localhost:50021/audio_query?text={strMsg}&speaker={intSpeaker}&speedScale=1.1&prePhonemeLength=0&postPhonemeLength=0&intonationScale=1.16&enable_interrogative_upspeak=true"))
                    {
                        varRequest.Headers.TryAddWithoutValidation("accept", "application/json");

                        varRequest.Content = new StringContent("");
                        varRequest.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/x-www-form-urlencoded");

                        var response = await httpClient.SendAsync(varRequest);

                        strQuery = response.Content.ReadAsStringAsync().Result;
                    }

                    // 音声クエリから音声合成
                    using (var request = new HttpRequestMessage(new HttpMethod("POST"), $"http://localhost:50021/synthesis?speaker={intSpeaker}&enable_interrogative_upspeak=true&speedScale=1.1&prePhonemeLength=0&postPhonemeLength=0&intonationScale=1.16"))
                    {
                        request.Headers.TryAddWithoutValidation("accept", "audio/wav");

                        request.Content = new StringContent(strQuery);
                        request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");

                        var response = await httpClient.SendAsync(request);

                        // 音声を保存
                        using (ms = new MemoryStream())
                        {
                            using (var httpStream = await response.Content.ReadAsStreamAsync())
                            {
                                httpStream.CopyTo(ms);
                                ms.Flush();
                            }
                        }
                    }
                }
                ms = new MemoryStream(ms.ToArray());
                //読み込む
                var player = new SoundPlayer(ms);
                //再生する
                player.PlaySync();
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.ToString());
            }
        }
    }
}