Fixed whisper converting non ASCII to ??? via Base64 encoding

PaciStardust · Mar 1, 2024 · 31c409a · 31c409a
1 parent a474735
commit 31c409a
Show file tree

Hide file tree

Showing 3 changed files with 39 additions and 9 deletions.
diff --git a/HoscyWhisperServer/Program.cs b/HoscyWhisperServer/Program.cs
@@ -1,5 +1,6 @@
 using Newtonsoft.Json;
 using System.Diagnostics;
+using System.Text;
 using Whisper;
 
 namespace HoscyWhisperServer
@@ -167,7 +168,27 @@ private static void OnSpeechRecognized(object? sender, sSegment[] segments)
         }
 
         private static void SendMessage(MessageType type, string message)
-            => Console.WriteLine($"{type}|||{message}".Replace("\n", "[NL]"));
+        {
+            if (type == MessageType.Segments)
+            {
+                try
+                {
+                    var bytes = Encoding.UTF8.GetBytes(message);
+                    message = Convert.ToBase64String(bytes);
+                }
+                catch (Exception ex)
+                {
+                    type = MessageType.Error;
+                    message = ex.Message;
+                }
+            }
+            else
+            {
+                message = message.Replace("\n", "[NL]");
+            }
+
+            Console.WriteLine($"{type}|||{message}");
+        }
 
         private enum MessageType
         {

diff --git a/OscMultitool/Hoscy.csproj b/OscMultitool/Hoscy.csproj
@@ -7,8 +7,8 @@
     <UseWPF>true</UseWPF>
     <ApplicationIcon>Resources\hoscy_circle.ico</ApplicationIcon>
     <Platforms>AnyCPU;x64</Platforms>
-    <AssemblyVersion>0.9</AssemblyVersion>
-	<FileVersion>0.9</FileVersion>
+    <AssemblyVersion>0.9.1</AssemblyVersion>
+	<FileVersion>0.9.1</FileVersion>
     <RepositoryUrl>https://github.com/PaciStardust/HOSCY</RepositoryUrl>
     <PackageProjectUrl>https://github.com/PaciStardust/HOSCY</PackageProjectUrl>
     <PackageReadmeFile>README.md</PackageReadmeFile>

diff --git a/OscMultitool/Services/Speech/Recognizers/RecognizerWhisper.cs b/OscMultitool/Services/Speech/Recognizers/RecognizerWhisper.cs
@@ -171,17 +171,26 @@ private void ProcessOutputRecieved(object sender, DataReceivedEventArgs e)
             if (indexOfSeparator != -1)
             {
                 var flag = e.Data.AsSpan()[..indexOfSeparator];
-                var text = e.Data.AsSpan()[(indexOfSeparator + _separator.Length)..];
+                var text = e.Data[(indexOfSeparator + _separator.Length)..];
 
                 if (SpanCompare(flag, "Segments"))
                 {
-                    var segments = JsonConvert.DeserializeObject<(string, TimeSpan, TimeSpan)[]>(text.ToString());
-                    HandleSegments(segments);
+                    try
+                    {
+                        var bytes = Convert.FromBase64String(text);
+                        var decoded = Encoding.UTF8.GetString(bytes);
+                        var segments = JsonConvert.DeserializeObject<(string, TimeSpan, TimeSpan)[]>(decoded);
+                        HandleSegments(segments);
+                    }
+                    catch (Exception ex)
+                    {
+                        Logger.Error(ex, "Failed to decode base64 " + text);
+                    }
                     return;
                 }
                 if (SpanCompare(flag, "Speech"))
                 {
-                    HandleSpeechActivityUpdated(SpanCompare(text, "T"));
+                    HandleSpeechActivityUpdated(text == "T");
                     return;
                 }
 
@@ -196,13 +205,13 @@ private void ProcessOutputRecieved(object sender, DataReceivedEventArgs e)
                 }
                 if (severity is not null)
                 {
-                    Logger.Log(text.ToString().Replace("[NL]", "\n"), severity.Value);
+                    Logger.Log(text.Replace("[NL]", "\n"), severity.Value);
                     return;
                 }
 
                 if (SpanCompare(flag, "Loaded"))
                 {
-                    if(DateTime.TryParse(text.ToString(), out var started))
+                    if(DateTime.TryParse(text, out var started))
                         _timeStarted = started;
                     else
                         _timeStarted = DateTime.MinValue;