Ability to embed twitch badges and bit emotes

TwitchDownloaderCLI/Modes/Arguments/ChatDownloadArgs.cs

@@ -19,16 +19,16 @@ public class ChatDownloadArgs
         [Option('e', "ending", HelpText = "Time in seconds to crop ending.")]
         public int CropEndingTime { get; set; }
 
-        [Option('E', "embed-emotes", Default = false, HelpText = "Embed emotes into the chat download.")]
-        public bool EmbedEmotes { get; set; }
+        [Option('E', "embed-images", Default = false, HelpText = "Embed first party emotes, badges, and cheermotes into the chat download for offline rendering.")]
+        public bool EmbedData { get; set; }
 
-        [Option("bttv", Default = true, HelpText = "Enable BTTV embedding in chat download. Requires -E / --embed-emotes!")]
+        [Option("bttv", Default = true, HelpText = "Enable BTTV embedding in chat download. Requires -E / --embed-images!")]
         public bool? BttvEmotes { get; set; }
 
-        [Option("ffz", Default = true, HelpText = "Enable FFZ embedding in chat download. Requires -E / --embed-emotes!")]
+        [Option("ffz", Default = true, HelpText = "Enable FFZ embedding in chat download. Requires -E / --embed-images!")]
         public bool? FfzEmotes { get; set; }
 
-        [Option("stv", Default = true, HelpText = "Enable 7tv embedding in chat download. Requires -E / --embed-emotes!")]
+        [Option("stv", Default = true, HelpText = "Enable 7tv embedding in chat download. Requires -E / --embed-images!")]
         public bool? StvEmotes { get; set; }
 
         [Option("timestamp", Default = false, HelpText = "Enable timestamps for .txt chat downloads.")]

TwitchDownloaderCLI/Modes/Arguments/ChatDownloadUpdaterArgs.cs

@@ -0,0 +1,33 @@
+using CommandLine;
+
+namespace TwitchDownloaderCLI.Modes.Arguments
+{
+
+    [Verb("chatupdate", HelpText = "Updates the embeded emotes, badges, and bits of a chat download.")]
+    public class ChatDownloadUpdaterArgs
+    {
+        [Option('i', "input", Required = true, HelpText = "Path to input file. Valid extensions are json")]
+        public string InputFile { get; set; }
+
+        [Option('o', "output", Required = true, HelpText = "Path to output file. Extension should match the input.")]
+        public string OutputFile { get; set; }
+
+        [Option('E', "embed-missing", Default = true, HelpText = "Embed missing emotes, badges, and bits. Already embedded images will be untouched")]
+        public bool EmbedMissing { get; set; }
+
+        [Option('U', "update-old", Default = false, HelpText = "Update old emotes, badges, and bits to the current. All embedded images will be overwritten")]
+        public bool UpdateOldEmbeds { get; set; }
+
+        [Option("bttv", Default = true, HelpText = "Enable BTTV embedding in chat download.")]
+        public bool BttvEmotes { get; set; }
+
+        [Option("ffz", Default = true, HelpText = "Enable FFZ embedding in chat download.")]
+        public bool FfzEmotes { get; set; }
+
+        [Option("stv", Default = true, HelpText = "Enable 7tv embedding in chat download.")]
+        public bool StvEmotes { get; set; }
+
+        [Option("temp-path", Default = "", HelpText = "Path to temporary folder to use for cache.")]
+        public string TempFolder { get; set; }
+    }
+}

TwitchDownloaderCLI/Modes/Arguments/ChatRenderArgs.cs

@@ -81,6 +81,9 @@ public class ChatRenderArgs
         [Option("badge-filter", Default = 0, HelpText = "Bitmask of types of Chat Badges to filter out. Add the numbers of the types of badges you want to filter. For example, 6 = no broadcaster or moderator badges.\r\nKey: Other = 1, Broadcaster = 2, Moderator = 4, VIP = 8, Subscriber = 16, Predictions = 32, NoAudio/NoVideo = 64, PrimeGaming = 128")]
         public int BadgeFilterMask { get; set; }
 
+        [Option("offline", Default = false, HelpText = "Render completely offline, using only resources embedded emotes, badges, and bits in the input json.")]
+        public bool Offline { get; set; }
+
         [Option("ffmpeg-path", HelpText = "Path to ffmpeg executable.")]
         public string FfmpegPath { get; set; }
 

TwitchDownloaderCLI/Modes/DownloadChat.cs

@@ -33,7 +33,7 @@ internal static void Download(ChatDownloadArgs inputOptions)
                 CropEnding = inputOptions.CropEndingTime > 0.0,
                 CropEndingTime = inputOptions.CropEndingTime,
                 Timestamp = inputOptions.Timestamp,
-                EmbedEmotes = inputOptions.EmbedEmotes,
+                EmbedData = inputOptions.EmbedData,
                 Filename = inputOptions.OutputFile,
                 TimeFormat = inputOptions.TimeFormat,
                 ConnectionCount = inputOptions.ChatConnections,

TwitchDownloaderCLI/Modes/DownloadChatUpdater.cs

@@ -0,0 +1,170 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text.Json;
+using System.Threading.Tasks;
+using TwitchDownloaderCLI.Modes.Arguments;
+using TwitchDownloaderCore;
+using TwitchDownloaderCore.Options;
+using TwitchDownloaderCore.TwitchObjects;
+
+namespace TwitchDownloaderCLI.Modes
+{
+    internal class DownloadChatUpdater
+    {
+        internal static void Update(ChatDownloadUpdaterArgs inputOptions)
+        {
+            DownloadFormat inFormat = Path.GetExtension(inputOptions.InputFile)!.ToLower() switch
+            {
+                ".json" => DownloadFormat.Json,
+                ".html" => DownloadFormat.Html,
+                ".htm" => DownloadFormat.Html,
+                _ => DownloadFormat.Text
+            };
+            DownloadFormat outFormat = Path.GetExtension(inputOptions.OutputFile)!.ToLower() switch
+            {
+                ".json" => DownloadFormat.Json,
+                ".html" => DownloadFormat.Html,
+                ".htm" => DownloadFormat.Html,
+                _ => DownloadFormat.Text
+            };
+            // Check that both input and output are json
+            if (inFormat != DownloadFormat.Json || outFormat != DownloadFormat.Json)
+            {
+                Console.WriteLine("[ERROR] - {0} format must be be json!", inFormat != DownloadFormat.Json ? "Input" : "Output");
+                Environment.Exit(1);
+            }
+            if (!File.Exists(inputOptions.InputFile))
+            {
+                Console.WriteLine("[ERROR] - Input file does not exist!");
+                Environment.Exit(1);
+            }
+            if (!inputOptions.EmbedMissing && !inputOptions.UpdateOldEmbeds)
+            {
+                Console.WriteLine("[ERROR] - Please enable either EmbedMissingEmotes or UpdateOldEmotes");
+                Environment.Exit(1);
+            }
+
+            // Read in the old input file
+            ChatRoot chatRoot = Task.Run(() => ChatRenderer.ParseJsonStatic(inputOptions.InputFile)).Result;
+            if (chatRoot.streamer == null)
+            {
+                chatRoot.streamer = new Streamer();
+                chatRoot.streamer.id = int.Parse(chatRoot.comments.First().channel_id);
+                chatRoot.streamer.name = Task.Run(() => TwitchHelper.GetStreamerName(chatRoot.streamer.id)).Result;
+            }
+            if (chatRoot.embeddedData == null)
+            {
+                chatRoot.embeddedData = new EmbeddedData();
+            }
+
+            string cacheFolder = Path.Combine(string.IsNullOrWhiteSpace(inputOptions.TempFolder) ? Path.GetTempPath() : inputOptions.TempFolder, "TwitchDownloader", "chatupdatecache");
+
+            // Clear working directory if it already exists
+            if (Directory.Exists(cacheFolder))
+                Directory.Delete(cacheFolder, true);
+
+            // Thirdparty emotes
+            if (chatRoot.embeddedData.thirdParty == null || inputOptions.UpdateOldEmbeds)
+            {
+                chatRoot.embeddedData.thirdParty = new List<EmbedEmoteData>();
+            }
+            Console.WriteLine("Input third party emote count: " + chatRoot.embeddedData.thirdParty.Count);
+            List<TwitchEmote> thirdPartyEmotes = new List<TwitchEmote>();
+            thirdPartyEmotes = Task.Run(() => TwitchHelper.GetThirdPartyEmotes(chatRoot.streamer.id, cacheFolder, bttv: inputOptions.BttvEmotes, ffz: inputOptions.FfzEmotes, stv: inputOptions.StvEmotes, embeddedData: chatRoot.embeddedData)).Result;
+            foreach (TwitchEmote emote in thirdPartyEmotes)
+            {
+                EmbedEmoteData newEmote = new EmbedEmoteData();
+                newEmote.id = emote.Id;
+                newEmote.imageScale = emote.ImageScale;
+                newEmote.data = emote.ImageData;
+                newEmote.name = emote.Name;
+                newEmote.width = emote.Width / emote.ImageScale;
+                newEmote.height = emote.Height / emote.ImageScale;
+                chatRoot.embeddedData.thirdParty.Add(newEmote);
+            }
+            Console.WriteLine("Output third party emote count: " + chatRoot.embeddedData.thirdParty.Count);
+
+            // Firstparty emotes
+            if (chatRoot.embeddedData.firstParty == null || inputOptions.UpdateOldEmbeds)
+            {
+                chatRoot.embeddedData.firstParty = new List<EmbedEmoteData>();
+            }
+            Console.WriteLine("Input first party emote count: " + chatRoot.embeddedData.firstParty.Count);
+            List<TwitchEmote> firstPartyEmotes = new List<TwitchEmote>();
+            firstPartyEmotes = Task.Run(() => TwitchHelper.GetEmotes(chatRoot.comments, cacheFolder, embeddedData: chatRoot.embeddedData)).Result;
+            foreach (TwitchEmote emote in firstPartyEmotes)
+            {
+                EmbedEmoteData newEmote = new EmbedEmoteData();
+                newEmote.id = emote.Id;
+                newEmote.imageScale = emote.ImageScale;
+                newEmote.data = emote.ImageData;
+                newEmote.width = emote.Width / emote.ImageScale;
+                newEmote.height = emote.Height / emote.ImageScale;
+                chatRoot.embeddedData.firstParty.Add(newEmote);
+            }
+            Console.WriteLine("Output third party emote count: " + chatRoot.embeddedData.firstParty.Count);
+
+            // Twitch badges
+            if (chatRoot.embeddedData.twitchBadges == null || inputOptions.UpdateOldEmbeds)
+            {
+                chatRoot.embeddedData.twitchBadges = new List<EmbedChatBadge>();
+            }
+            Console.WriteLine("Input twitch badge count: " + chatRoot.embeddedData.twitchBadges.Count);
+            List<ChatBadge> twitchBadges = new List<ChatBadge>();
+            twitchBadges = Task.Run(() => TwitchHelper.GetChatBadges(chatRoot.streamer.id, cacheFolder, embeddedData: chatRoot.embeddedData)).Result;
+            foreach (ChatBadge badge in twitchBadges)
+            {
+                EmbedChatBadge newBadge = new EmbedChatBadge();
+                newBadge.name = badge.Name;
+                newBadge.versions = badge.VersionsData;
+                chatRoot.embeddedData.twitchBadges.Add(newBadge);
+            }
+            Console.WriteLine("Output twitch badge count: " + chatRoot.embeddedData.twitchBadges.Count);
+
+            // Twitch bits / cheers
+            if (chatRoot.embeddedData.twitchBits == null || inputOptions.UpdateOldEmbeds)
+            {
+                chatRoot.embeddedData.twitchBits = new List<EmbedCheerEmote>();
+            }
+            Console.WriteLine("Input twitch bit count: " + chatRoot.embeddedData.twitchBits.Count);
+            List<CheerEmote> twitchBits = new List<CheerEmote>();
+            twitchBits = Task.Run(() => TwitchHelper.GetBits(cacheFolder, chatRoot.streamer.id.ToString(), embeddedData: chatRoot.embeddedData)).Result;
+            foreach (CheerEmote bit in twitchBits)
+            {
+                EmbedCheerEmote newBit = new EmbedCheerEmote();
+                newBit.prefix = bit.prefix;
+                newBit.tierList = new Dictionary<int, EmbedEmoteData>();
+                foreach (KeyValuePair<int, TwitchEmote> emotePair in bit.tierList)
+                {
+                    EmbedEmoteData newEmote = new EmbedEmoteData();
+                    newEmote.id = emotePair.Value.Id;
+                    newEmote.imageScale = emotePair.Value.ImageScale;
+                    newEmote.data = emotePair.Value.ImageData;
+                    newEmote.name = emotePair.Value.Name;
+                    newEmote.width = emotePair.Value.Width / emotePair.Value.ImageScale;
+                    newEmote.height = emotePair.Value.Height / emotePair.Value.ImageScale;
+                    newBit.tierList.Add(emotePair.Key, newEmote);
+                }
+                chatRoot.embeddedData.twitchBits.Add(newBit);
+            }
+            Console.WriteLine("Input twitch bit count: " + chatRoot.embeddedData.twitchBits.Count);
+
+            // Finally save the output to file!
+            // TODO: maybe in the future we could also export as HTML here too?
+            if (outFormat == DownloadFormat.Json)
+            {
+                using (TextWriter writer = File.CreateText(inputOptions.OutputFile))
+                {
+                    var serializer = new Newtonsoft.Json.JsonSerializer();
+                    serializer.Serialize(writer, chatRoot);
+                }
+            }
+
+            // Clear our working directory, it's highly unlikely we would reuse it anyways
+            if (Directory.Exists(cacheFolder))
+                Directory.Delete(cacheFolder, true);
+        }
+    }
+}

TwitchDownloaderCLI/Modes/RenderChat.cs

@@ -58,7 +58,8 @@ internal static void Render(ChatRenderArgs inputOptions)
                 TempFolder = inputOptions.TempFolder,
                 SubMessages = (bool)inputOptions.SubMessages,
                 ChatBadges = (bool)inputOptions.ChatBadges,
-                Timestamp = inputOptions.Timestamp
+                Timestamp = inputOptions.Timestamp,
+                Offline = (bool)inputOptions.Offline,
             };
 
             if (renderOptions.GenerateMask && renderOptions.BackgroundColor.Alpha == 255)

TwitchDownloaderCLI/Program.cs

@@ -32,22 +32,21 @@ static void Main(string[] args)
             }
 
             string[] preParsedArgs;
-            if (args.Any(x => x.Equals("-m") || x.Equals("--mode")))
+            if (args.Any(x => x is "-m" or "--mode" or "--embed-emotes"))
             {
-                // Old -m/--mode syntax was used, print an info message and convert to verb syntax
-                Console.WriteLine("[INFO] The program has switched from --mode <mode> to verbs (like \"git <verb>\"), consider using verbs instead." +
-                    " Run \"{0} help\" for more information.", processFileName);
-                preParsedArgs = PreParseArgs.Process(PreParseArgs.ConvertFromOldSyntax(args));
+                // A legacy syntax was used, convert to new syntax
+                preParsedArgs = PreParseArgs.Process(PreParseArgs.ConvertFromOldSyntax(args, processFileName));
             }
             else
             {
                 preParsedArgs = PreParseArgs.Process(args);
             }
 
-            Parser.Default.ParseArguments<VideoDownloadArgs, ClipDownloadArgs, ChatDownloadArgs, ChatRenderArgs, FfmpegArgs, CacheArgs>(preParsedArgs)
+            Parser.Default.ParseArguments<VideoDownloadArgs, ClipDownloadArgs, ChatDownloadArgs, ChatDownloadUpdaterArgs, ChatRenderArgs, FfmpegArgs, CacheArgs>(preParsedArgs)
                 .WithParsed<VideoDownloadArgs>(DownloadVideo.Download)
                 .WithParsed<ClipDownloadArgs>(DownloadClip.Download)
                 .WithParsed<ChatDownloadArgs>(DownloadChat.Download)
+                .WithParsed<ChatDownloadUpdaterArgs>(DownloadChatUpdater.Update)
                 .WithParsed<ChatRenderArgs>(RenderChat.Render)
                 .WithParsed<FfmpegArgs>(FfmpegHandler.ParseArgs)
                 .WithParsed<CacheArgs>(CacheHandler.ParseArgs)

TwitchDownloaderCLI/README.md

@@ -5,6 +5,7 @@ A cross platform command line tool that can do the main functions of the GUI pro
  - [Arguments for mode clipdownload](#arguments-for-mode-clipdownload)
  - [Arguments for mode chatdownload](#arguments-for-mode-chatdownload)
  - [Arguments for mode chatrender](#arguments-for-mode-chatrender)
+ - [Arguments for mode chatupdate](#arguments-for-mode-chatupdate)
  - [Arguments for mode ffmpeg](#arguments-for-mode-ffmpeg)
  - [Arguments for mode cache](#arguments-for-mode-cache)
  - [Example commands](#example-commands)
@@ -73,17 +74,17 @@ Time in seconds to crop beginning. For example if I had a 10 second stream but o
 **-e/-\-ending**
 Time in seconds to crop ending. For example if I had a 10 second stream but only wanted the first 4 seconds of it I would use `-e 4` to end on the 4th second.
 
-**-E/-\-embed-emotes**
-(Default: false) Embeds emotes into the JSON file so in the future when an emote is removed from Twitch or a 3rd party, it will still render correctly. Useful for archival purposes, file size will be larger.
+**-E/-\-embed-images**
+(Default: false) Embed first party emotes, badges, and cheermotes into the download file for offline rendering. Useful for archival purposes, file size will be larger.
 
 **-\-bttv**
-(Default: true) BTTV emote embedding. Requires `-E / --embed-emotes`.
+(Default: true) BTTV emote embedding. Requires `-E / --embed-images`.
 
 **-\-ffz**
-(Default: true) FFZ emote embedding. Requires `-E / --embed-emotes`.
+(Default: true) FFZ emote embedding. Requires `-E / --embed-images`.
 
 **-\-stv**
-(Default: true) 7TV emote embedding. Requires `-E / --embed-emotes`.
+(Default: true) 7TV emote embedding. Requires `-E / --embed-images`.
 
 **-\-timestamp**
 (Default: false) Enable timestamps
@@ -162,7 +163,7 @@ File the program will output to.
 (Default: 0.2) Time in seconds to update chat render output.
 
 **-\-input-args**
- (Default: -framerate {fps} -f rawvideo -analyzeduration {max_int} -probesize {max_int} -pix_fmt bgra -video_size {width}x{height} -i -) Input (pass1) arguments for ffmpeg chat render.
+(Default: -framerate {fps} -f rawvideo -analyzeduration {max_int} -probesize {max_int} -pix_fmt bgra -video_size {width}x{height} -i -) Input (pass1) arguments for ffmpeg chat render.
 
 **-\-output-args**
 (Default: -c:v libx264 -preset veryfast -crf 18 -pix_fmt yuv420p "{save_path}") Output (pass2) arguments for ffmpeg chat render.
@@ -182,13 +183,43 @@ Predictions = `32`,
 NoAudioVisual = `64`,
 PrimeGaming = `128`
 
+**-\-offline**
+Render completely offline, using only resources embedded emotes, badges, and bits in the input json.
+
 **-\-ffmpeg-path**
 Path to ffmpeg executable.
 
 **-\-temp-path**
 Path to temporary folder for cache.
 
 
+## Arguments for mode chatupdate
+
+**-i/-\-input (REQUIRED)**
+Path to input file. Valid extensions are json
+
+**-o/-\-output (REQUIRED)**
+Path to output file. Extension should match the input.
+
+**-E/-\-embed-missing**
+(Default: true) Embed missing emotes, badges, and bits. Already embedded images will be untouched.
+
+**-U/-\-update-old**
+(Default: false) Update old emotes, badges, and bits to the current. All embedded images will be overwritten!
+
+**-\-bttv**
+(Default: true) Enable embedding BTTV emotes.
+
+**-\-ffz**
+(Default: true) Enable embedding FFZ emotes.
+
+**-\-stv**
+(Default: true) Enable embedding 7TV emotes.
+
+**-\-temp-path**
+Path to temporary folder for cache.
+
+
 ## Arguments for mode ffmpeg
 Manage standalone ffmpeg
 
@@ -219,7 +250,7 @@ Download a Chat (plain text with timestamps)
     TwitchDownloaderCLI chatdownload --id 612942303 --timestamp-format Relative -o chat.txt
 Download a Chat (JSON with embeded emotes from Twitch and Bttv)
 
-    TwitchDownloaderCLI chatdownload --id 612942303 --embed-emotes --bttv=true --ffz=false --stv=false -o chat.json
+    TwitchDownloaderCLI chatdownload --id 612942303 --embed-images --bttv=true --ffz=false --stv=false -o chat.json
 Render a chat with defaults
 
     TwitchDownloaderCLI chatrender -i chat.json -o chat.mp4