Skip to content

Commit

Permalink
Add additional information to chat jsons (#440)
Browse files Browse the repository at this point in the history
* Add video/clip ID to json

* Also store title and creation date in json

* Add video length parameter

* Put ChatRoot in namespace

* Swap GQL and ChatRoot video class names

* Add ChatRootInfo property for tracking chatroot version and creation date

* Typo

* I should think of all possible scenarios before committing

* Make ChatRoot version not a string for easier parsing

* Trim entries and index out of bounds protection, add int based constructor

* Add ChatRootVersion.ToString() override

* Move Gql responses to dedicated namespace, Move ChatRootInfo to dedicated class, add ChatRootVersion operators and methods

* Remove string constructor due to complexity and unclear usage
  • Loading branch information
ScrubN committed Dec 11, 2022
1 parent 23984a5 commit 84bd684
Show file tree
Hide file tree
Showing 16 changed files with 277 additions and 169 deletions.
49 changes: 30 additions & 19 deletions TwitchDownloaderCore/ChatDownloader.cs
@@ -1,7 +1,4 @@
using NeoSmart.Unicode;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using SkiaSharp;
using System;
using System.Collections.Generic;
using System.IO;
Expand All @@ -13,6 +10,7 @@
using System.Web;
using TwitchDownloaderCore.Options;
using TwitchDownloaderCore.TwitchObjects;
using TwitchDownloaderCore.TwitchObjects.Gql;

namespace TwitchDownloaderCore
{
Expand Down Expand Up @@ -235,47 +233,60 @@ private List<Comment> ConvertComments(CommentVideo video)

public async Task DownloadAsync(IProgress<ProgressReport> progress, CancellationToken cancellationToken)
{
DownloadType downloadType = downloadOptions.Id.All(x => Char.IsDigit(x)) ? DownloadType.Video : DownloadType.Clip;
string videoId = "";
DownloadType downloadType = downloadOptions.Id.All(x => char.IsDigit(x)) ? DownloadType.Video : DownloadType.Clip;

List<Comment> comments = new List<Comment>();
ChatRoot chatRoot = new ChatRoot() { streamer = new Streamer(), video = new VideoTime(), comments = comments };
ChatRoot chatRoot = new ChatRoot() { FileInfo = new ChatRootInfo() { Version = new ChatRootVersion(1, 1, 0) }, streamer = new Streamer(), video = new Video(), comments = comments };

string videoId = "";
string videoTitle = "";
DateTime videoCreatedAt;
double videoStart = 0.0;
double videoEnd = 0.0;
double videoDuration = 0.0;
double videoTotalLength;
int connectionCount = downloadOptions.ConnectionCount;

if (downloadType == DownloadType.Video)
{
videoId = downloadOptions.Id;
GqlVideoResponse taskInfo = await TwitchHelper.GetVideoInfo(Int32.Parse(videoId));
chatRoot.streamer.name = taskInfo.data.video.owner.displayName;
chatRoot.streamer.id = int.Parse(taskInfo.data.video.owner.id);
GqlVideoResponse taskVideoInfo = await TwitchHelper.GetVideoInfo(int.Parse(videoId));
chatRoot.streamer.name = taskVideoInfo.data.video.owner.displayName;
chatRoot.streamer.id = int.Parse(taskVideoInfo.data.video.owner.id);
videoTitle = taskVideoInfo.data.video.title;
videoCreatedAt = taskVideoInfo.data.video.createdAt;
videoStart = downloadOptions.CropBeginning ? downloadOptions.CropBeginningTime : 0.0;
videoEnd = downloadOptions.CropEnding ? downloadOptions.CropEndingTime : taskInfo.data.video.lengthSeconds;
videoEnd = downloadOptions.CropEnding ? downloadOptions.CropEndingTime : taskVideoInfo.data.video.lengthSeconds;
videoTotalLength = taskVideoInfo.data.video.lengthSeconds;
}
else
{
GqlClipResponse taskInfo = await TwitchHelper.GetClipInfo(downloadOptions.Id);
GqlClipResponse taskClipInfo = await TwitchHelper.GetClipInfo(downloadOptions.Id);

if (taskInfo.data.clip.video == null || taskInfo.data.clip.videoOffsetSeconds == null)
if (taskClipInfo.data.clip.video == null || taskClipInfo.data.clip.videoOffsetSeconds == null)
throw new Exception("Invalid VOD for clip, deleted/expired VOD possibly?");

videoId = taskInfo.data.clip.video.id;
videoId = taskClipInfo.data.clip.video.id;
downloadOptions.CropBeginning = true;
downloadOptions.CropBeginningTime = (int)taskInfo.data.clip.videoOffsetSeconds;
downloadOptions.CropBeginningTime = (int)taskClipInfo.data.clip.videoOffsetSeconds;
downloadOptions.CropEnding = true;
downloadOptions.CropEndingTime = downloadOptions.CropBeginningTime + taskInfo.data.clip.durationSeconds;
chatRoot.streamer.name = taskInfo.data.clip.broadcaster.displayName;
chatRoot.streamer.id = int.Parse(taskInfo.data.clip.broadcaster.id);
videoStart = (int)taskInfo.data.clip.videoOffsetSeconds;
videoEnd = (int)taskInfo.data.clip.videoOffsetSeconds + taskInfo.data.clip.durationSeconds;
downloadOptions.CropEndingTime = downloadOptions.CropBeginningTime + taskClipInfo.data.clip.durationSeconds;
chatRoot.streamer.name = taskClipInfo.data.clip.broadcaster.displayName;
chatRoot.streamer.id = int.Parse(taskClipInfo.data.clip.broadcaster.id);
videoTitle = taskClipInfo.data.clip.title;
videoCreatedAt = taskClipInfo.data.clip.createdAt;
videoStart = (int)taskClipInfo.data.clip.videoOffsetSeconds;
videoEnd = (int)taskClipInfo.data.clip.videoOffsetSeconds + taskClipInfo.data.clip.durationSeconds;
videoTotalLength = taskClipInfo.data.clip.durationSeconds;
connectionCount = 1;
}

chatRoot.video.title = videoTitle;
chatRoot.video.id = videoId;
chatRoot.video.created_at = videoCreatedAt;
chatRoot.video.start = videoStart;
chatRoot.video.end = videoEnd;
chatRoot.video.length = videoTotalLength;
videoDuration = videoEnd - videoStart;

SortedSet<Comment> commentsSet = new SortedSet<Comment>(new SortedCommentComparer());
Expand Down
5 changes: 1 addition & 4 deletions TwitchDownloaderCore/ChatRenderer.cs
Expand Up @@ -1100,10 +1100,7 @@ public static async Task<ChatRoot> ParseJsonStatic(string inputJson)

if (jsonDocument.RootElement.TryGetProperty("video", out JsonElement videoJson))
{
if (videoJson.TryGetProperty("start", out JsonElement videoStartJson) && videoJson.TryGetProperty("end", out JsonElement videoEndJson))
{
chatRoot.video = videoJson.Deserialize<VideoTime>();
}
chatRoot.video = videoJson.Deserialize<Video>();
}

if (jsonDocument.RootElement.TryGetProperty("embeddedData", out JsonElement embedDataJson))
Expand Down
1 change: 1 addition & 0 deletions TwitchDownloaderCore/TwitchHelper.cs
Expand Up @@ -16,6 +16,7 @@
using System.Xml.Linq;
using TwitchDownloaderCore.Properties;
using TwitchDownloaderCore.TwitchObjects;
using TwitchDownloaderCore.TwitchObjects.Gql;

namespace TwitchDownloaderCore
{
Expand Down
264 changes: 133 additions & 131 deletions TwitchDownloaderCore/TwitchObjects/ChatRoot.cs
@@ -1,135 +1,137 @@
using System;
using System.Collections.Generic;
using System.Text.Json.Serialization;

public class Streamer
{
public string name { get; set; }
public int id { get; set; }
}

public class Commenter
{
public string display_name { get; set; }
public string _id { get; set; }
public string name { get; set; }
public string type { get; set; }
public string bio { get; set; }
public DateTime created_at { get; set; }
public DateTime updated_at { get; set; }
public string logo { get; set; }
}

public class Emoticon
{
public string emoticon_id { get; set; }
public string emoticon_set_id { get; set; }
}

public class Fragment
{
public string text { get; set; }
public Emoticon emoticon { get; set; }
}

public class UserBadge
{
public string _id { get; set; }
public string version { get; set; }
}

public class Emoticon2
{
public string _id { get; set; }
public int begin { get; set; }
public int end { get; set; }
}

public class Message
{
public string body { get; set; }
public int bits_spent { get; set; }
public List<Fragment> fragments { get; set; }
public bool is_action { get; set; }
public List<UserBadge> user_badges { get; set; }
public string user_color { get; set; }
public UserNoticeParams user_notice_params { get; set; }
public List<Emoticon2> emoticons { get; set; }
}

public class UserNoticeParams
{
[JsonPropertyName("msg-id")]
public string msg_id { get; set; }
}

public class Comment
{
public string _id { get; set; }
public DateTime created_at { get; set; }
public DateTime updated_at { get; set; }
public string channel_id { get; set; }
public string content_type { get; set; }
public string content_id { get; set; }
public double content_offset_seconds { get; set; }
public Commenter commenter { get; set; }
public string source { get; set; }
public string state { get; set; }
public Message message { get; set; }
public bool more_replies { get; set; }
}

public class VideoTime
{
public double start { get; set; }
public double end { get; set; }
}

public class EmbedEmoteData
{
public string id { get; set; }
public int imageScale { get; set; }
public byte[] data { get; set; }
public string name { get; set; }
public string url { get; set; }
public int width { get; set; }
public int height { get; set; }
}

public class EmbedChatBadge
{
public string name { get; set; }
public Dictionary<string, byte[]> versions { get; set; }
}

public class EmbedCheerEmote
{
public string prefix { get; set; }
public Dictionary<int, EmbedEmoteData> tierList { get; set; }
}

public class EmbeddedData
{
public List<EmbedEmoteData> thirdParty { get; set; }
public List<EmbedEmoteData> firstParty { get; set; }
public List<EmbedChatBadge> twitchBadges { get; set; }
public List<EmbedCheerEmote> twitchBits { get; set; }
}

public class CommentResponse
{
public List<Comment> comments { get; set; }
public string _next { get; set; }
}

public class ChatRoot
{
[JsonPropertyOrder(0)]
public Streamer streamer { get; set; }
[JsonPropertyOrder(1)]
public VideoTime video { get; set; }
[JsonPropertyOrder(2)]
public List<Comment> comments { get; set; }
[JsonPropertyOrder(3)]
public EmbeddedData embeddedData { get; set; }
namespace TwitchDownloaderCore.TwitchObjects
{
public class Streamer
{
public string name { get; set; }
public int id { get; set; }
}

public class Commenter
{
public string display_name { get; set; }
public string _id { get; set; }
public string name { get; set; }
public string type { get; set; }
public string bio { get; set; }
public DateTime created_at { get; set; }
public DateTime updated_at { get; set; }
public string logo { get; set; }
}

public class Emoticon
{
public string emoticon_id { get; set; }
public string emoticon_set_id { get; set; }
}

public class Fragment
{
public string text { get; set; }
public Emoticon emoticon { get; set; }
}

public class UserBadge
{
public string _id { get; set; }
public string version { get; set; }
}

public class Emoticon2
{
public string _id { get; set; }
public int begin { get; set; }
public int end { get; set; }
}

public class Message
{
public string body { get; set; }
public int bits_spent { get; set; }
public List<Fragment> fragments { get; set; }
public bool is_action { get; set; }
public List<UserBadge> user_badges { get; set; }
public string user_color { get; set; }
public UserNoticeParams user_notice_params { get; set; }
public List<Emoticon2> emoticons { get; set; }
}

public class UserNoticeParams
{
public string msg_id { get; set; }
}

public class Comment
{
public string _id { get; set; }
public DateTime created_at { get; set; }
public DateTime updated_at { get; set; }
public string channel_id { get; set; }
public string content_type { get; set; }
public string content_id { get; set; }
public double content_offset_seconds { get; set; }
public Commenter commenter { get; set; }
public string source { get; set; }
public string state { get; set; }
public Message message { get; set; }
public bool more_replies { get; set; }
}

public class Video
{
public string title { get; set; }
public string id { get; set; }
public DateTime created_at { get; set; }
public double start { get; set; }
public double end { get; set; }
public double length { get; set; }
}

public class EmbedEmoteData
{
public string id { get; set; }
public int imageScale { get; set; }
public byte[] data { get; set; }
public string name { get; set; }
public string url { get; set; }
public int width { get; set; }
public int height { get; set; }
}

public class EmbedChatBadge
{
public string name { get; set; }
public Dictionary<string, byte[]> versions { get; set; }
}

public class EmbedCheerEmote
{
public string prefix { get; set; }
public Dictionary<int, EmbedEmoteData> tierList { get; set; }
}

public class EmbeddedData
{
public List<EmbedEmoteData> thirdParty { get; set; }
public List<EmbedEmoteData> firstParty { get; set; }
public List<EmbedChatBadge> twitchBadges { get; set; }
public List<EmbedCheerEmote> twitchBits { get; set; }
}

public class CommentResponse
{
public List<Comment> comments { get; set; }
public string _next { get; set; }
}

public class ChatRoot
{
public ChatRootInfo FileInfo { get; set; } = new ChatRootInfo();
public Streamer streamer { get; set; }
public Video video { get; set; }
public List<Comment> comments { get; set; }
public EmbeddedData embeddedData { get; set; }
}
}

0 comments on commit 84bd684

Please sign in to comment.