Add live chat replays, add top chat support

.gitignore

@@ -10,8 +10,3 @@
 
 # Created files
 debug.txt
-
-# Config
-!/example.yml
-*.yaml
-*.yml

README.md

@@ -4,8 +4,18 @@ A fast tool for exporting YouTube data using their undocumented JSON APIs.
 
 No API keys or logins required, and no stability guarantees given.
 
+If you find it useful, please give it a star!
+
 Please only use this tool to the extent permitted by the [YouTube ToS](https://www.youtube.com/static?template=terms).
 
+## Installation
+
+### From source
+
+Requires a Go 1.14+ toolchain.
+
+Run `go install ./cmd/ytpriv` to install to `$(go env GOPATH)/bin/ytpriv`.
+
 ## Features
 
 ```

channel.go

@@ -13,7 +13,7 @@ import (
 func (c *Client) RequestChannelOverview(channelID string) ChannelOverviewRequest {
 	const uri = "https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
 	req := fasthttp.AcquireRequest()
-	req.Header.SetMethod("POST")
+	req.Header.SetMethod(fasthttp.MethodPost)
 	req.Header.SetContentType("application/json")
 	setHeaders(&req.Header)
 	req.SetRequestURI(uri)
@@ -76,11 +76,11 @@ func ParseChannelOverview(res *fasthttp.Response) (*types.ChannelOverview, error
 func parseLink(overview *types.ChannelOverview, v *fastjson.Value) {
 	kind := string(v.GetStringBytes("title", "simpleText"))
 	endpoint := string(v.GetStringBytes("navigationEndpoint", "urlEndpoint", "url"))
-	endpointUri, err := url.Parse(endpoint)
+	endpointURI, err := url.Parse(endpoint)
 	if err != nil {
 		return
 	}
-	link := endpointUri.Query().Get("q")
+	link := endpointURI.Query().Get("q")
 	if link == "" {
 		return
 	}
@@ -103,7 +103,7 @@ func parseLink(overview *types.ChannelOverview, v *fastjson.Value) {
 func (c *Client) RequestChannelVideosStart(channelID string) ChannelVideosStartRequest {
 	const uri = "https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
 	req := fasthttp.AcquireRequest()
-	req.Header.SetMethod("POST")
+	req.Header.SetMethod(fasthttp.MethodPost)
 	req.Header.SetContentType("application/json")
 	setHeaders(&req.Header)
 	req.SetRequestURI(uri)

cmd/worker/worker.go

@@ -74,10 +74,10 @@ func main() {
 	assignments := hive.NewAssignmentsClient(client)
 	discovery := hive.NewDiscoveryClient(client)
 	simpleWorker := &worker.Simple{
-		Collection:    "yt.videos",
-		Assignments:   assignments,
-		Log:           log.Named("worker"),
-		Handler:       &Handler{
+		Collection:  "yt.videos",
+		Assignments: assignments,
+		Log:         log.Named("worker"),
+		Handler: &Handler{
 			Log:       log.Named("handler"),
 			Discovery: discovery,
 		},
@@ -99,7 +99,7 @@ func main() {
 			continue
 		}
 		seedPointers = append(seedPointers, &hive.ItemPointer{
-			Dst:       &hive.ItemLocator{
+			Dst: &hive.ItemLocator{
 				Collection: "yt.videos",
 				Id:         strconv.FormatInt(compact, 10),
 			},

cmd/ytpriv/channel.go

@@ -21,10 +21,10 @@ func init() {
 }
 
 var channelOverviewCmd = cobra.Command{
-	Use: "overview <channel>",
+	Use:   "overview <channel>",
 	Short: "Get overview of channel",
-	Args: cobra.ExactArgs(1),
-	Run: cmdFunc(doChannelOverview),
+	Args:  cobra.ExactArgs(1),
+	Run:   cmdFunc(doChannelOverview),
 }
 
 func init() {
@@ -68,10 +68,10 @@ func doChannelOverview(c *cobra.Command, args []string) error {
 }
 
 var channelVideosCmd = cobra.Command{
-	Use: "videos <channel ID>",
+	Use:   "videos <channel ID>",
 	Short: "Get full list of videos of channel",
-	Args: cobra.ExactArgs(1),
-	Run: cmdFunc(doChannelVideos),
+	Args:  cobra.ExactArgs(1),
+	Run:   cmdFunc(doChannelVideos),
 }
 
 func init() {
@@ -134,7 +134,7 @@ func doChannelVideosPage(c *cobra.Command, args []string) error {
 		panic(err)
 	}
 	// TODO This is a bit ugly
-	var req interface{
+	var req interface {
 		Do() (*types.ChannelVideosPage, error)
 		GetRequest() *fasthttp.Request
 	}

cmd/ytpriv/livestream.go

@@ -2,11 +2,14 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"os"
 	"time"
 
+	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/terorie/ytpriv"
+	"github.com/terorie/ytpriv/types"
 )
 
 var livestreamCmd = cobra.Command{
@@ -27,39 +30,86 @@ var livestreamChat = cobra.Command{
 
 func init() {
 	livestreamCmd.AddCommand(&livestreamChat)
+	flags := livestreamChat.Flags()
+	flags.Bool("top", false, "Top chat only")
 }
 
-func doLivestreamChat(_ *cobra.Command, args []string) error {
+func doLivestreamChat(c *cobra.Command, args []string) error {
+	flags := c.Flags()
+	top, err := flags.GetBool("top")
+	if err != nil {
+		panic(err.Error())
+	}
 	videoID := args[0]
-	videoID, err := yt.ExtractVideoID(videoID)
+	videoID, err = yt.ExtractVideoID(videoID)
 	if err != nil {
 		return err
 	}
-
-	out, cont, err := client.RequestLivechatStart(videoID).Do()
+	video, err := client.RequestVideo(videoID).Do()
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to get video info: %w", err)
+	}
+	var cont string
+	var live bool
+	if top {
+		live = video.TopChatContinuation != ""
+		if live {
+			cont = video.TopChatContinuation
+		} else {
+			cont = video.TopChatReplayContinuation
+		}
+	} else {
+		live = video.LiveChatContinuation != ""
+		if live {
+			cont = video.LiveChatContinuation
+		} else {
+			cont = video.LiveChatReplayContinuation
+		}
+	}
+	if cont == "" {
+		return fmt.Errorf("no live chat found")
 	}
-
 	enc := json.NewEncoder(os.Stdout)
 	enc.SetEscapeHTML(false)
-	for _, msg := range out {
-		if err := enc.Encode(&msg); err != nil {
-			panic(err)
+	if !live {
+		logrus.Info("Dumping live chat replay")
+		for i := 0; cont != ""; i++ {
+			logrus.WithField("page", i).Info("Dumping page")
+			var out []*types.LivechatMessage
+			out, cont, err = client.RequestLivechatReplay(cont).Do()
+			if err != nil {
+				return err
+			}
+			for _, msg := range out {
+				if err := enc.Encode(&msg); err != nil {
+					panic(err)
+				}
+			}
 		}
-	}
-
-	for cont.Continuation != "" {
-		time.Sleep(time.Duration(cont.Timeout) * time.Millisecond)
-		out, cont, err = client.RequestLivechatContinuation(cont.Continuation).Do()
-		if err != nil {
-			return err
+	} else {
+		logrus.Info("Dumping live chat")
+		liveCont := yt.LivechatContinuation{
+			Timeout:      0,
+			Continuation: cont,
 		}
-		for _, msg := range out {
-			if err := enc.Encode(&msg); err != nil {
-				panic(err)
+		for i := 0; liveCont.Continuation != ""; i++ {
+			logrus.
+				WithField("page", i).
+				WithField("timeout_ms", liveCont.Timeout).
+				Info("Dumping page")
+			time.Sleep(time.Duration(liveCont.Timeout) * time.Millisecond)
+			var out []*types.LivechatMessage
+			out, liveCont, err = client.RequestLivechat(liveCont.Continuation).Do()
+			if err != nil {
+				return err
+			}
+			for _, msg := range out {
+				if err := enc.Encode(&msg); err != nil {
+					panic(err)
+				}
 			}
 		}
 	}
+	logrus.Info("Reached end of chat")
 	return nil
 }

cmd/ytpriv/playlist.go

@@ -63,10 +63,10 @@ func doPlaylistVideos(_ *cobra.Command, args []string) error {
 }
 
 var playlistVideosPageCmd = cobra.Command{
-	Use: "videos_page <playlist ID or continuation>",
+	Use:   "videos_page <playlist ID or continuation>",
 	Short: "Get page of videos of playlist",
-	Args: cobra.ExactArgs(1),
-	Run: cmdFunc(doPlaylistVideosPage),
+	Args:  cobra.ExactArgs(1),
+	Run:   cmdFunc(doPlaylistVideosPage),
 }
 
 func init() {

comments.go

@@ -16,7 +16,7 @@ import (
 func (c *Client) RequestCommentPage(continuation *types.CommentContinuation) CommentPageRequest {
 	const commentURL = "https://www.youtube.com/comment_service_ajax?action_get_comments=1&pbj=1&ctoken=%[1]s&continuation=%[1]s"
 	req := fasthttp.AcquireRequest()
-	req.Header.SetMethod("POST")
+	req.Header.SetMethod(fasthttp.MethodPost)
 	req.Header.Set("Cookie", continuation.Cookie)
 	req.SetRequestURI(fmt.Sprintf(commentURL, continuation.Token))
 	req.SetBodyString("session_token=" + continuation.XSRF)