extractors/twitter: new api

extractors/twitter.go

@@ -3,7 +3,6 @@ package extractors
 import (
 	"encoding/json"
 	"fmt"
-	"html"
 	"strconv"
 	"strings"
 
@@ -12,45 +11,42 @@ import (
 	"github.com/iawia002/annie/utils"
 )
 
-const prefix = "https://twitter.com/i/videos/tweet/"
-
-type twitterUser struct {
-	Name string `json:"name"`
-}
-
 type twitter struct {
-	VideoURL string      `json:"video_url"`
-	TweetID  string      `json:"tweet_id"`
-	User     twitterUser `json:"user"`
+	Track struct {
+		URL string `json:"playbackUrl"`
+	} `json:"track"`
+	TweetID  string
+	Username string
 }
 
 // Twitter download function
 func Twitter(uri string) downloader.VideoData {
-	twitterData := getVideoURI(uri)
-	extractedData := download(twitterData, uri)
-	extractedData.Download(uri)
-	return extractedData
-}
-
-func getVideoURI(uri string) twitter {
-	// extract tweet id from url
+	html := request.Get(uri, uri, nil)
+	username := utils.MatchOneOf(html, `property="og:title"\s+content="(.+)"`)[1]
 	tweetID := utils.MatchOneOf(uri, `(status|statuses)/(\d+)`)[2]
-	webPlayerURL := prefix + tweetID
-	h := request.Get(webPlayerURL, uri, nil)
-	// get dataconfig attribute
-	jsonString := html.UnescapeString(utils.MatchOneOf(h, `data-config="({.+})`)[1])
+	api := fmt.Sprintf(
+		"https://api.twitter.com/1.1/videos/tweet/config/%s.json", tweetID,
+	)
+	headers := map[string]string{
+		"Authorization": "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw",
+	}
+	jsonString := request.Get(api, uri, headers)
 	var twitterData twitter
 	json.Unmarshal([]byte(jsonString), &twitterData)
-	return twitterData
+	twitterData.TweetID = tweetID
+	twitterData.Username = username
+	extractedData := download(twitterData, uri)
+	extractedData.Download(uri)
+	return extractedData
 }
 
 func download(data twitter, uri string) downloader.VideoData {
 	var size int64
 	var format = make(map[string]downloader.FormatData)
 	switch {
 	// if video file is m3u8 and ts
-	case strings.HasSuffix(data.VideoURL, "m3u8"):
-		m3u8urls := utils.M3u8URLs(data.VideoURL)
+	case strings.HasSuffix(data.Track.URL, "m3u8"):
+		m3u8urls := utils.M3u8URLs(data.Track.URL)
 		for index, m3u8 := range m3u8urls {
 			var totalSize int64
 			var urls []downloader.URLData
@@ -78,10 +74,10 @@ func download(data twitter, uri string) downloader.VideoData {
 		}
 
 	// if video file is mp4
-	case strings.HasSuffix(data.VideoURL, "mp4"):
-		size = request.Size(data.VideoURL, uri)
+	case strings.HasSuffix(data.Track.URL, "mp4"):
+		size = request.Size(data.Track.URL, uri)
 		urlData := downloader.URLData{
-			URL:  data.VideoURL,
+			URL:  data.Track.URL,
 			Size: size,
 			Ext:  "mp4",
 		}
@@ -93,7 +89,7 @@ func download(data twitter, uri string) downloader.VideoData {
 
 	extractedData := downloader.VideoData{
 		Site:    "Twitter twitter.com",
-		Title:   fmt.Sprintf("%s %s", data.User.Name, data.TweetID),
+		Title:   fmt.Sprintf("%s %s", data.Username, data.TweetID),
 		Type:    "video",
 		Formats: format,
 	}

extractors/twitter_test.go

@@ -17,23 +17,23 @@ func TestTwitter(t *testing.T) {
 			name: "normal test",
 			args: test.Args{
 				URL:     "https://twitter.com/justinbieber/status/898217160060698624",
-				Title:   "Justin Bieber 898217160060698624",
+				Title:   "Justin Bieber on Twitter 898217160060698624",
 				Quality: "720x1280",
 			},
 		},
 		{
 			name: "abnormal uri test1",
 			args: test.Args{
 				URL:     "https://twitter.com/twitter/statuses/898567934192177153",
-				Title:   "Justin Bieber 898567934192177153",
+				Title:   "Justin Bieber on Twitter 898567934192177153",
 				Quality: "1280x720",
 			},
 		},
 		{
 			name: "abnormal uri test2",
 			args: test.Args{
 				URL:     "https://twitter.com/kyoudera/status/971819131711373312/video/1/",
-				Title:   "ネメシス 京寺 971819131711373312",
+				Title:   "ネメシス 京寺 on Twitter 971819131711373312",
 				Quality: "1280x720",
 			},
 		},