Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

extractors/twitter: new api #158

Merged
merged 1 commit into from
Jun 1, 2018
Merged

extractors/twitter: new api #158

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 25 additions & 29 deletions extractors/twitter.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@ package extractors
import (
"encoding/json"
"fmt"
"html"
"strconv"
"strings"

Expand All @@ -12,45 +11,42 @@ import (
"github.com/iawia002/annie/utils"
)

const prefix = "https://twitter.com/i/videos/tweet/"

type twitterUser struct {
Name string `json:"name"`
}

type twitter struct {
VideoURL string `json:"video_url"`
TweetID string `json:"tweet_id"`
User twitterUser `json:"user"`
Track struct {
URL string `json:"playbackUrl"`
} `json:"track"`
TweetID string
Username string
}

// Twitter download function
func Twitter(uri string) downloader.VideoData {
twitterData := getVideoURI(uri)
extractedData := download(twitterData, uri)
extractedData.Download(uri)
return extractedData
}

func getVideoURI(uri string) twitter {
// extract tweet id from url
html := request.Get(uri, uri, nil)
username := utils.MatchOneOf(html, `property="og:title"\s+content="(.+)"`)[1]
tweetID := utils.MatchOneOf(uri, `(status|statuses)/(\d+)`)[2]
webPlayerURL := prefix + tweetID
h := request.Get(webPlayerURL, uri, nil)
// get dataconfig attribute
jsonString := html.UnescapeString(utils.MatchOneOf(h, `data-config="({.+})`)[1])
api := fmt.Sprintf(
"https://api.twitter.com/1.1/videos/tweet/config/%s.json", tweetID,
)
headers := map[string]string{
"Authorization": "Bearer AAAAAAAAAAAAAAAAAAAAAIK1zgAAAAAA2tUWuhGZ2JceoId5GwYWU5GspY4%3DUq7gzFoCZs1QfwGoVdvSac3IniczZEYXIcDyumCauIXpcAPorE",
}
jsonString := request.Get(api, uri, headers)
var twitterData twitter
json.Unmarshal([]byte(jsonString), &twitterData)
return twitterData
twitterData.TweetID = tweetID
twitterData.Username = username
extractedData := download(twitterData, uri)
extractedData.Download(uri)
return extractedData
}

func download(data twitter, uri string) downloader.VideoData {
var size int64
var format = make(map[string]downloader.FormatData)
switch {
// if video file is m3u8 and ts
case strings.HasSuffix(data.VideoURL, "m3u8"):
m3u8urls := utils.M3u8URLs(data.VideoURL)
case strings.HasSuffix(data.Track.URL, "m3u8"):
m3u8urls := utils.M3u8URLs(data.Track.URL)
for index, m3u8 := range m3u8urls {
var totalSize int64
var urls []downloader.URLData
Expand Down Expand Up @@ -78,10 +74,10 @@ func download(data twitter, uri string) downloader.VideoData {
}

// if video file is mp4
case strings.HasSuffix(data.VideoURL, "mp4"):
size = request.Size(data.VideoURL, uri)
case strings.HasSuffix(data.Track.URL, "mp4"):
size = request.Size(data.Track.URL, uri)
urlData := downloader.URLData{
URL: data.VideoURL,
URL: data.Track.URL,
Size: size,
Ext: "mp4",
}
Expand All @@ -93,7 +89,7 @@ func download(data twitter, uri string) downloader.VideoData {

extractedData := downloader.VideoData{
Site: "Twitter twitter.com",
Title: fmt.Sprintf("%s %s", data.User.Name, data.TweetID),
Title: fmt.Sprintf("%s %s", data.Username, data.TweetID),
Type: "video",
Formats: format,
}
Expand Down
6 changes: 3 additions & 3 deletions extractors/twitter_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,23 @@ func TestTwitter(t *testing.T) {
name: "normal test",
args: test.Args{
URL: "https://twitter.com/justinbieber/status/898217160060698624",
Title: "Justin Bieber 898217160060698624",
Title: "Justin Bieber on Twitter 898217160060698624",
Quality: "720x1280",
},
},
{
name: "abnormal uri test1",
args: test.Args{
URL: "https://twitter.com/twitter/statuses/898567934192177153",
Title: "Justin Bieber 898567934192177153",
Title: "Justin Bieber on Twitter 898567934192177153",
Quality: "1280x720",
},
},
{
name: "abnormal uri test2",
args: test.Args{
URL: "https://twitter.com/kyoudera/status/971819131711373312/video/1/",
Title: "ネメシス 京寺 971819131711373312",
Title: "ネメシス 京寺 on Twitter 971819131711373312",
Quality: "1280x720",
},
},
Expand Down