From 6be3882726177d99b550c0a3d4197ab1b1a2f104 Mon Sep 17 00:00:00 2001 From: joeke80215 Date: Sun, 22 Sep 2019 01:17:23 +0800 Subject: [PATCH] add dailymotion support #498 --- extractors/dailymotion/dailymotion.go | 170 +++++++++++++++++++++ extractors/dailymotion/dailymotion_test.go | 34 +++++ main.go | 3 + 3 files changed, 207 insertions(+) create mode 100644 extractors/dailymotion/dailymotion.go create mode 100644 extractors/dailymotion/dailymotion_test.go diff --git a/extractors/dailymotion/dailymotion.go b/extractors/dailymotion/dailymotion.go new file mode 100644 index 000000000..066446afe --- /dev/null +++ b/extractors/dailymotion/dailymotion.go @@ -0,0 +1,170 @@ +package dailymotion + +import ( + "encoding/json" + "errors" + "strings" + + "github.com/iawia002/annie/downloader" + "github.com/iawia002/annie/request" + "github.com/iawia002/annie/utils" +) + +const ( + qualityAuto = "auto" + quality144 = "144p" + quality240 = "240p" + quality380 = "380p" + quality480 = "480p" + quality720 = "720p" +) + +type qualities struct { + Auto []*src `json:"auto"` + Q1 []*src `json:"144"` + Q2 []*src `json:"240"` + Q3 []*src `json:"380"` + Q4 []*src `json:"480"` + Q5 []*src `json:"720"` +} + +type src struct { + Type string `json:"type"` + URL string `json:"url"` +} + +type htmlContext struct { + Metadata struct { + Qualities *qualities `json:"qualities"` + } `json:"metadata"` +} + +func getSrc(html string) (*qualities, error) { + htmlCtx := &htmlContext{} + if jsonSrc := utils.MatchOneOf(html, `var config = (.+?);`); len(jsonSrc) > 1 && jsonSrc[1] != "" { + if err := json.Unmarshal([]byte(jsonSrc[1]), htmlCtx); err != nil { + return nil, err + } + return htmlCtx.Metadata.Qualities, nil + } + return nil, errors.New("parse html fail") +} + +func handleM3u8U(url string) ([]downloader.URL, int64, error) { + urls, err := utils.M3u8URLs(url) + if err != nil { + return nil, 0, err + } + var totalSize int64 + data := make([]downloader.URL, len(urls)) + for i, u := range urls { + size, err := request.Size(u, url) + if err != nil { + return nil, 0, err + } + data[i] = downloader.URL{ + URL: u, + Size: size, + Ext: "ts", + } + totalSize += size + } + return data, totalSize, nil +} + +func handleMP4(srcURL, ref string) (downloader.URL, error) { + size, err := request.Size(srcURL, ref) + if err != nil { + return downloader.URL{}, err + } + return downloader.URL{ + URL: srcURL, + Size: size, + Ext: "mp4", + }, nil +} + +func handle(srcs []*src, streams map[string]downloader.Stream, quality, refURL string) error { + for _, src := range srcs { + if src.Type == "application/x-mpegURL" { + drs, totalSize, err := handleM3u8U(src.URL) + if err != nil { + return err + } + streams[quality] = downloader.Stream{ + URLs: drs, + Size: totalSize, + Quality: quality, + } + continue + } + if src.Type == "video/mp4" { + dr, err := handleMP4(src.URL, refURL) + if err != nil { + return err + } + streams[quality] = downloader.Stream{ + URLs: []downloader.URL{dr}, + Size: dr.Size, + Quality: quality, + } + } + } + return nil +} + +func prepareEmbedURL(url string) string { + if !strings.Contains(url, "https://www.dailymotion.com/embed/") { + newIDs := strings.Split(url, "/") + return "https://www.dailymotion.com/embed/video/" + newIDs[len(newIDs)-1] + } + return url +} + +// Extract is the main function for extracting data +func Extract(url string) ([]downloader.Data, error) { + url = prepareEmbedURL(url) + html, err := request.Get(url, url, nil) + if err != nil { + return nil, err + } + var title string + if desc := utils.MatchOneOf(html, `(.+?)`); desc != nil { + title = desc[1] + } else { + title = "dailymotion" + } + title = strings.Replace(title, "Dailymotion Video Player - ", "", 1) + streams := make(map[string]downloader.Stream) + qts, err := getSrc(html) + if err != nil { + return nil, err + } + if err = handle(qts.Auto, streams, qualityAuto, url); err != nil { + return nil, err + } + if err = handle(qts.Q1, streams, quality144, url); err != nil { + return nil, err + } + if err = handle(qts.Q2, streams, quality240, url); err != nil { + return nil, err + } + if err = handle(qts.Q3, streams, quality380, url); err != nil { + return nil, err + } + if err = handle(qts.Q4, streams, quality480, url); err != nil { + return nil, err + } + if err = handle(qts.Q5, streams, quality720, url); err != nil { + return nil, err + } + return []downloader.Data{ + { + Site: "Dailymotion dailymotion.com", + Title: title, + Type: "video", + Streams: streams, + URL: url, + }, + }, nil +} diff --git a/extractors/dailymotion/dailymotion_test.go b/extractors/dailymotion/dailymotion_test.go new file mode 100644 index 000000000..1331a9721 --- /dev/null +++ b/extractors/dailymotion/dailymotion_test.go @@ -0,0 +1,34 @@ +package dailymotion + +import ( + "testing" + + "github.com/iawia002/annie/config" + "github.com/iawia002/annie/test" +) + +func TestExtract(t *testing.T) { + config.InfoOnly = true + config.RetryTimes = 10 + tests := []struct { + name string + args test.Args + }{ + { + name: "normal test", + args: test.Args{ + URL: "https://www.dailymotion.com/video/x6rs1ug", + Quality: qualityAuto, + Title: "野火燒不盡 人類面臨嚴重氣候變遷問題", + Size: 4308, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + data, err := Extract(tt.args.URL) + test.CheckError(t, err) + test.Check(t, tt.args, data[0]) + }) + } +} diff --git a/main.go b/main.go index 63edfe6ab..8cedc6a23 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( "github.com/iawia002/annie/downloader" "github.com/iawia002/annie/extractors/bcy" "github.com/iawia002/annie/extractors/bilibili" + "github.com/iawia002/annie/extractors/dailymotion" "github.com/iawia002/annie/extractors/douyin" "github.com/iawia002/annie/extractors/douyu" "github.com/iawia002/annie/extractors/facebook" @@ -163,6 +164,8 @@ func download(videoURL string) bool { data, err = udn.Extract(videoURL) case "tiktok": data, err = tiktok.Extract(videoURL) + case "dailymotion": + data, err = dailymotion.Extract(videoURL) default: data, err = universal.Extract(videoURL) }