extractors/iqiyi: Add support

README.md

@@ -200,6 +200,7 @@ Site | URL | 🎬 Videos | 🌁 Images | 📚 Playlist
 pixivision | <https://www.pixivision.net> | | ✓ | |
 优酷 | <https://www.youku.com> | ✓ | | |
 YouTube | <https://www.youtube.com> | ✓ | | |
+爱奇艺 | <https://www.iqiyi.com> | ✓ | | |
 
 
 ## Contributing

extractors/bcy.go

@@ -4,7 +4,6 @@ import (
 	"github.com/iawia002/annie/downloader"
 	"github.com/iawia002/annie/parser"
 	"github.com/iawia002/annie/request"
-	"github.com/iawia002/annie/utils"
 )
 
 // Bcy download function
@@ -19,7 +18,7 @@ func Bcy(url string) downloader.VideoData {
 
 	data := downloader.VideoData{
 		Site:  "半次元 bcy.net",
-		Title: utils.FileName(title),
+		Title: title,
 		Type:  "image",
 		URLs:  urls,
 		Size:  0,

extractors/bilibili.go

@@ -1,16 +1,14 @@
 package extractors
 
 import (
-	"crypto/md5"
 	"encoding/json"
 	"fmt"
 	"log"
 	"strings"
 
-	"github.com/PuerkitoBio/goquery"
-
 	"github.com/iawia002/annie/config"
 	"github.com/iawia002/annie/downloader"
+	"github.com/iawia002/annie/parser"
 	"github.com/iawia002/annie/request"
 	"github.com/iawia002/annie/utils"
 )
@@ -22,12 +20,6 @@ const (
 	secKey string = "94aba54af9065f71de72f5508f1cd42e"
 )
 
-func getSign(params string) string {
-	sign := md5.New()
-	sign.Write([]byte(params + secKey))
-	return fmt.Sprintf("%x", sign.Sum(nil))
-}
-
 func genAPI(aid, cid string, bangumi bool) string {
 	var (
 		baseAPIURL string
@@ -64,7 +56,7 @@ func genAPI(aid, cid string, bangumi bool) string {
 	}
 	// bangumi utoken also need to put in params to sign, but the ordinary video doesn't need
 	api := fmt.Sprintf(
-		"%s%s&sign=%s", baseAPIURL, params, getSign(params),
+		"%s%s&sign=%s", baseAPIURL, params, utils.Md5(params+secKey),
 	)
 	if !bangumi && utoken != "" {
 		api = fmt.Sprintf("%s&utoken=%s", api, utoken)
@@ -141,21 +133,13 @@ func bilibiliDownload(url string, bangumi bool) downloader.VideoData {
 	json.Unmarshal([]byte(apiData), &dataDict)
 
 	// get the title
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
-	if err != nil {
-		log.Fatal(err)
-	}
-	var title string
-	title = strings.TrimSpace(doc.Find("h1").First().Text())
-	if title == "" {
-		// Some movie page got no h1 tag
-		title, _ = doc.Find("meta[property=\"og:title\"]").Attr("content")
-	}
+	doc := parser.GetDoc(html)
+	title := parser.Title(doc)
 
 	urls, size := genURL(dataDict.DURL)
 	data := downloader.VideoData{
 		Site:    "哔哩哔哩 bilibili.com",
-		Title:   utils.FileName(title),
+		Title:   title,
 		URLs:    urls,
 		Type:    "video",
 		Size:    size,

extractors/bilibili_test.go

@@ -53,6 +53,14 @@ func TestBilibili(t *testing.T) {
 			},
 			playlist: true,
 		},
+		{
+			name: "bangumi test",
+			args: test.Args{
+				URL:     "https://www.bilibili.com/bangumi/play/ss12044",
+				Bangumi: true,
+				Title:   "你的名字。",
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {

extractors/iqiyi.go

@@ -0,0 +1,93 @@
+package extractors
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/iawia002/annie/downloader"
+	"github.com/iawia002/annie/parser"
+	"github.com/iawia002/annie/request"
+	"github.com/iawia002/annie/utils"
+)
+
+type vidl struct {
+	M3utx      string `json:"m3utx"`
+	Vd         int    `json:"vd"` // quality number
+	ScreenSize string `json:"screenSize"`
+}
+
+type iqiyiData struct {
+	Vidl []vidl `json:"vidl"`
+}
+
+type iqiyi struct {
+	Code string    `json:"code"`
+	Data iqiyiData `json:"data"`
+}
+
+func getIqiyiData(tvid, vid string) iqiyi {
+	t := time.Now().Unix() * 1000
+	src := "76f90cbd92f94a2e925d83e8ccd22cb7"
+	key := "d5fb4bd9d50c4be6948c97edd7254b0e"
+	sc := utils.Md5(strconv.FormatInt(t, 10) + key + vid)
+	info := request.Get(fmt.Sprintf(
+		"http://cache.m.iqiyi.com/jp/tmts/%s/%s/?t=%d&sc=%s&src=%s",
+		tvid, vid, t, sc, src,
+	))
+	var data iqiyi
+	json.Unmarshal([]byte(info[len("var tvInfoJs="):]), &data)
+	return data
+}
+
+// Iqiyi download function
+func Iqiyi(url string) downloader.VideoData {
+	html := request.Get(url)
+	tvid := utils.MatchOneOf(
+		html,
+		`data-player-tvid="([^"]+)"`,
+		`param\[\'tvid\'\]\s*=\s*"(.+?)"`,
+	)[1]
+	vid := utils.MatchOneOf(
+		html,
+		`data-player-videoid="([^"]+)"`,
+		`param\[\'vid\'\]\s*=\s*"(.+?)"`,
+	)[1]
+	doc := parser.GetDoc(html)
+	title := strings.TrimSpace(doc.Find("h1 a").Text()) +
+		strings.TrimSpace(doc.Find("h1 span").Text())
+	videoDatas := getIqiyiData(tvid, vid)
+	if videoDatas.Code != "A00000" {
+		log.Fatal("Can't play this video")
+	}
+	videoData := videoDatas.Data.Vidl[0]
+	var urls []downloader.URLData
+	var urlData downloader.URLData
+	var totalSize int64
+	var size int64
+	for _, ts := range utils.M3u8Urls(videoData.M3utx) {
+		size, _ = strconv.ParseInt(
+			utils.MatchOneOf(ts, `contentlength=(\d+)`)[1], 10, 64,
+		)
+		urlData = downloader.URLData{
+			URL:  ts,
+			Size: size,
+			Ext:  "ts",
+		}
+		totalSize += size
+		urls = append(urls, urlData)
+	}
+	data := downloader.VideoData{
+		Site:    "爱奇艺 iqiyi.com",
+		Title:   title,
+		Type:    "video",
+		URLs:    urls,
+		Size:    totalSize,
+		Quality: videoData.ScreenSize,
+	}
+	data.Download(url)
+	return data
+}

extractors/iqiyi_test.go

@@ -0,0 +1,41 @@
+package extractors
+
+import (
+	"testing"
+
+	"github.com/iawia002/annie/config"
+	"github.com/iawia002/annie/test"
+)
+
+func TestIqiyi(t *testing.T) {
+	config.InfoOnly = true
+	tests := []struct {
+		name string
+		args test.Args
+	}{
+		{
+			name: "normal test",
+			args: test.Args{
+				URL:     "http://www.iqiyi.com/v_19rrbhikxo.html",
+				Title:   "热血街舞团：鹿晗个人宣传片震撼发布 执着前行终现万丈荣光",
+				Size:    11718040,
+				Quality: "1280x720",
+			},
+		},
+		{
+			name: "normal test",
+			args: test.Args{
+				URL:     "http://www.iqiyi.com/v_19rrbdmaj0.html",
+				Title:   "新一轮降水将至 冷空气影响中东部地区",
+				Size:    2838236,
+				Quality: "896x504",
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			data := Iqiyi(tt.args.URL)
+			test.Check(t, tt.args, data)
+		})
+	}
+}

extractors/pixivision.go

@@ -4,7 +4,6 @@ import (
 	"github.com/iawia002/annie/downloader"
 	"github.com/iawia002/annie/parser"
 	"github.com/iawia002/annie/request"
-	"github.com/iawia002/annie/utils"
 )
 
 // Pixivision download function
@@ -13,7 +12,7 @@ func Pixivision(url string) downloader.VideoData {
 	title, urls := parser.GetImages(url, html, "am__work__illust  ", nil)
 	data := downloader.VideoData{
 		Site:  "pixivision pixivision.net",
-		Title: utils.FileName(title),
+		Title: title,
 		Type:  "image",
 		URLs:  urls,
 		Size:  0,

extractors/youku.go

@@ -7,9 +7,8 @@ import (
 	"strings"
 	"time"
 
-	"github.com/PuerkitoBio/goquery"
-
 	"github.com/iawia002/annie/downloader"
+	"github.com/iawia002/annie/parser"
 	"github.com/iawia002/annie/request"
 	"github.com/iawia002/annie/utils"
 )
@@ -99,11 +98,8 @@ func genData(youkuData data) ([]downloader.URLData, int64, string) {
 func Youku(url string) downloader.VideoData {
 	html := request.Get(url)
 	// get the title
-	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
-	if err != nil {
-		log.Fatal(err)
-	}
-	title := strings.TrimSpace(doc.Find("h1").First().Text())
+	doc := parser.GetDoc(html)
+	title := parser.Title(doc)
 	vid := utils.MatchOneOf(url, `id_(.+?).html`)[1]
 	youkuData := youkuUps(vid)
 	if youkuData.Data.Error.Code != 0 {

extractors/youku_test.go

@@ -26,7 +26,7 @@ func TestYouku(t *testing.T) {
 			name: "normal test",
 			args: test.Args{
 				URL:     "http://v.youku.com/v_show/id_XMzQ1MTAzNjQwNA==.html",
-				Title:   "这！就是街舞 第一季：百强“互杀”队长不忍直视",
+				Title:   "这！就是街舞 第一季：第3期：百强“互杀”队长不忍直视",
 				Size:    1741044063,
 				Quality: "mp4hd3 1920x1080",
 			},

extractors/youtube.go

@@ -78,7 +78,7 @@ func Youtube(uri string) downloader.VideoData {
 	}
 	data := downloader.VideoData{
 		Site:    "YouTube youtube.com",
-		Title:   title,
+		Title:   utils.FileName(title),
 		Type:    "video",
 		URLs:    []downloader.URLData{urlData},
 		Size:    size,

main.go

@@ -57,6 +57,8 @@ func main() {
 		extractors.Youtube(videoURL)
 	case "youtu": // youtu.be
 		extractors.Youtube(videoURL)
+	case "iqiyi":
+		extractors.Iqiyi(videoURL)
 	default:
 		extractors.Universal(videoURL)
 	}

parser/parser.go

@@ -26,7 +26,7 @@ func GetImages(
 	url, html, imgClass string, urlHandler func(string) string,
 ) (string, []downloader.URLData) {
 	doc := GetDoc(html)
-	title := strings.TrimSpace(doc.Find("h1").First().Text())
+	title := Title(doc)
 	urls := []downloader.URLData{}
 	urlData := downloader.URLData{}
 	doc.Find(fmt.Sprintf("img[class=\"%s\"]", imgClass)).Each(
@@ -43,3 +43,16 @@ func GetImages(
 	)
 	return title, urls
 }
+
+// Title get title
+func Title(doc *goquery.Document) string {
+	var title string
+	title = strings.Replace(
+		strings.TrimSpace(doc.Find("h1").First().Text()), "\n", "", -1,
+	)
+	if title == "" {
+		// Bilibili: Some movie page got no h1 tag
+		title, _ = doc.Find("meta[property=\"og:title\"]").Attr("content")
+	}
+	return utils.FileName(title)
+}

utils/utils.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"crypto/md5"
 	"fmt"
 	"net/url"
 	"os"
@@ -95,3 +96,30 @@ func GetNameAndExt(uri string) (string, string) {
 	contentType := request.ContentType(uri, uri)
 	return filename[0], strings.Split(contentType, "/")[1]
 }
+
+// Md5 md5 hash
+func Md5(text string) string {
+	sign := md5.New()
+	sign.Write([]byte(text))
+	return fmt.Sprintf("%x", sign.Sum(nil))
+}
+
+// M3u8Urls get all urls from m3u8 url
+func M3u8Urls(uri string) []string {
+	html := request.Get(uri)
+	lines := strings.Split(html, "\n")
+	var urls []string
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if line != "" && !strings.HasPrefix(line, "#") {
+			if strings.HasPrefix(line, "http") {
+				urls = append(urls, line)
+			} else {
+				base, _ := url.Parse(uri)
+				u, _ := url.Parse(line)
+				urls = append(urls, fmt.Sprintf("%s", base.ResolveReference(u)))
+			}
+		}
+	}
+	return urls
+}