Merge pull request #1338 from iAsuma/contrib

重写iXigua(Toutiao)视频获取方式

README.md

@@ -655,6 +655,13 @@ $ lux -j "https://www.bilibili.com/video/av20203945"
 
 最好是每次下载都附带登录过的 Cookie 以避免部分 `ccode` 的问题
 
+### 西瓜/头条视频
+西瓜/头条视频必须带 Cookie 才能下载成功，西瓜和头条可共用西瓜视频的 Cookie，Cookie 的有效期可能较短，下载失败就更新 Cookie 尝试：
+
+```
+$ lux -c "msToken=yoEh0-qLUq4obZ8Sfxsem_CxCo9R3NM6ViTrWaRcM1...; ttwid=1%7C..." "https://m.toutiao.com/is/iYbTfJ79/"
+```
+
 ## Contributing
 
 Lux is an open source project and built on the top of open-source projects. Check out the [Contributing Guide](./CONTRIBUTING.md) to get started.

extractors/ixigua/ixigua.go

@@ -8,11 +8,12 @@ import (
 	"regexp"
 	"strings"
 
-	"github.com/itchyny/gojq"
+	browser "github.com/EDDYCJY/fake-useragent"
 	"github.com/pkg/errors"
 
 	"github.com/iawia002/lux/extractors"
 	"github.com/iawia002/lux/request"
+	"github.com/iawia002/lux/utils"
 )
 
 func init() {
@@ -40,8 +41,8 @@ func New() extractors.Extractor {
 // Extract is the main function to extract the data.
 func (e *extractor) Extract(url string, option extractors.Options) ([]*extractors.Data, error) {
 	headers := map[string]string{
-		"User-Agent":   "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:98.0) Gecko/20100101 Firefox/98.0",
-		"Content-Type": "application/json",
+		"User-Agent": browser.Chrome(),
+		"Cookie":     option.Cookie,
 	}
 
 	// ixigua 有三种格式的 URL
@@ -66,60 +67,42 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor
 		finalURL = resp.Request.URL.String()
 	}
 
-	finalURL = strings.ReplaceAll(finalURL, "https://www.toutiao.com/a", "https://www.ixigua.com/")
+	finalURL = strings.ReplaceAll(finalURL, "https://www.toutiao.com/video/", "https://www.ixigua.com/")
 
 	r := regexp.MustCompile(`(ixigua.com/)(\w+)?`)
 	id := r.FindSubmatch([]byte(finalURL))[2]
-	url2 := fmt.Sprintf("https://www.ixigua.com/api/public/videov2/brief/details?group_id=%s", string(id))
+	url2 := fmt.Sprintf("https://www.ixigua.com/%s", string(id))
 
 	body, err := request.Get(url2, url, headers)
 	if err != nil {
 		return nil, errors.WithStack(err)
 	}
 
-	var m interface{}
-	err = json.Unmarshal([]byte(body), &m)
-	if err != nil {
-		return nil, errors.WithStack(err)
+	videoListJson := utils.MatchOneOf(body, `window._SSR_HYDRATED_DATA=(\{.*?\})\<\/script\>`)
+	if videoListJson == nil || len(videoListJson) != 2 {
+		return nil, errors.WithStack(extractors.ErrBodyParseFailed)
 	}
 
-	query, err := gojq.Parse("{title: .data.title} + {qualities: [.data.videoResource.normal.video_list | .[] | {url: .main_url, size: .size, ext: .vtype, quality: .definition}]}")
-	if err != nil {
+	videoUrl := videoListJson[1]
+	videoUrl = strings.Replace(videoUrl, ":undefined", ":\"undefined\"", -1)
+
+	var data xiguanData
+	if err = json.Unmarshal([]byte(videoUrl), &data); err != nil {
 		return nil, errors.WithStack(err)
 	}
 
-	video := Video{}
-
-	iter := query.Run(m)
-	for {
-		v, ok := iter.Next()
-		if !ok {
-			break
-		}
-		if err, ok := v.(error); ok {
-			return nil, errors.WithStack(err)
-		}
-
-		jsonbody, err := json.Marshal(v)
-		if err != nil {
-			return nil, errors.WithStack(err)
-		}
-
-		if err := json.Unmarshal(jsonbody, &video); err != nil {
-			return nil, errors.WithStack(err)
-		}
-	}
+	title := data.AnyVideo.GidInformation.PackerData.Video.Title
+	videoList := data.AnyVideo.GidInformation.PackerData.Video.VideoResource.Normal.VideoList
 
 	streams := make(map[string]*extractors.Stream)
-	for _, quality := range video.Qualities {
-		streams[quality.Quality] = &extractors.Stream{
-			Size:    quality.Size,
-			Quality: quality.Quality,
+	for _, v := range videoList {
+		streams[v.Definition] = &extractors.Stream{
+			Quality: v.Definition,
 			Parts: []*extractors.Part{
 				{
-					URL:  base64Decode(quality.URL),
-					Size: quality.Size,
-					Ext:  quality.Ext,
+					URL:  base64Decode(v.MainUrl),
+					Size: v.Size,
+					Ext:  v.Vtype,
 				},
 			},
 		}
@@ -128,7 +111,7 @@ func (e *extractor) Extract(url string, option extractors.Options) ([]*extractor
 	return []*extractors.Data{
 		{
 			Site:    "西瓜视频 ixigua.com",
-			Title:   video.Title,
+			Title:   title,
 			Type:    extractors.DataTypeVideo,
 			Streams: streams,
 			URL:     url,

extractors/ixigua/types.go

@@ -0,0 +1,36 @@
+package ixigua
+
+type xiguanData struct {
+	AnyVideo struct {
+		GidInformation struct {
+			Gid        string `json:"gid"`
+			PackerData struct {
+				Video struct {
+					Title         string `json:"title"`
+					PosterUrl     string `json:"poster_url"`
+					VideoResource struct {
+						Vid    string `json:"vid"`
+						Normal struct {
+							VideoId   string `json:"video_id"`
+							VideoList map[string]struct {
+								Definition  string `json:"definition"`
+								Quality     string `json:"quality"`
+								Vtype       string `json:"vtype"`
+								Vwidth      int    `json:"vwidth"`
+								Vheight     int    `json:"vheight"`
+								Bitrate     int64  `json:"bitrate"`
+								RealBitrate int64  `json:"real_bitrate"`
+								Fps         int    `json:"fps"`
+								CodecType   string `json:"codec_type"`
+								Size        int64  `json:"size"`
+								MainUrl     string `json:"main_url"`
+								BackupUrl1  string `json:"backup_url_1"`
+							} `json:"video_list"`
+						} `json:"normal"`
+					} `json:"videoResource"`
+				} `json:"video"`
+				Key string `json:"key"`
+			} `json:"packerData"`
+		} `json:"gidInformation"`
+	} `json:"anyVideo"`
+}

extractors/xiaohongshu/xiaohongshu_test.go

@@ -15,7 +15,7 @@ func TestDownload(t *testing.T) {
 		{
 			name: "normal test",
 			args: test.Args{
-				URL:   "https://www.xiaohongshu.com/explore/64e9f1e50000000003023b3f?m_source=pinpai",
+				URL:   "https://www.xiaohongshu.com/explore/64e9f1e50000000003023b3f",
 				Title: "七星级大厨都不会告诉你的，五花肉的8种做法",
 				Size:  59410194,
 			},