From bd84e40127a74e8d78bf2769cda19228dc898bbf Mon Sep 17 00:00:00 2001 From: LazzyFatty <45315173+LazzyFatty@users.noreply.github.com> Date: Mon, 8 Nov 2021 20:02:37 +0800 Subject: [PATCH 01/16] Update douyin.go --- extractors/douyin/douyin.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index 972d414f3..4ba0c6787 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -1,7 +1,6 @@ package douyin import ( - "encoding/json" "errors" "net/http" "strings" @@ -49,8 +48,9 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er if err != nil { return nil, err } - var douyin douyinData - if err = json.Unmarshal([]byte(jsonData), &douyin); err != nil { + realURL := "https://aweme.snssdk.com/aweme/v1/play/?video_id=" + douyin.ItemList[0].Video.PlayAddr.Uri + "&ratio=720p&line=0" + size, err := request.Size(realURL, url) + if err != nil { return nil, err } From e711d7eb1e65b093e216a2e6b89daa84ea0dc20d Mon Sep 17 00:00:00 2001 From: LazzyFatty <45315173+LazzyFatty@users.noreply.github.com> Date: Mon, 8 Nov 2021 20:26:56 +0800 Subject: [PATCH 02/16] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=8A=96=E9=9F=B3?= =?UTF-8?q?=E8=A7=A3=E6=9E=90=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/douyin/douyin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index 4ba0c6787..efb7cf0f3 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -48,7 +48,7 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er if err != nil { return nil, err } - realURL := "https://aweme.snssdk.com/aweme/v1/play/?video_id=" + douyin.ItemList[0].Video.PlayAddr.Uri + "&ratio=720p&line=0" + realURL := "https://aweme.snssdk.com/aweme/v1/play/?video_id=" + douyin.ItemList[0].Video.PlayAddr.URI + "&ratio=720p&line=0" size, err := request.Size(realURL, url) if err != nil { return nil, err From e3a9aa1580c3b2a8a08cc470637bcafea31a7b88 Mon Sep 17 00:00:00 2001 From: LazzyFatty <45315173+LazzyFatty@users.noreply.github.com> Date: Tue, 9 Nov 2021 22:23:05 +0800 Subject: [PATCH 03/16] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9=E6=8A=96?= =?UTF-8?q?=E9=9F=B3APP=E5=88=86=E4=BA=AB=E9=93=BE=E6=8E=A5=EF=BC=88?= =?UTF-8?q?=E7=9F=AD=E9=93=BE=EF=BC=89=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/douyin/douyin.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index efb7cf0f3..8452381fb 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -5,6 +5,9 @@ import ( "net/http" "strings" + "net/http" + "strings" + "github.com/iawia002/annie/extractors/types" "github.com/iawia002/annie/request" "github.com/iawia002/annie/utils" @@ -19,6 +22,7 @@ func New() types.Extractor { // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { + var err error if strings.Contains(url, "v.douyin.com") { req, err := http.NewRequest("GET", url, nil) if err != nil { @@ -30,12 +34,8 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er }, } resp, err := c.Do(req) - if err != nil { - return nil, err - } url = resp.Header.Get("location") } - itemIds := utils.MatchOneOf(url, `/video/(\d+)`) if len(itemIds) == 0 { return nil, errors.New("unable to get video ID") From b54f2147224290214137db68ecf056290c4b1fff Mon Sep 17 00:00:00 2001 From: LazzyFatty <45315173+LazzyFatty@users.noreply.github.com> Date: Wed, 10 Nov 2021 12:18:41 +0800 Subject: [PATCH 04/16] Update extractors/douyin/douyin.go Co-authored-by: Xinzhao Xu --- extractors/douyin/douyin.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index 8452381fb..a16980037 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -5,9 +5,6 @@ import ( "net/http" "strings" - "net/http" - "strings" - "github.com/iawia002/annie/extractors/types" "github.com/iawia002/annie/request" "github.com/iawia002/annie/utils" From b46525db755cd2c353f2547930b097bd3e449679 Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Wed, 10 Nov 2021 14:59:08 +0800 Subject: [PATCH 05/16] Update douyin.go --- extractors/douyin/douyin.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index a16980037..c7c8d5157 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -31,6 +31,9 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er }, } resp, err := c.Do(req) + if err != nil { + return nil, err + } url = resp.Header.Get("location") } itemIds := utils.MatchOneOf(url, `/video/(\d+)`) From 5c7354dbff863b0ca928f66bc973005744dc3b90 Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Sun, 14 Nov 2021 23:44:43 +0800 Subject: [PATCH 06/16] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9=E6=8A=96?= =?UTF-8?q?=E9=9F=B3=E5=9B=BE=E9=9B=86=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/douyin/douyin.go | 23 ++++++++--------------- extractors/douyin/douyin_test.go | 4 ++-- 2 files changed, 10 insertions(+), 17 deletions(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index c7c8d5157..205cbc1ae 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -48,39 +48,32 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er if err != nil { return nil, err } - realURL := "https://aweme.snssdk.com/aweme/v1/play/?video_id=" + douyin.ItemList[0].Video.PlayAddr.URI + "&ratio=720p&line=0" - size, err := request.Size(realURL, url) - if err != nil { - return nil, err - } - urlData := make([]*types.Part, 0) - var douyinType types.DataType var totalSize int64 - // AwemeType: 2:image 4:video + var types string + //AwemeType: 2:image 4:video if douyin.ItemList[0].AwemeType == 2 { - douyinType = types.DataTypeImage + types = types.DataTypeImage for _, img := range douyin.ItemList[0].Images { - realURL := img.URLList[len(img.URLList)-1] - size, err := request.Size(realURL, url) + size, err := request.Size(img.URLList[0], url) if err != nil { return nil, err } totalSize += size - _, ext, err := utils.GetNameAndExt(realURL) + _, ext, err := utils.GetNameAndExt(img.URLList[0]) if err != nil { return nil, err } urlData = append(urlData, &types.Part{ - URL: realURL, + URL: img.URLList[0], Size: size, Ext: ext, }) } } else { - douyinType = types.DataTypeVideo + types = types.DataTypeVideo realURL := "https://aweme.snssdk.com/aweme/v1/play/?video_id=" + douyin.ItemList[0].Video.PlayAddr.URI + "&ratio=720p&line=0" - totalSize, err = request.Size(realURL, url) + totalSize, err := request.Size(realURL, url) if err != nil { return nil, err } diff --git a/extractors/douyin/douyin_test.go b/extractors/douyin/douyin_test.go index d743c0150..892e8174e 100644 --- a/extractors/douyin/douyin_test.go +++ b/extractors/douyin/douyin_test.go @@ -15,8 +15,8 @@ func TestDownload(t *testing.T) { { name: "normal test", args: test.Args{ - URL: "https://www.douyin.com/video/6967223681286278436?previous_page=main_page&tab_name=home", - Title: "是爱情，让父子相认#陈翔六点半 #关于爱情", + URL: "https://v.douyin.com/RHanxuR/", + Title: "树叶只有树 树却有很多树叶", }, }, { From 9a5b07288f8c7a027d01c00bddbb3c0616009fa2 Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Mon, 15 Nov 2021 00:52:30 +0800 Subject: [PATCH 07/16] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20douyin.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/douyin/douyin.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index 205cbc1ae..466f05a41 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -50,10 +50,8 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er } urlData := make([]*types.Part, 0) var totalSize int64 - var types string //AwemeType: 2:image 4:video if douyin.ItemList[0].AwemeType == 2 { - types = types.DataTypeImage for _, img := range douyin.ItemList[0].Images { size, err := request.Size(img.URLList[0], url) if err != nil { @@ -71,7 +69,6 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er }) } } else { - types = types.DataTypeVideo realURL := "https://aweme.snssdk.com/aweme/v1/play/?video_id=" + douyin.ItemList[0].Video.PlayAddr.URI + "&ratio=720p&line=0" totalSize, err := request.Size(realURL, url) if err != nil { From 3333063469acc049a9f0b6e57dac190a5d7cb033 Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Mon, 15 Nov 2021 19:45:03 +0800 Subject: [PATCH 08/16] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20douyin.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/douyin/douyin.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index 466f05a41..a59bdd9d5 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -49,9 +49,11 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er return nil, err } urlData := make([]*types.Part, 0) + var douyinType types.DataType var totalSize int64 //AwemeType: 2:image 4:video if douyin.ItemList[0].AwemeType == 2 { + douyinType = types.DataTypeImage for _, img := range douyin.ItemList[0].Images { size, err := request.Size(img.URLList[0], url) if err != nil { @@ -69,6 +71,7 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er }) } } else { + douyinType = types.DataTypeVideo realURL := "https://aweme.snssdk.com/aweme/v1/play/?video_id=" + douyin.ItemList[0].Video.PlayAddr.URI + "&ratio=720p&line=0" totalSize, err := request.Size(realURL, url) if err != nil { From b235cda4c1bcc1762f0a619f76bf31069902ecaf Mon Sep 17 00:00:00 2001 From: LazzyFatty <45315173+LazzyFatty@users.noreply.github.com> Date: Tue, 16 Nov 2021 13:21:17 +0800 Subject: [PATCH 09/16] Apply suggestions from code review Co-authored-by: Xinzhao Xu --- extractors/douyin/douyin.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index a59bdd9d5..039fa144d 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -51,7 +51,7 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er urlData := make([]*types.Part, 0) var douyinType types.DataType var totalSize int64 - //AwemeType: 2:image 4:video + // AwemeType: 2:image 4:video if douyin.ItemList[0].AwemeType == 2 { douyinType = types.DataTypeImage for _, img := range douyin.ItemList[0].Images { @@ -73,7 +73,7 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er } else { douyinType = types.DataTypeVideo realURL := "https://aweme.snssdk.com/aweme/v1/play/?video_id=" + douyin.ItemList[0].Video.PlayAddr.URI + "&ratio=720p&line=0" - totalSize, err := request.Size(realURL, url) + totalSize, err = request.Size(realURL, url) if err != nil { return nil, err } From f5a1cb26832c550e4af17a6163034d2e11dd2ae1 Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Tue, 16 Nov 2021 13:47:09 +0800 Subject: [PATCH 10/16] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9=E6=8A=96?= =?UTF-8?q?=E9=9F=B3=E5=9B=BE=E9=9B=86=E7=9A=84=E6=94=AF=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 2 +- extractors/douyin/douyin.go | 6 +++--- extractors/douyin/douyin_test.go | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d8f4444e1..2b829f2c4 100644 --- a/README.md +++ b/README.md @@ -595,7 +595,7 @@ $ annie -j "https://www.bilibili.com/video/av20203945" | Site | URL | 🎬 Videos | 🌁 Images | 📚 Playlist | 🍪 VIP adaptation | | ---------- | ---------------------------- | -------- | -------- | ---------- | ---------------- | -| 抖音 | | ✓ | ✓ | | | +| 抖音 | | ✓ | ✓ | | | | 哔哩哔哩 | | ✓ | | ✓ | ✓ | | 半次元 | | | ✓ | | | | pixivision | | | ✓ | | | diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index 039fa144d..cbfc33810 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -55,17 +55,17 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er if douyin.ItemList[0].AwemeType == 2 { douyinType = types.DataTypeImage for _, img := range douyin.ItemList[0].Images { - size, err := request.Size(img.URLList[0], url) + size, err := request.Size(img.URLList[3], url) if err != nil { return nil, err } totalSize += size - _, ext, err := utils.GetNameAndExt(img.URLList[0]) + _, ext, err := utils.GetNameAndExt(img.URLList[3]) if err != nil { return nil, err } urlData = append(urlData, &types.Part{ - URL: img.URLList[0], + URL: img.URLList[3], Size: size, Ext: ext, }) diff --git a/extractors/douyin/douyin_test.go b/extractors/douyin/douyin_test.go index 892e8174e..6330b61c1 100644 --- a/extractors/douyin/douyin_test.go +++ b/extractors/douyin/douyin_test.go @@ -14,6 +14,13 @@ func TestDownload(t *testing.T) { }{ { name: "normal test", + args: test.Args{ + URL: "https://www.douyin.com/video/6967223681286278436?previous_page=main_page&tab_name=home", + Title: "是爱情，让父子相认#陈翔六点半 #关于爱情", + }, + }, + { + name: "image test", args: test.Args{ URL: "https://v.douyin.com/RHanxuR/", Title: "树叶只有树 树却有很多树叶", From f48445ee6881d0d0243623b36b4f1935a8e67edd Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Tue, 16 Nov 2021 14:13:34 +0800 Subject: [PATCH 11/16] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20douyin.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/douyin/douyin.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index cbfc33810..755f716b5 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -55,17 +55,18 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er if douyin.ItemList[0].AwemeType == 2 { douyinType = types.DataTypeImage for _, img := range douyin.ItemList[0].Images { - size, err := request.Size(img.URLList[3], url) + realURL := img.URLList[len(img.URLList)-1] + size, err := request.Size(realURL, url) if err != nil { return nil, err } totalSize += size - _, ext, err := utils.GetNameAndExt(img.URLList[3]) + _, ext, err := utils.GetNameAndExt(realURL) if err != nil { return nil, err } urlData = append(urlData, &types.Part{ - URL: img.URLList[3], + URL: realURL, Size: size, Ext: ext, }) From b77d4cde9660298ed2d1d4accbd247a814bb2e65 Mon Sep 17 00:00:00 2001 From: LazzyFatty <45315173+LazzyFatty@users.noreply.github.com> Date: Tue, 16 Nov 2021 14:33:32 +0800 Subject: [PATCH 12/16] Apply suggestions from code review Co-authored-by: Xinzhao Xu --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b829f2c4..d8f4444e1 100644 --- a/README.md +++ b/README.md @@ -595,7 +595,7 @@ $ annie -j "https://www.bilibili.com/video/av20203945" | Site | URL | 🎬 Videos | 🌁 Images | 📚 Playlist | 🍪 VIP adaptation | | ---------- | ---------------------------- | -------- | -------- | ---------- | ---------------- | -| 抖音 | | ✓ | ✓ | | | +| 抖音 | | ✓ | ✓ | | | | 哔哩哔哩 | | ✓ | | ✓ | ✓ | | 半次元 | | | ✓ | | | | pixivision | | | ✓ | | | From 1f6675ae94d2fc0b2df4c1d3d6affd3a0118c987 Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Mon, 6 Dec 2021 19:24:51 +0800 Subject: [PATCH 13/16] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9B=E7=AB=998K?= =?UTF-8?q?=E8=B6=85=E9=AB=98=E6=B8=85=E8=A7=86=E9=A2=91=E7=9A=84=E6=94=AF?= =?UTF-8?q?=E6=8C=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/bilibili/bilibili.go | 18 +++++++++++------- extractors/bilibili/bilibili_test.go | 7 +++++++ extractors/bilibili/types.go | 1 + 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index 9d14d688a..1c01c928c 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -23,6 +23,10 @@ const referer = "https://www.bilibili.com" var utoken string +var headers = map[string]string{ + "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", +} + func genAPI(aid, cid, quality int, bvid string, bangumi bool, cookie string) (string, error) { var ( err error @@ -33,7 +37,7 @@ func genAPI(aid, cid, quality int, bvid string, bangumi bool, cookie string) (st utoken, err = request.Get( fmt.Sprintf("%said=%d&cid=%d", bilibiliTokenAPI, aid, cid), referer, - nil, + headers, ) if err != nil { return "", err @@ -60,7 +64,7 @@ func genAPI(aid, cid, quality int, bvid string, bangumi bool, cookie string) (st baseAPIURL = bilibiliBangumiAPI } else { params = fmt.Sprintf( - "avid=%d&cid=%d&bvid=%s&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=16", + "avid=%d&cid=%d&bvid=%s&qn=%d&type=&otype=json&fourk=1&fnver=0&fnval=2000", aid, cid, bvid, quality, ) baseAPIURL = bilibiliAPI @@ -239,7 +243,7 @@ func New() types.Extractor { // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { var err error - html, err := request.Get(url, referer, nil) + html, err := request.Get(url, referer, headers) if err != nil { return nil, err } @@ -265,7 +269,7 @@ func bilibiliDownload(options bilibiliOptions, extractOption types.Options) *typ // reuse html string, but this can't be reused in case of playlist html = options.html } else { - html, err = request.Get(options.url, referer, nil) + html, err = request.Get(options.url, referer, headers) if err != nil { return types.EmptyData(options.url, err) } @@ -273,12 +277,12 @@ func bilibiliDownload(options bilibiliOptions, extractOption types.Options) *typ // Get "accept_quality" and "accept_description" // "accept_description":["高清 1080P","高清 720P","清晰 480P","流畅 360P"], - // "accept_quality":[120,112,80,48,32,16], - api, err := genAPI(options.aid, options.cid, 120, options.bvid, options.bangumi, extractOption.Cookie) + // "accept_quality":[127，120,112,80,48,32,16], + api, err := genAPI(options.aid, options.cid, 127, options.bvid, options.bangumi, extractOption.Cookie) if err != nil { return types.EmptyData(options.url, err) } - jsonString, err := request.Get(api, referer, nil) + jsonString, err := request.Get(api, referer, headers) if err != nil { return types.EmptyData(options.url, err) } diff --git a/extractors/bilibili/bilibili_test.go b/extractors/bilibili/bilibili_test.go index e9fb01bd8..12fe09c28 100644 --- a/extractors/bilibili/bilibili_test.go +++ b/extractors/bilibili/bilibili_test.go @@ -60,6 +60,13 @@ func TestBilibili(t *testing.T) { Title: "你的名字。", }, }, + { + name: "8k test", + args: test.Args{ + URL: "https://www.bilibili.com/video/BV1qM4y1w716", + Title: "【8K演示片】B站首发！你的设备还顶得住吗？", + }, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/extractors/bilibili/types.go b/extractors/bilibili/types.go index 25f292e58..6d6ae43fe 100644 --- a/extractors/bilibili/types.go +++ b/extractors/bilibili/types.go @@ -71,6 +71,7 @@ type dash struct { } var qualityString = map[int]string{ + 127: "超高清 8K", 120: "超清 4K", 116: "高清 1080P60", 74: "高清 720P60", From 647ad216f70871589de1519bc54da279f7f65d8e Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Mon, 6 Dec 2021 20:39:57 +0800 Subject: [PATCH 14/16] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E6=97=B6=E7=9A=84=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/douyin/douyin.go | 7 ++++++- extractors/douyin/douyin_test.go | 7 ------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index 755f716b5..ffdd3cc10 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -1,6 +1,7 @@ package douyin import ( + "encoding/json" "errors" "net/http" "strings" @@ -19,7 +20,6 @@ func New() types.Extractor { // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { - var err error if strings.Contains(url, "v.douyin.com") { req, err := http.NewRequest("GET", url, nil) if err != nil { @@ -36,6 +36,11 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er } url = resp.Header.Get("location") } + var douyin douyinData + if err = json.Unmarshal([]byte(jsonData), &douyin); err != nil { + return nil, err + } + itemIds := utils.MatchOneOf(url, `/video/(\d+)`) if len(itemIds) == 0 { return nil, errors.New("unable to get video ID") diff --git a/extractors/douyin/douyin_test.go b/extractors/douyin/douyin_test.go index 6330b61c1..d743c0150 100644 --- a/extractors/douyin/douyin_test.go +++ b/extractors/douyin/douyin_test.go @@ -26,13 +26,6 @@ func TestDownload(t *testing.T) { Title: "树叶只有树 树却有很多树叶", }, }, - { - name: "image test", - args: test.Args{ - URL: "https://v.douyin.com/RHanxuR/", - Title: "树叶只有树 树却有很多树叶", - }, - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From af14da2ba8a5aa6276ae13b852b4992c86d571e8 Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Mon, 6 Dec 2021 20:41:26 +0800 Subject: [PATCH 15/16] =?UTF-8?q?=E4=BF=AE=E6=AD=A3=E6=8F=90=E4=BA=A4?= =?UTF-8?q?=E6=97=B6=E7=9A=84=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/douyin/douyin.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/extractors/douyin/douyin.go b/extractors/douyin/douyin.go index ffdd3cc10..972d414f3 100644 --- a/extractors/douyin/douyin.go +++ b/extractors/douyin/douyin.go @@ -36,10 +36,6 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er } url = resp.Header.Get("location") } - var douyin douyinData - if err = json.Unmarshal([]byte(jsonData), &douyin); err != nil { - return nil, err - } itemIds := utils.MatchOneOf(url, `/video/(\d+)`) if len(itemIds) == 0 { @@ -53,6 +49,11 @@ func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, er if err != nil { return nil, err } + var douyin douyinData + if err = json.Unmarshal([]byte(jsonData), &douyin); err != nil { + return nil, err + } + urlData := make([]*types.Part, 0) var douyinType types.DataType var totalSize int64 From ab2de55a34cba49ba9baad197a0b83d0a5fd39b0 Mon Sep 17 00:00:00 2001 From: LazzyFatty <15066105601@163.com> Date: Tue, 7 Dec 2021 12:53:55 +0800 Subject: [PATCH 16/16] =?UTF-8?q?=E6=9B=B4=E6=96=B0=20bilibili.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- extractors/bilibili/bilibili.go | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/extractors/bilibili/bilibili.go b/extractors/bilibili/bilibili.go index 1c01c928c..8573428ff 100644 --- a/extractors/bilibili/bilibili.go +++ b/extractors/bilibili/bilibili.go @@ -23,10 +23,6 @@ const referer = "https://www.bilibili.com" var utoken string -var headers = map[string]string{ - "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.81 Safari/537.36", -} - func genAPI(aid, cid, quality int, bvid string, bangumi bool, cookie string) (string, error) { var ( err error @@ -37,7 +33,7 @@ func genAPI(aid, cid, quality int, bvid string, bangumi bool, cookie string) (st utoken, err = request.Get( fmt.Sprintf("%said=%d&cid=%d", bilibiliTokenAPI, aid, cid), referer, - headers, + nil, ) if err != nil { return "", err @@ -243,7 +239,7 @@ func New() types.Extractor { // Extract is the main function to extract the data. func (e *extractor) Extract(url string, option types.Options) ([]*types.Data, error) { var err error - html, err := request.Get(url, referer, headers) + html, err := request.Get(url, referer, nil) if err != nil { return nil, err } @@ -269,20 +265,20 @@ func bilibiliDownload(options bilibiliOptions, extractOption types.Options) *typ // reuse html string, but this can't be reused in case of playlist html = options.html } else { - html, err = request.Get(options.url, referer, headers) + html, err = request.Get(options.url, referer, nil) if err != nil { return types.EmptyData(options.url, err) } } // Get "accept_quality" and "accept_description" - // "accept_description":["高清 1080P","高清 720P","清晰 480P","流畅 360P"], + // "accept_description":["超高清 8K","超清 4K","高清 1080P+","高清 1080P","高清 720P","清晰 480P","流畅 360P"], // "accept_quality":[127，120,112,80,48,32,16], api, err := genAPI(options.aid, options.cid, 127, options.bvid, options.bangumi, extractOption.Cookie) if err != nil { return types.EmptyData(options.url, err) } - jsonString, err := request.Get(api, referer, headers) + jsonString, err := request.Get(api, referer, nil) if err != nil { return types.EmptyData(options.url, err) }