extractors/bcy: Add support

README.md

@@ -114,10 +114,11 @@ Usage of annie:
 
 ## Supported Sites
 
-Site | URL | Videos
---- | --- | ------
-抖音 | <https://www.douyin.com> | ✓
-哔哩哔哩 | <https://www.bilibili.com> | ✓
+Site | URL | Videos | Images
+--- | --- | ------| -----
+抖音 | <https://www.douyin.com> | ✓ | |
+哔哩哔哩 | <https://www.bilibili.com> | ✓ | |
+半次元 | <https://bcy.net> | | ✓ |
 
 
 ## Development

downloader/downloader.go

@@ -4,8 +4,10 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/url"
 	"os"
 	"os/exec"
+	"strings"
 	"sync"
 	"time"
 
@@ -20,31 +22,51 @@ import (
 type URLData struct {
 	URL  string
 	Size int64
+	Ext  string
 }
 
 // VideoData data struct of video info
 type VideoData struct {
 	Site  string
 	Title string
-	URLs  []URLData // Some video files have multiple fragments
-	Size  int64
-	Ext   string
+	// [URLData: {URL, Size, Ext}, ...]
+	// Some video files have multiple fragments
+	// and support for downloading multiple image files at once
+	URLs []URLData
+	Size int64
+	Type string
+}
+
+// GetNameAndExt return the name and ext of the URL
+func (data URLData) GetNameAndExt() (string, string) {
+	u, _ := url.ParseRequestURI(data.URL)
+	s := strings.Split(u.Path, "/")
+	filename := strings.Split(s[len(s)-1], ".")
+	return filename[0], filename[1]
 }
 
 func (data VideoData) printInfo() {
 	fmt.Println()
 	fmt.Println(" Site:   ", data.Site)
 	fmt.Println("Title:   ", data.Title)
-	fmt.Println(" Type:   ", data.Ext)
+	fmt.Println(" Type:   ", data.Type)
 	fmt.Printf(" Size:    %.2f MiB (%d Bytes)\n", float64(data.Size)/(1024*1024), data.Size)
 	fmt.Println()
 }
 
+func (data *VideoData) calculateTotalSize() {
+	var size int64
+	for _, urlData := range data.URLs {
+		size += urlData.Size
+	}
+	data.Size = size
+}
+
 // urlSave save url file
 func (data VideoData) urlSave(
 	urlData URLData, refer, fileName string, bar *pb.ProgressBar,
 ) {
-	filePath := utils.FilePath(fileName, data.Ext, false)
+	filePath := utils.FilePath(fileName, urlData.Ext, false)
 	fileSize := utils.FileSize(filePath)
 	if fileSize == urlData.Size {
 		fmt.Printf("%s: file already exists, skipping\n", filePath)
@@ -84,6 +106,9 @@ func (data VideoData) urlSave(
 
 // Download download urls
 func (data VideoData) Download(refer string) {
+	if data.Size == 0 {
+		data.calculateTotalSize()
+	}
 	data.printInfo()
 	if config.InfoOnly {
 		return
@@ -104,7 +129,7 @@ func (data VideoData) Download(refer string) {
 		for index, url := range data.URLs {
 			wg.Add(1)
 			partFileName := fmt.Sprintf("%s[%d]", data.Title, index)
-			partFilePath := utils.FilePath(partFileName, data.Ext, false)
+			partFilePath := utils.FilePath(partFileName, url.Ext, false)
 			parts = append(parts, partFilePath)
 			go func(url URLData, refer, fileName string, bar *pb.ProgressBar) {
 				defer wg.Done()
@@ -114,6 +139,9 @@ func (data VideoData) Download(refer string) {
 		wg.Wait()
 		bar.Finish()
 
+		if data.Type != "video" {
+			return
+		}
 		// merge
 		// write ffmpeg input file list
 		mergeFile := data.Title + "-merge.txt"
@@ -122,7 +150,7 @@ func (data VideoData) Download(refer string) {
 			file.Write([]byte(fmt.Sprintf("file '%s'\n", part)))
 		}
 
-		filePath := utils.FilePath(data.Title, data.Ext, false)
+		filePath := utils.FilePath(data.Title, "mp4", false)
 		fmt.Printf("Merging video parts into %s\n", filePath)
 		cmd := exec.Command(
 			"ffmpeg", "-y", "-f", "concat", "-safe", "-1",

extractors/bcy.go

@@ -0,0 +1,43 @@
+package extractors
+
+import (
+	"log"
+	"strings"
+
+	"github.com/PuerkitoBio/goquery"
+
+	"github.com/iawia002/annie/downloader"
+	"github.com/iawia002/annie/request"
+	"github.com/iawia002/annie/utils"
+)
+
+// Bcy download function
+func Bcy(url string) downloader.VideoData {
+	html := request.Get(url)
+	doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
+	if err != nil {
+		log.Fatal(err)
+	}
+	title := strings.TrimSpace(doc.Find("h1").First().Text())
+	urls := []downloader.URLData{}
+	urlData := downloader.URLData{}
+	doc.Find("img[class=\"detail_std detail_clickable\"]").Each(
+		func(i int, s *goquery.Selection) {
+			urlData.URL, _ = s.Attr("src")
+			// https://img9.bcyimg.com/drawer/15294/post/1799t/1f5a87801a0711e898b12b640777720f.jpg/w650
+			urlData.URL = urlData.URL[:len(urlData.URL)-5]
+			urlData.Size = request.Size(urlData.URL, url)
+			_, urlData.Ext = urlData.GetNameAndExt()
+			urls = append(urls, urlData)
+		},
+	)
+	data := downloader.VideoData{
+		Site:  "半次元 bcy.net",
+		Title: utils.FileName(title),
+		Type:  "image",
+		URLs:  urls,
+		Size:  0,
+	}
+	data.Download(url)
+	return data
+}

extractors/bilibili.go

@@ -102,6 +102,7 @@ func genURL(durl []dURLData) ([]downloader.URLData, int64) {
 		url := downloader.URLData{
 			URL:  data.URL,
 			Size: data.Size,
+			Ext:  "flv",
 		}
 		urls = append(urls, url)
 	}
@@ -146,7 +147,7 @@ func Bilibili(url string) downloader.VideoData {
 		Site:  "哔哩哔哩 bilibili.com",
 		Title: utils.FileName(title),
 		URLs:  urls,
-		Ext:   "flv",
+		Type:  "video",
 		Size:  size,
 	}
 	data.Download(url)

extractors/douyin.go

@@ -33,11 +33,12 @@ func Douyin(url string) downloader.VideoData {
 	urlData := downloader.URLData{
 		URL:  dataDict.Video.RealPlayAddr,
 		Size: size,
+		Ext:  "mp4",
 	}
 	data := downloader.VideoData{
 		Site:  "抖音 douyin.com",
 		Title: utils.FileName(dataDict.Desc),
-		Ext:   "mp4",
+		Type:  "video",
 		URLs:  []downloader.URLData{urlData},
 		Size:  size,
 	}

main.go

@@ -43,6 +43,8 @@ func main() {
 		extractors.Douyin(videoURL)
 	case "bilibili":
 		extractors.Bilibili(videoURL)
+	case "bcy":
+		extractors.Bcy(videoURL)
 	default:
 		fmt.Println("unsupported URL")
 	}

utils/utils.go

@@ -49,3 +49,13 @@ func FilePath(name, ext string, escape bool) string {
 	}
 	return fileName
 }
+
+// StringInSlice if a string is in the list
+func StringInSlice(str string, list []string) bool {
+	for _, a := range list {
+		if a == str {
+			return true
+		}
+	}
+	return false
+}