xvideos support (#489)

* xvideos support

* fix flag

extractors/xvideos/xvideos.go

@@ -0,0 +1,119 @@
+package xvideos
+
+import (
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/iawia002/annie/downloader"
+	"github.com/iawia002/annie/request"
+	"github.com/iawia002/annie/utils"
+)
+
+const (
+	lowFlag      = "html5player.setVideoUrlLow('"
+	lowFinalFlag = `');
+	    html5player.setVideoUrlHigh(`
+	highFlag      = "html5player.setVideoUrlHigh('"
+	highFinalFlag = `');
+	    html5player.setVideoHLS(`
+	qualityLow  = "low"
+	qualityHigh = "high"
+)
+
+var (
+	lowFlagLength  = len(lowFlag)
+	highFlagLength = len(highFlag)
+)
+
+type src struct {
+	url     string
+	quality string
+}
+
+func getSrc(html string) []*src {
+	var wg sync.WaitGroup
+	wg.Add(4)
+
+	startIndexLow := 0
+	go func() {
+		startIndexLow = strings.Index(html, lowFlag)
+		startIndexLow += lowFlagLength
+		wg.Done()
+	}()
+	endIndexLow := 0
+	go func() {
+		endIndexLow = strings.Index(html, lowFinalFlag)
+		wg.Done()
+	}()
+
+	startIndexHigh := 0
+	go func() {
+		startIndexHigh = strings.Index(html, highFlag)
+		startIndexHigh += highFlagLength
+		wg.Done()
+	}()
+	endIndexHigh := 0
+	go func() {
+		endIndexHigh = strings.Index(html, highFinalFlag)
+		wg.Done()
+	}()
+	wg.Wait()
+
+	var srcs []*src
+	if startIndexLow != -1 {
+		srcs = append(srcs, &src{
+			url:     html[startIndexLow:endIndexLow],
+			quality: qualityLow,
+		})
+	}
+	if startIndexHigh != -1 {
+		srcs = append(srcs, &src{
+			url:     html[startIndexHigh:endIndexHigh],
+			quality: qualityHigh,
+		})
+	}
+	return srcs
+}
+
+// Extract is the main function for extracting data
+func Extract(url string) ([]downloader.Data, error) {
+	html, err := request.Get(url, url, nil)
+	if err != nil {
+		return downloader.EmptyList, err
+	}
+	var title string
+	desc := utils.MatchOneOf(html, `<title>(.+?)</title>`)
+	if desc != nil {
+		title = desc[1]
+	} else {
+		title = "xvideos"
+	}
+
+	streams := make(map[string]downloader.Stream)
+	for _, src := range getSrc(html) {
+		size, err := request.Size(src.url, url)
+		if err != nil {
+			return downloader.EmptyList, err
+		}
+		urlData := downloader.URL{
+			URL:  src.url,
+			Size: size,
+			Ext:  "mp4",
+		}
+		streams[src.quality] = downloader.Stream{
+			URLs:    []downloader.URL{urlData},
+			Size:    size,
+			Quality: fmt.Sprintf("%s", src.quality),
+		}
+	}
+	return []downloader.Data{
+		{
+			Site:    "xvideos",
+			Title:   title,
+			Type:    "video",
+			Streams: streams,
+			URL:     url,
+		},
+	}, nil
+}

extractors/xvideos/xvideos_test.go

@@ -0,0 +1,33 @@
+package xvideos
+
+import (
+	"testing"
+
+	"github.com/iawia002/annie/config"
+	"github.com/iawia002/annie/test"
+)
+
+func TestExtract(t *testing.T) {
+	config.InfoOnly = true
+	config.RetryTimes = 10
+	tests := []struct {
+		name string
+		args test.Args
+	}{
+		{
+			name: "normal test",
+			args: test.Args{
+				URL:   "https://www.xvideos.com/video29018757/asian_chick_enjoying_sex_debut._hd_full_at_nanairo.co",
+				Title: "Asian chick enjoying sex debut. HD FULL at: nanairo.co - XVIDEOS.COM",
+				Size:  16574766,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			data, err := Extract(tt.args.URL)
+			test.CheckError(t, err)
+			test.Check(t, tt.args, data[0])
+		})
+	}
+}

main.go

@@ -32,6 +32,7 @@ import (
 	"github.com/iawia002/annie/extractors/universal"
 	"github.com/iawia002/annie/extractors/vimeo"
 	"github.com/iawia002/annie/extractors/weibo"
+	"github.com/iawia002/annie/extractors/xvideos"
 	"github.com/iawia002/annie/extractors/yinyuetai"
 	"github.com/iawia002/annie/extractors/youku"
 	"github.com/iawia002/annie/extractors/youtube"
@@ -155,6 +156,8 @@ func download(videoURL string) bool {
 		data, err = geekbang.Extract(videoURL)
 	case "pornhub":
 		data, err = pornhub.Extract(videoURL)
+	case "xvideos":
+		data, err = xvideos.Extract(videoURL)
 	default:
 		data, err = universal.Extract(videoURL)
 	}