From d903a4c4dd605c8040994fa8ba2cf3aa69b85f83 Mon Sep 17 00:00:00 2001 From: fzxiao233 Date: Mon, 23 Aug 2021 20:48:56 +0800 Subject: [PATCH] Rebase and simplify Signed-off-by: fzxiao233 --- config/config.go | 12 +-- live/monitor/monitorUtils.go | 4 +- .../downloader/provbase/downloader.go | 3 +- .../videoworker/downloader/stealth/stealth.go | 77 ------------------ live/videoworker/videoProcesser.go | 15 ++-- live/wrapper.go | 13 +-- main.go | 81 +++---------------- utils/file.go | 23 ------ utils/utils.go | 11 +-- 9 files changed, 32 insertions(+), 207 deletions(-) delete mode 100644 live/videoworker/downloader/stealth/stealth.go delete mode 100644 utils/file.go diff --git a/config/config.go b/config/config.go index 067614e..1238590 100644 --- a/config/config.go +++ b/config/config.go @@ -14,7 +14,7 @@ import ( ) var Config *MainConfig -var ConfigChanged bool +var Changed bool type UsersConfig struct { TargetId string @@ -41,7 +41,7 @@ type MainConfig struct { LogLevel string RLogLevel string DownloadQuality string - DownloadDir []string + DownloadDir string UploadDir string Module []ModuleConfig ExpressPort string @@ -74,17 +74,17 @@ func initConfig() { os.Exit(1) } - ConfigChanged = true + Changed = true v.OnConfigChange(func(in fsnotify.Event) { - ConfigChanged = true + Changed = true }) } func ReloadConfig() (bool, error) { - if !ConfigChanged { + if !Changed { return false, nil } - ConfigChanged = false + Changed = false err := v.ReadInConfig() if err != nil { return true, err diff --git a/live/monitor/monitorUtils.go b/live/monitor/monitorUtils.go index b932743..9133096 100644 --- a/live/monitor/monitorUtils.go +++ b/live/monitor/monitorUtils.go @@ -11,7 +11,7 @@ import ( ) type VideoMonitor = base.VideoMonitor -type LiveTrace func(monitor VideoMonitor) *interfaces.LiveStatus +type LiveTrace func() *interfaces.LiveStatus // Monitor is responsible for checking if live starts & live's title/link changed func CreateVideoMonitor(module config.ModuleConfig) VideoMonitor { @@ -33,7 +33,7 @@ func CreateVideoMonitor(module config.ModuleConfig) VideoMonitor { } // sanitize everything in the videoinfo for downloader & plugins -func CleanVideoInfo(info *interfaces.VideoInfo) *interfaces.VideoInfo { +func GetCleanVideoInfo(info *interfaces.VideoInfo) *interfaces.VideoInfo { info.Title = utils.RemoveIllegalChar(info.Title) return info } diff --git a/live/videoworker/downloader/provbase/downloader.go b/live/videoworker/downloader/provbase/downloader.go index c93f547..1a404a0 100644 --- a/live/videoworker/downloader/provbase/downloader.go +++ b/live/videoworker/downloader/provbase/downloader.go @@ -18,14 +18,13 @@ type Downloader struct { } func (d *Downloader) DownloadVideo(video *interfaces.VideoInfo, proxy string, cookie string, filePath string) string { - //rl.Take() logger := log.WithField("video", video) logger.Infof("start to download") video.FilePath = filePath err := d.Prov.StartDownload(video, proxy, cookie, filePath) logger.Infof("finished with status: %s", err) if !utils.IsFileExist(filePath) { - logger.Infof("%s the video file don't exist", filePath) + logger.Infof("download failed: %s", err) return "" } logger.Infof("%s download successfully", filePath) diff --git a/live/videoworker/downloader/stealth/stealth.go b/live/videoworker/downloader/stealth/stealth.go deleted file mode 100644 index cccbf5f..0000000 --- a/live/videoworker/downloader/stealth/stealth.go +++ /dev/null @@ -1,77 +0,0 @@ -package stealth - -import "strings" - -type URLRewriter interface { - Rewrite(url string) (newUrl string, useMain, useAlt int) - Callback(url string, err error) -} - -type BilibiliRewriter struct { - needTxyunRewrite bool -} - -func (u *BilibiliRewriter) Rewrite(url string) (newUrl string, useMain, useAlt int) { - //onlyAlt = false - useMain = 1 - useAlt = 1 - newUrl = url - // for gotcha105 & gotcha104, never use altproxy when downloading - if strings.Contains(url, "gotcha105") { - useAlt = 0 - } else if strings.Contains(url, "gotcha103") { - newUrl = strings.Replace(url, "https://d1--cn-gotcha103.bilivideo.com", "http://shnode.misty.moe:49980", 1) - newUrl = strings.Replace(url, "http://d1--cn-gotcha103.bilivideo.com", "http://shnode.misty.moe:49980", 1) - useAlt = 0 - } else if strings.Contains(url, "gotcha104") { - if u.needTxyunRewrite { - newUrl = strings.Replace(url, "https://d1--cn-gotcha104.bilivideo.com", "https://3hq4yf8r2xgz9.cfc-execute.su.baidubce.com", 1) - newUrl = strings.Replace(newUrl, "http://d1--cn-gotcha104.bilivideo.com", "https://3hq4yf8r2xgz9.cfc-execute.su.baidubce.com", 1) - useMain = 1 - useAlt = 0 - } else { - useMain = 0 - useAlt = 1 - } - } else if strings.Contains(url, "baidubce") { - useAlt = 2 - useMain = 1 - } - return -} - -func (u *BilibiliRewriter) Callback(url string, err error) { - if err != nil && strings.HasSuffix(err.Error(), "403") { - if strings.Contains(url, "gotcha104") { - u.needTxyunRewrite = true - } - } -} - -type RewriterWrap struct { - Rewriters []URLRewriter -} - -func (u *RewriterWrap) Rewrite(url string) (newUrl string, useMain, useAlt int) { - for _, rewriter := range u.Rewriters { - newUrl, useMain, useAlt = rewriter.Rewrite(url) - if newUrl != url || useMain != 1 || useAlt != 1 { - break - } - } - return -} - -func (u *RewriterWrap) Callback(url string, err error) { - for _, rewriter := range u.Rewriters { - rewriter.Callback(url, err) - } -} - -func GetRewriter() URLRewriter { - return &RewriterWrap{ - Rewriters: []URLRewriter{ - &BilibiliRewriter{}, - }, - } -} diff --git a/live/videoworker/videoProcesser.go b/live/videoworker/videoProcesser.go index 8946ced..15a0469 100644 --- a/live/videoworker/videoProcesser.go +++ b/live/videoworker/videoProcesser.go @@ -41,16 +41,15 @@ func init() { limit = rate.NewLimiter(rate.Every(time.Second*5), 1) } -func StartProcessVideo(LiveTrace monitor.LiveTrace, Monitor monitor.VideoMonitor, Plugins PluginManager) *ProcessVideo { +func StartProcessVideo(LiveTrace monitor.LiveTrace, Monitor monitor.VideoMonitor, Plugins PluginManager) { p := &ProcessVideo{LiveTrace: LiveTrace, Monitor: Monitor, Plugins: Plugins} - liveStatus := LiveTrace(Monitor) + liveStatus := LiveTrace() if liveStatus.IsLive { p.LiveStatus = liveStatus p.appendTitleHistory(p.LiveStatus.Video.Title) limit.Wait(context.Background()) p.StartProcessVideo() } - return p } func (p *ProcessVideo) getLogger() *log.Entry { @@ -82,10 +81,10 @@ func (p *ProcessVideo) StartProcessVideo() { func (p *ProcessVideo) prepareDownload() error { var pathSlice []string if !config.Config.EnableTS2MP4 { - pathSlice = []string{utils.RandChooseStr(config.Config.DownloadDir), p.LiveStatus.Video.UsersConfig.Name, + pathSlice = []string{config.Config.DownloadDir, p.LiveStatus.Video.UsersConfig.Name, p.liveStartTime.Format("20060102 150405")} } else { - pathSlice = []string{utils.RandChooseStr(config.Config.DownloadDir), p.LiveStatus.Video.UsersConfig.Name} + pathSlice = []string{config.Config.DownloadDir, p.LiveStatus.Video.UsersConfig.Name} } dirpath := strings.Join(pathSlice, "/") ret, err := utils.MakeDir(dirpath) @@ -203,7 +202,7 @@ func (p *ProcessVideo) keepLiveAlive() { p.needStop = true if p.isNeedDownload() { close(p.triggerChan) - return // 需要下载时不由此控制end + return } p.finish <- 1 return @@ -219,7 +218,7 @@ func (p *ProcessVideo) appendTitleHistory(title string) { } func (p *ProcessVideo) isNewLive() bool { - newLiveStatus := p.LiveTrace(p.Monitor) + newLiveStatus := p.LiveTrace() logger := p.getLogger() if newLiveStatus.IsLive == false || p.LiveStatus.IsLive == false { logger.Infof("[isNewLive] live offline") @@ -256,7 +255,7 @@ func (p *ProcessVideo) getFullTitle() string { } func (p *ProcessVideo) postProcessing() string { - pathSlice := []string{config.Config.UploadDir, p.LiveStatus.Video.UsersConfig.Name} // , p.getFullTitle() + pathSlice := []string{config.Config.DownloadDir, p.LiveStatus.Video.UsersConfig.Name} // , p.getFullTitle() dirpath := strings.Join(pathSlice, "/") _, err := utils.MakeDir(filepath.Dir(dirpath)) if err != nil { diff --git a/live/wrapper.go b/live/wrapper.go index 6439272..ae4b1e3 100644 --- a/live/wrapper.go +++ b/live/wrapper.go @@ -9,20 +9,13 @@ import ( ) func StartMonitor(mon base.VideoMonitor, usersConfig config.UsersConfig, pm videoworker.PluginManager) { - //ticker := time.NewTicker(time.Second * time.Duration(utils.Config.CheckSec)) - //for { - //pm.AddPlugin(&plugins.PluginTranslationRecorder{}) - //pm.AddPlugin(&plugins.PluginUploader{}) - - var fun = func(mon base.VideoMonitor) *interfaces.LiveStatus { + var liveTrace = func() *interfaces.LiveStatus { return &interfaces.LiveStatus{ IsLive: mon.CheckLive(usersConfig), - Video: monitor.CleanVideoInfo(mon.CreateVideo(usersConfig)), + Video: monitor.GetCleanVideoInfo(mon.CreateVideo(usersConfig)), } } - videoworker.StartProcessVideo(fun, mon, pm) + videoworker.StartProcessVideo(liveTrace, mon, pm) return - //<-ticker.C - //} } diff --git a/main.go b/main.go index b92ff2c..5c14cd6 100644 --- a/main.go +++ b/main.go @@ -7,20 +7,13 @@ import ( "github.com/fzxiao233/Vtb_Record/live/monitor" "github.com/fzxiao233/Vtb_Record/live/plugins" "github.com/fzxiao233/Vtb_Record/live/videoworker" - "github.com/fzxiao233/Vtb_Record/utils" log "github.com/sirupsen/logrus" "math/rand" - "net/http" _ "net/http/pprof" - "os" - "os/signal" "sync" - "syscall" "time" ) -var SafeStop bool - func initPluginManager() videoworker.PluginManager { pm := videoworker.PluginManager{} pm.AddPlugin(&plugins.PluginCQBot{}) @@ -38,17 +31,14 @@ func arrangeTask() { go func() { ticker := time.NewTicker(time.Second * time.Duration(1)) for { - if config.ConfigChanged { - allDone := true - if allDone { - time.Sleep(4 * time.Second) // wait to ensure the config is fully written - ret, err := config.ReloadConfig() - if ret { - if err == nil { - log.Infof("\n\n\t\tConfig changed and load successfully!\n\n") - } else { - log.Warnf("Config changed but loading failed: %s", err) - } + if config.Changed { + time.Sleep(4 * time.Second) // wait to ensure the config is fully written + ret, err := config.ReloadConfig() + if ret { + if err == nil { + log.Infof("\n\n\t\tConfig changed and load successfully!\n\n") + } else { + log.Warnf("Config changed but loading failed: %s", err) } } } @@ -56,9 +46,6 @@ func arrangeTask() { } }() - for _, dir := range config.Config.DownloadDir { - utils.MakeDir(dir) - } var statusMx sync.Mutex for { @@ -94,63 +81,15 @@ func arrangeTask() { log.Tracef("checked %s", changed) if time.Now().Minute() > 55 || time.Now().Minute() < 5 || (time.Now().Minute() > 25 && time.Now().Minute() < 35) { time.Sleep(time.Duration(config.Config.CriticalCheckSec) * time.Second) - } - time.Sleep(time.Duration(config.Config.NormalCheckSec) * time.Second) - - if SafeStop { - break + } else { + time.Sleep(time.Duration(config.Config.NormalCheckSec) * time.Second) } } - for { - living := make([]string, 0, 128) - statusMx.Lock() - for _, mod := range status { - for name, val := range mod { - if val { - living = append(living, name) - } - } - } - statusMx.Unlock() - if len(living) == 0 { - break - } - log.Infof("Waiting to finish: current living %s", living) - time.Sleep(time.Second * 5) - } - log.Infof("All tasks finished! Wait an additional time to ensure everything's saved") - time.Sleep(time.Second * 300) - log.Infof("Everything finished, exiting now~~") -} - -func handleInterrupt() { - c := make(chan os.Signal) - signal.Notify(c, os.Interrupt, syscall.SIGTERM) - go func() { - <-c - log.Warnf("Ctrl+C pressed in Terminal!") - time.Sleep(5 * time.Second) // wait rclone upload finish.. - os.Exit(0) - }() -} - -func handleUpdate() { - c := make(chan os.Signal) - SIGUSR1 := syscall.Signal(10) - signal.Notify(c, SIGUSR1) - go func() { - <-c - log.Warnf("Received update signal! Waiting everything done!") - SafeStop = true - }() } func main() { - handleInterrupt() - handleUpdate() rand.Seed(time.Now().UnixNano()) - http.DefaultClient.Transport = http.DefaultTransport config.PrepareConfig() config.InitLog() go config.InitProfiling() diff --git a/utils/file.go b/utils/file.go deleted file mode 100644 index 4993e13..0000000 --- a/utils/file.go +++ /dev/null @@ -1,23 +0,0 @@ -package utils - -import ( - "context" - _ "github.com/rclone/rclone/backend/all" - "github.com/rclone/rclone/cmd" - "github.com/rclone/rclone/fs/operations" - "github.com/rclone/rclone/fs/sync" -) - -func MkdirAll(path string) error { - fdst := cmd.NewFsDir([]string{path}) - err := operations.Mkdir(context.Background(), fdst, "") - return err -} - -func MoveFiles(src string, dst string) error { - fsrc, srcFileName, fdst := cmd.NewFsSrcFileDst([]string{src, dst}) - if srcFileName == "" { - return sync.MoveDir(context.Background(), fdst, fsrc, false, false) - } - return operations.MoveFile(context.Background(), fdst, fsrc, srcFileName, srcFileName) -} diff --git a/utils/utils.go b/utils/utils.go index 9a0170e..1f4e5ea 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "github.com/mitchellh/mapstructure" - log "github.com/sirupsen/logrus" "io" "math/rand" "net/http" @@ -148,14 +147,10 @@ func GenerateFilepath(DownDir string, VideoTitle string) string { return ChangeName(aFilepath) } func MakeDir(dirPath string) (ret string, err error) { - //if !IsFileExist(dirPath) { - if true { - //err := os.MkdirAll(dirPath, 0775) - err = MkdirAll(dirPath) + if !IsFileExist(dirPath) { + err := os.MkdirAll(dirPath, 0775) if err != nil { - log.Errorf("mkdir error: %s, err: %s", dirPath, err) - ret = "" - return + return "", err } return dirPath, nil }