get upload time from https://www.invid-project.eu/tools-and-services/…

…invid-verification-plugin/

downloader/downloader.go

@@ -4,12 +4,16 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/lbryio/ytsync/v5/downloader/ytdl"
 
 	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/util"
+
 	"github.com/sirupsen/logrus"
 )
 
@@ -40,9 +44,76 @@ func GetVideoInformation(videoID string) (*ytdl.YtdlVideo, error) {
 	if err != nil {
 		return nil, errors.Err(err)
 	}
+
+	// now get an accurate time
+	tries := 0
+GetTime:
+	tries++
+	t, err := getUploadTime(videoID)
+	if err != nil {
+		if errors.Is(err, errNotScraped) && tries <= 3 {
+			triggerScrape(videoID)
+			time.Sleep(2 * time.Second) // let them scrape it
+			goto GetTime
+		}
+		//return video, errors.Err(err) // just swallow this error and do fallback below
+	}
+
+	if t != "" {
+		parsed, err := time.Parse("2006-01-02, 15:04:05 (MST)", t) // this will probably be UTC, but Go's timezone parsing is fucked up. it ignores the timezone in the date
+		if err != nil {
+			return nil, errors.Err(err)
+		}
+		video.UploadDateForReal = parsed
+	} else {
+		_ = util.SendToSlack(":warning: Could not get accurate time for %s. Falling back to estimated time.", videoID)
+		// fall back to UploadDate from youtube-dl
+		video.UploadDateForReal, err = time.Parse("20060102", video.UploadDate)
+		if err != nil {
+			return nil, err
+		}
+	}
+
 	return video, nil
 }
 
+var errNotScraped = errors.Base("not yet scraped by caa.iti.gr")
+
+func triggerScrape(videoID string) error {
+	res, err := http.Get("https://caa.iti.gr/verify_videoV3?twtimeline=0&url=https://www.youtube.com/watch?v=" + videoID)
+	if err != nil {
+		return errors.Err(err)
+	}
+	defer res.Body.Close()
+
+	return nil
+	//https://caa.iti.gr/caa/api/v4/videos/reports/h-tuxHS5lSM
+}
+
+func getUploadTime(videoID string) (string, error) {
+	res, err := http.Get("https://caa.iti.gr/get_verificationV3?url=https://www.youtube.com/watch?v=" + videoID)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+	defer res.Body.Close()
+
+	var uploadTime struct {
+		Time    string `json:"video_upload_time"`
+		Message string `json:"message"`
+		Status  string `json:"status"`
+	}
+	err = json.NewDecoder(res.Body).Decode(&uploadTime)
+	if err != nil {
+		return "", errors.Err(err)
+	}
+
+	if uploadTime.Status == "ERROR1" {
+		return "", errNotScraped
+	}
+
+	return uploadTime.Time, nil
+}
+
 func run(args []string, withStdErr, withStdOut bool) ([]string, error) {
 	cmd := exec.Command("youtube-dl", args...)
 	logrus.Printf("Running command youtube-dl %s", strings.Join(args, " "))

downloader/ytdl/Video.go

@@ -1,7 +1,12 @@
 package ytdl
 
+import (
+	"time"
+)
+
 type YtdlVideo struct {
 	UploadDate         string      `json:"upload_date"`
+	UploadDateForReal  time.Time   // you need to manually set this since the value in the API doesn't include the time
 	Extractor          string      `json:"extractor"`
 	Series             interface{} `json:"series"`
 	Format             string      `json:"format"`

sources/youtubeVideo.go

@@ -92,16 +92,12 @@ var youtubeCategories = map[string]string{
 func NewYoutubeVideo(directory string, videoData *ytdl.YtdlVideo, playlistPosition int64, awsConfig aws.Config, stopGroup *stop.Group, pool *ip_manager.IPPool) (*YoutubeVideo, error) {
 	// youtube-dl returns times in local timezone sometimes. this could break in the future
 	// maybe we can file a PR to choose the timezone we want from youtube-dl
-	publishedAt, err := time.ParseInLocation("20060102", videoData.UploadDate, time.Local)
-	if err != nil {
-		return nil, errors.Err(err)
-	}
 	return &YoutubeVideo{
 		id:               videoData.ID,
 		title:            videoData.Title,
 		description:      videoData.Description,
 		playlistPosition: playlistPosition,
-		publishedAt:      publishedAt,
+		publishedAt:      videoData.UploadDateForReal,
 		dir:              directory,
 		youtubeInfo:      videoData,
 		awsConfig:        awsConfig,