Skip to content
Permalink
Browse files

Refactor ytdl

  • Loading branch information
mxpv committed Mar 7, 2020
1 parent a6c21e6 commit c6e602e5f6d08bbc6635a9a6cbe951f8e904a90e
@@ -3,6 +3,7 @@ package main
import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
@@ -20,10 +21,11 @@ import (
"github.com/mxpv/podsync/pkg/feed"
"github.com/mxpv/podsync/pkg/link"
"github.com/mxpv/podsync/pkg/model"
"github.com/mxpv/podsync/pkg/ytdl"
)

type Downloader interface {
Download(ctx context.Context, feedConfig *config.Feed, episode *model.Episode, feedPath string) (string, error)
Download(ctx context.Context, feedConfig *config.Feed, episode *model.Episode) (io.ReadCloser, error)
}

type Updater struct {
@@ -175,16 +177,16 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed,
}

// Download episode to disk
// We download the episode to a temp directory first to avoid downloading this file by clients
// while still being processed by youtube-dl (e.g. a file is being downloaded from YT or encoding in progress)

logger.Infof("! downloading episode %s", episode.VideoURL)
output, err := u.downloader.Download(ctx, feedConfig, episode, episodePath)
tempFile, err := u.downloader.Download(ctx, feedConfig, episode)
if err != nil {
logger.WithError(err).Errorf("youtube-dl error: %s", output)

// YouTube might block host with HTTP Error 429: Too Many Requests
// We still need to generate XML, so just stop sending download requests and
// retry next time
if strings.Contains(output, "HTTP Error 429") {
if err == ytdl.ErrTooManyRequests {
break
}

@@ -198,20 +200,22 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed,
continue
}

logger.Debugf("copying file to: %s", episodePath)
fileSize, err := copyFile(tempFile, episodePath)

tempFile.Close()

if err != nil {
logger.WithError(err).Error("failed to copy file")
return err
}
logger.Debugf("copied %d bytes", episode.Size)

// Update file status in database

logger.Infof("successfully downloaded file %q", episode.ID)

if err := u.db.UpdateEpisode(feedID, episode.ID, func(episode *model.Episode) error {
// Record file size of newly downloaded file
size, err := u.fileSize(episodePath)
if err != nil {
logger.WithError(err).Error("failed to get episode file size")
} else {
logger.Debugf("file size: %d bytes", episode.Size)
episode.Size = size
}

episode.Size = fileSize
episode.Status = model.EpisodeDownloaded
return nil
}); err != nil {
@@ -225,6 +229,22 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed,
return nil
}

func copyFile(source io.Reader, destinationPath string) (int64, error) {
dest, err := os.Create(destinationPath)
if err != nil {
return 0, errors.Wrap(err, "failed to create destination file")
}

defer dest.Close()

written, err := io.Copy(dest, source)
if err != nil {
return 0, errors.Wrap(err, "failed to copy data")
}

return written, nil
}

func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error {
feed, err := u.db.GetFeed(ctx, feedConfig.ID)
if err != nil {
@@ -363,15 +383,6 @@ func (u *Updater) episodeName(feedConfig *config.Feed, episode *model.Episode) s
return fmt.Sprintf("%s.%s", episode.ID, ext)
}

func (u *Updater) fileSize(path string) (int64, error) {
info, err := os.Stat(path)
if err != nil {
return 0, err
}

return info.Size(), nil
}

func (u *Updater) makeBuilder(ctx context.Context, cfg *config.Feed) (feed.Builder, error) {
var (
provider feed.Builder
@@ -1,9 +1,14 @@
package main

import (
"bytes"
"io/ioutil"
"os"
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/mxpv/podsync/pkg/config"
)
@@ -24,3 +29,22 @@ func TestUpdater_hostname(t *testing.T) {
u.config.Server.Hostname = "https://localhost:8080/"
assert.Equal(t, "https://localhost:8080", u.hostname())
}

func TestCopyFile(t *testing.T) {
reader := bytes.NewReader([]byte{1, 2, 4})

tmpDir, err := ioutil.TempDir("", "podsync-test-")
require.NoError(t, err)

defer os.RemoveAll(tmpDir)

file := filepath.Join(tmpDir, "1")

size, err := copyFile(reader, file)
assert.NoError(t, err)
assert.EqualValues(t, 3, size)

stat, err := os.Stat(file)
assert.NoError(t, err)
assert.EqualValues(t, 3, stat.Size())
}

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

0 comments on commit c6e602e

Please sign in to comment.
You can’t perform that action at this time.