Skip to content

Commit

Permalink
Use fs storage interface to access episodes
Browse files Browse the repository at this point in the history
  • Loading branch information
@mxpv
mxpv committed Mar 8, 2020
1 parent c6e602e commit 38f5c3c
Show file tree
Hide file tree
Showing 6 changed files with 297 additions and 138 deletions.
8 changes: 7 additions & 1 deletion cmd/podsync/main.go
View file
Expand Up @@ -16,6 +16,7 @@ import (


"github.com/mxpv/podsync/pkg/config" "github.com/mxpv/podsync/pkg/config"
"github.com/mxpv/podsync/pkg/db" "github.com/mxpv/podsync/pkg/db"
"github.com/mxpv/podsync/pkg/fs"
"github.com/mxpv/podsync/pkg/ytdl" "github.com/mxpv/podsync/pkg/ytdl"
) )


Expand Down Expand Up @@ -94,9 +95,14 @@ func main() {
log.WithError(err).Fatal("failed to open database") log.WithError(err).Fatal("failed to open database")
} }


storage, err := fs.NewLocal(cfg.Server.DataDir, cfg.Server.Hostname)
if err != nil {
log.WithError(err).Fatal("failed to open storage")
}

// Run updater thread // Run updater thread
log.Debug("creating updater") log.Debug("creating updater")
updater, err := NewUpdater(cfg, downloader, database) updater, err := NewUpdater(cfg, downloader, database, storage)
if err != nil { if err != nil {
log.WithError(err).Fatal("failed to create updater") log.WithError(err).Fatal("failed to create updater")
} }
Expand Down
128 changes: 41 additions & 87 deletions cmd/podsync/updater.go
View file
@@ -1,15 +1,13 @@
package main package main


import ( import (
"bytes"
"context" "context"
"fmt" "fmt"
"io" "io"
"io/ioutil"
"os" "os"
"path/filepath"
"regexp" "regexp"
"strconv" "strconv"
"strings"
"time" "time"


itunes "github.com/mxpv/podcast" itunes "github.com/mxpv/podcast"
Expand All @@ -19,6 +17,7 @@ import (
"github.com/mxpv/podsync/pkg/config" "github.com/mxpv/podsync/pkg/config"
"github.com/mxpv/podsync/pkg/db" "github.com/mxpv/podsync/pkg/db"
"github.com/mxpv/podsync/pkg/feed" "github.com/mxpv/podsync/pkg/feed"
"github.com/mxpv/podsync/pkg/fs"
"github.com/mxpv/podsync/pkg/link" "github.com/mxpv/podsync/pkg/link"
"github.com/mxpv/podsync/pkg/model" "github.com/mxpv/podsync/pkg/model"
"github.com/mxpv/podsync/pkg/ytdl" "github.com/mxpv/podsync/pkg/ytdl"
Expand All @@ -32,13 +31,15 @@ type Updater struct {
config *config.Config config *config.Config
downloader Downloader downloader Downloader
db db.Storage db db.Storage
fs fs.Storage
} }


func NewUpdater(config *config.Config, downloader Downloader, db db.Storage) (*Updater, error) { func NewUpdater(config *config.Config, downloader Downloader, db db.Storage, fs fs.Storage) (*Updater, error) {
return &Updater{ return &Updater{
config: config, config: config,
downloader: downloader, downloader: downloader,
db: db, db: db,
fs: fs,
}, nil }, nil
} }


Expand All @@ -50,18 +51,11 @@ func (u *Updater) Update(ctx context.Context, feedConfig *config.Feed) error {
}).Infof("-> updating %s", feedConfig.URL) }).Infof("-> updating %s", feedConfig.URL)
started := time.Now() started := time.Now()


// Make sure feed directory exists
feedPath := filepath.Join(u.config.Server.DataDir, feedConfig.ID)
log.Debugf("creating directory for feed %q", feedPath)
if err := os.MkdirAll(feedPath, 0755); err != nil {
return errors.Wrapf(err, "failed to create directory for feed %q", feedConfig.ID)
}

if err := u.updateFeed(ctx, feedConfig); err != nil { if err := u.updateFeed(ctx, feedConfig); err != nil {
return err return err
} }


if err := u.downloadEpisodes(ctx, feedConfig, feedPath); err != nil { if err := u.downloadEpisodes(ctx, feedConfig); err != nil {
return err return err
} }


Expand Down Expand Up @@ -100,7 +94,7 @@ func (u *Updater) updateFeed(ctx context.Context, feedConfig *config.Feed) error
return nil return nil
} }


func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed, targetDir string) error { func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed) error {
var ( var (
feedID = feedConfig.ID feedID = feedConfig.ID
downloadList []*model.Episode downloadList []*model.Episode
Expand Down Expand Up @@ -146,21 +140,19 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed,
// Download pending episodes // Download pending episodes


for idx, episode := range downloadList { for idx, episode := range downloadList {
logger := log.WithFields(log.Fields{ var (
"index": idx, logger = log.WithFields(log.Fields{"index": idx, "episode_id": episode.ID})
"episode_id": episode.ID, episodeName = u.episodeName(feedConfig, episode)
}) )

// Check whether episode exists on disk


episodePath := filepath.Join(targetDir, u.episodeName(feedConfig, episode)) // Check whether episode already exists
stat, err := os.Stat(episodePath) size, err := u.fs.Size(ctx, feedID, episodeName)
if err == nil { if err == nil {
logger.Infof("episode %q already exists on disk (%s)", episode.ID, episodePath) logger.Infof("episode %q already exists on disk", episode.ID)


// File already exists, update file status and disk size // File already exists, update file status and disk size
if err := u.db.UpdateEpisode(feedID, episode.ID, func(episode *model.Episode) error { if err := u.db.UpdateEpisode(feedID, episode.ID, func(episode *model.Episode) error {
episode.Size = stat.Size() episode.Size = size
episode.Status = model.EpisodeDownloaded episode.Status = model.EpisodeDownloaded
return nil return nil
}); err != nil { }); err != nil {
Expand Down Expand Up @@ -200,16 +192,14 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed,
continue continue
} }


logger.Debugf("copying file to: %s", episodePath) logger.Debug("copying file")
fileSize, err := copyFile(tempFile, episodePath) fileSize, err := u.fs.Create(ctx, feedID, episodeName, tempFile)

tempFile.Close() tempFile.Close()

if err != nil { if err != nil {
logger.WithError(err).Error("failed to copy file") logger.WithError(err).Error("failed to copy file")
return err return err
} }
logger.Debugf("copied %d bytes", episode.Size) logger.Debugf("copied %d bytes", fileSize)


// Update file status in database // Update file status in database


Expand All @@ -229,22 +219,6 @@ func (u *Updater) downloadEpisodes(ctx context.Context, feedConfig *config.Feed,
return nil return nil
} }


func copyFile(source io.Reader, destinationPath string) (int64, error) {
dest, err := os.Create(destinationPath)
if err != nil {
return 0, errors.Wrap(err, "failed to create destination file")
}

defer dest.Close()

written, err := io.Copy(dest, source)
if err != nil {
return 0, errors.Wrap(err, "failed to copy data")
}

return written, nil
}

func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error { func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error {
feed, err := u.db.GetFeed(ctx, feedConfig.ID) feed, err := u.db.GetFeed(ctx, feedConfig.ID)
if err != nil { if err != nil {
Expand All @@ -253,23 +227,24 @@ func (u *Updater) buildXML(ctx context.Context, feedConfig *config.Feed) error {


// Build iTunes XML feed with data received from builder // Build iTunes XML feed with data received from builder
log.Debug("building iTunes podcast feed") log.Debug("building iTunes podcast feed")
podcast, err := u.buildPodcast(feed, feedConfig) podcast, err := u.buildPodcast(ctx, feed, feedConfig)
if err != nil { if err != nil {
return err return err
} }


// Save XML to disk var (
xmlName := fmt.Sprintf("%s.xml", feedConfig.ID) reader = bytes.NewReader([]byte(podcast.String()))
xmlPath := filepath.Join(u.config.Server.DataDir, xmlName) xmlName = fmt.Sprintf("%s.xml", feedConfig.ID)
log.Debugf("saving feed XML file to %s", xmlPath) )
if err := ioutil.WriteFile(xmlPath, []byte(podcast.String()), 0600); err != nil {
return errors.Wrapf(err, "failed to write XML feed to disk") if _, err := u.fs.Create(ctx, "", xmlName, reader); err != nil {
return errors.Wrap(err, "failed to upload new XML feed")
} }


return nil return nil
} }


func (u *Updater) buildPodcast(feed *model.Feed, cfg *config.Feed) (*itunes.Podcast, error) { func (u *Updater) buildPodcast(ctx context.Context, feed *model.Feed, cfg *config.Feed) (*itunes.Podcast, error) {
const ( const (
podsyncGenerator = "Podsync generator (support us at https://github.com/mxpv/podsync)" podsyncGenerator = "Podsync generator (support us at https://github.com/mxpv/podsync)"
defaultCategory = "TV & Film" defaultCategory = "TV & Film"
Expand Down Expand Up @@ -320,7 +295,19 @@ func (u *Updater) buildPodcast(feed *model.Feed, cfg *config.Feed) (*itunes.Podc
item.AddSummary(episode.Description) item.AddSummary(episode.Description)
item.AddImage(episode.Thumbnail) item.AddImage(episode.Thumbnail)
item.AddDuration(episode.Duration) item.AddDuration(episode.Duration)
item.AddEnclosure(u.makeEnclosure(feed, episode, cfg))
enclosureType := itunes.MP4
if feed.Format == model.FormatAudio {
enclosureType = itunes.MP4
}

episodeName := u.episodeName(cfg, episode)
downloadURL, err := u.fs.URL(ctx, cfg.ID, episodeName)
if err != nil {
return nil, errors.Wrapf(err, "failed to obtain download URL for: %s", episodeName)
}

item.AddEnclosure(downloadURL, enclosureType, episode.Size)


// p.AddItem requires description to be not empty, use workaround // p.AddItem requires description to be not empty, use workaround
if item.Description == "" { if item.Description == "" {
Expand All @@ -333,47 +320,14 @@ func (u *Updater) buildPodcast(feed *model.Feed, cfg *config.Feed) (*itunes.Podc
item.IExplicit = "no" item.IExplicit = "no"
} }


_, err := p.AddItem(item) if _, err := p.AddItem(item); err != nil {
if err != nil {
return nil, errors.Wrapf(err, "failed to add item to podcast (id %q)", episode.ID) return nil, errors.Wrapf(err, "failed to add item to podcast (id %q)", episode.ID)
} }
} }


return &p, nil return &p, nil
} }


func (u *Updater) makeEnclosure(
feed *model.Feed,
episode *model.Episode,
cfg *config.Feed,
) (string, itunes.EnclosureType, int64) {
ext := "mp4"
contentType := itunes.MP4
if feed.Format == model.FormatAudio {
ext = "mp3"
contentType = itunes.MP3
}

url := fmt.Sprintf(
"%s/%s/%s.%s",
u.hostname(),
cfg.ID,
episode.ID,
ext,
)

return url, contentType, episode.Size
}

func (u *Updater) hostname() string {
hostname := strings.TrimSuffix(u.config.Server.Hostname, "/")
if !strings.HasPrefix(hostname, "http") {
hostname = fmt.Sprintf("http://%s", hostname)
}

return hostname
}

func (u *Updater) episodeName(feedConfig *config.Feed, episode *model.Episode) string { func (u *Updater) episodeName(feedConfig *config.Feed, episode *model.Episode) string {
ext := "mp4" ext := "mp4"
if feedConfig.Format == model.FormatAudio { if feedConfig.Format == model.FormatAudio {
Expand Down
50 changes: 0 additions & 50 deletions cmd/podsync/updater_test.go
View file

This file was deleted.

0 comments on commit 38f5c3c

Please sign in to comment.