feat: Better queue fetching asset (handle cache) (#7)

* feat: enhance localResources to handle caching

* Fix: cyclo error in fetchAssets

* fix: call to FindSourceClip

* wip: handle cache (almost finished)

* feat: remote local resource if fetched from external

README.md

@@ -54,7 +54,7 @@ At the end of the road this section should either disappear or be full of `Yes`
 | Timeline | background | Yes ✅ |  |
 | Timeline | fonts | Not yet |  |
 | Timeline | tracks | Yes ✅ |  |
-| Timeline | cache | Not yet |  |
+| Timeline | cache | Yes ✅ |  |
 | Track ✅ | all ✅ | Yes ✅ |  |
 | Clip | asset | Partial 🛠 | Only `VideoAsset` are started |
 | Clip | start | Yes ✅ |  |

go/ffmpeg.go

@@ -470,24 +470,24 @@ func (s *FFMPEG) generateOutputName() string {
 	return file.Name()
 }
 
-func (s *FFMPEG) ToFFMPEG(queue *RenderQueue) error {
+func (s *FFMPEG) ToFFMPEG(renderQueue *RenderQueue, queue *ProcessingQueue) error {
 	_ = s.AddDefaultParams()
-	_ = s.SetOutputFormat(queue.Data.Output.Format)
-	if queue.Data.Output.Fps != nil {
-		_ = s.SetOutputFps(*queue.Data.Output.Fps)
+	_ = s.SetOutputFormat(renderQueue.Data.Output.Format)
+	if renderQueue.Data.Output.Fps != nil {
+		_ = s.SetOutputFps(*renderQueue.Data.Output.Fps)
 	}
-	_ = s.SetDefaultBackground(queue.Data.Timeline.Background)
+	_ = s.SetDefaultBackground(renderQueue.Data.Timeline.Background)
 
 	// Handle Sources
 	var sourceClip = 0
 	s.fillerCounter = 0
 
-	for trackNumber, track := range queue.Data.Timeline.Tracks {
+	for trackNumber, track := range renderQueue.Data.Timeline.Tracks {
 		var lastStart float32
 		var clipNumber = 0
 
 		_ = s.AddTrack(trackNumber)
-		for _, clip := range track.Clips {
+		for iClip, clip := range track.Clips {
 			// for cIndex, clip := range track.Clips {
 			// fmt.Println(cIndex)
 
@@ -501,7 +501,11 @@ func (s *FFMPEG) ToFFMPEG(queue *RenderQueue) error {
 				clipNumber = clipNumber + 1
 			}
 			// fmt.Println(sourceClip, s.fillerCounter)
-			_ = s.AddSource(queue.LocalResources[sourceClip])
+
+			sourceFileName := queue.FindSourceClip(trackNumber, iClip)
+			if sourceFileName != "" {
+				_ = s.AddSource(sourceFileName)
+			}
 
 			_ = clip.ToFFMPEG(s, sourceClip, trackNumber, clipNumber)
 

go/model_ffmpeg.go

@@ -45,7 +45,7 @@ type FFMPEGCommand interface {
 	GetResolution() string
 	GenerateFiller(string) string
 	GenerateBackground() string
-	ToFFMPEG(*RenderQueue) error
+	ToFFMPEG(*RenderQueue, *ProcessingQueue) error
 	GetOutputName() string
 	GetDuration() float32
 }

go/model_localresource.go

@@ -0,0 +1,34 @@
+/*
+shottower
+Copyright (C) 2022 Rémy Boulanouar
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+package openapi
+
+import "time"
+
+type LocalResource struct {
+	Downloaded       time.Time `json:"downloaded"`
+	OriginalURL      string
+	LocalURL         string
+	KeepCache        bool
+	IsRemoteResource bool
+	Used             []*LocalResourceTrackInfo
+}
+
+type LocalResourceTrackInfo struct {
+	Track int
+	Clip  int
+}

go/model_render_queue.go

@@ -31,6 +31,5 @@ type RenderQueue struct {
 	FileName string
 
 	InternalStatus RenderResponseStatus
-	LocalResources []string
 	FFMPEGCommand  FFMPEGCommand
 }

go/model_timeline.go

@@ -26,7 +26,9 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package openapi
 
-import "github.com/creasty/defaults"
+import (
+	"github.com/creasty/defaults"
+)
 
 // Timeline - A timeline represents the contents of a video edit over time, an audio edit over time, in seconds, or an image layout. A timeline consists of layers called tracks. Tracks are composed of titles, images, audio, html or video segments referred to as clips which are placed along the track at specific starting point and lasting for a specific amount of time.
 type Timeline struct {
@@ -42,7 +44,7 @@ type Timeline struct {
 	Tracks []Track `json:"tracks"`
 
 	// Disable the caching of ingested source footage and assets. See  [caching](https://shotstack.io/docs/guide/architecting-an-application/caching) for more details.
-	Cache bool `json:"cache,omitempty"`
+	Cache bool `json:"cache,omitempty" default:"true"`
 }
 
 // AssertTimelineRequired checks if the required fields are not zero-ed

go/queue.go

@@ -25,17 +25,23 @@ import (
 	"log"
 	"net/http"
 	"net/url"
+	"os"
 	"os/exec"
 	"path/filepath"
 	"time"
 )
 
 type ProcessingQueue struct {
 	currentQueue *RenderQueue
+
+	LocalResources map[string]*LocalResource
 }
 
 func NewProcessingQueuer() ProcessingQueuer {
-	return &ProcessingQueue{}
+	processingQueue := &ProcessingQueue{}
+	processingQueue.LocalResources = make(map[string]*LocalResource)
+
+	return processingQueue
 }
 
 func (s *ProcessingQueue) StartProcessQueue(editAPI EditAPIServicer) {
@@ -44,6 +50,38 @@ func (s *ProcessingQueue) StartProcessQueue(editAPI EditAPIServicer) {
 	go time.AfterFunc(1*time.Second, func() {
 		s.ProcessQueue(editAPI)
 	})
+
+	go time.AfterFunc(1*time.Hour, func() {
+		s.CleanCache()
+	})
+}
+
+func (s *ProcessingQueue) CleanCache() {
+	for _, resource := range s.LocalResources {
+		if !resource.KeepCache {
+			// Remove local asset only if remote resource
+			if resource.IsRemoteResource {
+				_ = os.Remove(resource.LocalURL)
+			}
+
+			s.LocalResources[resource.OriginalURL] = nil
+		}
+	}
+
+	go time.AfterFunc(1*time.Hour, func() {
+		s.CleanCache()
+	})
+}
+
+func (s *ProcessingQueue) FindSourceClip(trackNumber int, clipNumber int) string {
+	for _, resource := range s.LocalResources {
+		for _, used := range resource.Used {
+			if used.Track == trackNumber && used.Clip == clipNumber {
+				return resource.LocalURL
+			}
+		}
+	}
+	return ""
 }
 
 func (s *ProcessingQueue) ProcessQueue(editAPI EditAPIServicer) {
@@ -138,50 +176,76 @@ func (s *ProcessingQueue) GenerateParameters(queue *RenderQueue) []string {
 	queue.Status = Rendering
 	queue.InternalStatus = Generating
 
-	_ = queue.FFMPEGCommand.ToFFMPEG(queue)
+	_ = queue.FFMPEGCommand.ToFFMPEG(queue, s)
 
 	queue.InternalStatus = Generated
 
 	return queue.FFMPEGCommand.ToString()
 }
 
+func (s *ProcessingQueue) FetchVideoAssets(trackNumber int, clipNumber int, clip Clip, useCache bool) bool {
+	var hasError bool
+
+	var asset = clip.Asset.(*VideoAsset)
+
+	if s.LocalResources[asset.Src] == nil {
+		var fileName string
+		var remote bool
+		url, _ := url.Parse(asset.Src)
+
+		if url.Scheme == "file" {
+			fileName = asset.Src[7:]
+		} else {
+			var err error
+			fileName, err = s.DownloadFile(asset.Src)
+			if err != nil {
+				fmt.Println("Error while downloading asset", err)
+				hasError = true
+			}
+			remote = true
+		}
+
+		if !hasError {
+			fmt.Println("Asset downloaded: "+asset.Src, fileName)
+			localResource := &LocalResource{
+				Downloaded:       time.Now(),
+				OriginalURL:      asset.Src,
+				LocalURL:         fileName,
+				KeepCache:        useCache,
+				IsRemoteResource: remote,
+			}
+			s.LocalResources[asset.Src] = localResource
+		}
+	}
+
+	if !hasError {
+		s.LocalResources[asset.Src].Used = append(
+			s.LocalResources[asset.Src].Used,
+			&LocalResourceTrackInfo{
+				Track: trackNumber,
+				Clip:  clipNumber,
+			})
+	}
+
+	return hasError
+}
+
 func (s *ProcessingQueue) FetchAssets(queue *RenderQueue) {
 	queue.Status = Fetching
 	queue.InternalStatus = Fetching
 
 	var hasError bool
-	var assetFiles = make(map[string]string)
 
-	for _, track := range queue.Data.Timeline.Tracks {
-		for _, clip := range track.Clips {
+	useCache := queue.Data.Timeline.Cache
+
+	for tIndex, track := range queue.Data.Timeline.Tracks {
+		for cIndex, clip := range track.Clips {
 			// fmt.Println(tIndex, cIndex, clip.Asset.Type)
 
 			var typeAsset = GetAssetType(clip.Asset)
 			switch typeAsset { // nolint:exhaustive
 			case VideoAssetType:
-				var asset = clip.Asset.(*VideoAsset)
-				var fileName = assetFiles[asset.Src]
-
-				if fileName == "" {
-					url, _ := url.Parse(asset.Src)
-
-					if url.Scheme == "file" {
-						fileName = asset.Src[7:]
-					} else {
-						var err error
-						fileName, err = s.DownloadFile(asset.Src)
-						if err != nil {
-							fmt.Println("Error while downloading asset", err)
-							hasError = true
-						}
-					}
-				}
-
-				if !hasError {
-					fmt.Println("Asset downloaded: "+asset.Src, fileName)
-					queue.LocalResources = append(queue.LocalResources, fileName)
-					assetFiles[asset.Src] = fileName
-				}
+				hasError = s.FetchVideoAssets(tIndex, cIndex, clip, useCache)
 
 			// case "image":
 			// 	fmt.Println("Image")