From 1e09bef17e7c03b69b52ee33ec97449ca9323b22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Boulanouar?= Date: Mon, 1 Aug 2022 16:08:16 +0200 Subject: [PATCH] feat: Better queue fetching asset (handle cache) (#7) * feat: enhance localResources to handle caching * Fix: cyclo error in fetchAssets * fix: call to FindSourceClip * wip: handle cache (almost finished) * feat: remote local resource if fetched from external --- README.md | 2 +- go/ffmpeg.go | 20 ++++--- go/model_ffmpeg.go | 2 +- go/model_localresource.go | 34 +++++++++++ go/model_render_queue.go | 1 - go/model_timeline.go | 6 +- go/queue.go | 120 +++++++++++++++++++++++++++++--------- 7 files changed, 144 insertions(+), 41 deletions(-) create mode 100644 go/model_localresource.go diff --git a/README.md b/README.md index 4ab791b..e1f4a8a 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ At the end of the road this section should either disappear or be full of `Yes` | Timeline | background | Yes ✅ | | | Timeline | fonts | Not yet | | | Timeline | tracks | Yes ✅ | | -| Timeline | cache | Not yet | | +| Timeline | cache | Yes ✅ | | | Track ✅ | all ✅ | Yes ✅ | | | Clip | asset | Partial 🛠 | Only `VideoAsset` are started | | Clip | start | Yes ✅ | | diff --git a/go/ffmpeg.go b/go/ffmpeg.go index df04415..4f2e836 100644 --- a/go/ffmpeg.go +++ b/go/ffmpeg.go @@ -470,24 +470,24 @@ func (s *FFMPEG) generateOutputName() string { return file.Name() } -func (s *FFMPEG) ToFFMPEG(queue *RenderQueue) error { +func (s *FFMPEG) ToFFMPEG(renderQueue *RenderQueue, queue *ProcessingQueue) error { _ = s.AddDefaultParams() - _ = s.SetOutputFormat(queue.Data.Output.Format) - if queue.Data.Output.Fps != nil { - _ = s.SetOutputFps(*queue.Data.Output.Fps) + _ = s.SetOutputFormat(renderQueue.Data.Output.Format) + if renderQueue.Data.Output.Fps != nil { + _ = s.SetOutputFps(*renderQueue.Data.Output.Fps) } - _ = s.SetDefaultBackground(queue.Data.Timeline.Background) + _ = s.SetDefaultBackground(renderQueue.Data.Timeline.Background) // Handle Sources var sourceClip = 0 s.fillerCounter = 0 - for trackNumber, track := range queue.Data.Timeline.Tracks { + for trackNumber, track := range renderQueue.Data.Timeline.Tracks { var lastStart float32 var clipNumber = 0 _ = s.AddTrack(trackNumber) - for _, clip := range track.Clips { + for iClip, clip := range track.Clips { // for cIndex, clip := range track.Clips { // fmt.Println(cIndex) @@ -501,7 +501,11 @@ func (s *FFMPEG) ToFFMPEG(queue *RenderQueue) error { clipNumber = clipNumber + 1 } // fmt.Println(sourceClip, s.fillerCounter) - _ = s.AddSource(queue.LocalResources[sourceClip]) + + sourceFileName := queue.FindSourceClip(trackNumber, iClip) + if sourceFileName != "" { + _ = s.AddSource(sourceFileName) + } _ = clip.ToFFMPEG(s, sourceClip, trackNumber, clipNumber) diff --git a/go/model_ffmpeg.go b/go/model_ffmpeg.go index 75d1b52..8c0cf27 100644 --- a/go/model_ffmpeg.go +++ b/go/model_ffmpeg.go @@ -45,7 +45,7 @@ type FFMPEGCommand interface { GetResolution() string GenerateFiller(string) string GenerateBackground() string - ToFFMPEG(*RenderQueue) error + ToFFMPEG(*RenderQueue, *ProcessingQueue) error GetOutputName() string GetDuration() float32 } diff --git a/go/model_localresource.go b/go/model_localresource.go new file mode 100644 index 0000000..8d21585 --- /dev/null +++ b/go/model_localresource.go @@ -0,0 +1,34 @@ +/* +shottower +Copyright (C) 2022 Rémy Boulanouar + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +package openapi + +import "time" + +type LocalResource struct { + Downloaded time.Time `json:"downloaded"` + OriginalURL string + LocalURL string + KeepCache bool + IsRemoteResource bool + Used []*LocalResourceTrackInfo +} + +type LocalResourceTrackInfo struct { + Track int + Clip int +} diff --git a/go/model_render_queue.go b/go/model_render_queue.go index 34b0a92..26d875d 100644 --- a/go/model_render_queue.go +++ b/go/model_render_queue.go @@ -31,6 +31,5 @@ type RenderQueue struct { FileName string InternalStatus RenderResponseStatus - LocalResources []string FFMPEGCommand FFMPEGCommand } diff --git a/go/model_timeline.go b/go/model_timeline.go index 6ff8298..8dfc200 100644 --- a/go/model_timeline.go +++ b/go/model_timeline.go @@ -26,7 +26,9 @@ along with this program. If not, see . package openapi -import "github.com/creasty/defaults" +import ( + "github.com/creasty/defaults" +) // Timeline - A timeline represents the contents of a video edit over time, an audio edit over time, in seconds, or an image layout. A timeline consists of layers called tracks. Tracks are composed of titles, images, audio, html or video segments referred to as clips which are placed along the track at specific starting point and lasting for a specific amount of time. type Timeline struct { @@ -42,7 +44,7 @@ type Timeline struct { Tracks []Track `json:"tracks"` // Disable the caching of ingested source footage and assets. See [caching](https://shotstack.io/docs/guide/architecting-an-application/caching) for more details. - Cache bool `json:"cache,omitempty"` + Cache bool `json:"cache,omitempty" default:"true"` } // AssertTimelineRequired checks if the required fields are not zero-ed diff --git a/go/queue.go b/go/queue.go index 7c827dd..a334e6f 100644 --- a/go/queue.go +++ b/go/queue.go @@ -25,6 +25,7 @@ import ( "log" "net/http" "net/url" + "os" "os/exec" "path/filepath" "time" @@ -32,10 +33,15 @@ import ( type ProcessingQueue struct { currentQueue *RenderQueue + + LocalResources map[string]*LocalResource } func NewProcessingQueuer() ProcessingQueuer { - return &ProcessingQueue{} + processingQueue := &ProcessingQueue{} + processingQueue.LocalResources = make(map[string]*LocalResource) + + return processingQueue } func (s *ProcessingQueue) StartProcessQueue(editAPI EditAPIServicer) { @@ -44,6 +50,38 @@ func (s *ProcessingQueue) StartProcessQueue(editAPI EditAPIServicer) { go time.AfterFunc(1*time.Second, func() { s.ProcessQueue(editAPI) }) + + go time.AfterFunc(1*time.Hour, func() { + s.CleanCache() + }) +} + +func (s *ProcessingQueue) CleanCache() { + for _, resource := range s.LocalResources { + if !resource.KeepCache { + // Remove local asset only if remote resource + if resource.IsRemoteResource { + _ = os.Remove(resource.LocalURL) + } + + s.LocalResources[resource.OriginalURL] = nil + } + } + + go time.AfterFunc(1*time.Hour, func() { + s.CleanCache() + }) +} + +func (s *ProcessingQueue) FindSourceClip(trackNumber int, clipNumber int) string { + for _, resource := range s.LocalResources { + for _, used := range resource.Used { + if used.Track == trackNumber && used.Clip == clipNumber { + return resource.LocalURL + } + } + } + return "" } func (s *ProcessingQueue) ProcessQueue(editAPI EditAPIServicer) { @@ -138,50 +176,76 @@ func (s *ProcessingQueue) GenerateParameters(queue *RenderQueue) []string { queue.Status = Rendering queue.InternalStatus = Generating - _ = queue.FFMPEGCommand.ToFFMPEG(queue) + _ = queue.FFMPEGCommand.ToFFMPEG(queue, s) queue.InternalStatus = Generated return queue.FFMPEGCommand.ToString() } +func (s *ProcessingQueue) FetchVideoAssets(trackNumber int, clipNumber int, clip Clip, useCache bool) bool { + var hasError bool + + var asset = clip.Asset.(*VideoAsset) + + if s.LocalResources[asset.Src] == nil { + var fileName string + var remote bool + url, _ := url.Parse(asset.Src) + + if url.Scheme == "file" { + fileName = asset.Src[7:] + } else { + var err error + fileName, err = s.DownloadFile(asset.Src) + if err != nil { + fmt.Println("Error while downloading asset", err) + hasError = true + } + remote = true + } + + if !hasError { + fmt.Println("Asset downloaded: "+asset.Src, fileName) + localResource := &LocalResource{ + Downloaded: time.Now(), + OriginalURL: asset.Src, + LocalURL: fileName, + KeepCache: useCache, + IsRemoteResource: remote, + } + s.LocalResources[asset.Src] = localResource + } + } + + if !hasError { + s.LocalResources[asset.Src].Used = append( + s.LocalResources[asset.Src].Used, + &LocalResourceTrackInfo{ + Track: trackNumber, + Clip: clipNumber, + }) + } + + return hasError +} + func (s *ProcessingQueue) FetchAssets(queue *RenderQueue) { queue.Status = Fetching queue.InternalStatus = Fetching var hasError bool - var assetFiles = make(map[string]string) - for _, track := range queue.Data.Timeline.Tracks { - for _, clip := range track.Clips { + useCache := queue.Data.Timeline.Cache + + for tIndex, track := range queue.Data.Timeline.Tracks { + for cIndex, clip := range track.Clips { // fmt.Println(tIndex, cIndex, clip.Asset.Type) var typeAsset = GetAssetType(clip.Asset) switch typeAsset { // nolint:exhaustive case VideoAssetType: - var asset = clip.Asset.(*VideoAsset) - var fileName = assetFiles[asset.Src] - - if fileName == "" { - url, _ := url.Parse(asset.Src) - - if url.Scheme == "file" { - fileName = asset.Src[7:] - } else { - var err error - fileName, err = s.DownloadFile(asset.Src) - if err != nil { - fmt.Println("Error while downloading asset", err) - hasError = true - } - } - } - - if !hasError { - fmt.Println("Asset downloaded: "+asset.Src, fileName) - queue.LocalResources = append(queue.LocalResources, fileName) - assetFiles[asset.Src] = fileName - } + hasError = s.FetchVideoAssets(tIndex, cIndex, clip, useCache) // case "image": // fmt.Println("Image")