feat: Add video asset crop (#8)

* Fix: soundtrack effects value

* feat: code marshall of all asset type

* feat: add image and title fetching assets

* Fix ensure Audio track

* Fetch all Clip.Asset type

* feat: Fetch without handling

* fix: Avoid useless map for audio

* fix: sourceClip issue bad increment

* Fix: lint issue on ffmpeg.toString

* (wip) Prepare failing tests for overlay all tracks

* fix: OverlayAllTracks to concat only handled tracks

* Add Proper Crop

* doc: update readme

.golangci.yml

@@ -67,5 +67,8 @@ issues:
     - linters:
         - goconst
       text: "string `video` has"
+    - linters:
+        - goconst
+      text: "string `audio` has"
   include:
     # - EXC0002

README.md

@@ -68,11 +68,8 @@ At the end of the road this section should either disappear or be full of `Yes`
 | Clip | filter | Not yet |  |
 | Clip | opacity | Not yet |  |
 | Clip | transform | Not yet |  |
-| Clip [`VideoAsset`] | src | Yes ✅ |  |
-| Clip [`VideoAsset`] | trim | Yes ✅ |  |
-| Clip [`VideoAsset`] | volume | Yes ✅ |  |
-| Clip [`VideoAsset`] | crop | Not yet |  |
-| Clip [`ImageAsset`] | src | Partial 🛠 |  |
+| Clip [`VideoAsset`] | all ✅ | Yes ✅ |  |
+| Clip [`ImageAsset`] | src | Partial 🛠 | Download asset only |
 | Clip [`ImageAsset`] | crop | Not yet |  |
 | Output | format | Partial 🛠 | Only `mp4` at the moment  |
 | Output | resolution | Yes ✅ |  |

go/ffmpeg.go

@@ -21,10 +21,10 @@ import (
 	"errors"
 	"fmt"
 	"io/ioutil"
-	"math"
 	"strings"
 
 	"github.com/spf13/cast"
+	"golang.org/x/exp/slices"
 )
 
 type FFMPEGSource struct {
@@ -289,6 +289,10 @@ func (s *FFMPEG) computeOverlayPosition(position string) string {
 	if position == "" {
 		return "x=0:y:0"
 	}
+	if strings.Contains(position, ":") {
+		return position
+	}
+
 	var x = "(main_w-overlay_w)/2"
 	var y = "(main_h-overlay_h)/2"
 
@@ -333,6 +337,19 @@ func (s *FFMPEG) ClipTrim(sourceClip int, trackNumber int, clipNumber int, start
 		", setpts=PTS-STARTPTS " + s.trackName("video", trackNumber, clipNumber, -1) + ";"
 }
 
+func (s *FFMPEG) ClipCropOverlayPosition(cropInfos *Crop) string {
+	return "x=main_w*" + cast.ToString(cropInfos.Left) + ":y=main_h*" + cast.ToString(cropInfos.Top)
+}
+
+func (s *FFMPEG) ClipCrop(sourceClip int, trackNumber int, clipNumber int, cropInfos *Crop) string {
+	return "[" +
+		cast.ToString(sourceClip) +
+		":v] crop=in_w-" + cast.ToString(cropInfos.Left) + "*in_w-" + cast.ToString(cropInfos.Right) + "*in_w:" +
+		"in_h-" + cast.ToString(cropInfos.Top) + "*in_h-" + cast.ToString(cropInfos.Bottom) + "*in_h:" +
+		cast.ToString(cropInfos.Left) + "*in_w:" + cast.ToString(cropInfos.Top) + "*in_h " +
+		s.trackName("video", trackNumber, clipNumber, -1) + ";"
+}
+
 func (s *FFMPEG) ClipResize(sourceClip int, trackNumber int, clipNumber int, scaleRatio float32) string {
 	return "[" +
 		cast.ToString(sourceClip) +
@@ -509,7 +526,9 @@ func (s *FFMPEG) ToFFMPEG(renderQueue *RenderQueue, queue *ProcessingQueue) erro
 
 			_ = clip.ToFFMPEG(s, sourceClip, trackNumber, clipNumber)
 
-			sourceClip = sourceClip + 1
+			if sourceFileName != "" {
+				sourceClip = sourceClip + 1
+			}
 			clipNumber = clipNumber + 1
 			lastStart = clip.Start + clip.Length
 		}
@@ -523,6 +542,36 @@ func (s *FFMPEG) ToFFMPEG(renderQueue *RenderQueue, queue *ProcessingQueue) erro
 	return nil
 }
 
+func (s *FFMPEG) OverlayAllTracks(missingVideoTracks []string) string {
+	// Overlay of all tracks
+	var vTracks = " [bg]"
+	var curOverlayName string
+	var previousOverlayName string
+
+	var availableTracks []string
+	for i := len(s.tracks) - 1; i >= 0; i-- {
+		if !slices.Contains(missingVideoTracks, "[vtrack"+cast.ToString(i)+"]") {
+			availableTracks = append(availableTracks, cast.ToString(i))
+		}
+	}
+
+	for i := 0; i <= len(availableTracks)-1; i++ {
+		curOverlayName = "[overlay" + cast.ToString(i) + "]"
+		if previousOverlayName != "" {
+			vTracks = vTracks + previousOverlayName
+		}
+		vTracks = vTracks + "[vtrack" + cast.ToString(availableTracks[i]) + "] overlay=shortest=1:x=0:y=0 "
+		if i == len(availableTracks)-1 {
+			vTracks = vTracks + "[vtracks];"
+		} else {
+			vTracks = vTracks + curOverlayName + ";"
+		}
+		previousOverlayName = curOverlayName
+	}
+
+	return vTracks
+}
+
 func (s *FFMPEG) ToString() []string {
 	var parameters = make([]string, 0)
 	if s.defaultParams {
@@ -590,28 +639,17 @@ func (s *FFMPEG) ToString() []string {
 	filterComplex = filterComplex + "[" + cast.ToString(maxSource+addedSources) + "] concat=n=1:v=1,setpts=PTS-STARTPTS,format=yuv420p [bg];"
 
 	// Add all tracks infos
-	for _, track := range s.tracks {
+	var missingVideoTracks []string
+	for i, track := range s.tracks {
 		filterComplex = filterComplex + strings.Join(track.video, " ") + strings.Join(track.audio, " ")
-	}
-
-	// Overlay of all tracks
-	var vTracks = " [bg]"
-	var curOverlayName string
-	var previousOverlayName string
-	for i := len(s.tracks) - 1; i >= 0; i-- {
-		curOverlayName = "[overlay" + cast.ToString(math.Abs(cast.ToFloat64(i-(len(s.tracks)-1)))) + "]"
-		if previousOverlayName != "" {
-			vTracks = vTracks + previousOverlayName
-		}
-		vTracks = vTracks + "[vtrack" + cast.ToString(i) + "] overlay=shortest=1:x=0:y=0 "
-		if i == 0 {
-			vTracks = vTracks + "[vtracks];"
-		} else {
-			vTracks = vTracks + curOverlayName + ";"
+		trackName := "[vtrack" + cast.ToString(i) + "]"
+		if !strings.Contains(filterComplex, trackName) {
+			missingVideoTracks = append(missingVideoTracks, trackName)
 		}
-		previousOverlayName = curOverlayName
 	}
-	filterComplex = filterComplex + vTracks
+
+	// Overlay all video tracks
+	filterComplex = filterComplex + s.OverlayAllTracks(missingVideoTracks)
 
 	// Handle audio tracks
 	var aTracks string
@@ -633,11 +671,13 @@ func (s *FFMPEG) ToString() []string {
 	}
 	parameters = append(parameters, filterComplex)
 
-	// Map result
+	// Map results
 	parameters = append(parameters, "-map")
 	parameters = append(parameters, "[vtracks]")
-	parameters = append(parameters, "-map")
-	parameters = append(parameters, "[atracks]")
+	if aTracks != "" {
+		parameters = append(parameters, "-map")
+		parameters = append(parameters, "[atracks]")
+	}
 
 	// Handle output
 	parameters = append(parameters, "-s")

go/ffmpeg_test.go

@@ -136,6 +136,42 @@ var _ = Describe("Ffmpeg", func() {
 			Expect(ff.GetResolution()).To(Equal("30x20"))
 		})
 	})
+	Describe("OverlayAllTracks", func() {
+		var ff openapi.FFMPEGCommand
+		var missingVideoTracks []string
+		BeforeEach(func() {
+			ff = openapi.NewFFMPEGCommand()
+			missingVideoTracks = []string{}
+		})
+
+		It("Overlay one handled tracks out of one", func() {
+
+			_ = ff.AddTrack(0)
+			res := ff.OverlayAllTracks(missingVideoTracks)
+			Expect(res).To(Equal(" [bg][vtrack0] overlay=shortest=1:x=0:y=0 [vtracks];"))
+		})
+		It("Overlay one handled tracks out of two", func() {
+			missingVideoTracks = append(missingVideoTracks, "[vtrack0]")
+			_ = ff.AddTrack(0)
+			_ = ff.AddTrack(1)
+			res := ff.OverlayAllTracks(missingVideoTracks)
+			Expect(res).To(Equal(" [bg][vtrack1] overlay=shortest=1:x=0:y=0 [vtracks];"))
+		})
+		It("Overlay two handled tracks out of two", func() {
+			_ = ff.AddTrack(0)
+			_ = ff.AddTrack(1)
+			res := ff.OverlayAllTracks(missingVideoTracks)
+			Expect(res).To(Equal(" [bg][vtrack1] overlay=shortest=1:x=0:y=0 [overlay0];[overlay0][vtrack0] overlay=shortest=1:x=0:y=0 [vtracks];"))
+		})
+		It("Overlay two handled tracks out of three", func() {
+			missingVideoTracks = append(missingVideoTracks, "[vtrack1]")
+			_ = ff.AddTrack(0)
+			_ = ff.AddTrack(1)
+			_ = ff.AddTrack(2)
+			res := ff.OverlayAllTracks(missingVideoTracks)
+			Expect(res).To(Equal(" [bg][vtrack2] overlay=shortest=1:x=0:y=0 [overlay0];[overlay0][vtrack0] overlay=shortest=1:x=0:y=0 [vtracks];"))
+		})
+	})
 	Describe("SetDefaultBackground/GenerateBackground", func() {
 		var ff openapi.FFMPEGCommand
 		BeforeEach(func() {

go/model_asset.go

@@ -64,6 +64,16 @@ func NewAsset(typeAsset string, obj map[string]interface{}) interface{} {
 	switch typeAsset {
 	case "video":
 		return NewVideoAsset(obj)
+	case "image":
+		return NewImageAsset(obj)
+	case "title":
+		return NewTitleAsset(obj)
+	case "html":
+		return NewHTMLAsset(obj)
+	case "audio":
+		return NewAudioAsset(obj)
+	case "luma":
+		return NewLumaAsset(obj)
 	}
 
 	return nil
@@ -93,6 +103,16 @@ func AssertAssetRequired(obj interface{}) error {
 	switch GetAssetType(obj) { // nolint:exhaustive
 	case VideoAssetType:
 		return AssertVideoAssetRequired(obj.(VideoAsset))
+	case ImageAssetType:
+		return AssertImageAssetRequired(obj.(ImageAsset))
+	case TitleAssetType:
+		return AssertTitleAssetRequired(obj.(TitleAsset))
+	case HTMLAssetType:
+		return AssertHTMLAssetRequired(obj.(HTMLAsset))
+	case AudioAssetType:
+		return AssertAudioAssetRequired(obj.(AudioAsset))
+	case LumaAssetType:
+		return AssertLumaAssetRequired(obj.(LumaAsset))
 	}
 	return nil
 }

go/model_audio_asset.go

@@ -26,6 +26,8 @@ along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
 package openapi
 
+import "github.com/spf13/cast"
+
 // AudioAsset - The AudioAsset is used to add sound effects and audio at specific intervals on the timeline. The src must be a publicly accessible URL to an audio resource such  as an mp3 file.
 type AudioAsset struct {
 
@@ -45,6 +47,26 @@ type AudioAsset struct {
 	Effect string `json:"effect,omitempty"`
 }
 
+func NewAudioAsset(m map[string]interface{}) *AudioAsset {
+	audioAsset := &AudioAsset{
+		Type: m["type"].(string),
+	}
+
+	if m["src"] != nil {
+		audioAsset.Src = m["src"].(string)
+	}
+	if m["trim"] != nil {
+		audioAsset.Trim = cast.ToFloat32(m["trim"].(float64))
+	}
+	if m["volume"] != nil {
+		audioAsset.Volume = cast.ToFloat32(m["volume"].(float64))
+	}
+	if m["effect"] != nil {
+		audioAsset.Effect = m["effect"].(string)
+	}
+	return audioAsset
+}
+
 // AssertAudioAssetRequired checks if the required fields are not zero-ed
 func AssertAudioAssetRequired(obj AudioAsset) error {
 	elements := map[string]interface{}{

go/model_clip.go

@@ -154,43 +154,47 @@ func (s *Clip) checkEnumValues() error {
 func (s *Clip) ToFFMPEG(FFMPEGCommand FFMPEGCommand, sourceClip int, trackNumber int, currentClip int) error {
 	var effects []string
 	var audioEffects []string
-	var handled bool
 
 	var typeAsset = GetAssetType(s.Asset)
 	switch typeAsset { // nolint:exhaustive
 	case VideoAssetType:
 		var currentAsset = s.Asset.(*VideoAsset)
 
+		// v1. Burn
 		if currentAsset.Subtitle != nil {
 			effects = append(effects, FFMPEGCommand.ClipSubtitleBurn(sourceClip, trackNumber, currentClip, currentAsset.Subtitle.Index))
 		}
+		// v2. Trim
 		if currentAsset.Trim != 0 {
-			handled = true
 			effects = append(effects, FFMPEGCommand.ClipTrim(sourceClip, trackNumber, currentClip, currentAsset.Trim, currentAsset.Trim+s.Length))
+
+			// a1. Trim
 			if currentAsset.Volume != 0 {
 				audioEffects = append(audioEffects, FFMPEGCommand.ClipAudioTrim(sourceClip, trackNumber, currentClip, currentAsset.Trim, currentAsset.Trim+s.Length))
 			}
 		} else {
-			handled = true
 			effects = append(effects, FFMPEGCommand.ClipTrim(sourceClip, trackNumber, currentClip, 0, s.Length))
 		}
+
+		// a2. Volume + Timing
 		if currentAsset.Volume != 0 {
 			audioEffects = append(audioEffects, FFMPEGCommand.ClipAudioVolume(sourceClip, trackNumber, currentClip, currentAsset.Volume))
 			audioEffects = append(audioEffects, FFMPEGCommand.ClipAudioDelay(sourceClip, trackNumber, currentClip, s.Start*1000))
 		}
 
-	case ImageAssetType:
-		handled = true
-		effects = append(effects, FFMPEGCommand.ClipImage(sourceClip, trackNumber, currentClip, 0, s.Length))
+		// v3. Crop
+		if currentAsset.Crop != nil {
+			effects = append(effects, FFMPEGCommand.ClipCrop(sourceClip, trackNumber, currentClip, currentAsset.Crop))
+			effects = append(effects, FFMPEGCommand.ClipFillerOverlay(sourceClip, trackNumber, currentClip, FFMPEGCommand.ClipCropOverlayPosition(currentAsset.Crop)))
+		}
+
+	// case ImageAssetType:
+	// handled = true
+	// effects = append(effects, FFMPEGCommand.ClipImage(sourceClip, trackNumber, currentClip, 0, s.Length))
 	default:
 		fmt.Println("Type not handled for converting to FFMPEG", typeAsset.String())
 	}
 
-	if !handled {
-		// TODO: Insert yellow image instead for duration (Or Not handled)
-		effects = append(effects, FFMPEGCommand.ClipRaw(sourceClip, trackNumber, currentClip))
-	}
-
 	// Resize clip to ensure concat will work
 	if s.Scale != nil {
 		effects = append(effects, FFMPEGCommand.ClipResize(sourceClip, trackNumber, currentClip, *s.Scale))

go/model_crop.go

@@ -32,16 +32,16 @@ import "github.com/spf13/cast"
 type Crop struct {
 
 	// Crop from the top of the asset
-	Top float32 `json:"top,omitempty"`
+	Top float32 `json:"top"`
 
 	// Crop from the bottom of the asset
-	Bottom float32 `json:"bottom,omitempty"`
+	Bottom float32 `json:"bottom"`
 
 	// Crop from the left of the asset
-	Left float32 `json:"left,omitempty"`
+	Left float32 `json:"left"`
 
 	// Crop from the left of the asset
-	Right float32 `json:"right,omitempty"`
+	Right float32 `json:"right"`
 }
 
 func NewCrop(m map[string]interface{}) *Crop {

go/model_ffmpeg.go

@@ -25,6 +25,8 @@ type FFMPEGCommand interface {
 	CloseTrack(int) error
 	AddClip(int, string) error
 	ClipTrim(int, int, int, float32, float32) string
+	ClipCrop(int, int, int, *Crop) string
+	ClipCropOverlayPosition(cropInfos *Crop) string
 	ClipImage(int, int, int, float32, float32) string
 	ClipMerge(int, int, int, []string) string
 	ClipRaw(int, int, int) string
@@ -48,4 +50,5 @@ type FFMPEGCommand interface {
 	ToFFMPEG(*RenderQueue, *ProcessingQueue) error
 	GetOutputName() string
 	GetDuration() float32
+	OverlayAllTracks([]string) string
 }