[feature] Enable basic video support (mp4 only) (#1274)

* [feature] basic video support

* fix missing semicolon

* replace text shadow with stacked icons

Co-authored-by: f0x <f0x@cthu.lu>

README.md

@@ -42,7 +42,7 @@ Here's a screenshot of the instance landing page!
 - [Credits](#credits)
   - [Libraries](#libraries)
   - [Image Attribution](#image-attribution)
-  - [Developers](#developers)
+  - [Team](#team)
   - [Special Thanks](#special-thanks)
 - [Sponsorship + Funding](#sponsorship--funding)
   - [OpenCollective](#opencollective)
@@ -210,6 +210,7 @@ For bugs and feature requests, please check to see if there's [already an issue]
 
 The following libraries and frameworks are used by GoToSocial, with gratitude 💕
 
+- [abema/go-mp4](https://github.com/abema/go-mp4); mp4 parsing. [MIT License](https://spdx.org/licenses/MIT.html).
 - [buckket/go-blurhash](https://github.com/buckket/go-blurhash); used for generating image blurhashes. [GPL-3.0 License](https://spdx.org/licenses/GPL-3.0-only.html).
 - [coreos/go-oidc](https://github.com/coreos/go-oidc); OIDC client library. [Apache-2.0 License](https://spdx.org/licenses/Apache-2.0.html).
 - [disintegration/imaging](https://github.com/disintegration/imaging); image resizing. [MIT License](https://spdx.org/licenses/MIT.html).

go.mod

@@ -13,6 +13,7 @@ require (
 	codeberg.org/gruf/go-mutexes v1.1.4
 	codeberg.org/gruf/go-runners v1.3.1
 	codeberg.org/gruf/go-store/v2 v2.0.10
+	github.com/abema/go-mp4 v0.8.0
 	github.com/buckket/go-blurhash v1.1.0
 	github.com/coreos/go-oidc/v3 v3.4.0
 	github.com/cornelk/hashmap v1.0.8

go.sum

@@ -110,6 +110,8 @@ github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030I
 github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/abema/go-mp4 v0.8.0 h1:JHYkOvTfBpTnqJHiFFOXe8d6wiFy5MtDnA10fgccNqY=
+github.com/abema/go-mp4 v0.8.0/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
 github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
 github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
 github.com/andybalholm/brotli v1.0.0/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y=
@@ -491,6 +493,8 @@ github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108
 github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw=
+github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0=
 github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8=
 github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
 github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
@@ -568,6 +572,7 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs=
 github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
+github.com/sunfish-shogi/bufseekio v0.0.0-20210207115823-a4185644b365/go.mod h1:dEzdXgvImkQ3WLI+0KQpmEx8T/C/ma9KeS3AfmU899I=
 github.com/superseriousbusiness/activity v1.2.1-gts h1:wh7v0zYa1mJmqB35PSfvgl4cs51Dh5PyfKvcZLSxMQU=
 github.com/superseriousbusiness/activity v1.2.1-gts/go.mod h1:AZw0Xb4Oju8rmaJCZ21gc5CPg47MmNgyac+Hx5jo8VM=
 github.com/superseriousbusiness/exif-terminator v0.5.0 h1:57SO/geyaOl2v/lJSQLVcQbdghpyFuK8ZTtaHL81fUQ=
@@ -1177,11 +1182,14 @@ gopkg.in/mcuadros/go-syslog.v2 v2.3.0 h1:kcsiS+WsTKyIEPABJBJtoG0KkOS6yzvJ+/eZlhD
 gopkg.in/mcuadros/go-syslog.v2 v2.3.0/go.mod h1:l5LPIyOOyIdQquNg+oU6Z3524YwrcqEm0aKH+5zpt2U=
 gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI=
 gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/src-d/go-billy.v4 v4.3.2 h1:0SQA1pRztfTFx2miS8sA97XvooFeNOmvUenF4o0EcVg=
+gopkg.in/src-d/go-billy.v4 v4.3.2/go.mod h1:nDjArDMp+XMs1aFAESLRjfGSgfvoYN0hDfzEk0GjC98=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=

internal/media/image.go

@@ -38,16 +38,7 @@ const (
 	thumbnailMaxHeight = 512
 )
 
-type imageMeta struct {
-	width    int
-	height   int
-	size     int
-	aspect   float64
-	blurhash string // defined only for calls to deriveThumbnail if createBlurhash is true
-	small    []byte // defined only for calls to deriveStaticEmoji or deriveThumbnail
-}
-
-func decodeGif(r io.Reader) (*imageMeta, error) {
+func decodeGif(r io.Reader) (*mediaMeta, error) {
 	gif, err := gif.DecodeAll(r)
 	if err != nil {
 		return nil, err
@@ -59,15 +50,15 @@ func decodeGif(r io.Reader) (*imageMeta, error) {
 	size := width * height
 	aspect := float64(width) / float64(height)
 
-	return &imageMeta{
+	return &mediaMeta{
 		width:  width,
 		height: height,
 		size:   size,
 		aspect: aspect,
 	}, nil
 }
 
-func decodeImage(r io.Reader, contentType string) (*imageMeta, error) {
+func decodeImage(r io.Reader, contentType string) (*mediaMeta, error) {
 	var i image.Image
 	var err error
 
@@ -96,24 +87,53 @@ func decodeImage(r io.Reader, contentType string) (*imageMeta, error) {
 	size := width * height
 	aspect := float64(width) / float64(height)
 
-	return &imageMeta{
+	return &mediaMeta{
 		width:  width,
 		height: height,
 		size:   size,
 		aspect: aspect,
 	}, nil
 }
 
-// deriveThumbnail returns a byte slice and metadata for a thumbnail
-// of a given jpeg, png, gif or webp, or an error if something goes wrong.
+// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png.
+func deriveStaticEmoji(r io.Reader, contentType string) (*mediaMeta, error) {
+	var i image.Image
+	var err error
+
+	switch contentType {
+	case mimeImagePng:
+		i, err = StrippedPngDecode(r)
+		if err != nil {
+			return nil, err
+		}
+	case mimeImageGif:
+		i, err = gif.Decode(r)
+		if err != nil {
+			return nil, err
+		}
+	default:
+		return nil, fmt.Errorf("content type %s not allowed for emoji", contentType)
+	}
+
+	out := &bytes.Buffer{}
+	if err := png.Encode(out, i); err != nil {
+		return nil, err
+	}
+	return &mediaMeta{
+		small: out.Bytes(),
+	}, nil
+}
+
+// deriveThumbnailFromImage returns a byte slice and metadata for a thumbnail
+// of a given piece of media, or an error if something goes wrong.
 //
 // If createBlurhash is true, then a blurhash will also be generated from a tiny
 // version of the image. This costs precious CPU cycles, so only use it if you
 // really need a blurhash and don't have one already.
 //
 // If createBlurhash is false, then the blurhash field on the returned ImageAndMeta
 // will be an empty string.
-func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*imageMeta, error) {
+func deriveThumbnailFromImage(r io.Reader, contentType string, createBlurhash bool) (*mediaMeta, error) {
 	var i image.Image
 	var err error
 
@@ -126,7 +146,7 @@ func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*ima
 		})
 		i, err = imaging.Decode(strippedPngReader, imaging.AutoOrientation(true))
 	default:
-		err = fmt.Errorf("content type %s can't be thumbnailed", contentType)
+		err = fmt.Errorf("content type %s can't be thumbnailed as an image", contentType)
 	}
 
 	if err != nil {
@@ -149,7 +169,7 @@ func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*ima
 	size := thumbX * thumbY
 	aspect := float64(thumbX) / float64(thumbY)
 
-	im := &imageMeta{
+	im := &mediaMeta{
 		width:  thumbX,
 		height: thumbY,
 		size:   size,
@@ -178,32 +198,3 @@ func deriveThumbnail(r io.Reader, contentType string, createBlurhash bool) (*ima
 
 	return im, nil
 }
-
-// deriveStaticEmojji takes a given gif or png of an emoji, decodes it, and re-encodes it as a static png.
-func deriveStaticEmoji(r io.Reader, contentType string) (*imageMeta, error) {
-	var i image.Image
-	var err error
-
-	switch contentType {
-	case mimeImagePng:
-		i, err = StrippedPngDecode(r)
-		if err != nil {
-			return nil, err
-		}
-	case mimeImageGif:
-		i, err = gif.Decode(r)
-		if err != nil {
-			return nil, err
-		}
-	default:
-		return nil, fmt.Errorf("content type %s not allowed for emoji", contentType)
-	}
-
-	out := &bytes.Buffer{}
-	if err := png.Encode(out, i); err != nil {
-		return nil, err
-	}
-	return &imageMeta{
-		small: out.Bytes(),
-	}, nil
-}

internal/media/manager_test.go

@@ -376,6 +376,78 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlocking() {
 	suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
 }
 
+func (suite *ManagerTestSuite) TestSlothVineProcessBlocking() {
+	ctx := context.Background()
+
+	data := func(_ context.Context) (io.ReadCloser, int64, error) {
+		// load bytes from a test video
+		b, err := os.ReadFile("./test/test-mp4-original.mp4")
+		if err != nil {
+			panic(err)
+		}
+		return io.NopCloser(bytes.NewBuffer(b)), int64(len(b)), nil
+	}
+
+	accountID := "01FS1X72SK9ZPW0J1QQ68BD264"
+
+	// process the media with no additional info provided
+	processingMedia, err := suite.manager.ProcessMedia(ctx, data, nil, accountID, nil)
+	suite.NoError(err)
+	// fetch the attachment id from the processing media
+	attachmentID := processingMedia.AttachmentID()
+
+	// do a blocking call to fetch the attachment
+	attachment, err := processingMedia.LoadAttachment(ctx)
+	suite.NoError(err)
+	suite.NotNil(attachment)
+
+	// make sure it's got the stuff set on it that we expect
+	// the attachment ID and accountID we expect
+	suite.Equal(attachmentID, attachment.ID)
+	suite.Equal(accountID, attachment.AccountID)
+
+	// file meta should be correctly derived from the video
+	suite.EqualValues(gtsmodel.Original{
+		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
+	}, attachment.FileMeta.Original)
+	suite.EqualValues(gtsmodel.Small{
+		Width: 338, Height: 240, Size: 81120, Aspect: 1.4083333333333334,
+	}, attachment.FileMeta.Small)
+	suite.Equal("video/mp4", attachment.File.ContentType)
+	suite.Equal("image/jpeg", attachment.Thumbnail.ContentType)
+	suite.Equal(312413, attachment.File.FileSize)
+	suite.Equal("", attachment.Blurhash)
+
+	// now make sure the attachment is in the database
+	dbAttachment, err := suite.db.GetAttachmentByID(ctx, attachmentID)
+	suite.NoError(err)
+	suite.NotNil(dbAttachment)
+
+	// make sure the processed file is in storage
+	processedFullBytes, err := suite.storage.Get(ctx, attachment.File.Path)
+	suite.NoError(err)
+	suite.NotEmpty(processedFullBytes)
+
+	// load the processed bytes from our test folder, to compare
+	processedFullBytesExpected, err := os.ReadFile("./test/test-mp4-processed.mp4")
+	suite.NoError(err)
+	suite.NotEmpty(processedFullBytesExpected)
+
+	// the bytes in storage should be what we expected
+	suite.Equal(processedFullBytesExpected, processedFullBytes)
+
+	// now do the same for the thumbnail and make sure it's what we expected
+	processedThumbnailBytes, err := suite.storage.Get(ctx, attachment.Thumbnail.Path)
+	suite.NoError(err)
+	suite.NotEmpty(processedThumbnailBytes)
+
+	processedThumbnailBytesExpected, err := os.ReadFile("./test/test-mp4-thumbnail.jpg")
+	suite.NoError(err)
+	suite.NotEmpty(processedThumbnailBytesExpected)
+
+	suite.Equal(processedThumbnailBytesExpected, processedThumbnailBytes)
+}
+
 func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingNoContentLengthGiven() {
 	ctx := context.Background()