add git.iim.gay/grufwub/go-store for storage backend, replacing blob.…

…Storage

Signed-off-by: kim (grufwub) <grufwub@gmail.com>

go.mod

@@ -3,6 +3,7 @@ module github.com/superseriousbusiness/gotosocial
 go 1.17
 
 require (
+	git.iim.gay/grufwub/go-store v0.4.1
 	github.com/ReneKroon/ttlcache v1.7.0
 	github.com/buckket/go-blurhash v1.1.0
 	github.com/coreos/go-oidc/v3 v3.0.0
@@ -41,6 +42,13 @@ require (
 )
 
 require (
+	git.iim.gay/grufwub/fastpath v0.2.2 // indirect
+	git.iim.gay/grufwub/go-bufpool v0.2.1 // indirect
+	git.iim.gay/grufwub/go-bytes v0.7.0 // indirect
+	git.iim.gay/grufwub/go-errors v0.2.3 // indirect
+	git.iim.gay/grufwub/go-hashenc v0.3.0 // indirect
+	git.iim.gay/grufwub/go-mutexes v0.5.0 // indirect
+	git.iim.gay/grufwub/go-nowish v0.3.4 // indirect
 	github.com/aymerick/douceur v0.2.0 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -61,6 +69,7 @@ require (
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.3 // indirect
 	github.com/gorilla/context v1.1.1 // indirect
 	github.com/gorilla/css v1.0.0 // indirect
 	github.com/gorilla/securecookie v1.1.1 // indirect

go.sum

@@ -31,6 +31,23 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+git.iim.gay/grufwub/fastpath v0.2.2 h1:ST89k6XukDE2xN5zZ0VGi7Jo2A/DPhik9dr3/ny4QGg=
+git.iim.gay/grufwub/fastpath v0.2.2/go.mod h1:HYwrf4ku2CfP/1D3ItBqBbFHayuWkiV6BOwYQQ86dbo=
+git.iim.gay/grufwub/go-bufpool v0.2.1 h1:TRwEzLtdvutp3+W/5wQYxmW6lAz9twQKmsfiUO0XfSU=
+git.iim.gay/grufwub/go-bufpool v0.2.1/go.mod h1:X/QI5K0hxyhWPRtu+D+ZRzLqEOqD1Ns/rpNNWq2R1DU=
+git.iim.gay/grufwub/go-bytes v0.7.0 h1:tGwIrVOANiaifx7702yDdkp5uMVYGk+rheZd6ZuSvEo=
+git.iim.gay/grufwub/go-bytes v0.7.0/go.mod h1:ADc2p+R74ZjQf0FzIaIUnLyuHdbGCk7ldlaiHauAs+U=
+git.iim.gay/grufwub/go-errors v0.2.3 h1:MSvzN/tW5Vj5Rm4EmBfdVpzyYjAY92ANE+ESNpZnjvw=
+git.iim.gay/grufwub/go-errors v0.2.3/go.mod h1:rfKZpjI7A67zJfzpt5zfwAUMA7gec0EHXZEuYv/A9UI=
+git.iim.gay/grufwub/go-fixedmap v0.1.3/go.mod h1:KB4nV2+NeBMVFvFwpdgsP74AsMiDeX68oD20wiC2S3I=
+git.iim.gay/grufwub/go-hashenc v0.3.0 h1:2etpzwoUTPTLvlWZ9u9a+FUCGBcVU37x5zM5XZ0kahQ=
+git.iim.gay/grufwub/go-hashenc v0.3.0/go.mod h1:wjztiGUzaZsEw5kKE6gz/UOFN2cbcDnGiUSUjOLXi4o=
+git.iim.gay/grufwub/go-mutexes v0.5.0 h1:HojjhBWI1ry1TTvijczhm2oQ5CxLbxzihawqYk2Umto=
+git.iim.gay/grufwub/go-mutexes v0.5.0/go.mod h1:xMhjpEP5UsCuFQD4qCIcq4pJLf7vMXZ56TD/u+wWJ4Y=
+git.iim.gay/grufwub/go-nowish v0.3.4 h1:VgUzSEO7xJsJFN2HPbPYHT79s3pUkd5Z8hQOPecZzFU=
+git.iim.gay/grufwub/go-nowish v0.3.4/go.mod h1:oII7zlMQMFclFzgmI1qRd7DdQXKNHWcJYnwHFgdgiRI=
+git.iim.gay/grufwub/go-store v0.4.1 h1:orSqupN2iTm8P0sUpGWlCl6qckM2phkFnsB3TFTo6aQ=
+git.iim.gay/grufwub/go-store v0.4.1/go.mod h1:NaSfOLKNzjj9lUQ0MA/gLGEeRiXhwOx26zR+l/SC9VM=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
@@ -194,6 +211,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
 github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
+github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=

internal/api/client/account/account_test.go

@@ -6,11 +6,11 @@ import (
 	"net/http"
 	"net/http/httptest"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/account"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -28,7 +28,7 @@ type AccountStandardTestSuite struct {
 	db        db.DB
 	log       *logrus.Logger
 	tc        typeutils.TypeConverter
-	storage   blob.Storage
+	storage   *kv.KVStore
 	federator federation.Federator
 	processor processing.Processor
 

internal/api/client/fileserver/servefile_test.go

@@ -26,12 +26,12 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/fileserver"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -49,7 +49,7 @@ type ServeFileTestSuite struct {
 	config       *config.Config
 	db           db.DB
 	log          *logrus.Logger
-	storage      blob.Storage
+	storage      *kv.KVStore
 	federator    federation.Federator
 	tc           typeutils.TypeConverter
 	processor    processing.Processor
@@ -152,7 +152,7 @@ func (suite *ServeFileTestSuite) TestServeOriginalFileSuccessful() {
 	assert.NoError(suite.T(), err)
 	assert.NotNil(suite.T(), b)
 
-	fileInStorage, err := suite.storage.RetrieveFileFrom(targetAttachment.File.Path)
+	fileInStorage, err := suite.storage.Get(targetAttachment.File.Path)
 	assert.NoError(suite.T(), err)
 	assert.NotNil(suite.T(), fileInStorage)
 	assert.Equal(suite.T(), b, fileInStorage)

internal/api/client/media/mediacreate_test.go

@@ -28,13 +28,13 @@ import (
 	"net/http/httptest"
 	"testing"
 
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/gin-gonic/gin"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/suite"
 	mediamodule "github.com/superseriousbusiness/gotosocial/internal/api/client/media"
 	"github.com/superseriousbusiness/gotosocial/internal/api/model"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -52,7 +52,7 @@ type MediaCreateTestSuite struct {
 	config       *config.Config
 	db           db.DB
 	log          *logrus.Logger
-	storage      blob.Storage
+	storage      *kv.KVStore
 	federator    federation.Federator
 	tc           typeutils.TypeConverter
 	mediaHandler media.Handler
@@ -118,7 +118,6 @@ func (suite *MediaCreateTestSuite) TearDownTest() {
 */
 
 func (suite *MediaCreateTestSuite) TestStatusCreatePOSTImageHandlerSuccessful() {
-
 	// set up the context for the request
 	t := suite.testTokens["local_account_1"]
 	oauthToken := oauth.DBTokenToToken(t)
@@ -130,10 +129,15 @@ func (suite *MediaCreateTestSuite) TestStatusCreatePOSTImageHandlerSuccessful()
 	ctx.Set(oauth.SessionAuthorizedAccount, suite.testAccounts["local_account_1"])
 
 	// see what's in storage *before* the request
-	storageKeysBeforeRequest, err := suite.storage.ListKeys()
+	storageKeysBeforeRequest := []string{}
+	iter, err := suite.storage.Iterator(nil)
 	if err != nil {
 		panic(err)
 	}
+	for iter.Next() {
+		storageKeysBeforeRequest = append(storageKeysBeforeRequest, iter.Key())
+	}
+	iter.Release()
 
 	// create the request
 	buf, w, err := testrig.CreateMultipartFormData("file", "../../../../testrig/media/test-jpeg.jpg", map[string]string{
@@ -150,10 +154,15 @@ func (suite *MediaCreateTestSuite) TestStatusCreatePOSTImageHandlerSuccessful()
 	suite.mediaModule.MediaCreatePOSTHandler(ctx)
 
 	// check what's in storage *after* the request
-	storageKeysAfterRequest, err := suite.storage.ListKeys()
+	storageKeysAfterRequest := []string{}
+	iter, err = suite.storage.Iterator(nil)
 	if err != nil {
 		panic(err)
 	}
+	for iter.Next() {
+		storageKeysAfterRequest = append(storageKeysAfterRequest, iter.Key())
+	}
+	iter.Release()
 
 	// check response
 	suite.EqualValues(http.StatusOK, recorder.Code)

internal/api/client/status/status_test.go

@@ -19,10 +19,10 @@
 package status_test
 
 import (
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/client/status"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -41,7 +41,7 @@ type StatusStandardTestSuite struct {
 	tc        typeutils.TypeConverter
 	federator federation.Federator
 	processor processing.Processor
-	storage   blob.Storage
+	storage   *kv.KVStore
 
 	// standard suite models
 	testTokens       map[string]*gtsmodel.Token

internal/api/s2s/user/user_test.go

@@ -1,11 +1,11 @@
 package user_test
 
 import (
+	"git.iim.gay/grufwub/go-store/kv"
 	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/suite"
 	"github.com/superseriousbusiness/gotosocial/internal/api/s2s/user"
 	"github.com/superseriousbusiness/gotosocial/internal/api/security"
-	"github.com/superseriousbusiness/gotosocial/internal/blob"
 	"github.com/superseriousbusiness/gotosocial/internal/config"
 	"github.com/superseriousbusiness/gotosocial/internal/db"
 	"github.com/superseriousbusiness/gotosocial/internal/federation"
@@ -24,7 +24,7 @@ type UserStandardTestSuite struct {
 	tc             typeutils.TypeConverter
 	federator      federation.Federator
 	processor      processing.Processor
-	storage        blob.Storage
+	storage        *kv.KVStore
 	securityModule *security.Module
 
 	// standard suite models