[chore] cleanup storage implementation, no need for multiple interfac…

…e types (#1131) Signed-off-by: kim <grufwub@gmail.com> Signed-off-by: kim <grufwub@gmail.com>
superseriousbusiness · Nov 24, 2022 · fcb9c0b · fcb9c0b
1 parent c9d893f
commit fcb9c0b
Show file tree

Hide file tree

Showing 35 changed files with 129 additions and 178 deletions.
diff --git a/cmd/gotosocial/action/testrig/testrig.go b/cmd/gotosocial/action/testrig/testrig.go
@@ -71,7 +71,7 @@ var Start action.GTSAction = func(ctx context.Context) error {
 	dbService := testrig.NewTestDB()
 	testrig.StandardDBSetup(dbService, nil)
 	router := testrig.NewTestRouter(dbService)
-	var storageBackend storage.Driver
+	var storageBackend *storage.Driver
 	if os.Getenv("GTS_STORAGE_BACKEND") == "s3" {
 		storageBackend = testrig.NewS3Storage()
 	} else {

diff --git a/internal/api/client/account/account_test.go b/internal/api/client/account/account_test.go
@@ -45,7 +45,7 @@ type AccountStandardTestSuite struct {
 	// standard suite interfaces
 	suite.Suite
 	db           db.DB
-	storage      storage.Driver
+	storage      *storage.Driver
 	mediaManager media.Manager
 	federator    federation.Federator
 	processor    processing.Processor

diff --git a/internal/api/client/admin/admin_test.go b/internal/api/client/admin/admin_test.go
@@ -45,7 +45,7 @@ type AdminStandardTestSuite struct {
 	// standard suite interfaces
 	suite.Suite
 	db           db.DB
-	storage      storage.Driver
+	storage      *storage.Driver
 	mediaManager media.Manager
 	federator    federation.Federator
 	processor    processing.Processor

diff --git a/internal/api/client/auth/auth_test.go b/internal/api/client/auth/auth_test.go
@@ -48,7 +48,7 @@ import (
 type AuthStandardTestSuite struct {
 	suite.Suite
 	db           db.DB
-	storage      storage.Driver
+	storage      *storage.Driver
 	mediaManager media.Manager
 	federator    federation.Federator
 	processor    processing.Processor

diff --git a/internal/api/client/fileserver/servefile_test.go b/internal/api/client/fileserver/servefile_test.go
@@ -48,7 +48,7 @@ type ServeFileTestSuite struct {
 	// standard suite interfaces
 	suite.Suite
 	db           db.DB
-	storage      storage.Driver
+	storage      *storage.Driver
 	federator    federation.Federator
 	tc           typeutils.TypeConverter
 	processor    processing.Processor

diff --git a/internal/api/client/followrequest/followrequest_test.go b/internal/api/client/followrequest/followrequest_test.go
@@ -43,7 +43,7 @@ import (
 type FollowRequestStandardTestSuite struct {
 	suite.Suite
 	db           db.DB
-	storage      storage.Driver
+	storage      *storage.Driver
 	mediaManager media.Manager
 	federator    federation.Federator
 	processor    processing.Processor

diff --git a/internal/api/client/instance/instance_test.go b/internal/api/client/instance/instance_test.go
@@ -44,7 +44,7 @@ type InstanceStandardTestSuite struct {
 	// standard suite interfaces
 	suite.Suite
 	db           db.DB
-	storage      storage.Driver
+	storage      *storage.Driver
 	mediaManager media.Manager
 	federator    federation.Federator
 	processor    processing.Processor

diff --git a/internal/api/client/media/mediacreate_test.go b/internal/api/client/media/mediacreate_test.go
@@ -54,7 +54,7 @@ type MediaCreateTestSuite struct {
 	// standard suite interfaces
 	suite.Suite
 	db           db.DB
-	storage      *storage.Local
+	storage      *storage.Driver
 	mediaManager media.Manager
 	federator    federation.Federator
 	tc           typeutils.TypeConverter

diff --git a/internal/api/client/media/mediaupdate_test.go b/internal/api/client/media/mediaupdate_test.go
@@ -52,7 +52,7 @@ type MediaUpdateTestSuite struct {
 	// standard suite interfaces
 	suite.Suite
 	db           db.DB
-	storage      storage.Driver
+	storage      *storage.Driver
 	federator    federation.Federator
 	tc           typeutils.TypeConverter
 	mediaManager media.Manager

diff --git a/internal/api/client/status/status_test.go b/internal/api/client/status/status_test.go
@@ -43,7 +43,7 @@ type StatusStandardTestSuite struct {
 	federator    federation.Federator
 	emailSender  email.Sender
 	processor    processing.Processor
-	storage      storage.Driver
+	storage      *storage.Driver
 
 	// standard suite models
 	testTokens       map[string]*gtsmodel.Token

diff --git a/internal/api/client/user/user_test.go b/internal/api/client/user/user_test.go
@@ -42,7 +42,7 @@ type UserStandardTestSuite struct {
 	federator    federation.Federator
 	emailSender  email.Sender
 	processor    processing.Processor
-	storage      storage.Driver
+	storage      *storage.Driver
 
 	testTokens       map[string]*gtsmodel.Token
 	testClients      map[string]*gtsmodel.Client

diff --git a/internal/api/s2s/emoji/emojiget_test.go b/internal/api/s2s/emoji/emojiget_test.go
@@ -50,7 +50,7 @@ type EmojiGetTestSuite struct {
 	federator      federation.Federator
 	emailSender    email.Sender
 	processor      processing.Processor
-	storage        storage.Driver
+	storage        *storage.Driver
 	oauthServer    oauth.Server
 	securityModule *security.Module
 

diff --git a/internal/api/s2s/user/user_test.go b/internal/api/s2s/user/user_test.go
@@ -45,7 +45,7 @@ type UserStandardTestSuite struct {
 	federator      federation.Federator
 	emailSender    email.Sender
 	processor      processing.Processor
-	storage        storage.Driver
+	storage        *storage.Driver
 	oauthServer    oauth.Server
 	securityModule *security.Module
 

diff --git a/internal/api/s2s/webfinger/webfinger_test.go b/internal/api/s2s/webfinger/webfinger_test.go
@@ -50,7 +50,7 @@ type WebfingerStandardTestSuite struct {
 	federator      federation.Federator
 	emailSender    email.Sender
 	processor      processing.Processor
-	storage        storage.Driver
+	storage        *storage.Driver
 	oauthServer    oauth.Server
 	securityModule *security.Module
 

diff --git a/internal/federation/dereferencing/dereferencer_test.go b/internal/federation/dereferencing/dereferencer_test.go
@@ -33,7 +33,7 @@ import (
 type DereferencerStandardTestSuite struct {
 	suite.Suite
 	db      db.DB
-	storage storage.Driver
+	storage *storage.Driver
 
 	testRemoteStatuses    map[string]vocab.ActivityStreamsNote
 	testRemotePeople      map[string]vocab.ActivityStreamsPerson

diff --git a/internal/federation/federator_test.go b/internal/federation/federator_test.go
@@ -31,7 +31,7 @@ import (
 type FederatorStandardTestSuite struct {
 	suite.Suite
 	db             db.DB
-	storage        storage.Driver
+	storage        *storage.Driver
 	tc             typeutils.TypeConverter
 	testAccounts   map[string]*gtsmodel.Account
 	testStatuses   map[string]*gtsmodel.Status

diff --git a/internal/media/manager.go b/internal/media/manager.go
@@ -100,7 +100,7 @@ type Manager interface {
 
 type manager struct {
 	db           db.DB
-	storage      storage.Driver
+	storage      *storage.Driver
 	emojiWorker  *concurrency.WorkerPool[*ProcessingEmoji]
 	mediaWorker  *concurrency.WorkerPool[*ProcessingMedia]
 	stopCronJobs func() error
@@ -112,7 +112,7 @@ type manager struct {
 // a limited number of media will be processed in parallel. The numbers of workers
 // is determined from the $GOMAXPROCS environment variable (usually no. CPU cores).
 // See internal/concurrency.NewWorkerPool() documentation for further information.
-func NewManager(database db.DB, storage storage.Driver) (Manager, error) {
+func NewManager(database db.DB, storage *storage.Driver) (Manager, error) {
 	m := &manager{
 		db:      database,
 		storage: storage,

diff --git a/internal/media/manager_test.go b/internal/media/manager_test.go
@@ -927,14 +927,19 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() {
 	temp := fmt.Sprintf("%s/gotosocial-test", os.TempDir())
 	defer os.RemoveAll(temp)
 
-	diskStorage, err := kv.OpenDisk(temp, &storage.DiskConfig{
+	disk, err := storage.OpenDisk(temp, &storage.DiskConfig{
 		LockFile: path.Join(temp, "store.lock"),
 	})
 	if err != nil {
 		panic(err)
 	}
 
-	diskManager, err := media.NewManager(suite.db, &gtsstorage.Local{KVStore: diskStorage})
+	storage := &gtsstorage.Driver{
+		KVStore: kv.New(disk),
+		Storage: disk,
+	}
+
+	diskManager, err := media.NewManager(suite.db, storage)
 	if err != nil {
 		panic(err)
 	}
@@ -974,7 +979,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() {
 	suite.NotNil(dbAttachment)
 
 	// make sure the processed file is in storage
-	processedFullBytes, err := diskStorage.Get(ctx, attachment.File.Path)
+	processedFullBytes, err := storage.Get(ctx, attachment.File.Path)
 	suite.NoError(err)
 	suite.NotEmpty(processedFullBytes)
 
@@ -987,7 +992,7 @@ func (suite *ManagerTestSuite) TestSimpleJpegProcessBlockingWithDiskStorage() {
 	suite.Equal(processedFullBytesExpected, processedFullBytes)
 
 	// now do the same for the thumbnail and make sure it's what we expected
-	processedThumbnailBytes, err := diskStorage.Get(ctx, attachment.Thumbnail.Path)
+	processedThumbnailBytes, err := storage.Get(ctx, attachment.Thumbnail.Path)
 	suite.NoError(err)
 	suite.NotEmpty(processedThumbnailBytes)
 

diff --git a/internal/media/media_test.go b/internal/media/media_test.go
@@ -31,7 +31,7 @@ type MediaStandardTestSuite struct {
 	suite.Suite
 
 	db              db.DB
-	storage         storage.Driver
+	storage         *storage.Driver
 	manager         media.Manager
 	testAttachments map[string]*gtsmodel.MediaAttachment
 	testAccounts    map[string]*gtsmodel.Account

diff --git a/internal/media/processingemoji.go b/internal/media/processingemoji.go
@@ -68,7 +68,7 @@ type ProcessingEmoji struct {
 	*/
 
 	database db.DB
-	storage  storage.Driver
+	storage  *storage.Driver
 
 	err error // error created during processing, if any
 

diff --git a/internal/media/processingmedia.go b/internal/media/processingmedia.go
@@ -62,7 +62,7 @@ type ProcessingMedia struct {
 	*/
 
 	database db.DB
-	storage  storage.Driver
+	storage  *storage.Driver
 
 	err error // error created during processing, if any
 

diff --git a/internal/media/util.go b/internal/media/util.go
@@ -163,7 +163,7 @@ func (r *lengthReader) Read(b []byte) (int, error) {
 // putStream either puts a file with a known fileSize into storage directly, and returns the
 // fileSize unchanged, or it wraps the reader with a lengthReader and returns the discovered
 // fileSize.
-func putStream(ctx context.Context, storage storage.Driver, key string, r io.Reader, fileSize int64) (int64, error) {
+func putStream(ctx context.Context, storage *storage.Driver, key string, r io.Reader, fileSize int64) (int64, error) {
 	if fileSize > 0 {
 		return fileSize, storage.PutStream(ctx, key, r)
 	}

diff --git a/internal/processing/account/account_test.go b/internal/processing/account/account_test.go
@@ -43,7 +43,7 @@ type AccountStandardTestSuite struct {
 	suite.Suite
 	db                  db.DB
 	tc                  typeutils.TypeConverter
-	storage             storage.Driver
+	storage             *storage.Driver
 	mediaManager        media.Manager
 	oauthServer         oauth.Server
 	fromClientAPIChan   chan messages.FromClientAPI

diff --git a/internal/processing/media/media.go b/internal/processing/media/media.go
@@ -51,12 +51,12 @@ type processor struct {
 	tc                  typeutils.TypeConverter
 	mediaManager        media.Manager
 	transportController transport.Controller
-	storage             storage.Driver
+	storage             *storage.Driver
 	db                  db.DB
 }
 
 // New returns a new media processor.
-func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage storage.Driver) Processor {
+func New(db db.DB, tc typeutils.TypeConverter, mediaManager media.Manager, transportController transport.Controller, storage *storage.Driver) Processor {
 	return &processor{
 		tc:                  tc,
 		mediaManager:        mediaManager,

diff --git a/internal/processing/media/media_test.go b/internal/processing/media/media_test.go
@@ -37,7 +37,7 @@ type MediaStandardTestSuite struct {
 	suite.Suite
 	db                  db.DB
 	tc                  typeutils.TypeConverter
-	storage             storage.Driver
+	storage             *storage.Driver
 	mediaManager        media.Manager
 	transportController transport.Controller
 

diff --git a/internal/processing/processor.go b/internal/processing/processor.go
@@ -273,7 +273,7 @@ type processor struct {
 	tc              typeutils.TypeConverter
 	oauthServer     oauth.Server
 	mediaManager    media.Manager
-	storage         storage.Driver
+	storage         *storage.Driver
 	statusTimelines timeline.Manager
 	db              db.DB
 	filter          visibility.Filter
@@ -297,7 +297,7 @@ func NewProcessor(
 	federator federation.Federator,
 	oauthServer oauth.Server,
 	mediaManager media.Manager,
-	storage storage.Driver,
+	storage *storage.Driver,
 	db db.DB,
 	emailSender email.Sender,
 	clientWorker *concurrency.WorkerPool[messages.FromClientAPI],

diff --git a/internal/processing/processor_test.go b/internal/processing/processor_test.go
@@ -39,7 +39,7 @@ type ProcessingStandardTestSuite struct {
 	// standard suite interfaces
 	suite.Suite
 	db                  db.DB
-	storage             storage.Driver
+	storage             *storage.Driver
 	mediaManager        media.Manager
 	typeconverter       typeutils.TypeConverter
 	httpClient          *testrig.MockHTTPClient

diff --git a/internal/processing/status/status_test.go b/internal/processing/status/status_test.go
@@ -41,7 +41,7 @@ type StatusStandardTestSuite struct {
 	db            db.DB
 	typeConverter typeutils.TypeConverter
 	tc            transport.Controller
-	storage       storage.Driver
+	storage       *storage.Driver
 	mediaManager  media.Manager
 	federator     federation.Federator
 	clientWorker  *concurrency.WorkerPool[messages.FromClientAPI]

diff --git a/internal/storage/local.go b/internal/storage/local.go
diff --git a/internal/storage/s3.go b/internal/storage/s3.go