From f6c17372da5aece53315c388d2f392fa2dcb040a Mon Sep 17 00:00:00 2001 From: Anders Johnsen Date: Mon, 25 Sep 2023 12:55:44 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A5=20Cleanup=20unused=20components?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/rig-admin/cmd/database.go | 341 ----------------- cmd/rig-admin/cmd/storage.go | 312 --------------- cmd/rig/cmd/database/connect.go | 50 --- cmd/rig/cmd/database/create.go | 44 --- cmd/rig/cmd/database/create_credential.go | 39 -- cmd/rig/cmd/database/create_table.go | 39 -- cmd/rig/cmd/database/delete.go | 30 -- cmd/rig/cmd/database/delete_credential.go | 39 -- cmd/rig/cmd/database/delete_table.go | 38 -- cmd/rig/cmd/database/get.go | 39 -- cmd/rig/cmd/database/list.go | 42 -- cmd/rig/cmd/database/list_tables.go | 52 --- cmd/rig/cmd/database/setup.go | 147 ------- cmd/rig/cmd/storage/common.go | 25 -- cmd/rig/cmd/storage/copy.go | 339 ---------------- cmd/rig/cmd/storage/create_bucket.go | 97 ----- cmd/rig/cmd/storage/create_provider.go | 186 --------- cmd/rig/cmd/storage/delete_bucket.go | 37 -- cmd/rig/cmd/storage/delete_object.go | 43 --- cmd/rig/cmd/storage/delete_provider.go | 32 -- cmd/rig/cmd/storage/get_bucket.go | 49 --- cmd/rig/cmd/storage/get_object.go | 59 --- cmd/rig/cmd/storage/get_provider.go | 35 -- cmd/rig/cmd/storage/list_providers.go | 62 --- cmd/rig/cmd/storage/ls.go | 141 ------- cmd/rig/cmd/storage/setup.go | 207 ---------- cmd/rig/cmd/storage/unlink_bucket.go | 37 -- internal/config/defaults.go | 2 - internal/config/types.go | 2 - internal/handler/api/database/create.go | 24 -- .../handler/api/database/create_credential.go | 25 -- internal/handler/api/database/create_table.go | 21 - internal/handler/api/database/delete.go | 21 - .../handler/api/database/delete_credential.go | 22 -- internal/handler/api/database/delete_table.go | 21 - internal/handler/api/database/get.go | 24 -- internal/handler/api/database/get_by_name.go | 21 - internal/handler/api/database/get_endpoint.go | 22 -- internal/handler/api/database/handler.go | 30 -- internal/handler/api/database/list.go | 24 -- internal/handler/api/database/list_tables.go | 25 -- internal/handler/api/database/update.go | 24 -- internal/handler/api/storage/copy_object.go | 18 - internal/handler/api/storage/create_bucket.go | 25 -- .../handler/api/storage/create_provider.go | 21 - internal/handler/api/storage/delete_bucket.go | 19 - internal/handler/api/storage/delete_object.go | 19 - .../handler/api/storage/delete_provider.go | 24 -- .../handler/api/storage/download_object.go | 33 -- internal/handler/api/storage/get_bucket.go | 21 - internal/handler/api/storage/get_object.go | 21 - internal/handler/api/storage/get_provider.go | 27 -- internal/handler/api/storage/handler.go | 33 -- internal/handler/api/storage/list_buckets.go | 27 -- internal/handler/api/storage/list_objects.go | 27 -- .../handler/api/storage/list_providers.go | 28 -- .../handler/api/storage/lookup_provider.go | 22 -- internal/handler/api/storage/unlink_bucket.go | 19 - internal/handler/api/storage/upload_object.go | 53 --- internal/handler/api/storage_http/download.go | 68 ---- internal/handler/api/storage_http/upload.go | 60 --- internal/handler/module.go | 7 - internal/repository/database.go | 23 -- internal/repository/database/mongo/client.go | 52 --- internal/repository/database/mongo/create.go | 24 -- internal/repository/database/mongo/delete.go | 26 -- internal/repository/database/mongo/get.go | 23 -- .../repository/database/mongo/get_by_name.go | 33 -- internal/repository/database/mongo/list.go | 58 --- .../database/mongo/schema/database.go | 43 --- internal/repository/database/mongo/update.go | 33 -- internal/repository/module.go | 30 -- internal/repository/storage.go | 21 - internal/repository/storage/mongo/client.go | 66 ---- internal/repository/storage/mongo/create.go | 28 -- internal/repository/storage/mongo/delete.go | 29 -- internal/repository/storage/mongo/get.go | 31 -- internal/repository/storage/mongo/list.go | 54 --- internal/repository/storage/mongo/lookup.go | 31 -- .../storage/mongo/lookup_by_bucket.go | 32 -- .../storage/mongo/schema/provider.go | 60 --- internal/repository/storage/mongo/update.go | 26 -- internal/service/database/create.go | 51 --- .../service/database/create_credential.go | 87 ----- internal/service/database/create_table.go | 38 -- internal/service/database/delete.go | 50 --- .../service/database/delete_credential.go | 50 --- internal/service/database/delete_table.go | 28 -- internal/service/database/get.go | 49 --- internal/service/database/get_by_name.go | 49 --- .../service/database/get_database_endpoint.go | 41 -- internal/service/database/list_credentials.go | 16 - internal/service/database/list_databases.go | 17 - internal/service/database/list_tables.go | 16 - internal/service/database/service.go | 105 ----- internal/service/database/update.go | 23 -- internal/service/module.go | 4 - internal/service/storage/service.go | 361 ------------------ internal/service/storage/storage_provider.go | 255 ------------- 99 files changed, 5404 deletions(-) delete mode 100644 cmd/rig-admin/cmd/database.go delete mode 100644 cmd/rig-admin/cmd/storage.go delete mode 100644 cmd/rig/cmd/database/connect.go delete mode 100644 cmd/rig/cmd/database/create.go delete mode 100644 cmd/rig/cmd/database/create_credential.go delete mode 100644 cmd/rig/cmd/database/create_table.go delete mode 100644 cmd/rig/cmd/database/delete.go delete mode 100644 cmd/rig/cmd/database/delete_credential.go delete mode 100644 cmd/rig/cmd/database/delete_table.go delete mode 100644 cmd/rig/cmd/database/get.go delete mode 100644 cmd/rig/cmd/database/list.go delete mode 100644 cmd/rig/cmd/database/list_tables.go delete mode 100644 cmd/rig/cmd/database/setup.go delete mode 100644 cmd/rig/cmd/storage/common.go delete mode 100644 cmd/rig/cmd/storage/copy.go delete mode 100644 cmd/rig/cmd/storage/create_bucket.go delete mode 100644 cmd/rig/cmd/storage/create_provider.go delete mode 100644 cmd/rig/cmd/storage/delete_bucket.go delete mode 100644 cmd/rig/cmd/storage/delete_object.go delete mode 100644 cmd/rig/cmd/storage/delete_provider.go delete mode 100644 cmd/rig/cmd/storage/get_bucket.go delete mode 100644 cmd/rig/cmd/storage/get_object.go delete mode 100644 cmd/rig/cmd/storage/get_provider.go delete mode 100644 cmd/rig/cmd/storage/list_providers.go delete mode 100644 cmd/rig/cmd/storage/ls.go delete mode 100644 cmd/rig/cmd/storage/setup.go delete mode 100644 cmd/rig/cmd/storage/unlink_bucket.go delete mode 100644 internal/handler/api/database/create.go delete mode 100644 internal/handler/api/database/create_credential.go delete mode 100644 internal/handler/api/database/create_table.go delete mode 100644 internal/handler/api/database/delete.go delete mode 100644 internal/handler/api/database/delete_credential.go delete mode 100644 internal/handler/api/database/delete_table.go delete mode 100644 internal/handler/api/database/get.go delete mode 100644 internal/handler/api/database/get_by_name.go delete mode 100644 internal/handler/api/database/get_endpoint.go delete mode 100644 internal/handler/api/database/handler.go delete mode 100644 internal/handler/api/database/list.go delete mode 100644 internal/handler/api/database/list_tables.go delete mode 100644 internal/handler/api/database/update.go delete mode 100644 internal/handler/api/storage/copy_object.go delete mode 100644 internal/handler/api/storage/create_bucket.go delete mode 100644 internal/handler/api/storage/create_provider.go delete mode 100644 internal/handler/api/storage/delete_bucket.go delete mode 100644 internal/handler/api/storage/delete_object.go delete mode 100644 internal/handler/api/storage/delete_provider.go delete mode 100644 internal/handler/api/storage/download_object.go delete mode 100644 internal/handler/api/storage/get_bucket.go delete mode 100644 internal/handler/api/storage/get_object.go delete mode 100644 internal/handler/api/storage/get_provider.go delete mode 100644 internal/handler/api/storage/handler.go delete mode 100644 internal/handler/api/storage/list_buckets.go delete mode 100644 internal/handler/api/storage/list_objects.go delete mode 100644 internal/handler/api/storage/list_providers.go delete mode 100644 internal/handler/api/storage/lookup_provider.go delete mode 100644 internal/handler/api/storage/unlink_bucket.go delete mode 100644 internal/handler/api/storage/upload_object.go delete mode 100644 internal/handler/api/storage_http/download.go delete mode 100644 internal/handler/api/storage_http/upload.go delete mode 100644 internal/repository/database.go delete mode 100644 internal/repository/database/mongo/client.go delete mode 100644 internal/repository/database/mongo/create.go delete mode 100644 internal/repository/database/mongo/delete.go delete mode 100644 internal/repository/database/mongo/get.go delete mode 100644 internal/repository/database/mongo/get_by_name.go delete mode 100644 internal/repository/database/mongo/list.go delete mode 100644 internal/repository/database/mongo/schema/database.go delete mode 100644 internal/repository/database/mongo/update.go delete mode 100644 internal/repository/storage.go delete mode 100644 internal/repository/storage/mongo/client.go delete mode 100644 internal/repository/storage/mongo/create.go delete mode 100644 internal/repository/storage/mongo/delete.go delete mode 100644 internal/repository/storage/mongo/get.go delete mode 100644 internal/repository/storage/mongo/list.go delete mode 100644 internal/repository/storage/mongo/lookup.go delete mode 100644 internal/repository/storage/mongo/lookup_by_bucket.go delete mode 100644 internal/repository/storage/mongo/schema/provider.go delete mode 100644 internal/repository/storage/mongo/update.go delete mode 100644 internal/service/database/create.go delete mode 100644 internal/service/database/create_credential.go delete mode 100644 internal/service/database/create_table.go delete mode 100644 internal/service/database/delete.go delete mode 100644 internal/service/database/delete_credential.go delete mode 100644 internal/service/database/delete_table.go delete mode 100644 internal/service/database/get.go delete mode 100644 internal/service/database/get_by_name.go delete mode 100644 internal/service/database/get_database_endpoint.go delete mode 100644 internal/service/database/list_credentials.go delete mode 100644 internal/service/database/list_databases.go delete mode 100644 internal/service/database/list_tables.go delete mode 100644 internal/service/database/service.go delete mode 100644 internal/service/database/update.go delete mode 100644 internal/service/storage/service.go delete mode 100644 internal/service/storage/storage_provider.go diff --git a/cmd/rig-admin/cmd/database.go b/cmd/rig-admin/cmd/database.go deleted file mode 100644 index 74a79ab78..000000000 --- a/cmd/rig-admin/cmd/database.go +++ /dev/null @@ -1,341 +0,0 @@ -package cmd - -import ( - "context" - "errors" - "fmt" - "io" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig-go-api/model" - service_database "github.com/rigdev/rig/internal/service/database" - "github.com/rigdev/rig/pkg/uuid" - "github.com/spf13/cobra" - "go.uber.org/zap" -) - -var databaseId string - -func init() { - database := &cobra.Command{ - Use: "database", - } - database.PersistentFlags().StringVar(&databaseId, "database", "", "uuid of your database") - - createDatabase := &cobra.Command{ - Use: "create-database ", - RunE: register(CreateDatabase), - Args: cobra.ExactArgs(2), - } - database.AddCommand(createDatabase) - - createDatabaseCredential := &cobra.Command{ - Use: "create-credential ", - RunE: register(CreateDatabaseCredential), - Args: cobra.ExactArgs(1), - } - database.AddCommand(createDatabaseCredential) - - deleteDatabaseCredential := &cobra.Command{ - Use: "delete-credential ", - RunE: register(DeleteDatabaseCredential), - Args: cobra.ExactArgs(1), - } - database.AddCommand(deleteDatabaseCredential) - - listCredentials := &cobra.Command{ - Use: "list-credentials", - RunE: register(ListDatabaseCredentials), - } - database.AddCommand(listCredentials) - - getDatabase := &cobra.Command{ - Use: "get-database", - RunE: register(GetDatabase), - } - database.AddCommand(getDatabase) - - connectDatabase := &cobra.Command{ - Use: "connect ", - RunE: register(ConnectDatabase), - Args: cobra.ExactArgs(2), - } - database.AddCommand(connectDatabase) - - deleteDatabase := &cobra.Command{ - Use: "delete-database", - RunE: register(DeleteDatabase), - } - database.AddCommand(deleteDatabase) - - listDatabases := &cobra.Command{ - Use: "list-databases", - RunE: register(ListDatabases), - } - database.AddCommand(listDatabases) - - createTable := &cobra.Command{ - Use: "create-table ", - RunE: register(CreateTable), - Args: cobra.ExactArgs(1), - } - database.AddCommand(createTable) - - listTables := &cobra.Command{ - Use: "list-tables", - RunE: register(ListTables), - } - database.AddCommand(listTables) - - deleteTable := &cobra.Command{ - Use: "delete-table ", - RunE: register(DeleteTable), - Args: cobra.ExactArgs(1), - } - database.AddCommand(deleteTable) - - rootCmd.AddCommand(database) -} - -func getDbType(databaseType string) (database.Type, error) { - if databaseType == "" { - return database.Type_TYPE_UNSPECIFIED, errors.New("type is required") - } - var dbType database.Type - switch databaseType { - case "mongo": - dbType = database.Type_TYPE_MONGO - case "postgres": - dbType = database.Type_TYPE_POSTGRES - default: - return database.Type_TYPE_UNSPECIFIED, fmt.Errorf("invalid database type: %v (insert mongo or postgres)", databaseType) - } - return dbType, nil -} - -func CreateDatabase(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - databaseName := args[0] - if databaseName == "" { - return errors.New("database name is required") - } - dbType, err := getDbType(args[1]) - if err != nil { - return err - } - databaseID, db, err := ds.Create(ctx, dbType, []*database.Update{ - {Field: &database.Update_Name{Name: databaseName}}, - }) - if err != nil { - return err - } - logger.Info("created database", zap.String("name", databaseName), zap.String("type", db.GetType().String()), zap.String("id", databaseID.String())) - return nil -} - -func CreateDatabaseCredential(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - if databaseId == "" { - return errors.New("missing required database id") - } - credentialName := args[0] - if credentialName == "" { - return errors.New("credential name is required") - } - - dbId, err := uuid.Parse(databaseId) - if err != nil { - return err - } - - clientId, clientSecret, err := ds.CreateCredential(ctx, credentialName, dbId) - if err != nil { - return err - } - logger.Info("created credential", zap.String("clientId", clientId), zap.String("secret", clientSecret)) - return nil -} - -func DeleteDatabaseCredential(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - if databaseId == "" { - return errors.New("missing required database id") - } - credentialsName := args[0] - if credentialsName == "" { - return errors.New("credentials name is required") - } - - dbId, err := uuid.Parse(databaseId) - if err != nil { - return err - } - - if err := ds.DeleteCredential(ctx, credentialsName, dbId); err != nil { - return err - } - logger.Info("deleted credential", zap.String("clientId", credentialsName)) - return nil -} - -func ListDatabaseCredentials(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - if databaseId == "" { - return errors.New("missing required database id") - } - - dbId, err := uuid.Parse(databaseId) - if err != nil { - return err - } - - credentials, err := ds.ListCredentials(ctx, dbId) - if err != nil { - return err - } - logger.Info("found credentials", zap.Int("amount", len(credentials))) - for _, credential := range credentials { - logger.Info("credential", zap.String("name", credential.GetName()), zap.String("clientId", credential.ClientId), zap.String("secret", string(credential.Secret))) - } - return nil -} - -func GetDatabase(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - if databaseId == "" { - return errors.New("missing required database id") - } - - dbId, err := uuid.Parse(databaseId) - if err != nil { - return err - } - - database, err := ds.Get(ctx, dbId) - if err != nil { - return err - } - logger.Info("got database", zap.String("name", database.Name)) - return nil -} - -func ConnectDatabase(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - if databaseId == "" { - return errors.New("missing required database id") - } - - dbId, err := uuid.Parse(databaseId) - if err != nil { - return err - } - - clientId := args[0] - if clientId == "" { - return errors.New("clientId is required") - } - clientSecret := args[1] - if clientSecret == "" { - return errors.New("clientSecret is required") - } - endpoint, _, err := ds.GetDatabaseEndpoint(ctx, dbId, clientId, clientSecret) - if err != nil { - return err - } - fmt.Printf(` - -connect to mongo by running: -mongosh "%s" - - `, endpoint) - - return nil -} - -func DeleteDatabase(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - if databaseId == "" { - return errors.New("missing required database id") - } - - dbId, err := uuid.Parse(databaseId) - if err != nil { - return err - } - - if err := ds.Delete(ctx, dbId); err != nil { - return err - } - logger.Info("deleted database") - return nil -} - -func ListDatabases(ctx context.Context, cmd *cobra.Command, ds *service_database.Service, logger *zap.Logger) error { - it, count, err := ds.List(ctx, &model.Pagination{}) - if err != nil { - return err - } - logger.Info(fmt.Sprintf("found %d databases: ", count)) - for { - database, err := it.Next() - if err == io.EOF { - break - } else if err != nil { - return err - } - logger.Info("", zap.String("name", database.Name), zap.String("type", database.GetType().String()), zap.Time("createdAt", database.GetInfo().GetCreatedAt().AsTime())) - } - return nil -} - -func CreateTable(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - if databaseId == "" { - return errors.New("missing required database id") - } - dbId, err := uuid.Parse(databaseId) - if err != nil { - return err - } - - tableName := args[0] - if tableName == "" { - return errors.New("table name is required") - } - if err := ds.CreateTable(ctx, dbId, tableName); err != nil { - return err - } - logger.Info("created table", zap.String("name", tableName)) - return nil -} - -func ListTables(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - if databaseId == "" { - return errors.New("missing required database id") - } - dbId, err := uuid.Parse(databaseId) - if err != nil { - return err - } - - tables, err := ds.ListTables(ctx, dbId) - if err != nil { - return err - } - for _, table := range tables { - logger.Info("table", zap.String("name", table.GetName())) - } - return nil -} - -func DeleteTable(ctx context.Context, cmd *cobra.Command, args []string, ds *service_database.Service, logger *zap.Logger) error { - if databaseId == "" { - return errors.New("missing required database id") - } - dbId, err := uuid.Parse(databaseId) - if err != nil { - return err - } - - tableName := args[0] - if tableName == "" { - return errors.New("table name is required") - } - if err := ds.DeleteTable(ctx, dbId, tableName); err != nil { - return err - } - logger.Info("deleted table", zap.String("name", tableName)) - return nil -} diff --git a/cmd/rig-admin/cmd/storage.go b/cmd/rig-admin/cmd/storage.go deleted file mode 100644 index d4887f5fe..000000000 --- a/cmd/rig-admin/cmd/storage.go +++ /dev/null @@ -1,312 +0,0 @@ -package cmd - -import ( - "context" - "errors" - "io" - "os" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/uuid" - storage_service "github.com/rigdev/rig/internal/service/storage" - "github.com/spf13/cobra" - "go.uber.org/zap" -) - -var recursive bool - -func init() { - storage := &cobra.Command{ - Use: "storage", - } - - createBucket := &cobra.Command{ - Use: "create-bucket ", - RunE: register(StorageCreateBucket), - } - storage.AddCommand(createBucket) - - getBucket := &cobra.Command{ - Use: "get-bucket ", - RunE: register(StorageGetBucket), - } - storage.AddCommand(getBucket) - - deleteBucket := &cobra.Command{ - Use: "delete-bucket ", - RunE: register(StorageDeleteBucket), - } - storage.AddCommand(deleteBucket) - - listBuckets := &cobra.Command{ - Use: "list-buckets", - RunE: register(StorageListBuckets), - } - storage.AddCommand(listBuckets) - - getObject := &cobra.Command{ - Use: "get-object ", - RunE: register(StorageGetObject), - } - storage.AddCommand(getObject) - - listObject := &cobra.Command{ - Use: "list-objects ", - RunE: register(StorageListObjects), - } - listObject.Flags().BoolVarP(&recursive, "recursive", "r", false, "recursive") - storage.AddCommand(listObject) - - deleteObject := &cobra.Command{ - Use: "delete-object ", - RunE: register(StorageDeleteObject), - } - storage.AddCommand(deleteObject) - - copyObject := &cobra.Command{ - Use: "copy-object ", - RunE: register(StorageCopyObject), - } - - storage.AddCommand(copyObject) - - uploadObject := &cobra.Command{ - Use: "upload ", - RunE: register(StorageUploadFile), - } - storage.AddCommand(uploadObject) - - downloadObject := &cobra.Command{ - Use: "download ", - RunE: register(StorageDownloadFile), - } - storage.AddCommand(downloadObject) - - rootCmd.AddCommand(storage) -} - -func StorageCreateBucket(ctx context.Context, cmd *cobra.Command, args []string, ss *storage_service.Service, logger *zap.Logger) error { - bucketName := args[0] - if bucketName == "" { - return errors.New("name is required") - } - - providerBucketName := args[1] - if providerBucketName == "" { - return errors.New("provider name is required") - } - - bucketRegion := args[2] - if bucketName == "" { - return errors.New("region is required") - } - providerID, err := uuid.Parse(args[3]) - if err != nil { - return err - } - err = ss.CreateBucket(ctx, bucketName, providerBucketName, bucketRegion, providerID) - if err != nil { - return err - } - logger.Info("created bucket", zap.String("name", bucketName)) - return nil -} - -func StorageGetBucket(ctx context.Context, cmd *cobra.Command, args []string, ss *storage_service.Service, logger *zap.Logger) error { - bucketName := args[0] - if bucketName == "" { - return errors.New("name is required") - } - bucket, err := ss.GetBucket(ctx, bucketName) - if err != nil { - return err - } - logger.Info("got bucket", zap.String("name", bucket.Name)) - return nil -} - -func StorageDeleteBucket(ctx context.Context, cmd *cobra.Command, args []string, ss *storage_service.Service, logger *zap.Logger) error { - bucketName := args[0] - if bucketName == "" { - return errors.New("name is required") - } - err := ss.DeleteBucket(ctx, bucketName) - if err != nil { - return err - } - logger.Info("deleted bucket", zap.String("name", bucketName)) - return nil -} - -func StorageListBuckets(ctx context.Context, cmd *cobra.Command, ss *storage_service.Service, logger *zap.Logger) error { - it, err := ss.ListBuckets(ctx) - if err != nil { - return err - } - for { - bucket, err := it.Next() - if err == io.EOF { - break - } else if err != nil { - return err - } - logger.Info("got bucket", zap.String("name", bucket.Name)) - } - return nil -} - -func StorageGetObject(ctx context.Context, cmd *cobra.Command, args []string, ss *storage_service.Service, logger *zap.Logger) error { - bucketName := args[0] - path := args[1] - if bucketName == "" { - return errors.New("name is required") - } - if path == "" { - return errors.New("path is required") - } - object, err := ss.GetObject(ctx, bucketName, path) - if err != nil { - return err - } - logger.Info("got object", zap.String("name", object.GetPath()), zap.Int("size", int(object.GetSize())), zap.String("etag", object.GetEtag())) - return nil -} - -func StorageDeleteObject(ctx context.Context, cmd *cobra.Command, args []string, ss *storage_service.Service, logger *zap.Logger) error { - bucketName := args[0] - path := args[1] - if bucketName == "" { - return errors.New("name is required") - } - if path == "" { - return errors.New("path is required") - } - err := ss.DeleteObject(ctx, bucketName, path) - if err != nil { - return err - } - logger.Info("deleted object", zap.String("name", path)) - return nil -} - -func StorageCopyObject(ctx context.Context, cmd *cobra.Command, args []string, ss *storage_service.Service, logger *zap.Logger) error { - bucketName := args[0] - path := args[1] - dstBucketName := args[2] - dstObjectpath := args[3] - if bucketName == "" { - return errors.New("name is required") - } - if path == "" { - return errors.New("path is required") - } - if dstBucketName == "" { - return errors.New("dstbucket is required") - } - if dstObjectpath == "" { - return errors.New("dstpath is required") - } - err := ss.CopyObject(ctx, bucketName, path, dstBucketName, dstObjectpath) - if err != nil { - return err - } - logger.Info("copied object from" + bucketName + ":" + path + " to " + dstBucketName + ":" + dstObjectpath) - return nil -} - -func StorageListObjects(ctx context.Context, cmd *cobra.Command, args []string, ss *storage_service.Service, logger *zap.Logger) error { - bucketName := args[0] - - if bucketName == "" { - return errors.New("name is required") - } - _, it, err := ss.ListObjects(ctx, bucketName, "", "/", "", "", recursive, 100) - if err != nil { - return err - } - for { - object, err := it.Next() - if err == io.EOF { - break - } else if err != nil { - return err - } - switch v := object.GetResult().(type) { - case *storage.ListObjectsResponse_Result_Folder: - logger.Info("got folder", zap.String("name", v.Folder)) - case *storage.ListObjectsResponse_Result_Object: - logger.Info("got object", zap.String("name", v.Object.GetPath()), zap.Int("size", int(v.Object.GetSize())), zap.String("etag", v.Object.GetEtag())) - } - } - return nil -} - -func StorageUploadFile(ctx context.Context, cmd *cobra.Command, args []string, ss *storage_service.Service, logger *zap.Logger) error { - bucketName := args[0] - path := args[1] - filePath := args[2] - - if bucketName == "" { - return errors.New("name is required") - } - if path == "" { - return errors.New("path is required") - } - if filePath == "" { - return errors.New("file is required") - } - file, err := os.Open(filePath) - if err != nil { - return err - } - defer file.Close() - - fileStat, err := file.Stat() - if err != nil { - return err - } - - metadata := &storage.UploadObjectRequest_Metadata{ - Bucket: bucketName, - Path: path, - Size: uint64(fileStat.Size()), - } - _, _, err = ss.UploadObject(ctx, file, metadata) - if err != nil { - return err - } - logger.Info("uploaded file", zap.String("name", path), zap.Int64("size", fileStat.Size())) - return nil -} - -func StorageDownloadFile(ctx context.Context, cmd *cobra.Command, args []string, ss *storage_service.Service, logger *zap.Logger) error { - bucketName := args[0] - path := args[1] - filePath := args[2] - - if bucketName == "" { - return errors.New("name is required") - } - if path == "" { - return errors.New("path is required") - } - if filePath == "" { - return errors.New("file is required") - } - file, err := os.Create(filePath) - if err != nil { - return err - } - defer file.Close() - - object, err := ss.DownloadObject(ctx, bucketName, path) - if err != nil { - return err - } - written, err := io.Copy(file, object) - if err != nil { - return err - } - logger.Info("downloaded file", zap.String("name", path), zap.Int64("size", written)) - return nil -} diff --git a/cmd/rig/cmd/database/connect.go b/cmd/rig/cmd/database/connect.go deleted file mode 100644 index 70cb0cf33..000000000 --- a/cmd/rig/cmd/database/connect.go +++ /dev/null @@ -1,50 +0,0 @@ -package database - -import ( - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) connect(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - _, id, err := common.GetDatabase(ctx, identifier, c.Rig) - if err != nil { - return err - } - - if clientID == "" { - clientID, err = common.PromptInput("Client ID:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } - - if clientSecret == "" { - clientSecret, err = common.PromptInput("Client Secret:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } - - endpointRes, err := c.Rig.Database().GetEndpoint(ctx, &connect.Request[database.GetEndpointRequest]{ - Msg: &database.GetEndpointRequest{ - DatabaseId: id, - ClientId: clientID, - ClientSecret: clientSecret, - }, - }) - if err != nil { - return err - } - - endpoint := endpointRes.Msg.GetEndpoint() - - cmd.Printf("Connect to mongo by running: \n mongosh \"%s\"\n", endpoint) - return nil -} diff --git a/cmd/rig/cmd/database/create.go b/cmd/rig/cmd/database/create.go deleted file mode 100644 index 1f6ecf307..000000000 --- a/cmd/rig/cmd/database/create.go +++ /dev/null @@ -1,44 +0,0 @@ -package database - -import ( - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) create(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - var err error - if name == "" { - name, err = common.PromptInput("Database name:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } - - if dbTypeString == "" { - _, dbTypeString, err = common.PromptSelect("Database type:", []string{"mongo", "postgres"}) - if err != nil { - return err - } - } - - dbType, err := getDbType(dbTypeString) - if err != nil { - return err - } - - res, err := c.Rig.Database().Create(ctx, &connect.Request[database.CreateRequest]{Msg: &database.CreateRequest{ - Initializers: []*database.Update{ - {Field: &database.Update_Name{Name: name}}, - }, - Type: dbType, - }}) - if err != nil { - return err - } - - cmd.Printf("created database %s of type %s with id %s\n", name, dbTypeString, res.Msg.GetDatabase().GetDatabaseId()) - return nil -} diff --git a/cmd/rig/cmd/database/create_credential.go b/cmd/rig/cmd/database/create_credential.go deleted file mode 100644 index 5c1c27019..000000000 --- a/cmd/rig/cmd/database/create_credential.go +++ /dev/null @@ -1,39 +0,0 @@ -package database - -import ( - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) createCredentials(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - _, id, err := common.GetDatabase(ctx, identifier, c.Rig) - if err != nil { - return err - } - - if name == "" { - name, err = common.PromptInput("Credential Name:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } - res, err := c.Rig.Database().CreateCredential(ctx, &connect.Request[database.CreateCredentialRequest]{ - Msg: &database.CreateCredentialRequest{ - DatabaseId: id, - Name: name, - }, - }) - if err != nil { - return err - } - - cmd.Printf("created credential - clientID: %s, secret: %s\n", res.Msg.GetClientId(), res.Msg.GetClientSecret()) - return nil -} diff --git a/cmd/rig/cmd/database/create_table.go b/cmd/rig/cmd/database/create_table.go deleted file mode 100644 index 7147eb75f..000000000 --- a/cmd/rig/cmd/database/create_table.go +++ /dev/null @@ -1,39 +0,0 @@ -package database - -import ( - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) createTable(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - _, id, err := common.GetDatabase(ctx, identifier, c.Rig) - if err != nil { - return err - } - - if name == "" { - name, err = common.PromptInput("Table Name:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } - - if _, err := c.Rig.Database().CreateTable(ctx, &connect.Request[database.CreateTableRequest]{ - Msg: &database.CreateTableRequest{ - DatabaseId: id, - TableName: name, - }, - }); err != nil { - return err - } - - cmd.Printf("created table %s\n", name) - return nil -} diff --git a/cmd/rig/cmd/database/delete.go b/cmd/rig/cmd/database/delete.go deleted file mode 100644 index 95d03d323..000000000 --- a/cmd/rig/cmd/database/delete.go +++ /dev/null @@ -1,30 +0,0 @@ -package database - -import ( - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) delete(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - db, id, err := common.GetDatabase(ctx, identifier, c.Rig) - if err != nil { - return err - } - - if _, err := c.Rig.Database().Delete(ctx, &connect.Request[database.DeleteRequest]{ - Msg: &database.DeleteRequest{ - DatabaseId: id, - }, - }); err != nil { - return err - } - cmd.Println("deleted database", db.GetName()) - return nil -} diff --git a/cmd/rig/cmd/database/delete_credential.go b/cmd/rig/cmd/database/delete_credential.go deleted file mode 100644 index e3ded666e..000000000 --- a/cmd/rig/cmd/database/delete_credential.go +++ /dev/null @@ -1,39 +0,0 @@ -package database - -import ( - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) deleteCredential(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - _, id, err := common.GetDatabase(ctx, identifier, c.Rig) - if err != nil { - return err - } - - if name == "" { - name, err = common.PromptInput("Credential Name:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } - - if _, err := c.Rig.Database().DeleteCredential(ctx, &connect.Request[database.DeleteCredentialRequest]{ - Msg: &database.DeleteCredentialRequest{ - DatabaseId: id, - CredentialName: name, - }, - }); err != nil { - return err - } - - cmd.Printf("deleted credential %s\n", name) - return nil -} diff --git a/cmd/rig/cmd/database/delete_table.go b/cmd/rig/cmd/database/delete_table.go deleted file mode 100644 index eafe3c347..000000000 --- a/cmd/rig/cmd/database/delete_table.go +++ /dev/null @@ -1,38 +0,0 @@ -package database - -import ( - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) deleteTable(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - _, id, err := common.GetDatabase(ctx, identifier, c.Rig) - if err != nil { - return err - } - - if name == "" { - name, err = common.PromptInput("Table Name:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } - - if _, err := c.Rig.Database().DeleteTable(ctx, &connect.Request[database.DeleteTableRequest]{ - Msg: &database.DeleteTableRequest{ - DatabaseId: id, - TableName: name, - }, - }); err != nil { - return err - } - cmd.Println("deleted table\n", name) - return nil -} diff --git a/cmd/rig/cmd/database/get.go b/cmd/rig/cmd/database/get.go deleted file mode 100644 index c9ee201ff..000000000 --- a/cmd/rig/cmd/database/get.go +++ /dev/null @@ -1,39 +0,0 @@ -package database - -import ( - "github.com/jedib0t/go-pretty/v6/table" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) get(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - db, id, err := common.GetDatabase(ctx, identifier, c.Rig) - if err != nil { - return err - } - - if outputJSON { - cmd.Println(common.ProtoToPrettyJson(db)) - return nil - } - - // print a table with a column for attributes, and a column for values - t := table.NewWriter() - t.AppendHeader(table.Row{"Attribute", "Value"}) - t.AppendRows([]table.Row{ - {"ID", id}, - {"Name", db.GetName()}, - {"Type", db.GetType()}, - {"Num Tables", len(db.GetInfo().GetTables())}, - {"Num Creds", len(db.GetInfo().GetCredentials())}, - {"Created At", db.GetInfo().GetCreatedAt().AsTime().Format("2006-01-02 15:04:05")}, - }) - cmd.Println(t.Render()) - - return nil -} diff --git a/cmd/rig/cmd/database/list.go b/cmd/rig/cmd/database/list.go deleted file mode 100644 index e6532a530..000000000 --- a/cmd/rig/cmd/database/list.go +++ /dev/null @@ -1,42 +0,0 @@ -package database - -import ( - "fmt" - - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/table" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) list(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - res, err := c.Rig.Database().List(ctx, &connect.Request[database.ListRequest]{ - Msg: &database.ListRequest{ - Pagination: &model.Pagination{ - Offset: uint32(offset), - Limit: uint32(limit), - }, - }, - }) - if err != nil { - return err - } - - if outputJSON { - for _, db := range res.Msg.GetDatabases() { - cmd.Println(common.ProtoToPrettyJson(db)) - } - return nil - } - - t := table.NewWriter() - t.AppendHeader(table.Row{fmt.Sprintf("DBs (%d)", res.Msg.GetTotal()), "Name", "Type"}) - for i, db := range res.Msg.GetDatabases() { - t.AppendRow(table.Row{i + 1, db.GetName(), db.GetType()}) - } - cmd.Println(t.Render()) - return nil -} diff --git a/cmd/rig/cmd/database/list_tables.go b/cmd/rig/cmd/database/list_tables.go deleted file mode 100644 index bc6bb2bfc..000000000 --- a/cmd/rig/cmd/database/list_tables.go +++ /dev/null @@ -1,52 +0,0 @@ -package database - -import ( - "fmt" - - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/table" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) listTables(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - _, id, err := common.GetDatabase(ctx, identifier, c.Rig) - if err != nil { - return err - } - - res, err := c.Rig.Database().ListTables(ctx, &connect.Request[database.ListTablesRequest]{ - Msg: &database.ListTablesRequest{ - DatabaseId: id, - Pagination: &model.Pagination{ - Offset: uint32(offset), - Limit: uint32(limit), - }, - }, - }) - if err != nil { - return err - } - - if outputJSON { - for _, tb := range res.Msg.GetTables() { - cmd.Println(common.ProtoToPrettyJson(tb)) - } - return nil - } - - t := table.NewWriter() - t.AppendHeader(table.Row{fmt.Sprintf("Tables (%d)", res.Msg.GetTotal()), "Name"}) - for i, tb := range res.Msg.GetTables() { - t.AppendRow(table.Row{i + 1, tb.GetName()}) - } - cmd.Println(t.Render()) - return nil -} diff --git a/cmd/rig/cmd/database/setup.go b/cmd/rig/cmd/database/setup.go deleted file mode 100644 index 62fbf1bb4..000000000 --- a/cmd/rig/cmd/database/setup.go +++ /dev/null @@ -1,147 +0,0 @@ -package database - -import ( - "context" - "errors" - "fmt" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig-go-sdk" - "github.com/rigdev/rig/cmd/rig/cmd/cmd_config" - "github.com/spf13/cobra" - "go.uber.org/fx" -) - -var ( - outputJSON bool -) - -var ( - offset int - limit int -) - -var ( - name string - dbTypeString string - clientID string - clientSecret string -) - -type Cmd struct { - fx.In - - Ctx context.Context - Rig rig.Client - Cfg *cmd_config.Config -} - -func (c Cmd) Setup(parent *cobra.Command) { - database := &cobra.Command{ - Use: "database", - } - - createDatabase := &cobra.Command{ - Use: "create", - RunE: c.create, - Args: cobra.NoArgs, - } - createDatabase.Flags().StringVarP(&name, "name", "n", "", "name of the database") - createDatabase.Flags().StringVarP(&dbTypeString, "type", "t", "", "type of the database (mongo, postgres)") - database.AddCommand(createDatabase) - - createDatabaseCredential := &cobra.Command{ - Use: "create-credentials [id | db-name]", - RunE: c.createCredentials, - Args: cobra.MaximumNArgs(1), - } - createDatabaseCredential.Flags().StringVarP(&name, "name", "n", "", "name of the credentials") - database.AddCommand(createDatabaseCredential) - - deleteCredential := &cobra.Command{ - Use: "delete-credentials [id | db-name]", - RunE: c.deleteCredential, - Args: cobra.MaximumNArgs(1), - } - deleteCredential.Flags().StringVarP(&name, "name", "n", "", "name of the credentials") - database.AddCommand(deleteCredential) - - getDatabase := &cobra.Command{ - Use: "get [id | name]", - RunE: c.get, - Args: cobra.MaximumNArgs(1), - } - getDatabase.Flags().BoolVar(&outputJSON, "json", false, "output as json") - database.AddCommand(getDatabase) - - connect := &cobra.Command{ - Use: "connect [id | name]", - RunE: c.connect, - Args: cobra.MaximumNArgs(1), - } - connect.Flags().StringVarP(&clientID, "client-id", "i", "", "client id") - connect.Flags().StringVarP(&clientSecret, "client-secret", "s", "", "client secret") - database.AddCommand(connect) - - delete := &cobra.Command{ - Use: "delete [id | name]", - RunE: c.delete, - Args: cobra.MaximumNArgs(1), - } - database.AddCommand(delete) - - list := &cobra.Command{ - Use: "list", - Aliases: []string{"ls"}, - RunE: c.list, - Args: cobra.NoArgs, - } - list.Flags().BoolVar(&outputJSON, "json", false, "output as json") - list.Flags().IntVarP(&offset, "offset", "o", 0, "offset") - list.Flags().IntVarP(&limit, "limit", "l", 10, "limit") - database.AddCommand(list) - - createTable := &cobra.Command{ - Use: "create-table [id | db-name]", - RunE: c.createTable, - Args: cobra.MaximumNArgs(1), - } - createTable.Flags().StringVarP(&name, "name", "n", "", "name of the table") - database.AddCommand(createTable) - - listTables := &cobra.Command{ - Use: "list-tables [id | name]", - RunE: c.listTables, - Args: cobra.MaximumNArgs(1), - } - listTables.Flags().BoolVar(&outputJSON, "json", false, "output as json") - listTables.Flags().IntVarP(&offset, "offset", "o", 0, "offset") - listTables.Flags().IntVarP(&limit, "limit", "l", 10, "limit") - database.AddCommand(listTables) - - deleteTable := &cobra.Command{ - Use: "delete-table [id | db-name]", - RunE: c.deleteTable, - Args: cobra.MaximumNArgs(1), - } - deleteTable.Flags().StringVarP(&name, "name", "n", "", "name of the table") - database.AddCommand(deleteTable) - - parent.AddCommand(database) -} - -func getDbType(databaseType string) (database.Type, error) { - if databaseType == "" { - return database.Type_TYPE_UNSPECIFIED, errors.New("type is required") - } - var dbType database.Type - switch databaseType { - case "mongo": - dbType = database.Type_TYPE_MONGO - case "postgres": - dbType = database.Type_TYPE_POSTGRES - default: - return database.Type_TYPE_UNSPECIFIED, fmt.Errorf("invalid database type: %v (insert mongo or postgres)", databaseType) - } - return dbType, nil -} diff --git a/cmd/rig/cmd/storage/common.go b/cmd/rig/cmd/storage/common.go deleted file mode 100644 index 94ccc8e57..000000000 --- a/cmd/rig/cmd/storage/common.go +++ /dev/null @@ -1,25 +0,0 @@ -package storage - -import ( - "net/url" - - "github.com/rigdev/rig/pkg/errors" -) - -func isRigUri(raw string) bool { - _, _, err := parseRigUri(raw) - return err == nil -} - -func parseRigUri(raw string) (string, string, error) { - uri, err := url.Parse(raw) - if err != nil { - return "", "", err - } - - if uri.Scheme != "rig" { - return "", "", errors.InvalidArgumentErrorf("expect file of format `rig://bucket/path/to/file`") - } - - return uri.Host, uri.Path, nil -} diff --git a/cmd/rig/cmd/storage/copy.go b/cmd/rig/cmd/storage/copy.go deleted file mode 100644 index 3e6d77b6b..000000000 --- a/cmd/rig/cmd/storage/copy.go +++ /dev/null @@ -1,339 +0,0 @@ -package storage - -import ( - "context" - "io" - "io/fs" - "log" - "net/http" - "os" - "path" - "path/filepath" - "regexp" - "strings" - - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/progress" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/iterator" - "github.com/spf13/cobra" - "golang.org/x/sync/semaphore" -) - -var excludeList = []string{ - ".git", - ".DS_Store", -} - -func (c Cmd) cp(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - rawFrom := args[0] - rawTo := args[1] - - pw := progress.NewWriter() - pw.SetOutputWriter(cmd.OutOrStderr()) - pw.SetStyle(progress.StyleCircle) - pw.SetNumTrackersExpected(3) - go pw.Render() - - if isRigUri(rawFrom) { - bucket, prefix, err := parseRigUri(rawFrom) - if err != nil { - return err - } - results := []*storage.ListObjectsResponse_Result{} - base := "" - if prefix != "" { - base = filepath.Base(prefix) - } - if !strings.Contains(base, ".") { - res, err := c.Rig.Storage().ListObjects(ctx, &connect.Request[storage.ListObjectsRequest]{ - Msg: &storage.ListObjectsRequest{ - Bucket: bucket, - Prefix: prefix, - Recursive: storageRecursive, - }, - }) - if err != nil { - return err - } - results = res.Msg.GetResults() - } else { - toBase := filepath.Base(rawTo) - if !strings.Contains(toBase, ".") { - rawTo = path.Join(rawTo, base) - } - - res, err := c.Rig.Storage().GetObject(ctx, &connect.Request[storage.GetObjectRequest]{ - Msg: &storage.GetObjectRequest{ - Bucket: bucket, - Path: prefix, - }, - }) - if err != nil { - return err - } - results = append(results, &storage.ListObjectsResponse_Result{ - Result: &storage.ListObjectsResponse_Result_Object{ - Object: res.Msg.GetObject(), - }, - }) - } - if isRigUri(rawTo) { - // Copy. - dstBucket, dstPrefix, err := parseRigUri(rawTo) - if err != nil { - return err - } - var p int64 = 3 - sem := semaphore.NewWeighted(p) - for _, o := range results { - obj := o.GetObject() - if obj == nil { - continue - } - p := strings.TrimPrefix(obj.GetPath(), prefix) - - sem.Acquire(ctx, 1) - - go func() { - t := &progress.Tracker{ - Message: obj.GetPath(), - Units: progress.UnitsBytes, - Total: int64(obj.GetSize()), - } - pw.AppendTracker(t) - - if _, err := c.Rig.Storage().CopyObject(ctx, &connect.Request[storage.CopyObjectRequest]{ - Msg: &storage.CopyObjectRequest{ - FromBucket: bucket, - FromPath: obj.GetPath(), - ToBucket: dstBucket, - ToPath: path.Join(dstPrefix, p), - }, - }); err != nil { - log.Fatal(err) - } - t.Increment(t.Total) - sem.Release(1) - }() - } - sem.Acquire(ctx, p) - pw.Stop() - return nil - } else { - // Download. - var p int64 = 3 - sem := semaphore.NewWeighted(p) - for _, o := range results { - obj := o.GetObject() - if obj == nil { - continue - } - p := strings.TrimPrefix(obj.GetPath(), prefix) - - sem.Acquire(ctx, 1) - go func() { - t := &progress.Tracker{ - Message: obj.GetPath(), - Units: progress.UnitsBytes, - Total: int64(obj.GetSize()), - } - pw.AppendTracker(t) - - if err := c.downloadFile(ctx, cmd, t, bucket, path.Join(rawTo, p)); err != nil { - log.Fatal(err) - } - - sem.Release(1) - }() - } - - sem.Acquire(ctx, p) - pw.Stop() - return nil - } - } else if isRigUri(rawTo) { - // Upload. - bucket, prefix, err := parseRigUri(rawTo) - if err != nil { - return err - } - - itFiles := iterator.NewProducer[string]() - - go func() { - defer itFiles.Done() - if err := filepath.WalkDir(rawFrom, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - if regexp.MustCompile(strings.Join(excludeList, "|")).Match([]byte(path)) { - return nil - } - if path != rawFrom && !storageRecursive && d.IsDir() { - return filepath.SkipDir - } - if d.Type().IsRegular() { - itFiles.Value(path) - } - return nil - }); err != nil { - itFiles.Error(err) - } - }() - - it := iterator.Map[string](itFiles, func(filePath string) (*progress.Tracker, error) { - t := &progress.Tracker{ - Message: filePath, - Units: progress.UnitsBytes, - } - pw.AppendTracker(t) - return t, nil - }) - - defer it.Close() - - var p int64 = 3 - - sem := semaphore.NewWeighted(p) - for { - t, err := it.Next() - if err == io.EOF { - break - } else if err != nil { - return err - } - - p := strings.TrimPrefix(t.Message, rawFrom) - - sem.Acquire(ctx, 1) - - go func() { - if err := c.uploadFile(ctx, cmd, t, bucket, path.Join(prefix, p)); err != nil { - log.Fatal(err) - } - - sem.Release(1) - }() - } - - sem.Acquire(ctx, p) - pw.Stop() - return nil - } else { - return errors.InvalidArgumentErrorf("one of `from` and `to` must be a storage path") - } -} - -func (c Cmd) uploadFile(ctx context.Context, cmd *cobra.Command, t *progress.Tracker, bucket, path string) error { - from := t.Message - - f, err := os.Open(from) - if err != nil { - return err - } - - defer f.Close() - - s, err := f.Stat() - if err != nil { - return err - } - - size := s.Size() - - t.UpdateTotal(size) - - mimeData := make([]byte, 512) - n, err := f.Read(mimeData) - if err != nil { - return err - } - - if _, err := f.Seek(0, 0); err != nil { - return err - } - - m := &storage.UploadObjectRequest_Metadata{ - Bucket: bucket, - Path: path, - Size: uint64(size), - ContentType: http.DetectContentType(mimeData[:n]), - } - - // Upload. - cc := c.Rig.Storage().UploadObject(ctx) - if err := cc.Send(&storage.UploadObjectRequest{Request: &storage.UploadObjectRequest_Metadata_{Metadata: m}}); err != nil { - return err - } - - buffer := make([]byte, 64*1024) - for { - n, err := f.Read(buffer) - if err == io.EOF { - _, err := cc.CloseAndReceive() - if err != nil { - return err - } - - return nil - } else if err != nil { - return err - } - - if err := cc.Send(&storage.UploadObjectRequest{Request: &storage.UploadObjectRequest_Chunk{Chunk: buffer[:n]}}); err != nil { - return err - } - - t.Increment(int64(n)) - } -} - -func (c Cmd) downloadFile(ctx context.Context, cmd *cobra.Command, t *progress.Tracker, bucket, path string) error { - from := t.Message - // Create the directories if they don't exist. - if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil { - return err - } - - f, err := os.Create(path) - if err != nil { - return err - } - - defer f.Close() - - // Download. - cc, err := c.Rig.Storage().DownloadObject(ctx, &connect.Request[storage.DownloadObjectRequest]{ - Msg: &storage.DownloadObjectRequest{ - Bucket: bucket, - Path: from, - }, - }) - if err != nil { - return err - } - defer cc.Close() - - for cc.Receive() { - res := cc.Msg() - n, err := f.Write(res.GetChunk()) - if err != nil { - return err - } - t.Increment(int64(n)) - } - // For some reason the EOF error does not match io.EOF, but instead is unknown at just says unknown: EOF - if cc.Err() == io.EOF { - return nil - } - if errors.IsUnknown(cc.Err()) { - return nil - } else if cc.Err() != nil { - return cc.Err() - } else { - return nil - } -} diff --git a/cmd/rig/cmd/storage/create_bucket.go b/cmd/rig/cmd/storage/create_bucket.go deleted file mode 100644 index 1d49b5dfb..000000000 --- a/cmd/rig/cmd/storage/create_bucket.go +++ /dev/null @@ -1,97 +0,0 @@ -package storage - -import ( - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/list" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/cmd/common" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" - "github.com/spf13/cobra" -) - -func (c Cmd) createBucket(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - l := list.NewWriter() - l.SetStyle(list.StyleConnectedRounded) - - var pid string - var err error - if len(args) == 1 { - id, err := uuid.Parse(args[0]) - if err != nil { - res, err := c.Rig.Storage().LookupProvider(ctx, &connect.Request[storage.LookupProviderRequest]{ - Msg: &storage.LookupProviderRequest{ - Name: args[0], - }, - }) - if err != nil { - return err - } - pid = res.Msg.GetProviderId() - } else { - pid = id.String() - } - } else { - res, err := c.Rig.Storage().ListProviders(ctx, &connect.Request[storage.ListProvidersRequest]{ - Msg: &storage.ListProvidersRequest{}, - }) - if err != nil { - return err - } - - if len(res.Msg.GetProviders()) == 0 { - return errors.NotFoundErrorf("no providers found") - } - - if len(res.Msg.GetProviders()) == 1 { - pid = res.Msg.GetProviders()[0].GetProviderId() - } else { - // Ask the user to choose a provider. - providerNames := make([]string, len(res.Msg.GetProviders())) - for i, p := range res.Msg.GetProviders() { - providerNames[i] = p.GetName() - } - i, _, err := common.PromptSelect("Select provider:", providerNames) - if err != nil { - return err - } - - pid = res.Msg.GetProviders()[i].GetProviderId() - } - } - - if name == "" { - name, err = common.PromptInput("Bucket name:", ValidateBucketNameOpt) - if err != nil { - return err - } - } else if err := ValidateBucketName(name); err != nil { - return err - } - - if providerBucketName == "" { - providerBucketName, err = common.PromptInput("Provider bucket name:", ValidateBucketNameOpt, common.InputDefaultOpt(name)) - if err != nil { - return err - } - } else if err := ValidateBucketName(providerBucketName); err != nil { - return err - } - - _, err = c.Rig.Storage().CreateBucket(ctx, &connect.Request[storage.CreateBucketRequest]{ - Msg: &storage.CreateBucketRequest{ - Bucket: name, - ProviderBucket: providerBucketName, - Region: region, - ProviderId: pid, - }, - }) - if err != nil { - return err - } - - cmd.Println("Bucket created: ", name) - - return nil -} diff --git a/cmd/rig/cmd/storage/create_provider.go b/cmd/rig/cmd/storage/create_provider.go deleted file mode 100644 index c0ad56aaa..000000000 --- a/cmd/rig/cmd/storage/create_provider.go +++ /dev/null @@ -1,186 +0,0 @@ -package storage - -import ( - "bufio" - "fmt" - "io" - "os" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) createProvider(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - var err error - if name == "" { - name, err = common.PromptInput("Provider identifier:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } - - var config *storage.Config - var providerType string - if GCS { - providerType = "Google Cloud Storage" - config, err = getGCSConfig(credsFilePath) - if err != nil { - return err - } - } else if S3 { - providerType = "Amazon S3" - config = &storage.Config{ - Config: &storage.Config_S3{ - S3: &storage.S3Config{ - Credentials: &model.ProviderCredentials{ - PublicKey: accessKey, - PrivateKey: secretKey, - }, - Region: region, - }, - }, - } - } else if Minio { - providerType = "Minio" - config = &storage.Config{ - Config: &storage.Config_Minio{ - Minio: &storage.MinioConfig{ - Endpoint: endpoint, - Credentials: &model.ProviderCredentials{ - PublicKey: accessKey, - PrivateKey: secretKey, - }, - }, - }, - } - } else { - fields := []string{ - "Google Cloud Storage", - "Amazon S3", - "Minio", - } - var i int - i, providerType, err = common.PromptSelect("Provider type:", fields) - if err != nil { - return err - } - - switch i { - case 0: - // GCS - path, err := common.PromptInput("Credentials Path:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - - config, err = getGCSConfig(path) - if err != nil { - return err - } - case 1: - // S3 - accessKey, err := common.PromptInput("Access Key:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - - secretKey, err := common.PromptInput("Secret Key:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - - region, err := common.PromptInput("Region:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - - config = &storage.Config{ - Config: &storage.Config_S3{ - S3: &storage.S3Config{ - Credentials: &model.ProviderCredentials{ - PublicKey: accessKey, - PrivateKey: secretKey, - }, - Region: region, - }, - }, - } - - case 2: - // Minio - - accessKey, err := common.PromptInput("Access Key:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - - secretKey, err := common.PromptInput("Secret Key:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - - endpoint, err := common.PromptInput("Endpoint:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - - config = &storage.Config{ - Config: &storage.Config_Minio{ - Minio: &storage.MinioConfig{ - Endpoint: endpoint, - Credentials: &model.ProviderCredentials{ - PublicKey: accessKey, - PrivateKey: secretKey, - }, - }, - }, - } - } - } - - _, err = c.Rig.Storage().CreateProvider(ctx, &connect.Request[storage.CreateProviderRequest]{ - Msg: &storage.CreateProviderRequest{ - Name: name, - Config: config, - LinkBuckets: linkBuckets, - }, - }) - if err != nil { - return err - } - - cmd.Println(fmt.Sprintf("%s provder %s created", providerType, name)) - return nil -} - -func getGCSConfig(path string) (*storage.Config, error) { - // load json credentials file from path - f, err := os.Open(path) - if err != nil { - return nil, err - } - defer f.Close() - - stat, err := f.Stat() - if err != nil { - return nil, err - } - - buf := make([]byte, stat.Size()) - _, err = bufio.NewReader(f).Read(buf) - if err != nil && err != io.EOF { - return nil, err - } - - return &storage.Config{ - Config: &storage.Config_Gcs{ - Gcs: &storage.GcsConfig{ - Config: buf, - }, - }, - }, nil -} diff --git a/cmd/rig/cmd/storage/delete_bucket.go b/cmd/rig/cmd/storage/delete_bucket.go deleted file mode 100644 index 9a4f0202c..000000000 --- a/cmd/rig/cmd/storage/delete_bucket.go +++ /dev/null @@ -1,37 +0,0 @@ -package storage - -import ( - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/list" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) deleteBucket(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - l := list.NewWriter() - l.SetStyle(list.StyleConnectedRounded) - var bucket string - var err error - if len(args) < 1 { - bucket, err = common.PromptInput("Bucket name:", ValidateBucketNameOpt) - if err != nil { - return err - } - } else { - bucket = args[0] - } - - _, err = c.Rig.Storage().DeleteBucket(ctx, &connect.Request[storage.DeleteBucketRequest]{ - Msg: &storage.DeleteBucketRequest{ - Bucket: bucket, - }, - }) - if err != nil { - return err - } - - cmd.Println("Bucket deleted: ", bucket) - return nil -} diff --git a/cmd/rig/cmd/storage/delete_object.go b/cmd/rig/cmd/storage/delete_object.go deleted file mode 100644 index b5fa89061..000000000 --- a/cmd/rig/cmd/storage/delete_object.go +++ /dev/null @@ -1,43 +0,0 @@ -package storage - -import ( - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/cmd/common" - "github.com/rigdev/rig/pkg/errors" - "github.com/spf13/cobra" -) - -func (c Cmd) deleteObject(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - var path string - var err error - if len(args) < 1 { - path, err = common.PromptInput("Object path:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } else { - path = args[0] - } - if isRigUri(path) { - bucket, prefix, err := parseRigUri(path) - if err != nil { - return err - } - _, err = c.Rig.Storage().DeleteObject(ctx, &connect.Request[storage.DeleteObjectRequest]{ - Msg: &storage.DeleteObjectRequest{ - Bucket: bucket, - Path: prefix, - }, - }) - if err != nil { - return err - } - } else { - return errors.InvalidArgumentErrorf("invalid path: %s", path) - } - - cmd.Println("Object deleted at: ", path) - return nil -} diff --git a/cmd/rig/cmd/storage/delete_provider.go b/cmd/rig/cmd/storage/delete_provider.go deleted file mode 100644 index 82007987a..000000000 --- a/cmd/rig/cmd/storage/delete_provider.go +++ /dev/null @@ -1,32 +0,0 @@ -package storage - -import ( - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) deleteProvider(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - g, uid, err := common.GetStorageProvider(ctx, identifier, c.Rig) - if err != nil { - return err - } - - _, err = c.Rig.Storage().DeleteProvider(ctx, &connect.Request[storage.DeleteProviderRequest]{ - Msg: &storage.DeleteProviderRequest{ - ProviderId: uid, - }, - }) - if err != nil { - return err - } - - cmd.Println("Provider deleted: ", g.GetName()) - return nil -} diff --git a/cmd/rig/cmd/storage/get_bucket.go b/cmd/rig/cmd/storage/get_bucket.go deleted file mode 100644 index 5d6ba68b7..000000000 --- a/cmd/rig/cmd/storage/get_bucket.go +++ /dev/null @@ -1,49 +0,0 @@ -package storage - -import ( - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/table" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) getBucket(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - var bucket string - var err error - if len(args) < 1 { - bucket, err = common.PromptInput("Bucket name:", ValidateBucketNameOpt) - if err != nil { - return err - } - } else { - bucket = args[0] - } - - res, err := c.Rig.Storage().GetBucket(ctx, &connect.Request[storage.GetBucketRequest]{ - Msg: &storage.GetBucketRequest{ - Bucket: bucket, - }, - }) - if err != nil { - return err - } - - if outputJson { - cmd.Println(common.ProtoToPrettyJson(res.Msg.GetBucket())) - return nil - } - - t := table.NewWriter() - t.AppendHeader(table.Row{"Attribute", "Value"}) - t.AppendRows([]table.Row{ - {"Name", res.Msg.GetBucket().GetName()}, - {"Provider name", res.Msg.GetBucket().GetProviderBucket()}, - {"Region", res.Msg.GetBucket().GetRegion()}, - {"Created", res.Msg.GetBucket().GetCreatedAt().AsTime().Format("2006-01-02 15:04:05")}, - }) - - cmd.Println(t.Render()) - return nil -} diff --git a/cmd/rig/cmd/storage/get_object.go b/cmd/rig/cmd/storage/get_object.go deleted file mode 100644 index f4bd975b4..000000000 --- a/cmd/rig/cmd/storage/get_object.go +++ /dev/null @@ -1,59 +0,0 @@ -package storage - -import ( - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/table" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/cmd/common" - "github.com/rigdev/rig/pkg/errors" - "github.com/spf13/cobra" -) - -func (c Cmd) getObject(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - var path string - var err error - if len(args) < 1 { - path, err = common.PromptInput("Object path:", common.ValidateNonEmptyOpt) - if err != nil { - return err - } - } else { - path = args[0] - } - if isRigUri(path) { - bucket, prefix, err := parseRigUri(path) - if err != nil { - return err - } - res, err := c.Rig.Storage().GetObject(ctx, &connect.Request[storage.GetObjectRequest]{ - Msg: &storage.GetObjectRequest{ - Bucket: bucket, - Path: prefix, - }, - }) - if err != nil { - return err - } - - if outputJson { - cmd.Println(common.ProtoToPrettyJson(res.Msg.GetObject())) - return nil - } - - t := table.NewWriter() - t.AppendHeader(table.Row{"Attribute", "Value"}) - t.AppendRows([]table.Row{ - {"Name", res.Msg.GetObject().GetPath()}, - {"Content type", res.Msg.GetObject().GetContentType()}, - {"Etag", res.Msg.GetObject().GetEtag()}, - {"Size", res.Msg.GetObject().GetSize()}, - {"Uploaded at", res.Msg.GetObject().GetLastModified().AsTime().Format("2006-01-02 15:04:05")}, - }) - cmd.Println(t.Render()) - - } else { - return errors.InvalidArgumentErrorf("invalid path: %s", path) - } - return nil -} diff --git a/cmd/rig/cmd/storage/get_provider.go b/cmd/rig/cmd/storage/get_provider.go deleted file mode 100644 index 2447c0708..000000000 --- a/cmd/rig/cmd/storage/get_provider.go +++ /dev/null @@ -1,35 +0,0 @@ -package storage - -import ( - "github.com/jedib0t/go-pretty/v6/table" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) getProvider(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - identifier := "" - if len(args) > 0 { - identifier = args[0] - } - g, uid, err := common.GetStorageProvider(ctx, identifier, c.Rig) - if err != nil { - return err - } - - if outputJson { - cmd.Println(common.ProtoToPrettyJson(g)) - return nil - } - - t := table.NewWriter() - t.AppendHeader(table.Row{"Attribute", "Value"}) - t.AppendRows([]table.Row{ - {"ID", uid}, - {"Name", g.GetName()}, - {"#Buckets", len(g.GetBuckets())}, - {"Created at", g.GetCreatedAt().AsTime().Format("2006-01-02 15:04:05")}, - }) - cmd.Println(t.Render()) - return nil -} diff --git a/cmd/rig/cmd/storage/list_providers.go b/cmd/rig/cmd/storage/list_providers.go deleted file mode 100644 index 895e7d16c..000000000 --- a/cmd/rig/cmd/storage/list_providers.go +++ /dev/null @@ -1,62 +0,0 @@ -package storage - -import ( - "fmt" - - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/table" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/cmd/common" - "github.com/rigdev/rig/pkg/errors" - "github.com/spf13/cobra" -) - -func (c Cmd) listProviders(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - Pagination := &model.Pagination{ - Offset: uint32(offset), - Limit: uint32(limit), - } - - resp, err := c.Rig.Storage().ListProviders(ctx, &connect.Request[storage.ListProvidersRequest]{ - Msg: &storage.ListProvidersRequest{ - Pagination: Pagination, - }, - }) - if err != nil { - return err - } - - if outputJson { - for _, u := range resp.Msg.GetProviders() { - cmd.Println(common.ProtoToPrettyJson(u)) - } - return nil - } - - t := table.NewWriter() - t.AppendHeader(table.Row{fmt.Sprintf("Providers (%d)", resp.Msg.GetTotal()), "Name", "ID", "Backend", "#Buckets"}) - for i, u := range resp.Msg.GetProviders() { - typ, err := getProviderType(u.GetConfig()) - if err != nil { - return err - } - t.AppendRow(table.Row{i + 1, u.GetName(), u.GetProviderId(), typ, len(u.GetBuckets())}) - } - cmd.Println(t.Render()) - return nil -} - -func getProviderType(p *storage.Config) (string, error) { - switch p.GetConfig().(type) { - case *storage.Config_S3: - return "s3", nil - case *storage.Config_Gcs: - return "gcs", nil - case *storage.Config_Minio: - return "minio", nil - default: - return "", errors.InvalidArgumentErrorf("unknown provider type") - } -} diff --git a/cmd/rig/cmd/storage/ls.go b/cmd/rig/cmd/storage/ls.go deleted file mode 100644 index 474a9ae9c..000000000 --- a/cmd/rig/cmd/storage/ls.go +++ /dev/null @@ -1,141 +0,0 @@ -package storage - -import ( - "fmt" - "path" - "strings" - "time" - - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/list" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) ls(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - l := list.NewWriter() - l.SetStyle(list.StyleConnectedRounded) - - if len(args) == 0 { - // List buckets. - token := "" - for { - res, err := c.Rig.Storage().ListBuckets(ctx, &connect.Request[storage.ListBucketsRequest]{ - Msg: &storage.ListBucketsRequest{ - Token: token, - }, - }) - if err != nil { - return err - } - - if outputJson { - for _, b := range res.Msg.GetBuckets() { - cmd.Println(common.ProtoToPrettyJson(b)) - } - return nil - } - - for _, b := range res.Msg.GetBuckets() { - l.AppendItem(fmt.Sprint("rig://", b.GetName(), "/ - (created_at=", b.GetCreatedAt().AsTime().Format(time.RFC3339), ")")) - cmd.Println(l.Render()) - l.Reset() - l.SetStyle(list.StyleConnectedRounded) - } - - token = res.Msg.GetToken() - if token == "" { - break - } - } - - return nil - } - - bucket, prefix, err := parseRigUri(args[0]) - if err != nil { - return err - } - - l.AppendItem(fmt.Sprint("rig://", path.Join(bucket, prefix))) - l.Indent() - - // List files. - token := "" - for { - res, err := c.Rig.Storage().ListObjects(ctx, &connect.Request[storage.ListObjectsRequest]{ - Msg: &storage.ListObjectsRequest{ - Token: token, - Bucket: bucket, - Prefix: prefix, - Recursive: storageRecursive, - }, - }) - - if outputJson { - for _, o := range res.Msg.GetResults() { - cmd.Println(common.ProtoToPrettyJson(o)) - } - token = res.Msg.GetToken() - if token == "" { - return nil - } - continue - } - - if err != nil { - return err - } - - for _, r := range res.Msg.GetResults() { - switch v := r.GetResult().(type) { - case *storage.ListObjectsResponse_Result_Folder: - path := path.Clean(path.Join("/", v.Folder)) - path = strings.TrimPrefix(path, prefix) - l.AppendItem(path) - case *storage.ListObjectsResponse_Result_Object: - prefix = listItem(l, prefix, v.Object) - } - } - - token = res.Msg.GetToken() - if token == "" { - break - } - } - - cmd.Println(l.Render()) - return nil -} - -func listItem(l list.Writer, prefix string, item *storage.Object) string { - fullPath := path.Clean(path.Join("/", item.GetPath())) - // First find longest common prefix. - for !strings.HasPrefix(fullPath, prefix) { - prefix = path.Dir(prefix) - l.UnIndent() - } - - uniqueSuffix := fullPath - uniqueSuffix = strings.TrimPrefix(uniqueSuffix, prefix) - - d, b := path.Split(uniqueSuffix) - - for _, s := range strings.Split(d, "/") { - if s == "" { - continue - } - l.AppendItem(fmt.Sprint(s, "/")) - l.Indent() - } - - if item.ContentType != "" { - l.AppendItem(fmt.Sprint(b, " - ", item.GetContentType())) - } else { - l.AppendItem(b) - } - - return path.Dir(fullPath) -} diff --git a/cmd/rig/cmd/storage/setup.go b/cmd/rig/cmd/storage/setup.go deleted file mode 100644 index f2d69b7aa..000000000 --- a/cmd/rig/cmd/storage/setup.go +++ /dev/null @@ -1,207 +0,0 @@ -package storage - -import ( - "context" - - "github.com/erikgeiser/promptkit/textinput" - "github.com/rigdev/rig-go-sdk" - "github.com/rigdev/rig/pkg/errors" - "github.com/spf13/cobra" - "go.uber.org/fx" -) - -var ( - offset int - limit int -) - -var ( - storageRecursive bool - linkBuckets bool - outputJson bool - - GCS bool - S3 bool - Minio bool -) - -var ( - name string - credsFilePath string - accessKey string - secretKey string - region string - endpoint string - providerBucketName string -) - -type Cmd struct { - fx.In - - Ctx context.Context - Rig rig.Client -} - -func (c Cmd) Setup(parent *cobra.Command) { - storage := &cobra.Command{ - Use: "storage", - } - - cp := &cobra.Command{ - Use: "copy from to", - Aliases: []string{"cp"}, - Short: "Copy files to and from buckets", - Args: cobra.ExactArgs(2), - RunE: c.cp, - } - cp.PersistentFlags().BoolVarP(&storageRecursive, "recursive", "r", false, "if copy should be recursive") - storage.AddCommand(cp) - - ls := &cobra.Command{ - Use: "list [path]", - Aliases: []string{"ls"}, - Short: "List buckets and objects", - Args: cobra.MaximumNArgs(1), - RunE: c.ls, - } - ls.PersistentFlags().BoolVarP(&storageRecursive, "recursive", "r", false, "if listing should be recursive. Does only work for listing within a single bucket") - ls.Flags().BoolVar(&outputJson, "json", false, "output as json") - storage.AddCommand(ls) - - createBucket := &cobra.Command{ - Use: "create-bucket [provider-name]", - Short: "Create a new bucket", - Args: cobra.MaximumNArgs(1), - RunE: c.createBucket, - } - createBucket.Flags().StringVarP(&name, "name", "n", "", "name of the bucket") - createBucket.Flags().StringVarP(&providerBucketName, "provider-bucket-name", "p", "", "name of the bucket on the provider") - createBucket.Flags().StringVarP(®ion, "region", "r", "", "region of the bucket") - storage.AddCommand(createBucket) - - deleteBucket := &cobra.Command{ - Use: "delete-bucket [bucket-name]", - Short: "Delete a bucket", - Args: cobra.MaximumNArgs(1), - RunE: c.deleteBucket, - } - storage.AddCommand(deleteBucket) - - unlinkBucket := &cobra.Command{ - Use: "unlink-bucket [bucket-name]", - Short: "Unlink a bucket", - Args: cobra.MaximumNArgs(1), - RunE: c.unlinkBucket, - } - storage.AddCommand(unlinkBucket) - - getObject := &cobra.Command{ - Use: "get-object [path]", - Short: "Get an object", - Args: cobra.MaximumNArgs(1), - RunE: c.getObject, - } - getObject.Flags().BoolVar(&outputJson, "json", false, "output as json") - storage.AddCommand(getObject) - - getBucket := &cobra.Command{ - Use: "get-bucket [bucket]", - Short: "Get a bucket", - Args: cobra.MaximumNArgs(1), - RunE: c.getBucket, - } - getBucket.Flags().BoolVar(&outputJson, "json", false, "output as json") - storage.AddCommand(getBucket) - - deleteObject := &cobra.Command{ - Use: "delete-object [path]", - Short: "Delete an object", - Args: cobra.MaximumNArgs(1), - RunE: c.deleteObject, - } - storage.AddCommand(deleteObject) - - createProvider := &cobra.Command{ - Use: "create-provider", - Short: "Create a new provider", - Args: cobra.MaximumNArgs(1), - RunE: c.createProvider, - } - createProvider.Flags().StringVarP(&name, "name", "n", "", "name of the provider") - - createProvider.Flags().BoolVar(&GCS, "gcs", false, "if the provider should be a GCS provider") - createProvider.Flags().BoolVar(&S3, "s3", false, "if the provider should be a S3 provider") - createProvider.Flags().BoolVar(&Minio, "minio", false, "if the provider should be a Minio provider") - createProvider.MarkFlagsMutuallyExclusive("gcs", "s3", "minio") - - createProvider.Flags().StringVarP(&credsFilePath, "creds-file", "c", "", "path to the GCS credentials file") - createProvider.MarkFlagsRequiredTogether("gcs", "creds-file") - - createProvider.Flags().StringVarP(&accessKey, "access-key", "a", "", "access key for the provider") - createProvider.Flags().StringVarP(&secretKey, "secret-key", "s", "", "secret key for the provider") - createProvider.Flags().StringVarP(®ion, "region", "r", "", "region for the provider") - createProvider.Flags().StringVarP(&endpoint, "endpoint", "e", "", "endpoint for the provider") - - createProvider.MarkFlagsRequiredTogether("s3", "region") - createProvider.MarkFlagsRequiredTogether("minio", "endpoint") - createProvider.MarkFlagsRequiredTogether("access-key", "secret-key") - - createProvider.Flags().BoolVarP(&linkBuckets, "link-buckets", "l", false, "if buckets should be linked to the provider") - storage.AddCommand(createProvider) - - listProviders := &cobra.Command{ - Use: "list-providers", - Short: "List all providers", - Args: cobra.NoArgs, - RunE: c.listProviders, - } - listProviders.Flags().IntVarP(&limit, "limit", "l", 10, "limit the number of groups to return") - listProviders.Flags().IntVarP(&offset, "offset", "o", 0, "offset the number of groups to return") - listProviders.Flags().BoolVar(&outputJson, "json", false, "output as json") - storage.AddCommand(listProviders) - - parent.AddCommand(storage) - - GetProvider := &cobra.Command{ - Use: "get-provider [id | name]", - Short: "Get a provider", - Args: cobra.MaximumNArgs(1), - RunE: c.getProvider, - } - GetProvider.Flags().BoolVar(&outputJson, "json", false, "output as json") - storage.AddCommand(GetProvider) - - DeleteProvider := &cobra.Command{ - Use: "delete-provider [id | name]", - Short: "Delete a provider", - Args: cobra.MaximumNArgs(1), - RunE: c.deleteProvider, - } - storage.AddCommand(DeleteProvider) -} - -var ValidateBucketName = func(input string) error { - // check if length of input is greater than 3 and less than 63 - if len(input) < 3 || len(input) > 63 { - return errors.InvalidArgumentErrorf("bucket name must be between 3 and 63 characters long") - } - // check if input contains only lowercase letters, numbers, dashes and underscores - for _, c := range input { - if !((c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_') { - return errors.InvalidArgumentErrorf("bucket name must only contain lowercase letters, numbers, dashes and underscores") - } - } - // input must start and end with a lowercase letter or number - if !(input[0] >= 'a' && input[0] <= 'z') && !(input[0] >= '0' && input[0] <= '9') { - return errors.InvalidArgumentErrorf("bucket name must start with a lowercase letter or number") - } - - if !(input[len(input)-1] >= 'a' && input[len(input)-1] <= 'z') && !(input[len(input)-1] >= '0' && input[len(input)-1] <= '9') { - return errors.InvalidArgumentErrorf("bucket name must end with a lowercase letter or number") - } - return nil -} - -var ValidateBucketNameOpt = func(inp *textinput.TextInput) { - inp.Validate = ValidateBucketName -} diff --git a/cmd/rig/cmd/storage/unlink_bucket.go b/cmd/rig/cmd/storage/unlink_bucket.go deleted file mode 100644 index 013b1e1a9..000000000 --- a/cmd/rig/cmd/storage/unlink_bucket.go +++ /dev/null @@ -1,37 +0,0 @@ -package storage - -import ( - "github.com/bufbuild/connect-go" - "github.com/jedib0t/go-pretty/v6/list" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/cmd/common" - "github.com/spf13/cobra" -) - -func (c Cmd) unlinkBucket(cmd *cobra.Command, args []string) error { - ctx := c.Ctx - l := list.NewWriter() - l.SetStyle(list.StyleConnectedRounded) - var bucket string - var err error - if len(args) < 1 { - bucket, err = common.PromptInput("Bucket name:", ValidateBucketNameOpt) - if err != nil { - return err - } - } else { - bucket = args[0] - } - - _, err = c.Rig.Storage().UnlinkBucket(ctx, &connect.Request[storage.UnlinkBucketRequest]{ - Msg: &storage.UnlinkBucketRequest{ - Bucket: bucket, - }, - }) - if err != nil { - return err - } - - cmd.Println("Bucket unlinked: ", bucket) - return nil -} diff --git a/internal/config/defaults.go b/internal/config/defaults.go index e4196d3d6..8614421d9 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -45,10 +45,8 @@ func newDefault() Config { }, Repository: Repository{ - Storage: defaultRepositoryStore(), Capsule: defaultRepositoryStore(), ClusterConfig: defaultRepositoryStore(), - Database: defaultRepositoryStore(), ServiceAccount: defaultRepositoryStore(), Group: defaultRepositoryStore(), Project: defaultRepositoryStore(), diff --git a/internal/config/types.go b/internal/config/types.go index ac88df5ba..ca8defea6 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -80,9 +80,7 @@ type ClientMailjet struct { } type Repository struct { - Storage RepositoryStore `mapstructure:"storage"` Capsule RepositoryStore `mapstructure:"capsule"` - Database RepositoryStore `mapstructure:"database"` ServiceAccount RepositoryStore `mapstructure:"service_account"` Group RepositoryStore `mapstructure:"group"` Project RepositoryStore `mapstructure:"project"` diff --git a/internal/handler/api/database/create.go b/internal/handler/api/database/create.go deleted file mode 100644 index 864ab989b..000000000 --- a/internal/handler/api/database/create.go +++ /dev/null @@ -1,24 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" -) - -func (h *Handler) Create(ctx context.Context, req *connect.Request[database.CreateRequest]) (*connect.Response[database.CreateResponse], error) { - databaseID, db, err := h.ds.Create(ctx, req.Msg.GetType(), req.Msg.GetInitializers()) - if err != nil { - return nil, err - } - clientId, clientSecret, err := h.ds.CreateCredential(ctx, "Default Credential", databaseID) - if err != nil { - return nil, err - } - return connect.NewResponse(&database.CreateResponse{ - Database: db, - ClientId: clientId, - ClientSecret: clientSecret, - }), nil -} diff --git a/internal/handler/api/database/create_credential.go b/internal/handler/api/database/create_credential.go deleted file mode 100644 index e4ff26375..000000000 --- a/internal/handler/api/database/create_credential.go +++ /dev/null @@ -1,25 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h *Handler) CreateCredential(ctx context.Context, req *connect.Request[database.CreateCredentialRequest]) (*connect.Response[database.CreateCredentialResponse], error) { - dbId, err := uuid.Parse(req.Msg.GetDatabaseId()) - if err != nil { - return nil, err - } - - clientId, clientSecret, err := h.ds.CreateCredential(ctx, req.Msg.GetName(), dbId) - if err != nil { - return nil, err - } - return connect.NewResponse(&database.CreateCredentialResponse{ - ClientId: clientId, - ClientSecret: clientSecret, - }), nil -} diff --git a/internal/handler/api/database/create_table.go b/internal/handler/api/database/create_table.go deleted file mode 100644 index d30acadeb..000000000 --- a/internal/handler/api/database/create_table.go +++ /dev/null @@ -1,21 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h *Handler) CreateTable(ctx context.Context, req *connect.Request[database.CreateTableRequest]) (*connect.Response[database.CreateTableResponse], error) { - dbId, err := uuid.Parse(req.Msg.GetDatabaseId()) - if err != nil { - return nil, err - } - - if err := h.ds.CreateTable(ctx, dbId, req.Msg.GetTableName()); err != nil { - return nil, err - } - return connect.NewResponse(&database.CreateTableResponse{}), nil -} diff --git a/internal/handler/api/database/delete.go b/internal/handler/api/database/delete.go deleted file mode 100644 index 43e578837..000000000 --- a/internal/handler/api/database/delete.go +++ /dev/null @@ -1,21 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - api_database "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h *Handler) Delete(ctx context.Context, req *connect.Request[api_database.DeleteRequest]) (*connect.Response[api_database.DeleteResponse], error) { - dbId, err := uuid.Parse(req.Msg.GetDatabaseId()) - if err != nil { - return nil, err - } - - if err := h.ds.Delete(ctx, dbId); err != nil { - return nil, err - } - return connect.NewResponse(&api_database.DeleteResponse{}), nil -} diff --git a/internal/handler/api/database/delete_credential.go b/internal/handler/api/database/delete_credential.go deleted file mode 100644 index b20ad577c..000000000 --- a/internal/handler/api/database/delete_credential.go +++ /dev/null @@ -1,22 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h *Handler) DeleteCredential(ctx context.Context, req *connect.Request[database.DeleteCredentialRequest]) (*connect.Response[database.DeleteCredentialResponse], error) { - dbId, err := uuid.Parse(req.Msg.GetDatabaseId()) - if err != nil { - return nil, err - } - - err = h.ds.DeleteCredential(ctx, req.Msg.GetCredentialName(), dbId) - if err != nil { - return nil, err - } - return connect.NewResponse(&database.DeleteCredentialResponse{}), nil -} diff --git a/internal/handler/api/database/delete_table.go b/internal/handler/api/database/delete_table.go deleted file mode 100644 index e6c0ba85b..000000000 --- a/internal/handler/api/database/delete_table.go +++ /dev/null @@ -1,21 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h *Handler) DeleteTable(ctx context.Context, req *connect.Request[database.DeleteTableRequest]) (*connect.Response[database.DeleteTableResponse], error) { - dbId, err := uuid.Parse(req.Msg.GetDatabaseId()) - if err != nil { - return nil, err - } - - if err := h.ds.DeleteTable(ctx, dbId, req.Msg.GetTableName()); err != nil { - return nil, err - } - return connect.NewResponse(&database.DeleteTableResponse{}), nil -} diff --git a/internal/handler/api/database/get.go b/internal/handler/api/database/get.go deleted file mode 100644 index d5be9d2e9..000000000 --- a/internal/handler/api/database/get.go +++ /dev/null @@ -1,24 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h *Handler) Get(ctx context.Context, req *connect.Request[database.GetRequest]) (*connect.Response[database.GetResponse], error) { - dbId, err := uuid.Parse(req.Msg.GetDatabaseId()) - if err != nil { - return nil, err - } - - db, err := h.ds.Get(ctx, dbId) - if err != nil { - return nil, err - } - return connect.NewResponse(&database.GetResponse{ - Database: db, - }), nil -} diff --git a/internal/handler/api/database/get_by_name.go b/internal/handler/api/database/get_by_name.go deleted file mode 100644 index 8e6754639..000000000 --- a/internal/handler/api/database/get_by_name.go +++ /dev/null @@ -1,21 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" -) - -func (h *Handler) GetByName(ctx context.Context, req *connect.Request[database.GetByNameRequest]) (*connect.Response[database.GetByNameResponse], error) { - db, err := h.ds.GetByName(ctx, req.Msg.GetName()) - if err != nil { - return nil, err - } - - return connect.NewResponse( - &database.GetByNameResponse{ - Database: db, - }, - ), nil -} diff --git a/internal/handler/api/database/get_endpoint.go b/internal/handler/api/database/get_endpoint.go deleted file mode 100644 index c3093a218..000000000 --- a/internal/handler/api/database/get_endpoint.go +++ /dev/null @@ -1,22 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h *Handler) GetEndpoint(ctx context.Context, req *connect.Request[database.GetEndpointRequest]) (*connect.Response[database.GetEndpointResponse], error) { - dbId, err := uuid.Parse(req.Msg.GetDatabaseId()) - if err != nil { - return nil, err - } - - endpoint, dbName, err := h.ds.GetDatabaseEndpoint(ctx, dbId, req.Msg.GetClientId(), req.Msg.GetClientSecret()) - if err != nil { - return nil, err - } - return connect.NewResponse(&database.GetEndpointResponse{Endpoint: endpoint, DatabaseName: dbName}), nil -} diff --git a/internal/handler/api/database/handler.go b/internal/handler/api/database/handler.go deleted file mode 100644 index ccab6f23b..000000000 --- a/internal/handler/api/database/handler.go +++ /dev/null @@ -1,30 +0,0 @@ -package database - -import ( - "net/http" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database/databaseconnect" - "github.com/rigdev/rig/internal/repository" - "github.com/rigdev/rig/internal/service/database" -) - -type Handler struct { - ds *database.Service - pr repository.Project -} - -func New(ds *database.Service, pr repository.Project) *Handler { - return &Handler{ - ds: ds, - pr: pr, - } -} - -func (h *Handler) ServiceName() string { - return databaseconnect.ServiceName -} - -func (h *Handler) Build(opts ...connect.HandlerOption) (string, http.Handler) { - return databaseconnect.NewServiceHandler(h, opts...) -} diff --git a/internal/handler/api/database/list.go b/internal/handler/api/database/list.go deleted file mode 100644 index 0e450d687..000000000 --- a/internal/handler/api/database/list.go +++ /dev/null @@ -1,24 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/iterator" -) - -func (h *Handler) List(ctx context.Context, req *connect.Request[database.ListRequest]) (*connect.Response[database.ListResponse], error) { - it, total, err := h.ds.List(ctx, req.Msg.GetPagination()) - if err != nil { - return nil, err - } - ds, err := iterator.Collect(it) - if err != nil { - return nil, err - } - return connect.NewResponse(&database.ListResponse{ - Databases: ds, - Total: total, - }), nil -} diff --git a/internal/handler/api/database/list_tables.go b/internal/handler/api/database/list_tables.go deleted file mode 100644 index de0348de7..000000000 --- a/internal/handler/api/database/list_tables.go +++ /dev/null @@ -1,25 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h *Handler) ListTables(ctx context.Context, req *connect.Request[database.ListTablesRequest]) (*connect.Response[database.ListTablesResponse], error) { - dbId, err := uuid.Parse(req.Msg.GetDatabaseId()) - if err != nil { - return nil, err - } - - tables, err := h.ds.ListTables(ctx, dbId) - if err != nil { - return nil, err - } - return connect.NewResponse(&database.ListTablesResponse{ - Tables: tables, - Total: uint64(len(tables)), - }), nil -} diff --git a/internal/handler/api/database/update.go b/internal/handler/api/database/update.go deleted file mode 100644 index 85f95c691..000000000 --- a/internal/handler/api/database/update.go +++ /dev/null @@ -1,24 +0,0 @@ -package database - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h *Handler) Update(ctx context.Context, req *connect.Request[database.UpdateRequest]) (*connect.Response[database.UpdateResponse], error) { - dbId, err := uuid.Parse(req.Msg.GetDatabaseId()) - if err != nil { - return nil, err - } - - db, err := h.ds.Update(ctx, dbId, req.Msg.GetUpdates()) - if err != nil { - return nil, err - } - return connect.NewResponse(&database.UpdateResponse{ - Database: db, - }), nil -} diff --git a/internal/handler/api/storage/copy_object.go b/internal/handler/api/storage/copy_object.go deleted file mode 100644 index d71b849a1..000000000 --- a/internal/handler/api/storage/copy_object.go +++ /dev/null @@ -1,18 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" -) - -func (h *Handler) CopyObject(ctx context.Context, req *connect.Request[storage.CopyObjectRequest]) (*connect.Response[storage.CopyObjectResponse], error) { - err := h.ss.CopyObject(ctx, req.Msg.GetToBucket(), req.Msg.GetToPath(), req.Msg.GetFromBucket(), req.Msg.GetFromPath()) - if err != nil { - return nil, err - } - return &connect.Response[storage.CopyObjectResponse]{ - Msg: &storage.CopyObjectResponse{}, - }, nil -} diff --git a/internal/handler/api/storage/create_bucket.go b/internal/handler/api/storage/create_bucket.go deleted file mode 100644 index 1ac01181a..000000000 --- a/internal/handler/api/storage/create_bucket.go +++ /dev/null @@ -1,25 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/uuid" -) - -// CreateBucket implements storageconnect.ServiceHandler -func (h *Handler) CreateBucket(ctx context.Context, req *connect.Request[storage.CreateBucketRequest]) (*connect.Response[storage.CreateBucketResponse], error) { - pid, err := uuid.Parse(req.Msg.GetProviderId()) - if err != nil { - return nil, err - } - - err = h.ss.CreateBucket(ctx, req.Msg.GetBucket(), req.Msg.GetProviderBucket(), req.Msg.GetRegion(), pid) - if err != nil { - return nil, err - } - return &connect.Response[storage.CreateBucketResponse]{ - Msg: &storage.CreateBucketResponse{}, - }, nil -} diff --git a/internal/handler/api/storage/create_provider.go b/internal/handler/api/storage/create_provider.go deleted file mode 100644 index 2411aa3bc..000000000 --- a/internal/handler/api/storage/create_provider.go +++ /dev/null @@ -1,21 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" -) - -func (h Handler) CreateProvider(ctx context.Context, req *connect.Request[storage.CreateProviderRequest]) (*connect.Response[storage.CreateProviderResponse], error) { - _, p, err := h.ss.CreateProvider(ctx, req.Msg.GetName(), req.Msg.GetConfig(), req.Msg.GetLinkBuckets()) - if err != nil { - return nil, err - } - - return &connect.Response[storage.CreateProviderResponse]{ - Msg: &storage.CreateProviderResponse{ - Provider: p, - }, - }, nil -} diff --git a/internal/handler/api/storage/delete_bucket.go b/internal/handler/api/storage/delete_bucket.go deleted file mode 100644 index 13d1b6efa..000000000 --- a/internal/handler/api/storage/delete_bucket.go +++ /dev/null @@ -1,19 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" -) - -// DeleteBucket implements storageconnect.ServiceHandler -func (h *Handler) DeleteBucket(ctx context.Context, req *connect.Request[storage.DeleteBucketRequest]) (*connect.Response[storage.DeleteBucketResponse], error) { - err := h.ss.DeleteBucket(ctx, req.Msg.Bucket) - if err != nil { - return nil, err - } - return &connect.Response[storage.DeleteBucketResponse]{ - Msg: &storage.DeleteBucketResponse{}, - }, nil -} diff --git a/internal/handler/api/storage/delete_object.go b/internal/handler/api/storage/delete_object.go deleted file mode 100644 index 039fd11a9..000000000 --- a/internal/handler/api/storage/delete_object.go +++ /dev/null @@ -1,19 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" -) - -// DeleteObject implements storageconnect.ServiceHandler -func (h *Handler) DeleteObject(ctx context.Context, req *connect.Request[storage.DeleteObjectRequest]) (*connect.Response[storage.DeleteObjectResponse], error) { - err := h.ss.DeleteObject(ctx, req.Msg.Bucket, req.Msg.Path) - if err != nil { - return nil, err - } - return &connect.Response[storage.DeleteObjectResponse]{ - Msg: &storage.DeleteObjectResponse{}, - }, nil -} diff --git a/internal/handler/api/storage/delete_provider.go b/internal/handler/api/storage/delete_provider.go deleted file mode 100644 index 6af5e815a..000000000 --- a/internal/handler/api/storage/delete_provider.go +++ /dev/null @@ -1,24 +0,0 @@ -package storage - -import ( - "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h Handler) DeleteProvider(ctx context.Context, req *connect.Request[storage.DeleteProviderRequest]) (*connect.Response[storage.DeleteProviderResponse], error) { - pid, err := uuid.Parse(req.Msg.GetProviderId()) - if err != nil { - return nil, err - } - - if err := h.ss.DeleteProvider(ctx, pid); err != nil { - return nil, err - } - - return &connect.Response[storage.DeleteProviderResponse]{ - Msg: &storage.DeleteProviderResponse{}, - }, nil -} diff --git a/internal/handler/api/storage/download_object.go b/internal/handler/api/storage/download_object.go deleted file mode 100644 index 5e2b71669..000000000 --- a/internal/handler/api/storage/download_object.go +++ /dev/null @@ -1,33 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" -) - -// DownloadObject implements storageconnect.ServiceHandler -func (h *Handler) DownloadObject(ctx context.Context, req *connect.Request[storage.DownloadObjectRequest], stream *connect.ServerStream[storage.DownloadObjectResponse]) error { - reader, err := h.ss.DownloadObject(ctx, req.Msg.Bucket, req.Msg.Path) - if err != nil { - return err - } - defer reader.Close() - buf := make([]byte, 64*1024) - for { - n, err := reader.Read(buf) - if err != nil { - return err - } - if n == 0 { - break - } - if err := stream.Send(&storage.DownloadObjectResponse{ - Chunk: buf[:n], - }); err != nil { - return err - } - } - return nil -} diff --git a/internal/handler/api/storage/get_bucket.go b/internal/handler/api/storage/get_bucket.go deleted file mode 100644 index 336c63d47..000000000 --- a/internal/handler/api/storage/get_bucket.go +++ /dev/null @@ -1,21 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" -) - -// GetBucket implements storageconnect.ServiceHandler -func (h *Handler) GetBucket(ctx context.Context, req *connect.Request[storage.GetBucketRequest]) (*connect.Response[storage.GetBucketResponse], error) { - bucket, err := h.ss.GetBucket(ctx, req.Msg.Bucket) - if err != nil { - return nil, err - } - return &connect.Response[storage.GetBucketResponse]{ - Msg: &storage.GetBucketResponse{ - Bucket: bucket, - }, - }, nil -} diff --git a/internal/handler/api/storage/get_object.go b/internal/handler/api/storage/get_object.go deleted file mode 100644 index 231929e39..000000000 --- a/internal/handler/api/storage/get_object.go +++ /dev/null @@ -1,21 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" -) - -// GetObject implements storageconnect.ServiceHandler -func (h *Handler) GetObject(ctx context.Context, req *connect.Request[storage.GetObjectRequest]) (*connect.Response[storage.GetObjectResponse], error) { - object, err := h.ss.GetObject(ctx, req.Msg.Bucket, req.Msg.Path) - if err != nil { - return nil, err - } - return &connect.Response[storage.GetObjectResponse]{ - Msg: &storage.GetObjectResponse{ - Object: object, - }, - }, nil -} diff --git a/internal/handler/api/storage/get_provider.go b/internal/handler/api/storage/get_provider.go deleted file mode 100644 index c5095b824..000000000 --- a/internal/handler/api/storage/get_provider.go +++ /dev/null @@ -1,27 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/uuid" -) - -func (h Handler) GetProvider(ctx context.Context, req *connect.Request[storage.GetProviderRequest]) (*connect.Response[storage.GetProviderResponse], error) { - pid, err := uuid.Parse(req.Msg.GetProviderId()) - if err != nil { - return nil, err - } - - p, err := h.ss.GetProvider(ctx, pid) - if err != nil { - return nil, err - } - - return &connect.Response[storage.GetProviderResponse]{ - Msg: &storage.GetProviderResponse{ - Provider: p, - }, - }, nil -} diff --git a/internal/handler/api/storage/handler.go b/internal/handler/api/storage/handler.go deleted file mode 100644 index 3380d6079..000000000 --- a/internal/handler/api/storage/handler.go +++ /dev/null @@ -1,33 +0,0 @@ -package storage - -import ( - "net/http" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage/storageconnect" - storage_service "github.com/rigdev/rig/internal/service/storage" - "go.uber.org/fx" -) - -type Handler struct { - ss *storage_service.Service -} - -type storageParams struct { - fx.In - Serv *storage_service.Service -} - -func New(p storageParams) *Handler { - return &Handler{ - ss: p.Serv, - } -} - -func (h *Handler) ServiceName() string { - return storageconnect.ServiceName -} - -func (h *Handler) Build(opts ...connect.HandlerOption) (string, http.Handler) { - return storageconnect.NewServiceHandler(h, opts...) -} diff --git a/internal/handler/api/storage/list_buckets.go b/internal/handler/api/storage/list_buckets.go deleted file mode 100644 index 996bad71a..000000000 --- a/internal/handler/api/storage/list_buckets.go +++ /dev/null @@ -1,27 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/iterator" -) - -// ListBuckets implements storageconnect.ServiceHandler -func (h *Handler) ListBuckets(ctx context.Context, req *connect.Request[storage.ListBucketsRequest]) (*connect.Response[storage.ListBucketsResponse], error) { - it, err := h.ss.ListBuckets(ctx) - if err != nil { - return nil, err - } - res, err := iterator.Collect(it) - if err != nil { - return nil, err - } - - return &connect.Response[storage.ListBucketsResponse]{ - Msg: &storage.ListBucketsResponse{ - Buckets: res, - }, - }, nil -} diff --git a/internal/handler/api/storage/list_objects.go b/internal/handler/api/storage/list_objects.go deleted file mode 100644 index dfea7897c..000000000 --- a/internal/handler/api/storage/list_objects.go +++ /dev/null @@ -1,27 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/iterator" -) - -// ListObjects implements storageconnect.ServiceHandler -func (h *Handler) ListObjects(ctx context.Context, req *connect.Request[storage.ListObjectsRequest]) (*connect.Response[storage.ListObjectsResponse], error) { - token, it, err := h.ss.ListObjects(ctx, req.Msg.GetBucket(), req.Msg.GetToken(), req.Msg.GetPrefix(), req.Msg.GetStartPath(), req.Msg.GetEndPath(), req.Msg.GetRecursive(), req.Msg.GetLimit()) - if err != nil { - return nil, err - } - res, err := iterator.Collect(it) - if err != nil { - return nil, err - } - return &connect.Response[storage.ListObjectsResponse]{ - Msg: &storage.ListObjectsResponse{ - Token: token, - Results: res, - }, - }, nil -} diff --git a/internal/handler/api/storage/list_providers.go b/internal/handler/api/storage/list_providers.go deleted file mode 100644 index 3d65ad1dc..000000000 --- a/internal/handler/api/storage/list_providers.go +++ /dev/null @@ -1,28 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/iterator" -) - -func (h Handler) ListProviders(ctx context.Context, req *connect.Request[storage.ListProvidersRequest]) (*connect.Response[storage.ListProvidersResponse], error) { - it, total, err := h.ss.ListProviders(ctx, req.Msg.GetPagination()) - if err != nil { - return nil, err - } - - ps, err := iterator.Collect(it) - if err != nil { - return nil, err - } - - return &connect.Response[storage.ListProvidersResponse]{ - Msg: &storage.ListProvidersResponse{ - Providers: ps, - Total: total, - }, - }, nil -} diff --git a/internal/handler/api/storage/lookup_provider.go b/internal/handler/api/storage/lookup_provider.go deleted file mode 100644 index 6ac46ef5f..000000000 --- a/internal/handler/api/storage/lookup_provider.go +++ /dev/null @@ -1,22 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" -) - -func (h Handler) LookupProvider(ctx context.Context, req *connect.Request[storage.LookupProviderRequest]) (*connect.Response[storage.LookupProviderResponse], error) { - providerID, provider, err := h.ss.LookupProvider(ctx, req.Msg.GetName()) - if err != nil { - return nil, err - } - - return &connect.Response[storage.LookupProviderResponse]{ - Msg: &storage.LookupProviderResponse{ - ProviderId: providerID.String(), - Provider: provider, - }, - }, nil -} diff --git a/internal/handler/api/storage/unlink_bucket.go b/internal/handler/api/storage/unlink_bucket.go deleted file mode 100644 index afffacdd1..000000000 --- a/internal/handler/api/storage/unlink_bucket.go +++ /dev/null @@ -1,19 +0,0 @@ -package storage - -import ( - context "context" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" -) - -func (h Handler) UnlinkBucket(ctx context.Context, req *connect.Request[storage.UnlinkBucketRequest]) (*connect.Response[storage.UnlinkBucketResponse], error) { - err := h.ss.UnlinkBucket(ctx, req.Msg.GetBucket()) - if err != nil { - return nil, err - } - - return &connect.Response[storage.UnlinkBucketResponse]{ - Msg: &storage.UnlinkBucketResponse{}, - }, nil -} diff --git a/internal/handler/api/storage/upload_object.go b/internal/handler/api/storage/upload_object.go deleted file mode 100644 index ff3932248..000000000 --- a/internal/handler/api/storage/upload_object.go +++ /dev/null @@ -1,53 +0,0 @@ -package storage - -import ( - context "context" - "io" - - "github.com/bufbuild/connect-go" - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/errors" -) - -// UploadObject implements storageconnect.ServiceHandler -func (h *Handler) UploadObject(ctx context.Context, stream *connect.ClientStream[storage.UploadObjectRequest]) (*connect.Response[storage.UploadObjectResponse], error) { - if stream.Receive() { - switch v := stream.Msg().GetRequest().(type) { - case *storage.UploadObjectRequest_Metadata_: - r, w := io.Pipe() - go func() { - for stream.Receive() { - switch v := stream.Msg().GetRequest().(type) { - case *storage.UploadObjectRequest_Chunk: - _, err := w.Write(v.Chunk) - if err != nil { - w.CloseWithError(err) - return - } - default: - w.CloseWithError(errors.InvalidArgumentErrorf("invalid request")) - return - } - } - if err := stream.Err(); err != nil { - w.CloseWithError(err) - return - } else { - w.Close() - } - }() - _, _, err := h.ss.UploadObject(ctx, r, v.Metadata) - if err != nil { - return nil, err - } - return &connect.Response[storage.UploadObjectResponse]{}, nil - default: - return nil, errors.InvalidArgumentErrorf("invalid request") - } - } - if err := stream.Err(); err != nil { - return nil, err - } - - return nil, errors.InvalidArgumentErrorf("invalid request") -} diff --git a/internal/handler/api/storage_http/download.go b/internal/handler/api/storage_http/download.go deleted file mode 100644 index aceec4863..000000000 --- a/internal/handler/api/storage_http/download.go +++ /dev/null @@ -1,68 +0,0 @@ -package storage_http - -import ( - "fmt" - "io" - "net/http" - "strings" - - "github.com/go-chi/chi/v5" - storage_service "github.com/rigdev/rig/internal/service/storage" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/service" - "go.uber.org/fx" -) - -type DownloadHandler struct { - ss *storage_service.Service -} - -type downloadParams struct { - fx.In - Serv *storage_service.Service -} - -func NewDownloadHandler(p downloadParams) *DownloadHandler { - return &DownloadHandler{ - ss: p.Serv, - } -} - -func (h *DownloadHandler) Build() (string, string, service.HandlerFunc) { - return http.MethodGet, "/api/v1/storage/{bucket}/*", h.download -} - -func (h *DownloadHandler) download(w http.ResponseWriter, r *http.Request) error { - bucket := chi.URLParam(r, "bucket") - if bucket == "" { - return errors.InvalidArgumentErrorf("missing bucket name") - } - - objectPath := chi.URLParam(r, "*") - if objectPath == "" { - return errors.InvalidArgumentErrorf("missing object path") - } - - if strings.HasSuffix(objectPath, "/") { - return errors.InvalidArgumentErrorf("object path cannot be a folder") - } - - o, err := h.ss.GetObject(r.Context(), bucket, objectPath) - if err != nil { - return err - } - - dr, err := h.ss.DownloadObject(r.Context(), bucket, objectPath) - if err != nil { - return err - } - - w.Header().Set("Content-Length", fmt.Sprint(o.Size)) - w.Header().Set("Content-Type", o.GetContentType()) - - if _, err := io.Copy(w, dr); err != nil { - return err - } - - return nil -} diff --git a/internal/handler/api/storage_http/upload.go b/internal/handler/api/storage_http/upload.go deleted file mode 100644 index 88f44c19c..000000000 --- a/internal/handler/api/storage_http/upload.go +++ /dev/null @@ -1,60 +0,0 @@ -package storage_http - -import ( - "net/http" - "strings" - - "github.com/go-chi/chi/v5" - "github.com/rigdev/rig-go-api/api/v1/storage" - storage_service "github.com/rigdev/rig/internal/service/storage" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/service" - "go.uber.org/fx" -) - -type UploadHandler struct { - ss *storage_service.Service -} - -type uploadParams struct { - fx.In - Serv *storage_service.Service -} - -func NewUploadHandler(p uploadParams) *UploadHandler { - return &UploadHandler{ - ss: p.Serv, - } -} - -func (h *UploadHandler) Build() (string, string, service.HandlerFunc) { - return http.MethodPut, "/api/v1/storage/{bucket}/*", h.upload -} - -func (h *UploadHandler) upload(w http.ResponseWriter, r *http.Request) error { - bucket := chi.URLParam(r, "bucket") - if bucket == "" { - return errors.InvalidArgumentErrorf("missing bucket name") - } - - objectPath := chi.URLParam(r, "*") - if objectPath == "" { - return errors.InvalidArgumentErrorf("missing object path") - } - - if strings.HasSuffix(objectPath, "/") { - return errors.InvalidArgumentErrorf("object path cannot be a folder") - } - - if r.ContentLength < 0 { - return errors.InvalidArgumentErrorf("missing Content-Length header") - } - - _, _, err := h.ss.UploadObject(r.Context(), r.Body, &storage.UploadObjectRequest_Metadata{ - Bucket: bucket, - Path: objectPath, - Size: uint64(r.ContentLength), - ContentType: r.Header.Get("Content-Type"), - }) - return err -} diff --git a/internal/handler/module.go b/internal/handler/module.go index 05137e68e..5e1693e83 100644 --- a/internal/handler/module.go +++ b/internal/handler/module.go @@ -4,14 +4,11 @@ import ( "github.com/rigdev/rig/internal/handler/api/authentication" "github.com/rigdev/rig/internal/handler/api/capsule" "github.com/rigdev/rig/internal/handler/api/cluster" - "github.com/rigdev/rig/internal/handler/api/database" "github.com/rigdev/rig/internal/handler/api/group" "github.com/rigdev/rig/internal/handler/api/project" project_settings "github.com/rigdev/rig/internal/handler/api/project/settings" "github.com/rigdev/rig/internal/handler/api/service_account" "github.com/rigdev/rig/internal/handler/api/status_http" - "github.com/rigdev/rig/internal/handler/api/storage" - "github.com/rigdev/rig/internal/handler/api/storage_http" "github.com/rigdev/rig/internal/handler/api/user" user_settings "github.com/rigdev/rig/internal/handler/api/user/settings" "github.com/rigdev/rig/internal/handler/http" @@ -26,17 +23,13 @@ var Module = fx.Module( asGRPCHandler(authentication.New), asGRPCHandler(capsule.New), asGRPCHandler(cluster.New), - asGRPCHandler(database.New), asGRPCHandler(group.New), asGRPCHandler(project.New), asGRPCHandler(project_settings.New), asGRPCHandler(service_account.New), - asGRPCHandler(storage.New), asGRPCHandler(user.New), asGRPCHandler(user_settings.New), asHTTPHandler(http.New), - asHTTPHandler(storage_http.NewUploadHandler), - asHTTPHandler(storage_http.NewDownloadHandler), asHTTPHandler(status_http.NewStatusHandler), registry.NewServer, ), diff --git a/internal/repository/database.go b/internal/repository/database.go deleted file mode 100644 index bcd75b8c3..000000000 --- a/internal/repository/database.go +++ /dev/null @@ -1,23 +0,0 @@ -package repository - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/pkg/iterator" - "github.com/rigdev/rig/pkg/uuid" -) - -type Database interface { - Create(ctx context.Context, database *database.Database) (*database.Database, error) - Delete(ctx context.Context, databaseID uuid.UUID) error - GetByName(ctx context.Context, name string) (*database.Database, error) - - Get(ctx context.Context, databaseID uuid.UUID) (*database.Database, error) - List(ctx context.Context, pagination *model.Pagination) (iterator.Iterator[*database.Database], uint64, error) - - Update(ctx context.Context, database *database.Database) (*database.Database, error) - - BuildIndexes(ctx context.Context) error -} diff --git a/internal/repository/database/mongo/client.go b/internal/repository/database/mongo/client.go deleted file mode 100644 index c4655bb13..000000000 --- a/internal/repository/database/mongo/client.go +++ /dev/null @@ -1,52 +0,0 @@ -package mongo - -import ( - "context" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -var ( - databaseIDIndex = "database_id_idx" - nameIndex = "database_name_idx" -) - -type MongoRepository struct { - DatabaseCollection *mongo.Collection -} - -func (r *MongoRepository) BuildIndexes(ctx context.Context) error { - databaseIdIndexModel := mongo.IndexModel{ - Keys: bson.D{ - {Key: "database_id", Value: 1}, - }, - Options: options.Index().SetName(databaseIDIndex).SetUnique(true), - } - if _, err := r.DatabaseCollection.Indexes().CreateOne(ctx, databaseIdIndexModel); err != nil { - return err - } - nameIndexModel := mongo.IndexModel{ - Keys: bson.D{ - {Key: "project_id", Value: 1}, - {Key: "name", Value: 1}, - }, - Options: options.Index().SetName(nameIndex).SetUnique(true), - } - if _, err := r.DatabaseCollection.Indexes().CreateOne(ctx, nameIndexModel); err != nil { - return err - } - return nil -} - -func NewRepository(c *mongo.Client) (*MongoRepository, error) { - repo := &MongoRepository{ - DatabaseCollection: c.Database("rig").Collection("databases"), - } - err := repo.BuildIndexes(context.Background()) - if err != nil { - return nil, err - } - return repo, nil -} diff --git a/internal/repository/database/mongo/create.go b/internal/repository/database/mongo/create.go deleted file mode 100644 index 9f9417807..000000000 --- a/internal/repository/database/mongo/create.go +++ /dev/null @@ -1,24 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/internal/repository/database/mongo/schema" - "github.com/rigdev/rig/pkg/auth" -) - -func (m *MongoRepository) Create(ctx context.Context, database *database.Database) (*database.Database, error) { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return nil, err - } - d, err := schema.DatabaseFromProto(projectID, database) - if err != nil { - return nil, err - } - if _, err := m.DatabaseCollection.InsertOne(ctx, d); err != nil { - return nil, err - } - return database, nil -} diff --git a/internal/repository/database/mongo/delete.go b/internal/repository/database/mongo/delete.go deleted file mode 100644 index 158c8cf4f..000000000 --- a/internal/repository/database/mongo/delete.go +++ /dev/null @@ -1,26 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" - - "go.mongodb.org/mongo-driver/bson" -) - -func (m *MongoRepository) Delete(ctx context.Context, databaseID uuid.UUID) error { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return err - } - res, err := m.DatabaseCollection.DeleteOne(ctx, bson.M{"database_id": databaseID, "project_id": projectID}) - if err != nil { - return err - } - if res.DeletedCount == 0 { - return errors.NotFoundErrorf("database with id %s not found", databaseID) - } - return nil -} diff --git a/internal/repository/database/mongo/get.go b/internal/repository/database/mongo/get.go deleted file mode 100644 index 699302c33..000000000 --- a/internal/repository/database/mongo/get.go +++ /dev/null @@ -1,23 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/uuid" - "github.com/rigdev/rig/internal/repository/database/mongo/schema" - "go.mongodb.org/mongo-driver/bson" -) - -func (m *MongoRepository) Get(ctx context.Context, databaseID uuid.UUID) (*database.Database, error) { - var d schema.Database - projectId, err := auth.GetProjectID(ctx) - if err != nil { - return nil, err - } - if err := m.DatabaseCollection.FindOne(ctx, bson.M{"database_id": databaseID, "project_id": projectId}).Decode(&d); err != nil { - return nil, err - } - return d.ToProto() -} diff --git a/internal/repository/database/mongo/get_by_name.go b/internal/repository/database/mongo/get_by_name.go deleted file mode 100644 index 828628caa..000000000 --- a/internal/repository/database/mongo/get_by_name.go +++ /dev/null @@ -1,33 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/internal/repository/database/mongo/schema" - "github.com/rigdev/rig/pkg/auth" - "go.mongodb.org/mongo-driver/bson" -) - -func (m *MongoRepository) GetByName(ctx context.Context, name string) (*database.Database, error) { - projectId, err := auth.GetProjectID(ctx) - if err != nil { - return nil, err - } - - var database *schema.Database - err = m.DatabaseCollection.FindOne(ctx, bson.M{ - "name": name, - "project_id": projectId, - }).Decode(&database) - if err != nil { - return nil, err - } - - db, err := database.ToProto() - if err != nil { - return nil, err - } - - return db, nil -} diff --git a/internal/repository/database/mongo/list.go b/internal/repository/database/mongo/list.go deleted file mode 100644 index cedff370e..000000000 --- a/internal/repository/database/mongo/list.go +++ /dev/null @@ -1,58 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/internal/client/mongo" - "github.com/rigdev/rig/internal/repository/database/mongo/schema" - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/iterator" - "go.mongodb.org/mongo-driver/bson" -) - -// List implements group_repository.GroupRepository -func (m *MongoRepository) List(ctx context.Context, pagination *model.Pagination) (iterator.Iterator[*database.Database], uint64, error) { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return nil, 0, err - } - - filter := bson.M{"project_id": projectID} - - count, err := m.DatabaseCollection.CountDocuments(ctx, filter) - if err != nil { - return nil, 0, err - } - - cursor, err := m.DatabaseCollection.Find(ctx, filter, mongo.SortOptions(pagination)) - if err != nil { - return nil, 0, err - } - - it := iterator.NewProducer[*database.Database]() - go func() { - defer it.Done() - defer cursor.Close(ctx) - for cursor.Next(ctx) { - var d schema.Database - if err := cursor.Decode(&d); err != nil { - it.Error(err) - return - } - - e, err := d.ToProto() - if err != nil { - it.Error(err) - return - } - if err := it.Value(e); err != nil { - it.Error(err) - return - } - } - }() - - return it, uint64(count), nil -} diff --git a/internal/repository/database/mongo/schema/database.go b/internal/repository/database/mongo/schema/database.go deleted file mode 100644 index 3bbd45acf..000000000 --- a/internal/repository/database/mongo/schema/database.go +++ /dev/null @@ -1,43 +0,0 @@ -package schema - -import ( - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" - "google.golang.org/protobuf/proto" -) - -type Database struct { - DatabaseID uuid.UUID `bson:"database_id" json:"database_id"` - ProjectID uuid.UUID `bson:"project_id" json:"project_id"` - Name string `bson:"name" json:"name"` - DatabaseType database.Type `bson:"database_type" json:"database_type"` - Data []byte `bson:"data" json:"data"` -} - -func (d Database) ToProto() (*database.Database, error) { - p := &database.Info{} - if err := proto.Unmarshal(d.Data, p); err != nil { - return nil, err - } - return &database.Database{ - DatabaseId: d.DatabaseID.String(), - Name: d.Name, - Info: p, - Type: d.DatabaseType, - }, nil -} - -func DatabaseFromProto(projectID uuid.UUID, d *database.Database) (Database, error) { - bs, err := proto.Marshal(d.Info) - if err != nil { - return Database{}, err - } - - return Database{ - DatabaseID: uuid.UUID(d.GetDatabaseId()), - ProjectID: projectID, - Name: d.GetName(), - DatabaseType: d.Type, - Data: bs, - }, nil -} diff --git a/internal/repository/database/mongo/update.go b/internal/repository/database/mongo/update.go deleted file mode 100644 index c5ce1d211..000000000 --- a/internal/repository/database/mongo/update.go +++ /dev/null @@ -1,33 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/internal/repository/database/mongo/schema" - "github.com/rigdev/rig/pkg/auth" - "go.mongodb.org/mongo-driver/bson" -) - -// Update updates the name of a specific group. -func (m *MongoRepository) Update(ctx context.Context, database *database.Database) (*database.Database, error) { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return nil, err - } - d, err := schema.DatabaseFromProto(projectID, database) - if err != nil { - return nil, err - } - if err := m.DatabaseCollection.FindOneAndUpdate( - ctx, - bson.M{"project_id": projectID, "database_id": database.GetDatabaseId()}, - bson.M{ - "$set": d, - }, - ).Decode(&d); err != nil { - return nil, err - } - - return d.ToProto() -} diff --git a/internal/repository/module.go b/internal/repository/module.go index dabd03890..d7ba866ac 100644 --- a/internal/repository/module.go +++ b/internal/repository/module.go @@ -7,13 +7,11 @@ import ( "github.com/rigdev/rig/internal/config" capsule_mongo "github.com/rigdev/rig/internal/repository/capsule/mongo" cluster_config_mongo "github.com/rigdev/rig/internal/repository/cluster_config/mongo" - database_mongo "github.com/rigdev/rig/internal/repository/database/mongo" group_mongo "github.com/rigdev/rig/internal/repository/group/mongo" project_mongo "github.com/rigdev/rig/internal/repository/project/mongo" secret_mongo "github.com/rigdev/rig/internal/repository/secret/mongo" service_account_mongo "github.com/rigdev/rig/internal/repository/service_account/mongo" session_mongo "github.com/rigdev/rig/internal/repository/session/mongo" - storage_mongo "github.com/rigdev/rig/internal/repository/storage/mongo" user_mongo "github.com/rigdev/rig/internal/repository/user/mongo" verification_code_mongo "github.com/rigdev/rig/internal/repository/verification_code/mongo" "github.com/uptrace/bun" @@ -31,9 +29,7 @@ var Module = fx.Module( NewUser, NewVerificationCode, NewServiceAccount, - NewDatabase, NewCapsule, - NewStorage, NewSecret, NewClusterConfig, ), @@ -49,19 +45,6 @@ type params struct { Postgres *bun.DB `optional:"true"` } -func NewDatabase(p params) (Database, error) { - s := p.Cfg.Repository.Database.Store - switch s { - case storeTypeMongoDB: - if p.MongoClient == nil { - return nil, errNoMongoDBClient - } - return database_mongo.NewRepository(p.MongoClient) - default: - return nil, errInvalidStore("database", s) - } -} - func NewGroup(p params) (Group, error) { s := p.Cfg.Repository.Group.Store switch s { @@ -153,19 +136,6 @@ func NewCapsule(p params) (Capsule, error) { } } -func NewStorage(p params) (Storage, error) { - s := p.Cfg.Repository.Storage.Store - switch s { - case storeTypeMongoDB: - if p.MongoClient == nil { - return nil, errNoMongoDBClient - } - return storage_mongo.NewRepository(p.MongoClient) - default: - return nil, errInvalidStore("storage", s) - } -} - func NewSecret(p params) (Secret, error) { s := p.Cfg.Repository.Secret.Store switch s { diff --git a/internal/repository/storage.go b/internal/repository/storage.go deleted file mode 100644 index b40d5997f..000000000 --- a/internal/repository/storage.go +++ /dev/null @@ -1,21 +0,0 @@ -package repository - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/pkg/iterator" - "github.com/rigdev/rig/pkg/uuid" -) - -type Storage interface { - Create(ctx context.Context, providerID, secretID uuid.UUID, provider *storage.Provider) (*storage.Provider, error) - Delete(ctx context.Context, providerID uuid.UUID) error - Get(ctx context.Context, providerID uuid.UUID) (*storage.Provider, uuid.UUID, error) - List(ctx context.Context, pagination *model.Pagination) (iterator.Iterator[*storage.ProviderEntry], uint64, error) - Update(ctx context.Context, providerID uuid.UUID, provider *storage.Provider) (*storage.Provider, error) - LookupByBucket(ctx context.Context, bucket string) (uuid.UUID, *storage.Provider, uuid.UUID, error) - Lookup(ctx context.Context, name string) (uuid.UUID, *storage.Provider, uuid.UUID, error) - BuildIndexes(ctx context.Context) error -} diff --git a/internal/repository/storage/mongo/client.go b/internal/repository/storage/mongo/client.go deleted file mode 100644 index ccce40333..000000000 --- a/internal/repository/storage/mongo/client.go +++ /dev/null @@ -1,66 +0,0 @@ -package mongo - -import ( - "context" - - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" -) - -var ( - providerIDIndex = "provider_id_idx" - providerNameIndex = "provider_name_idx" - bucketNameIndex = "bucket_name_idx" -) - -type MongoRepository struct { - ProviderCollection *mongo.Collection -} - -func NewRepository(c *mongo.Client) (*MongoRepository, error) { - repo := &MongoRepository{ - ProviderCollection: c.Database("rig").Collection("providers"), - } - err := repo.BuildIndexes(context.Background()) - if err != nil { - return nil, err - } - return repo, nil -} - -func (r *MongoRepository) BuildIndexes(ctx context.Context) error { - providerIDIndexModel := mongo.IndexModel{ - Keys: bson.D{ - {Key: "project_id", Value: 1}, - {Key: "provider_id", Value: 1}, - }, - Options: options.Index().SetName(providerIDIndex).SetUnique(true), - } - if _, err := r.ProviderCollection.Indexes().CreateOne(ctx, providerIDIndexModel); err != nil { - return err - } - - providerNameIndexModel := mongo.IndexModel{ - Keys: bson.D{ - {Key: "project_id", Value: 1}, - {Key: "name", Value: 1}, - }, - Options: options.Index().SetName(providerNameIndex).SetUnique(true), - } - if _, err := r.ProviderCollection.Indexes().CreateOne(ctx, providerNameIndexModel); err != nil { - return err - } - - bucketNameIndexModel := mongo.IndexModel{ - Keys: bson.D{ - {Key: "project_id", Value: 1}, - {Key: "buckets.name", Value: 1}, - }, - Options: options.Index().SetName(bucketNameIndex).SetUnique(true).SetPartialFilterExpression(bson.M{"buckets.name": bson.M{"$exists": true}}), - } - if _, err := r.ProviderCollection.Indexes().CreateOne(ctx, bucketNameIndexModel); err != nil { - return err - } - return nil -} diff --git a/internal/repository/storage/mongo/create.go b/internal/repository/storage/mongo/create.go deleted file mode 100644 index 7775a97a1..000000000 --- a/internal/repository/storage/mongo/create.go +++ /dev/null @@ -1,28 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/uuid" - "github.com/rigdev/rig/internal/repository/storage/mongo/schema" -) - -func (m *MongoRepository) Create(ctx context.Context, providerId, secretID uuid.UUID, provider *storage.Provider) (*storage.Provider, error) { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return nil, err - } - - p, err := schema.ProviderFromProto(projectID, providerId, secretID, provider) - if err != nil { - return nil, err - } - - if _, err := m.ProviderCollection.InsertOne(ctx, p); err != nil { - return nil, err - } - - return provider, nil -} diff --git a/internal/repository/storage/mongo/delete.go b/internal/repository/storage/mongo/delete.go deleted file mode 100644 index 629a52f80..000000000 --- a/internal/repository/storage/mongo/delete.go +++ /dev/null @@ -1,29 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" - "go.mongodb.org/mongo-driver/bson" -) - -func (m *MongoRepository) Delete(ctx context.Context, providerID uuid.UUID) error { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return err - } - - filter := bson.M{"provider_id": providerID, "project_id": projectID} - res, err := m.ProviderCollection.DeleteOne(ctx, filter) - if err != nil { - return err - } - - if res.DeletedCount == 0 { - return errors.NotFoundErrorf("provider with id %s not found", providerID) - } - - return nil -} diff --git a/internal/repository/storage/mongo/get.go b/internal/repository/storage/mongo/get.go deleted file mode 100644 index 79cd3c7a2..000000000 --- a/internal/repository/storage/mongo/get.go +++ /dev/null @@ -1,31 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/uuid" - "github.com/rigdev/rig/internal/repository/storage/mongo/schema" - "go.mongodb.org/mongo-driver/bson" -) - -func (m *MongoRepository) Get(ctx context.Context, providerID uuid.UUID) (*storage.Provider, uuid.UUID, error) { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return nil, uuid.Nil, err - } - - filter := bson.M{"provider_id": providerID, "project_id": projectID} - var p schema.Provider - if err := m.ProviderCollection.FindOne(ctx, filter).Decode(&p); err != nil { - return nil, uuid.Nil, err - } - - provider, err := p.ToProto() - if err != nil { - return nil, uuid.Nil, err - } - - return provider, p.SecretID, nil -} diff --git a/internal/repository/storage/mongo/list.go b/internal/repository/storage/mongo/list.go deleted file mode 100644 index 2cbffbd61..000000000 --- a/internal/repository/storage/mongo/list.go +++ /dev/null @@ -1,54 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/iterator" - "github.com/rigdev/rig/internal/repository/storage/mongo/schema" - "go.mongodb.org/mongo-driver/bson" -) - -func (m *MongoRepository) List(ctx context.Context, pagination *model.Pagination) (iterator.Iterator[*storage.ProviderEntry], uint64, error) { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return nil, 0, err - } - - count, err := m.ProviderCollection.CountDocuments(ctx, bson.M{"project_id": projectID}) - if err != nil { - return nil, 0, err - } - - cursor, err := m.ProviderCollection.Find(ctx, bson.M{"project_id": projectID}) - if err != nil { - return nil, 0, err - } - - it := iterator.NewProducer[*storage.ProviderEntry]() - go func() { - defer it.Done() - defer cursor.Close(ctx) - for cursor.Next(ctx) { - var p schema.Provider - if err := cursor.Decode(&p); err != nil { - it.Error(err) - return - } - - e, err := p.ToProtoEntry() - if err != nil { - it.Error(err) - return - } - if err := it.Value(e); err != nil { - it.Error(err) - return - } - } - }() - - return it, uint64(count), nil -} diff --git a/internal/repository/storage/mongo/lookup.go b/internal/repository/storage/mongo/lookup.go deleted file mode 100644 index c27717531..000000000 --- a/internal/repository/storage/mongo/lookup.go +++ /dev/null @@ -1,31 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/uuid" - "github.com/rigdev/rig/internal/repository/storage/mongo/schema" - "go.mongodb.org/mongo-driver/bson" -) - -func (m *MongoRepository) Lookup(ctx context.Context, name string) (uuid.UUID, *storage.Provider, uuid.UUID, error) { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return uuid.Nil, nil, uuid.Nil, err - } - - filter := bson.M{"project_id": projectID, "name": name} - provider := &schema.Provider{} - if err := m.ProviderCollection.FindOne(ctx, filter).Decode(provider); err != nil { - return uuid.Nil, nil, uuid.Nil, err - } - - p, err := provider.ToProto() - if err != nil { - return uuid.Nil, nil, uuid.Nil, err - } - - return provider.ProviderID, p, provider.SecretID, nil -} diff --git a/internal/repository/storage/mongo/lookup_by_bucket.go b/internal/repository/storage/mongo/lookup_by_bucket.go deleted file mode 100644 index 9d837a4f2..000000000 --- a/internal/repository/storage/mongo/lookup_by_bucket.go +++ /dev/null @@ -1,32 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/uuid" - "github.com/rigdev/rig/internal/repository/storage/mongo/schema" - "go.mongodb.org/mongo-driver/bson" -) - -func (m *MongoRepository) LookupByBucket(ctx context.Context, bucket string) (uuid.UUID, *storage.Provider, uuid.UUID, error) { - projectID, err := auth.GetProjectID(ctx) - if err != nil { - return uuid.Nil, nil, uuid.Nil, err - } - - filter := bson.M{"project_id": projectID, "buckets.name": bucket} - provider := &schema.Provider{} - if err := m.ProviderCollection.FindOne(ctx, filter).Decode(provider); err != nil { - return uuid.Nil, nil, uuid.Nil, err - } - - p, err := provider.ToProto() - - if err != nil { - return uuid.Nil, nil, uuid.Nil, err - } - - return provider.ProviderID, p, provider.SecretID, nil -} diff --git a/internal/repository/storage/mongo/schema/provider.go b/internal/repository/storage/mongo/schema/provider.go deleted file mode 100644 index 888afe277..000000000 --- a/internal/repository/storage/mongo/schema/provider.go +++ /dev/null @@ -1,60 +0,0 @@ -package schema - -import ( - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/uuid" - "google.golang.org/protobuf/proto" -) - -type Provider struct { - ProjectID uuid.UUID `bson:"project_id" json:"project_id"` - ProviderID uuid.UUID `bson:"provider_id" json:"provider_id"` - SecretID uuid.UUID `bson:"secrets_id" json:"secrets_id"` - Name string `bson:"name" json:"name"` - Buckets []*storage.Bucket `bson:"buckets" json:"buckets"` - Data []byte `bson:"data" json:"data"` -} - -func (p *Provider) ToProto() (*storage.Provider, error) { - pr := &storage.Provider{} - if err := proto.Unmarshal(p.Data, pr); err != nil { - return nil, err - } - - pr.Buckets = p.Buckets - - return pr, nil -} - -func (p *Provider) ToProtoEntry() (*storage.ProviderEntry, error) { - pp, err := p.ToProto() - if err != nil { - return nil, err - } - - e := &storage.ProviderEntry{ - ProviderId: p.ProviderID.String(), - Name: p.Name, - Config: pp.GetConfig(), - Buckets: p.Buckets, - CreatedAt: pp.CreatedAt, - } - - return e, nil -} - -func ProviderFromProto(projectID, providerID, secretsID uuid.UUID, p *storage.Provider) (*Provider, error) { - bs, err := proto.Marshal(p) - if err != nil { - return nil, err - } - - return &Provider{ - ProjectID: projectID, - ProviderID: providerID, - SecretID: secretsID, - Name: p.GetName(), - Buckets: p.GetBuckets(), - Data: bs, - }, nil -} diff --git a/internal/repository/storage/mongo/update.go b/internal/repository/storage/mongo/update.go deleted file mode 100644 index 5924e96a9..000000000 --- a/internal/repository/storage/mongo/update.go +++ /dev/null @@ -1,26 +0,0 @@ -package mongo - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig/pkg/auth" - "github.com/rigdev/rig/pkg/uuid" - "go.mongodb.org/mongo-driver/bson" -) - -func (m *MongoRepository) Update(ctx context.Context, providerID uuid.UUID, provider *storage.Provider) (*storage.Provider, error) { - projectId, err := auth.GetProjectID(ctx) - if err != nil { - return nil, err - } - - filter := bson.M{"provider_id": providerID, "project_id": projectId} - update := bson.M{"$set": bson.M{"name": provider.Name, "buckets": provider.Buckets}} - - if _, err := m.ProviderCollection.UpdateOne(ctx, filter, update); err != nil { - return nil, err - } - - return provider, nil -} diff --git a/internal/service/database/create.go b/internal/service/database/create.go deleted file mode 100644 index 918df0087..000000000 --- a/internal/service/database/create.go +++ /dev/null @@ -1,51 +0,0 @@ -package database - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" - "google.golang.org/protobuf/types/known/timestamppb" -) - -func (s *Service) Create(ctx context.Context, dbType database.Type, initializers []*database.Update) (uuid.UUID, *database.Database, error) { - databaseID := uuid.New() - - switch dbType { - case database.Type_TYPE_MONGO: - if err := s.mongoEnabled(); err != nil { - return uuid.Nil, nil, err - } - case database.Type_TYPE_POSTGRES: - if err := s.postgresEnabled(); err != nil { - return uuid.Nil, nil, err - } - if _, err := s.postgres.Exec("create database " + formatDatabaseID(databaseID)); err != nil { - return uuid.Nil, nil, err - } - default: - return uuid.Nil, nil, errors.InvalidArgumentErrorf("invalid database type: %v", dbType) - } - - d := &database.Database{ - DatabaseId: databaseID.String(), - Type: dbType, - Info: &database.Info{ - CreatedAt: timestamppb.Now(), - Credentials: []*database.Credential{}, - }, - } - if err := applyUpdates(d, initializers); err != nil { - return uuid.Nil, nil, err - } - if d.Name == "" { - return uuid.Nil, nil, errors.InvalidArgumentErrorf("missing required database name") - } - var err error - d, err = s.dr.Create(ctx, d) - if err != nil { - return uuid.Nil, nil, err - } - return databaseID, d, nil -} diff --git a/internal/service/database/create_credential.go b/internal/service/database/create_credential.go deleted file mode 100644 index f026e5364..000000000 --- a/internal/service/database/create_credential.go +++ /dev/null @@ -1,87 +0,0 @@ -package database - -import ( - "context" - "crypto/rand" - "encoding/hex" - "fmt" - "io" - "strings" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" - "go.mongodb.org/mongo-driver/bson" - "google.golang.org/protobuf/types/known/timestamppb" -) - -func (s *Service) CreateCredential(ctx context.Context, credentialName string, databaseID uuid.UUID) (clientId string, clientSecret string, err error) { - db, err := s.Get(ctx, databaseID) - if err != nil { - return "", "", err - } - if credentialName == "" { - return "", "", errors.InvalidArgumentErrorf("credential name cannot be empty") - } - - if db.GetInfo().GetCredentials() == nil { - db.Info.Credentials = []*database.Credential{} - } - - for _, credential := range db.GetInfo().GetCredentials() { - if credential.GetName() == credentialName { - return "", "", errors.AlreadyExistsErrorf("credential with name %s already exists", credentialName) - } - } - - certificateID := uuid.New() - - clientSecretRaw := make([]byte, 32) - if _, err := io.ReadFull(rand.Reader, clientSecretRaw); err != nil { - return "", "", err - } - - clientSecret = fmt.Sprint("secret_", hex.EncodeToString(clientSecretRaw)) - clientID := formatClientID(certificateID) - - switch db.GetType() { - case database.Type_TYPE_MONGO: - if err := s.mongoEnabled(); err != nil { - return "", "", err - } - if err := s.mongo.Database("admin").RunCommand(context.Background(), bson.D{ - {Key: "createUser", Value: clientID}, - {Key: "pwd", Value: clientSecret}, - {Key: "roles", Value: []bson.M{{"role": "readWrite", "db": formatDatabaseID(databaseID)}}}, - }); err.Err() != nil { - return "", "", err.Err() - } - case database.Type_TYPE_POSTGRES: - if err := s.postgresEnabled(); err != nil { - return "", "", err - } - // TODO(Oscar): (First step might successed, but second step fail, before trying again) - if _, err := s.postgres.Exec(fmt.Sprintf("create user %s with encrypted password '%s'", clientID, clientSecret)); err != nil { - return "", "", err - } - if _, err := s.postgres.Exec(fmt.Sprintf("grant all privileges on database %s to %s", formatDatabaseID(databaseID), clientID)); err != nil { - return "", "", err - } - default: - return "", "", errors.InvalidArgumentErrorf("invalid database type: %v", db.GetType()) - } - c := &database.Credential{ - Name: credentialName, - ClientId: clientID, - CreatedAt: timestamppb.Now(), - Secret: clientSecret[0:10] + strings.Repeat("*", 32-10), - } - - db.Info.Credentials = append(db.Info.Credentials, c) - - if _, err := s.dr.Update(ctx, db); err != nil { - return "", "", err - } - - return formatClientID(certificateID), clientSecret, nil -} diff --git a/internal/service/database/create_table.go b/internal/service/database/create_table.go deleted file mode 100644 index 7751a1b95..000000000 --- a/internal/service/database/create_table.go +++ /dev/null @@ -1,38 +0,0 @@ -package database - -import ( - "context" - "fmt" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" -) - -func (s *Service) CreateTable(ctx context.Context, databaseID uuid.UUID, tableName string) error { - db, err := s.Get(ctx, databaseID) - if err != nil { - return err - } - switch db.Type { - case database.Type_TYPE_MONGO: - if err := s.mongoEnabled(); err != nil { - return err - } - if err := s.mongo.Database(formatDatabaseID(databaseID)).CreateCollection(ctx, tableName); err != nil { - return err - } - case database.Type_TYPE_POSTGRES: - if err := s.postgresEnabled(); err != nil { - return err - } - if _, err := s.postgres.Exec(fmt.Sprintf("create table %s ()", tableName)); err != nil { - return err - } - // TODO: parse table formats - // int2, int4, int8, float4, float8, numeric, json, text, varchar, uuid, date, time, timetz, timestamp, timestamptz, bool - default: - return errors.InternalErrorf("invalid database type: %v", db.Type) - } - return nil -} diff --git a/internal/service/database/delete.go b/internal/service/database/delete.go deleted file mode 100644 index 34f293735..000000000 --- a/internal/service/database/delete.go +++ /dev/null @@ -1,50 +0,0 @@ -package database - -import ( - "context" - "fmt" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" -) - -func (s *Service) Delete(ctx context.Context, databaseID uuid.UUID) error { - db, err := s.Get(ctx, databaseID) - if err != nil { - return err - } - switch db.Type { - case database.Type_TYPE_MONGO: - if err := s.mongoEnabled(); err != nil { - return err - } - // delete all database user in that database - for _, credential := range db.GetInfo().GetCredentials() { - if err := s.dropMongoUser(credential.ClientId, databaseID); err != nil { - return err - } - } - if err := s.mongo.Database(formatDatabaseID(databaseID)).Drop(ctx); err != nil { - return err - } - case database.Type_TYPE_POSTGRES: - if err := s.postgresEnabled(); err != nil { - return err - } - for _, credential := range db.GetInfo().GetCredentials() { - if err := s.dropPostgresUser(credential.ClientId, databaseID); err != nil { - return err - } - } - if _, err := s.postgres.Exec(fmt.Sprintf("drop database %s", formatDatabaseID(databaseID))); err != nil { - return err - } - default: - return errors.InternalErrorf("invalid database type: %v", db.Type) - } - if err := s.dr.Delete(ctx, databaseID); err != nil { - return err - } - return nil -} diff --git a/internal/service/database/delete_credential.go b/internal/service/database/delete_credential.go deleted file mode 100644 index be6d5b00c..000000000 --- a/internal/service/database/delete_credential.go +++ /dev/null @@ -1,50 +0,0 @@ -package database - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" -) - -func (s *Service) DeleteCredential(ctx context.Context, credentialsName string, databaseID uuid.UUID) error { - db, err := s.Get(ctx, databaseID) - if err != nil { - return err - } - isFound := false - for index, credential := range db.GetInfo().GetCredentials() { - if credential.GetName() == credentialsName { - switch db.GetType() { - case database.Type_TYPE_MONGO: - if err := s.mongoEnabled(); err != nil { - return err - } - if err := s.dropMongoUser(credential.ClientId, databaseID); err != nil { - return err - } - case database.Type_TYPE_POSTGRES: - if err := s.postgresEnabled(); err != nil { - return err - } - if err := s.dropPostgresUser(credential.ClientId, databaseID); err != nil { - return err - } - default: - return errors.InternalErrorf("invalid database type: %v", db.Type) - - } - db.Info.Credentials = append(db.Info.Credentials[:index], db.Info.Credentials[index+1:]...) - isFound = true - break - } - } - if !isFound { - return errors.NotFoundErrorf("could not find clientId credential") - } - if _, err := s.dr.Update(ctx, db); err != nil { - return err - } - return nil -} diff --git a/internal/service/database/delete_table.go b/internal/service/database/delete_table.go deleted file mode 100644 index 454f7cf72..000000000 --- a/internal/service/database/delete_table.go +++ /dev/null @@ -1,28 +0,0 @@ -package database - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" -) - -func (s *Service) DeleteTable(ctx context.Context, databaseID uuid.UUID, collectionName string) error { - db, err := s.Get(ctx, databaseID) - if err != nil { - return err - } - switch db.Type { - case database.Type_TYPE_MONGO: - if err := s.mongoEnabled(); err != nil { - return err - } - if err := s.mongo.Database(formatDatabaseID(databaseID)).Collection(collectionName).Drop(ctx); err != nil { - return err - } - default: - return errors.InternalErrorf("invalid database type: %v", db.Type) - } - return nil -} diff --git a/internal/service/database/get.go b/internal/service/database/get.go deleted file mode 100644 index 180a9f5bd..000000000 --- a/internal/service/database/get.go +++ /dev/null @@ -1,49 +0,0 @@ -package database - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" - "go.mongodb.org/mongo-driver/bson" -) - -func (s *Service) Get(ctx context.Context, databaseID uuid.UUID) (*database.Database, error) { - var ( - db *database.Database - err error - ) - db, err = s.dr.Get(ctx, databaseID) - if err != nil { - return nil, err - } - switch db.GetType() { - case database.Type_TYPE_MONGO: - names, err := s.mongo.Database(formatDatabaseID(databaseID)).ListCollectionNames(ctx, bson.M{}) - if err != nil { - return nil, err - } - db.Info.Tables = make([]*database.Table, len(names)) - for index, name := range names { - db.Info.Tables[index] = &database.Table{Name: name} - } - case database.Type_TYPE_POSTGRES: - res, err := s.postgres.Query("SELECT table_name FROM information_schema.tables") - if err != nil { - return nil, err - } - db.Info.Tables = []*database.Table{} - for res.Next() { - var table string - if err := res.Scan(&table); err != nil { - return nil, err - } - db.Info.Tables = append(db.Info.Tables, &database.Table{Name: table}) - } - - default: - return nil, errors.InvalidArgumentErrorf("invalid database type: %v", db.GetType()) - } - return db, nil -} diff --git a/internal/service/database/get_by_name.go b/internal/service/database/get_by_name.go deleted file mode 100644 index c9ff6d911..000000000 --- a/internal/service/database/get_by_name.go +++ /dev/null @@ -1,49 +0,0 @@ -package database - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" - "go.mongodb.org/mongo-driver/bson" -) - -func (s *Service) GetByName(ctx context.Context, name string) (*database.Database, error) { - db, err := s.dr.GetByName(ctx, name) - if err != nil { - return nil, err - } - - dbID := uuid.UUID(db.GetDatabaseId()) - - switch db.GetType() { - case database.Type_TYPE_MONGO: - names, err := s.mongo.Database(formatDatabaseID(dbID)).ListCollectionNames(ctx, bson.M{}) - if err != nil { - return nil, err - } - db.Info.Tables = make([]*database.Table, len(names)) - for index, name := range names { - db.Info.Tables[index] = &database.Table{Name: name} - } - case database.Type_TYPE_POSTGRES: - res, err := s.postgres.Query("SELECT table_name FROM information_schema.tables") - if err != nil { - return nil, err - } - db.Info.Tables = []*database.Table{} - for res.Next() { - var table string - if err := res.Scan(&table); err != nil { - return nil, err - } - db.Info.Tables = append(db.Info.Tables, &database.Table{Name: table}) - } - - default: - return nil, errors.InvalidArgumentErrorf("invalid database type: %v", db.GetType()) - } - - return db, nil -} diff --git a/internal/service/database/get_database_endpoint.go b/internal/service/database/get_database_endpoint.go deleted file mode 100644 index 0c6a1be9d..000000000 --- a/internal/service/database/get_database_endpoint.go +++ /dev/null @@ -1,41 +0,0 @@ -package database - -import ( - "context" - "fmt" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" -) - -func (s *Service) GetDatabaseEndpoint(ctx context.Context, databaseID uuid.UUID, clientID, clientSecret string) (string, string, error) { - db, err := s.Get(ctx, databaseID) - if err != nil { - return "", "", err - } - if clientID != "" { - found := false - for _, credential := range db.GetInfo().GetCredentials() { - if credential.GetClientId() == clientID { - found = true - break - } - } - if !found { - return "", "", errors.NotFoundErrorf("could not find clientID credential") - } - } - if clientID == "" && clientSecret == "" { - clientID = "username" - clientSecret = "password" - } - switch db.GetType() { - case database.Type_TYPE_MONGO: - return fmt.Sprintf("mongodb://%s:%s@%s/%s?authSource=admin", clientID, clientSecret, s.cfg.Client.Mongo.Host, formatDatabaseID(databaseID)), formatDatabaseID(databaseID), nil - case database.Type_TYPE_POSTGRES: - return "", "", errors.UnimplementedErrorf("not currently available for postgres") - default: - return "", "", errors.InternalErrorf("invalid database type: %v", db.GetType()) - } -} diff --git a/internal/service/database/list_credentials.go b/internal/service/database/list_credentials.go deleted file mode 100644 index 0f9f07b13..000000000 --- a/internal/service/database/list_credentials.go +++ /dev/null @@ -1,16 +0,0 @@ -package database - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (s *Service) ListCredentials(ctx context.Context, databaseID uuid.UUID) ([]*database.Credential, error) { - db, err := s.Get(ctx, databaseID) - if err != nil { - return nil, err - } - return db.GetInfo().GetCredentials(), nil -} diff --git a/internal/service/database/list_databases.go b/internal/service/database/list_databases.go deleted file mode 100644 index 0f144dea7..000000000 --- a/internal/service/database/list_databases.go +++ /dev/null @@ -1,17 +0,0 @@ -package database - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/pkg/iterator" -) - -func (s *Service) List(ctx context.Context, pagination *model.Pagination) (iterator.Iterator[*database.Database], uint64, error) { - it, total, err := s.dr.List(ctx, pagination) - if err != nil { - return nil, 0, err - } - return it, total, nil -} diff --git a/internal/service/database/list_tables.go b/internal/service/database/list_tables.go deleted file mode 100644 index ce7d4f6ac..000000000 --- a/internal/service/database/list_tables.go +++ /dev/null @@ -1,16 +0,0 @@ -package database - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (s *Service) ListTables(ctx context.Context, databaseID uuid.UUID) ([]*database.Table, error) { - db, err := s.Get(ctx, databaseID) - if err != nil { - return nil, err - } - return db.GetInfo().GetTables(), nil -} diff --git a/internal/service/database/service.go b/internal/service/database/service.go deleted file mode 100644 index af095eff6..000000000 --- a/internal/service/database/service.go +++ /dev/null @@ -1,105 +0,0 @@ -package database - -import ( - "context" - "database/sql" - "fmt" - "reflect" - "strings" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/internal/config" - "github.com/rigdev/rig/internal/repository" - "github.com/rigdev/rig/internal/service/user" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/uuid" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.uber.org/fx" - "go.uber.org/zap" -) - -type Service struct { - cfg config.Config - dr repository.Database - - mongo *mongo.Client - postgres *sql.DB - - logger *zap.Logger -} - -type newServiceParams struct { - fx.In - - Config config.Config - DatabaseRepo repository.Database - UserService user.Service - Logger *zap.Logger - Mongo *mongo.Client `optional:"true"` - Postgres *sql.DB `optional:"true"` -} - -func NewService(p newServiceParams) (*Service, error) { - return &Service{ - cfg: p.Config, - dr: p.DatabaseRepo, - logger: p.Logger, - mongo: p.Mongo, - postgres: p.Postgres, - }, nil -} - -func applyUpdates(d *database.Database, ds []*database.Update) error { - for _, up := range ds { - switch v := up.GetField().(type) { - case *database.Update_Name: - d.Name = v.Name - default: - return errors.InvalidArgumentErrorf("invalid database update type '%v'", reflect.TypeOf(up.GetField())) - } - } - return nil -} - -func formatClientID(certificateID uuid.UUID) string { - return fmt.Sprint("rig_", certificateID) -} - -func formatDatabaseID(databaseID uuid.UUID) string { - return fmt.Sprint("rigdb_", databaseID) -} - -func (s *Service) dropMongoUser(clientID string, databaseID uuid.UUID) error { - err := s.mongo.Database("admin").RunCommand(context.Background(), bson.D{{Key: "dropUser", Value: clientID}}).Err() - if err != nil && strings.Contains(err.Error(), "UserNotFound") { - return nil - } else if err != nil { - return err - } - return nil -} - -func (s *Service) dropPostgresUser(clientID string, databaseID uuid.UUID) error { - if _, err := s.postgres.Exec(fmt.Sprintf("revoke all privileges on database %s from %s", formatDatabaseID(databaseID), clientID)); err != nil { - return err - } - if _, err := s.postgres.Exec(fmt.Sprintf("drop user %s", clientID)); err != nil { - return err - } - return nil -} - -func (s *Service) mongoEnabled() error { - if s.mongo == nil { - return errors.FailedPreconditionErrorf("cannot register a MongoDB database without MongoDB being enabled") - } - return nil -} - -func (s *Service) postgresEnabled() error { - if s.postgres == nil { - return errors.FailedPreconditionErrorf("cannot register a Postgres database without Postgres being enabled") - } - return nil -} diff --git a/internal/service/database/update.go b/internal/service/database/update.go deleted file mode 100644 index 655e02281..000000000 --- a/internal/service/database/update.go +++ /dev/null @@ -1,23 +0,0 @@ -package database - -import ( - "context" - - "github.com/rigdev/rig-go-api/api/v1/database" - "github.com/rigdev/rig/pkg/uuid" -) - -func (s *Service) Update(ctx context.Context, databaseID uuid.UUID, updates []*database.Update) (*database.Database, error) { - db, err := s.Get(ctx, databaseID) - if err != nil { - return nil, err - } - if err := applyUpdates(db, updates); err != nil { - return nil, err - } - db, err = s.dr.Update(ctx, db) - if err != nil { - return nil, err - } - return db, nil -} diff --git a/internal/service/module.go b/internal/service/module.go index 685cd8eee..ab33132dc 100644 --- a/internal/service/module.go +++ b/internal/service/module.go @@ -4,12 +4,10 @@ import ( "github.com/rigdev/rig/internal/service/auth" "github.com/rigdev/rig/internal/service/capsule" "github.com/rigdev/rig/internal/service/cluster" - "github.com/rigdev/rig/internal/service/database" "github.com/rigdev/rig/internal/service/group" "github.com/rigdev/rig/internal/service/metrics" "github.com/rigdev/rig/internal/service/operator" "github.com/rigdev/rig/internal/service/project" - "github.com/rigdev/rig/internal/service/storage" "github.com/rigdev/rig/internal/service/user" "go.uber.org/fx" ) @@ -22,8 +20,6 @@ var Module = fx.Module( auth.NewService, project.NewService, group.NewService, - database.NewService, - storage.NewService, metrics.NewService, operator.New, cluster.NewService, diff --git a/internal/service/storage/service.go b/internal/service/storage/service.go deleted file mode 100644 index 2ddee0e0e..000000000 --- a/internal/service/storage/service.go +++ /dev/null @@ -1,361 +0,0 @@ -package storage - -import ( - "context" - "io" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig-go-api/api/v1/storage/settings" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/internal/repository" - "github.com/rigdev/rig/internal/service/project" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/iterator" - "github.com/rigdev/rig/pkg/uuid" - "go.uber.org/zap" - "google.golang.org/protobuf/types/known/timestamppb" -) - -type Service struct { - ps project.Service - rs repository.Storage - rsec repository.Secret - logger *zap.Logger -} - -func NewService(logger *zap.Logger, ps project.Service, rs repository.Storage, rsec repository.Secret) *Service { - return &Service{ - rs: rs, - ps: ps, - rsec: rsec, - logger: logger, - } -} - -func (s *Service) GetBucket(ctx context.Context, bucketName string) (*storage.Bucket, error) { - _, p, err := s.lookupProviderByBucket(ctx, bucketName) - if err != nil { - return nil, err - } - - for _, b := range p.Buckets { - if b.Name == bucketName { - return b, nil - } - } - return nil, errors.NotFoundErrorf("bucket %q not found", bucketName) -} - -func (s *Service) GetObject(ctx context.Context, bucketName, path string) (*storage.Object, error) { - _, p, err := s.lookupProviderByBucket(ctx, bucketName) - if err != nil { - return nil, err - } - - sg, err := s.getStorageGateway(ctx, p) - if err != nil { - return nil, err - } - - var providerBucketName string - for _, b := range p.Buckets { - if b.Name == bucketName { - providerBucketName = b.ProviderBucket - break - } - } - - return sg.GetObject(ctx, providerBucketName, path) -} - -func (s *Service) UploadObject(ctx context.Context, reader io.Reader, metadata *storage.UploadObjectRequest_Metadata) (string, uint64, error) { - _, p, err := s.lookupProviderByBucket(ctx, metadata.GetBucket()) - if err != nil { - return "", 0, err - } - - sg, err := s.getStorageGateway(ctx, p) - if err != nil { - return "", 0, err - } - - var providerBucketName string - for _, b := range p.Buckets { - if b.Name == metadata.GetBucket() { - providerBucketName = b.ProviderBucket - break - } - } - - if metadata.GetOnlyCreate() { - if _, err := s.GetObject(ctx, metadata.Bucket, metadata.Path); errors.IsNotFound(err) { - // Good, continue. - } else if err != nil { - return "", 0, err - } else { - return "", 0, errors.FailedPreconditionErrorf("only create is set, but file does exist") - } - } - - if metadata.GetOnlyReplace() { - if _, err := s.GetObject(ctx, metadata.Bucket, metadata.Path); errors.IsNotFound(err) { - return "", 0, errors.FailedPreconditionErrorf("only replace is set, but file does not exist") - } else if err != nil { - return "", 0, err - } - } - - return sg.UploadObject(ctx, reader, int64(metadata.GetSize()), providerBucketName, metadata.GetPath(), metadata.GetContentType()) -} - -func (s *Service) DownloadObject(ctx context.Context, bucketName, path string) (io.ReadSeekCloser, error) { - _, p, err := s.lookupProviderByBucket(ctx, bucketName) - if err != nil { - return nil, err - } - - sg, err := s.getStorageGateway(ctx, p) - if err != nil { - return nil, err - } - - var providerBucketName string - for _, b := range p.Buckets { - if b.Name == bucketName { - providerBucketName = b.ProviderBucket - break - } - } - - return sg.DownloadObject(ctx, providerBucketName, path) -} - -func (s *Service) DeleteObject(ctx context.Context, bucketName, path string) error { - _, p, err := s.lookupProviderByBucket(ctx, bucketName) - if err != nil { - return err - } - - sg, err := s.getStorageGateway(ctx, p) - if err != nil { - return err - } - - var providerBucketName string - for _, b := range p.Buckets { - if b.Name == bucketName { - providerBucketName = b.ProviderBucket - break - } - } - - return sg.DeleteObject(ctx, providerBucketName, path) -} - -func (s *Service) ListObjects(ctx context.Context, bucketName, token, prefix, startpath, endpath string, recursive bool, limit uint32) (string, iterator.Iterator[*storage.ListObjectsResponse_Result], error) { - _, p, err := s.lookupProviderByBucket(ctx, bucketName) - if err != nil { - return "", nil, err - } - - sg, err := s.getStorageGateway(ctx, p) - if err != nil { - return "", nil, err - } - - var providerBucketName string - for _, b := range p.Buckets { - if b.Name == bucketName { - providerBucketName = b.ProviderBucket - break - } - } - - return sg.ListObjects(ctx, providerBucketName, token, prefix, startpath, endpath, recursive, limit) -} - -func (s *Service) CreateBucket(ctx context.Context, name, providerName, region string, providerID uuid.UUID) error { - provider, err := s.GetProvider(ctx, providerID) - if err != nil { - return err - } - - switch provider.GetConfig().GetConfig().(type) { - case *storage.Config_Minio: - provider.GetConfig().GetMinio().Region = region - case *storage.Config_S3: - provider.GetConfig().GetS3().Region = region - } - - sg, err := s.getStorageGateway(ctx, provider) - if err != nil { - return err - } - - pn, err := sg.CreateBucket(ctx, providerName, region) - if err != nil { - return err - } - - bucket := &storage.Bucket{ - Name: name, - Region: region, - ProviderBucket: pn, - CreatedAt: timestamppb.Now(), - } - - provider.Buckets = append(provider.Buckets, bucket) - - _, err = s.rs.Update(ctx, providerID, provider) - return err -} - -func (s *Service) UnlinkBucket(ctx context.Context, bucket string) error { - pid, p, err := s.lookupProviderByBucket(ctx, bucket) - if err != nil { - return err - } - - for i, b := range p.Buckets { - if b.Name == bucket { - p.Buckets = append(p.Buckets[:i], p.Buckets[i+1:]...) - break - } - } - - _, err = s.rs.Update(ctx, pid, p) - return err -} - -func (s *Service) DeleteBucket(ctx context.Context, bucketName string) error { - pid, p, err := s.lookupProviderByBucket(ctx, bucketName) - if err != nil { - return err - } - - sg, err := s.getStorageGateway(ctx, p) - if err != nil { - return err - } - - var providerBucketName string - for i, b := range p.Buckets { - if b.Name == bucketName { - providerBucketName = b.ProviderBucket - p.Buckets = append(p.Buckets[:i], p.Buckets[i+1:]...) - break - } - } - - err = sg.DeleteBucket(ctx, providerBucketName) - if err != nil { - return err - } - - _, err = s.rs.Update(ctx, pid, p) - if err != nil { - return err - } - - return nil -} - -func (s *Service) ListBuckets(ctx context.Context) (iterator.Iterator[*storage.Bucket], error) { - pit, _, err := s.ListProviders(ctx, &model.Pagination{}) - if err != nil { - return nil, err - } - - bit := iterator.NewProducer[*storage.Bucket]() - go func() { - for { - defer pit.Close() - defer bit.Done() - provider, err := pit.Next() - if err == io.EOF { - bit.Error(nil) - return - } else if err != nil { - bit.Error(err) - return - } - for _, bucket := range provider.GetBuckets() { - if err := bit.Value(bucket); err != nil { - bit.Error(err) - return - } - } - } - }() - return bit, nil -} - -func (s *Service) CopyObject(ctx context.Context, dstBucket, dstPath, srcBucket, srcPath string) error { - _, srcProvider, err := s.lookupProviderByBucket(ctx, srcBucket) - if err != nil { - return err - } - - var srcProviderBucketName string - for _, b := range srcProvider.Buckets { - if b.Name == srcBucket { - srcProviderBucketName = b.ProviderBucket - break - } - } - - _, dstProvider, err := s.lookupProviderByBucket(ctx, dstBucket) - if err != nil { - return err - } - - var dstProviderBucketName string - for _, b := range dstProvider.Buckets { - if b.Name == dstBucket { - dstProviderBucketName = b.ProviderBucket - break - } - } - - if srcProvider.Name == dstProvider.Name { - sg, err := s.getStorageGateway(ctx, srcProvider) - if err != nil { - return err - } - - return sg.CopyObject(ctx, dstProviderBucketName, dstPath, srcProviderBucketName, srcPath) - } - - srcSg, err := s.getStorageGateway(ctx, srcProvider) - if err != nil { - return err - } - - dstSg, err := s.getStorageGateway(ctx, dstProvider) - if err != nil { - return err - } - - reader, err := srcSg.DownloadObject(ctx, srcProviderBucketName, srcPath) - if err != nil { - return err - } - - defer reader.Close() - - _, _, err = dstSg.UploadObject(ctx, reader, 0, dstProviderBucketName, dstPath, "") - if err != nil { - return err - } - - return nil -} - -func (s *Service) GetSettings(ctx context.Context) (*settings.Settings, error) { - res := &settings.Settings{} - err := s.ps.GetSettings(ctx, project.SettingsTypeStorage, res) - if err != nil { - return nil, err - } - return res, nil -} diff --git a/internal/service/storage/storage_provider.go b/internal/service/storage/storage_provider.go deleted file mode 100644 index cb6ea5483..000000000 --- a/internal/service/storage/storage_provider.go +++ /dev/null @@ -1,255 +0,0 @@ -package storage - -import ( - "context" - "encoding/json" - "fmt" - "io" - "reflect" - - "github.com/rigdev/rig-go-api/api/v1/storage" - "github.com/rigdev/rig-go-api/model" - "github.com/rigdev/rig/internal/client/gcs" - "github.com/rigdev/rig/internal/client/minio" - "github.com/rigdev/rig/internal/client/s3" - storage_gateway "github.com/rigdev/rig/internal/gateway/storage" - "github.com/rigdev/rig/pkg/errors" - "github.com/rigdev/rig/pkg/iterator" - "github.com/rigdev/rig/pkg/uuid" - "golang.org/x/exp/slices" - "google.golang.org/protobuf/types/known/timestamppb" -) - -func (s *Service) LookupProvider(ctx context.Context, name string) (uuid.UUID, *storage.Provider, error) { - pid, p, sid, err := s.rs.Lookup(ctx, name) - if err != nil { - return uuid.Nil, nil, err - } - - err = s.getProviderSecret(ctx, sid, p) - if err != nil { - return pid, p, err - } - - return pid, p, err -} - -func (s *Service) CreateProvider(ctx context.Context, name string, config *storage.Config, linkBuckets bool) (uuid.UUID, *storage.Provider, error) { - providerID := uuid.New() - - p := &storage.Provider{ - Name: name, - Config: config, - CreatedAt: timestamppb.Now(), - } - - sg, err := s.getStorageGateway(ctx, p) - if err != nil { - return uuid.Nil, nil, err - } - - err = sg.Test(ctx) - if err != nil { - return uuid.Nil, nil, err - } - - if linkBuckets { - bucketsIt, err := sg.ListBuckets(ctx) - if err != nil { - return uuid.Nil, nil, err - } - defer bucketsIt.Close() - - rigBucketsIt, err := s.ListBuckets(ctx) - if err != nil { - return uuid.Nil, nil, err - } - defer rigBucketsIt.Close() - - rigBuckets, err := iterator.Collect(rigBucketsIt) - if err != nil { - return uuid.Nil, nil, err - } - - buckets := []*storage.Bucket{} - for { - b, err := bucketsIt.Next() - if err == io.EOF { - p.Buckets = buckets - break - } else if err != nil { - return uuid.Nil, nil, err - } - - if !slices.ContainsFunc(rigBuckets, func(bb *storage.Bucket) bool { - return b.ProviderBucket == bb.Name - }) { - b.Name = b.ProviderBucket - buckets = append(buckets, b) - } else { - ptype, err := GetProviderType(p.GetConfig()) - if err != nil { - fmt.Println("Skipping bucket") - continue - } - b.Name = ptype + "-" + b.ProviderBucket - buckets = append(buckets, b) - } - } - } - - sID := uuid.New() - var secret []byte - - switch config.Config.(type) { - case *storage.Config_Gcs: - secret = config.GetGcs().GetConfig() - config.GetGcs().Config = nil - case *storage.Config_S3: - secret, err = json.Marshal(config.GetS3().GetCredentials()) - if err != nil { - return uuid.Nil, nil, err - } - - config.GetS3().Credentials = nil - case *storage.Config_Minio: - secret, err = json.Marshal(config.GetMinio().GetCredentials()) - if err != nil { - return uuid.Nil, nil, err - } - - config.GetMinio().Credentials = nil - } - - err = s.rsec.Create(ctx, sID, secret) - if err != nil { - return uuid.Nil, nil, err - } - - p, err = s.rs.Create(ctx, providerID, sID, p) - if err != nil { - return uuid.Nil, nil, err - } - - return providerID, p, nil -} - -func (s *Service) DeleteProvider(ctx context.Context, providerID uuid.UUID) error { - _, sid, err := s.rs.Get(ctx, providerID) - if err != nil { - return err - } - - err = s.rsec.Delete(ctx, sid) - if err != nil { - return err - } - - return s.rs.Delete(ctx, providerID) -} - -func (s *Service) ListProviders(ctx context.Context, pagination *model.Pagination) (iterator.Iterator[*storage.ProviderEntry], uint64, error) { - return s.rs.List(ctx, pagination) -} - -func (s *Service) GetProvider(ctx context.Context, providerID uuid.UUID) (*storage.Provider, error) { - p, sid, err := s.rs.Get(ctx, providerID) - if err != nil { - return nil, err - } - - err = s.getProviderSecret(ctx, sid, p) - if err != nil { - return nil, err - } - - return p, err -} - -func (s *Service) getStorageGateway(ctx context.Context, provider *storage.Provider) (storage_gateway.Gateway, error) { - switch v := provider.GetConfig().GetConfig().(type) { - case *storage.Config_Gcs: - return gcs.New(ctx, v.Gcs) - case *storage.Config_S3: - return s3.New(v.S3) - case *storage.Config_Minio: - return minio.New(v.Minio) - default: - return nil, errors.InvalidArgumentErrorf("invalid storage provider type '%v'", reflect.TypeOf(v)) - } -} - -func (s *Service) lookupProviderByBucket(ctx context.Context, bucketName string) (uuid.UUID, *storage.Provider, error) { - id, provider, sid, err := s.rs.LookupByBucket(ctx, bucketName) - if err != nil { - return uuid.Nil, nil, err - } - switch provider.GetConfig().GetConfig().(type) { - case *storage.Config_S3: - var region string - for _, b := range provider.Buckets { - if b.Name == bucketName { - region = b.Region - provider.Config.GetS3().Region = region - break - } - } - case *storage.Config_Minio: - var region string - for _, b := range provider.Buckets { - if b.Name == bucketName { - region = b.Region - provider.Config.GetMinio().Region = region - break - } - } - } - - err = s.getProviderSecret(ctx, sid, provider) - if err != nil { - return uuid.Nil, nil, err - } - - return id, provider, nil -} - -func GetProviderType(p *storage.Config) (string, error) { - switch p.GetConfig().(type) { - case *storage.Config_S3: - return "s3", nil - case *storage.Config_Gcs: - return "gcs", nil - case *storage.Config_Minio: - return "minio", nil - default: - return "", errors.InvalidArgumentErrorf("unknown provider type") - } -} - -func (s *Service) getProviderSecret(ctx context.Context, sid uuid.UUID, p *storage.Provider) error { - secret, err := s.rsec.Get(ctx, sid) - if err != nil { - return err - } - - switch p.GetConfig().GetConfig().(type) { - case *storage.Config_Gcs: - p.GetConfig().GetGcs().Config = secret - case *storage.Config_S3: - p.GetConfig().GetS3().Credentials = &model.ProviderCredentials{} - err = json.Unmarshal(secret, p.GetConfig().GetS3().Credentials) - if err != nil { - return err - } - case *storage.Config_Minio: - p.GetConfig().GetMinio().Credentials = &model.ProviderCredentials{} - err = json.Unmarshal(secret, p.GetConfig().GetMinio().GetCredentials()) - if err != nil { - return err - } - default: - return errors.InvalidArgumentErrorf("invalid storage provider type") - } - - return nil -}