feat(CAS): AWS S3 blob storage support

app/artifact-cas/cmd/main.go

@@ -25,8 +25,6 @@ import (
 	"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/conf"
 	"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/server"
 	backend "github.com/chainloop-dev/chainloop/internal/blobmanager"
-	"github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob"
-	"github.com/chainloop-dev/chainloop/internal/blobmanager/oci"
 	"github.com/chainloop-dev/chainloop/internal/credentials"
 	"github.com/chainloop-dev/chainloop/internal/credentials/manager"
 	"github.com/chainloop-dev/chainloop/internal/servicelogger"
@@ -64,16 +62,6 @@ type app struct {
 	backend.Providers
 }
 
-func loadCASBackendProviders(creader credentials.Reader) backend.Providers {
-	// Initialize CAS backend providers
-	ociProvider := oci.NewBackendProvider(creader)
-	azureBlobProvider := azureblob.NewBackendProvider(creader)
-	return backend.Providers{
-		ociProvider.ID():       ociProvider,
-		azureBlobProvider.ID(): azureBlobProvider,
-	}
-}
-
 func newApp(logger log.Logger, gs *grpc.Server, hs *http.Server, ms *server.HTTPMetricsServer, providers backend.Providers) *app {
 	return &app{
 		kratos.New(

app/artifact-cas/cmd/wire.go

@@ -24,6 +24,7 @@ import (
 	"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/conf"
 	"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/server"
 	"github.com/chainloop-dev/chainloop/app/artifact-cas/internal/service"
+	"github.com/chainloop-dev/chainloop/internal/blobmanager/loader"
 	"github.com/chainloop-dev/chainloop/internal/credentials"
 	"github.com/go-kratos/kratos/v2/log"
 	"github.com/google/wire"
@@ -35,7 +36,7 @@ func wireApp(*conf.Server, *conf.Auth, credentials.Reader, log.Logger) (*app, fu
 		wire.Build(
 			server.ProviderSet,
 			service.ProviderSet,
-			loadCASBackendProviders,
+			loader.LoadProviders,
 			newApp,
 			serviceOpts,
 		),

app/cli/cmd/casbackend.go

@@ -41,7 +41,7 @@ func newCASBackendAddCmd() *cobra.Command {
 	cmd.PersistentFlags().Bool("default", false, "set the backend as default in your organization")
 	cmd.PersistentFlags().String("description", "", "descriptive information for this registration")
 
-	cmd.AddCommand(newCASBackendAddOCICmd(), newCASBackendAddAzureBlobStorageCmd())
+	cmd.AddCommand(newCASBackendAddOCICmd(), newCASBackendAddAzureBlobStorageCmd(), newCASBackendAddAWSS3Cmd())
 	return cmd
 }
 
@@ -54,7 +54,7 @@ func newCASBackendUpdateCmd() *cobra.Command {
 	cmd.PersistentFlags().Bool("default", false, "set the backend as default in your organization")
 	cmd.PersistentFlags().String("description", "", "descriptive information for this registration")
 
-	cmd.AddCommand(newCASBackendUpdateOCICmd(), newCASBackendUpdateInlineCmd(), newCASBackendUpdateAzureBlobCmd())
+	cmd.AddCommand(newCASBackendUpdateOCICmd(), newCASBackendUpdateInlineCmd(), newCASBackendUpdateAzureBlobCmd(), newCASBackendUpdateAWSS3Cmd())
 	return cmd
 }
 

app/cli/cmd/casbackend_add_s3.go

@@ -0,0 +1,86 @@
+//
+// Copyright 2023 The Chainloop Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"github.com/chainloop-dev/chainloop/app/cli/internal/action"
+	"github.com/chainloop-dev/chainloop/internal/blobmanager/s3"
+	"github.com/go-kratos/kratos/v2/log"
+	"github.com/spf13/cobra"
+)
+
+func newCASBackendAddAWSS3Cmd() *cobra.Command {
+	var bucketName, accessKeyID, secretAccessKey, region string
+	cmd := &cobra.Command{
+		Use:   "aws-s3",
+		Short: "Register a AWS S3 storage bucket",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			isDefault, err := cmd.Flags().GetBool("default")
+			cobra.CheckErr(err)
+
+			description, err := cmd.Flags().GetString("description")
+			cobra.CheckErr(err)
+
+			if isDefault {
+				if confirmed, err := confirmDefaultCASBackendOverride(actionOpts, ""); err != nil {
+					return err
+				} else if !confirmed {
+					log.Info("Aborting...")
+					return nil
+				}
+			}
+
+			opts := &action.NewCASBackendAddOpts{
+				Location:    bucketName,
+				Provider:    s3.ProviderID,
+				Description: description,
+				Credentials: map[string]any{
+					"accessKeyID":     accessKeyID,
+					"secretAccessKey": secretAccessKey,
+					"region":          region,
+				},
+				Default: isDefault,
+			}
+
+			res, err := action.NewCASBackendAdd(actionOpts).Run(opts)
+			if err != nil {
+				return err
+			} else if res == nil {
+				return nil
+			}
+
+			return encodeOutput([]*action.CASBackendItem{res}, casBackendListTableOutput)
+		},
+	}
+
+	cmd.Flags().StringVar(&bucketName, "bucket", "", "S3 bucket name")
+	err := cmd.MarkFlagRequired("bucket")
+	cobra.CheckErr(err)
+
+	cmd.Flags().StringVar(&accessKeyID, "access-key-id", "", "AWS Access Key ID")
+	err = cmd.MarkFlagRequired("access-key-id")
+	cobra.CheckErr(err)
+
+	cmd.Flags().StringVar(&secretAccessKey, "secret-access-key", "", "AWS Secret Access Key")
+	err = cmd.MarkFlagRequired("secret-access-key")
+	cobra.CheckErr(err)
+
+	cmd.Flags().StringVar(&region, "region", "", "AWS region for the bucket")
+	err = cmd.MarkFlagRequired("region")
+	cobra.CheckErr(err)
+
+	return cmd
+}

app/cli/cmd/casbackend_update_s3.go

@@ -0,0 +1,92 @@
+//
+// Copyright 2023 The Chainloop Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"github.com/chainloop-dev/chainloop/app/cli/internal/action"
+	"github.com/go-kratos/kratos/v2/log"
+	"github.com/spf13/cobra"
+)
+
+func newCASBackendUpdateAWSS3Cmd() *cobra.Command {
+	var backendID, accessKeyID, secretAccessKey, region string
+	cmd := &cobra.Command{
+		Use:   "aws-s3",
+		Short: "Update a AWS S3 CAS Backend description, credentials or default status",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			// If we are setting the default, we list existing CAS backends
+			// and ask the user to confirm the rewrite
+			isDefault, err := cmd.Flags().GetBool("default")
+			cobra.CheckErr(err)
+
+			description, err := cmd.Flags().GetString("description")
+			cobra.CheckErr(err)
+
+			// If we are overriding the default we ask for confirmation
+			if isDefault {
+				if confirmed, err := confirmDefaultCASBackendOverride(actionOpts, backendID); err != nil {
+					return err
+				} else if !confirmed {
+					log.Info("Aborting...")
+					return nil
+				}
+			} else {
+				// If we are removing the default we ask for confirmation too
+				if confirmed, err := confirmDefaultCASBackendUnset(backendID, "You are setting the default CAS backend to false", actionOpts); err != nil {
+					return err
+				} else if !confirmed {
+					log.Info("Aborting...")
+					return nil
+				}
+			}
+
+			opts := &action.NewCASBackendUpdateOpts{
+				ID:          backendID,
+				Description: description,
+				Credentials: map[string]any{
+					"accessKeyID":     accessKeyID,
+					"secretAccessKey": secretAccessKey,
+					"region":          region,
+				},
+				Default: isDefault,
+			}
+
+			// this means that we are not updating credentials
+			if accessKeyID == "" && secretAccessKey == "" && region == "" {
+				opts.Credentials = nil
+			}
+
+			res, err := action.NewCASBackendUpdate(actionOpts).Run(opts)
+			if err != nil {
+				return err
+			} else if res == nil {
+				return nil
+			}
+
+			return encodeOutput([]*action.CASBackendItem{res}, casBackendListTableOutput)
+		},
+	}
+
+	cmd.Flags().StringVar(&backendID, "id", "", "CAS Backend ID")
+	err := cmd.MarkFlagRequired("id")
+	cobra.CheckErr(err)
+
+	cmd.Flags().StringVar(&accessKeyID, "access-key-id", "", "AWS Access Key ID")
+	cmd.Flags().StringVar(&secretAccessKey, "secret-access-key", "", "AWS Secret Access Key")
+	cmd.Flags().StringVar(&region, "region", "", "AWS region for the bucket")
+
+	return cmd
+}

app/controlplane/cmd/main.go

@@ -28,9 +28,6 @@ import (
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/server"
 	"github.com/chainloop-dev/chainloop/app/controlplane/plugins"
 	"github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1"
-	backends "github.com/chainloop-dev/chainloop/internal/blobmanager"
-	"github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob"
-	"github.com/chainloop-dev/chainloop/internal/blobmanager/oci"
 	"github.com/chainloop-dev/chainloop/internal/credentials"
 	"github.com/chainloop-dev/chainloop/internal/credentials/manager"
 	"github.com/chainloop-dev/chainloop/internal/servicelogger"
@@ -171,16 +168,6 @@ func maskArgs(keyvals []interface{}) {
 	}
 }
 
-func loadCASBackendProviders(creader credentials.Reader) backends.Providers {
-	// Initialize CAS backend providers
-	ociProvider := oci.NewBackendProvider(creader)
-	azureBlobProvider := azureblob.NewBackendProvider(creader)
-	return backends.Providers{
-		ociProvider.ID():       ociProvider,
-		azureBlobProvider.ID(): azureBlobProvider,
-	}
-}
-
 func initSentry(c *conf.Bootstrap, logger log.Logger) (cleanupFunc func(), err error) {
 	cleanupFunc = func() {
 		sentry.Flush(2 * time.Second)

app/controlplane/cmd/wire.go

@@ -28,6 +28,7 @@ import (
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/server"
 	"github.com/chainloop-dev/chainloop/app/controlplane/internal/service"
 	"github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1"
+	"github.com/chainloop-dev/chainloop/internal/blobmanager/loader"
 	"github.com/chainloop-dev/chainloop/internal/credentials"
 	"github.com/go-kratos/kratos/v2/log"
 	"github.com/google/wire"
@@ -40,7 +41,7 @@ func wireApp(*conf.Bootstrap, credentials.ReaderWriter, log.Logger, sdk.Availabl
 			server.ProviderSet,
 			data.ProviderSet,
 			biz.ProviderSet,
-			loadCASBackendProviders,
+			loader.LoadProviders,
 			service.ProviderSet,
 			wire.Bind(new(biz.CASClient), new(*biz.CASClientUseCase)),
 			serviceOpts,

app/controlplane/internal/biz/casbackend.go

@@ -26,6 +26,7 @@ import (
 	backend "github.com/chainloop-dev/chainloop/internal/blobmanager"
 	"github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob"
 	"github.com/chainloop-dev/chainloop/internal/blobmanager/oci"
+	"github.com/chainloop-dev/chainloop/internal/blobmanager/s3"
 	"github.com/chainloop-dev/chainloop/internal/credentials"
 	"github.com/chainloop-dev/chainloop/internal/servicelogger"
 	"github.com/go-kratos/kratos/v2/log"
@@ -470,7 +471,7 @@ func (uc *CASBackendUseCase) PerformValidation(ctx context.Context, id string) (
 
 // Implements https://pkg.go.dev/entgo.io/ent/schema/field#EnumValues
 func (CASBackendProvider) Values() (kinds []string) {
-	for _, s := range []CASBackendProvider{azureblob.ProviderID, oci.ProviderID, CASBackendInline} {
+	for _, s := range []CASBackendProvider{azureblob.ProviderID, oci.ProviderID, CASBackendInline, s3.ProviderID} {
 		kinds = append(kinds, string(s))
 	}
 

app/controlplane/internal/data/ent/migrate/schema.go

@@ -12,7 +12,7 @@ var (
 	CasBackendsColumns = []*schema.Column{
 		{Name: "id", Type: field.TypeUUID, Unique: true},
 		{Name: "location", Type: field.TypeString},
-		{Name: "provider", Type: field.TypeEnum, Enums: []string{"AzureBlob", "OCI", "INLINE"}},
+		{Name: "provider", Type: field.TypeEnum, Enums: []string{"AzureBlob", "OCI", "INLINE", "AWS-S3"}},
 		{Name: "description", Type: field.TypeString, Nullable: true},
 		{Name: "secret_name", Type: field.TypeString},
 		{Name: "created_at", Type: field.TypeTime, Default: "CURRENT_TIMESTAMP"},

go.mod

@@ -89,6 +89,7 @@ require (
 	github.com/cockroachdb/apd/v3 v3.2.0 // indirect
 	github.com/cpuguy83/dockercfg v0.3.1 // indirect
 	github.com/cyphar/filepath-securejoin v0.2.4 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/emirpasic/gods v1.18.1 // indirect
 	github.com/fatih/color v1.15.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
@@ -101,15 +102,20 @@ require (
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/yamux v0.1.1 // indirect
 	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.5 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.1 // indirect
 	github.com/mitchellh/go-testing-interface v1.14.1 // indirect
 	github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/package-url/packageurl-go v0.1.1 // indirect
 	github.com/pjbgf/sha1cd v0.3.0 // indirect
 	github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect
 	github.com/pkg/xattr v0.4.9 // indirect
+	github.com/rs/xid v1.5.0 // indirect
 	github.com/sergi/go-diff v1.3.1 // indirect
 	github.com/skeema/knownhosts v1.2.0 // indirect
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
@@ -132,6 +138,7 @@ require (
 	github.com/agext/levenshtein v1.2.3 // indirect
 	github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
+	github.com/aws/aws-sdk-go v1.45.25
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect
@@ -221,6 +228,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/miekg/pkcs11 v1.1.1 // indirect
+	github.com/minio/minio-go/v7 v7.0.63
 	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect