diff --git a/app/artifact-cas/cmd/main.go b/app/artifact-cas/cmd/main.go index 11784edcc..f7d9da389 100644 --- a/app/artifact-cas/cmd/main.go +++ b/app/artifact-cas/cmd/main.go @@ -25,8 +25,6 @@ import ( "github.com/chainloop-dev/chainloop/app/artifact-cas/internal/conf" "github.com/chainloop-dev/chainloop/app/artifact-cas/internal/server" backend "github.com/chainloop-dev/chainloop/internal/blobmanager" - "github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob" - "github.com/chainloop-dev/chainloop/internal/blobmanager/oci" "github.com/chainloop-dev/chainloop/internal/credentials" "github.com/chainloop-dev/chainloop/internal/credentials/manager" "github.com/chainloop-dev/chainloop/internal/servicelogger" @@ -64,16 +62,6 @@ type app struct { backend.Providers } -func loadCASBackendProviders(creader credentials.Reader) backend.Providers { - // Initialize CAS backend providers - ociProvider := oci.NewBackendProvider(creader) - azureBlobProvider := azureblob.NewBackendProvider(creader) - return backend.Providers{ - ociProvider.ID(): ociProvider, - azureBlobProvider.ID(): azureBlobProvider, - } -} - func newApp(logger log.Logger, gs *grpc.Server, hs *http.Server, ms *server.HTTPMetricsServer, providers backend.Providers) *app { return &app{ kratos.New( diff --git a/app/artifact-cas/cmd/wire.go b/app/artifact-cas/cmd/wire.go index d02fd236a..370c1505b 100644 --- a/app/artifact-cas/cmd/wire.go +++ b/app/artifact-cas/cmd/wire.go @@ -24,6 +24,7 @@ import ( "github.com/chainloop-dev/chainloop/app/artifact-cas/internal/conf" "github.com/chainloop-dev/chainloop/app/artifact-cas/internal/server" "github.com/chainloop-dev/chainloop/app/artifact-cas/internal/service" + "github.com/chainloop-dev/chainloop/internal/blobmanager/loader" "github.com/chainloop-dev/chainloop/internal/credentials" "github.com/go-kratos/kratos/v2/log" "github.com/google/wire" @@ -35,7 +36,7 @@ func wireApp(*conf.Server, *conf.Auth, credentials.Reader, log.Logger) (*app, fu wire.Build( server.ProviderSet, service.ProviderSet, - loadCASBackendProviders, + loader.LoadProviders, newApp, serviceOpts, ), diff --git a/app/artifact-cas/cmd/wire_gen.go b/app/artifact-cas/cmd/wire_gen.go index a447a17f3..dd928447b 100644 --- a/app/artifact-cas/cmd/wire_gen.go +++ b/app/artifact-cas/cmd/wire_gen.go @@ -10,6 +10,7 @@ import ( "github.com/chainloop-dev/chainloop/app/artifact-cas/internal/conf" "github.com/chainloop-dev/chainloop/app/artifact-cas/internal/server" "github.com/chainloop-dev/chainloop/app/artifact-cas/internal/service" + "github.com/chainloop-dev/chainloop/internal/blobmanager/loader" "github.com/chainloop-dev/chainloop/internal/credentials" "github.com/go-kratos/kratos/v2/log" ) @@ -22,7 +23,7 @@ import ( // wireApp init kratos application. func wireApp(confServer *conf.Server, auth *conf.Auth, reader credentials.Reader, logger log.Logger) (*app, func(), error) { - providers := loadCASBackendProviders(reader) + providers := loader.LoadProviders(reader) v := serviceOpts(logger) byteStreamService := service.NewByteStreamService(providers, v...) resourceService := service.NewResourceService(providers, v...) diff --git a/app/cli/cmd/casbackend.go b/app/cli/cmd/casbackend.go index c54704505..49c7947cb 100644 --- a/app/cli/cmd/casbackend.go +++ b/app/cli/cmd/casbackend.go @@ -41,7 +41,7 @@ func newCASBackendAddCmd() *cobra.Command { cmd.PersistentFlags().Bool("default", false, "set the backend as default in your organization") cmd.PersistentFlags().String("description", "", "descriptive information for this registration") - cmd.AddCommand(newCASBackendAddOCICmd(), newCASBackendAddAzureBlobStorageCmd()) + cmd.AddCommand(newCASBackendAddOCICmd(), newCASBackendAddAzureBlobStorageCmd(), newCASBackendAddAWSS3Cmd()) return cmd } @@ -54,7 +54,7 @@ func newCASBackendUpdateCmd() *cobra.Command { cmd.PersistentFlags().Bool("default", false, "set the backend as default in your organization") cmd.PersistentFlags().String("description", "", "descriptive information for this registration") - cmd.AddCommand(newCASBackendUpdateOCICmd(), newCASBackendUpdateInlineCmd(), newCASBackendUpdateAzureBlobCmd()) + cmd.AddCommand(newCASBackendUpdateOCICmd(), newCASBackendUpdateInlineCmd(), newCASBackendUpdateAzureBlobCmd(), newCASBackendUpdateAWSS3Cmd()) return cmd } diff --git a/app/cli/cmd/casbackend_add_s3.go b/app/cli/cmd/casbackend_add_s3.go new file mode 100644 index 000000000..0343ced63 --- /dev/null +++ b/app/cli/cmd/casbackend_add_s3.go @@ -0,0 +1,86 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/chainloop-dev/chainloop/app/cli/internal/action" + "github.com/chainloop-dev/chainloop/internal/blobmanager/s3" + "github.com/go-kratos/kratos/v2/log" + "github.com/spf13/cobra" +) + +func newCASBackendAddAWSS3Cmd() *cobra.Command { + var bucketName, accessKeyID, secretAccessKey, region string + cmd := &cobra.Command{ + Use: "aws-s3", + Short: "Register a AWS S3 storage bucket", + RunE: func(cmd *cobra.Command, args []string) error { + isDefault, err := cmd.Flags().GetBool("default") + cobra.CheckErr(err) + + description, err := cmd.Flags().GetString("description") + cobra.CheckErr(err) + + if isDefault { + if confirmed, err := confirmDefaultCASBackendOverride(actionOpts, ""); err != nil { + return err + } else if !confirmed { + log.Info("Aborting...") + return nil + } + } + + opts := &action.NewCASBackendAddOpts{ + Location: bucketName, + Provider: s3.ProviderID, + Description: description, + Credentials: map[string]any{ + "accessKeyID": accessKeyID, + "secretAccessKey": secretAccessKey, + "region": region, + }, + Default: isDefault, + } + + res, err := action.NewCASBackendAdd(actionOpts).Run(opts) + if err != nil { + return err + } else if res == nil { + return nil + } + + return encodeOutput([]*action.CASBackendItem{res}, casBackendListTableOutput) + }, + } + + cmd.Flags().StringVar(&bucketName, "bucket", "", "S3 bucket name") + err := cmd.MarkFlagRequired("bucket") + cobra.CheckErr(err) + + cmd.Flags().StringVar(&accessKeyID, "access-key-id", "", "AWS Access Key ID") + err = cmd.MarkFlagRequired("access-key-id") + cobra.CheckErr(err) + + cmd.Flags().StringVar(&secretAccessKey, "secret-access-key", "", "AWS Secret Access Key") + err = cmd.MarkFlagRequired("secret-access-key") + cobra.CheckErr(err) + + cmd.Flags().StringVar(®ion, "region", "", "AWS region for the bucket") + err = cmd.MarkFlagRequired("region") + cobra.CheckErr(err) + + return cmd +} diff --git a/app/cli/cmd/casbackend_update_s3.go b/app/cli/cmd/casbackend_update_s3.go new file mode 100644 index 000000000..fedde00b5 --- /dev/null +++ b/app/cli/cmd/casbackend_update_s3.go @@ -0,0 +1,92 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "github.com/chainloop-dev/chainloop/app/cli/internal/action" + "github.com/go-kratos/kratos/v2/log" + "github.com/spf13/cobra" +) + +func newCASBackendUpdateAWSS3Cmd() *cobra.Command { + var backendID, accessKeyID, secretAccessKey, region string + cmd := &cobra.Command{ + Use: "aws-s3", + Short: "Update a AWS S3 CAS Backend description, credentials or default status", + RunE: func(cmd *cobra.Command, args []string) error { + // If we are setting the default, we list existing CAS backends + // and ask the user to confirm the rewrite + isDefault, err := cmd.Flags().GetBool("default") + cobra.CheckErr(err) + + description, err := cmd.Flags().GetString("description") + cobra.CheckErr(err) + + // If we are overriding the default we ask for confirmation + if isDefault { + if confirmed, err := confirmDefaultCASBackendOverride(actionOpts, backendID); err != nil { + return err + } else if !confirmed { + log.Info("Aborting...") + return nil + } + } else { + // If we are removing the default we ask for confirmation too + if confirmed, err := confirmDefaultCASBackendUnset(backendID, "You are setting the default CAS backend to false", actionOpts); err != nil { + return err + } else if !confirmed { + log.Info("Aborting...") + return nil + } + } + + opts := &action.NewCASBackendUpdateOpts{ + ID: backendID, + Description: description, + Credentials: map[string]any{ + "accessKeyID": accessKeyID, + "secretAccessKey": secretAccessKey, + "region": region, + }, + Default: isDefault, + } + + // this means that we are not updating credentials + if accessKeyID == "" && secretAccessKey == "" && region == "" { + opts.Credentials = nil + } + + res, err := action.NewCASBackendUpdate(actionOpts).Run(opts) + if err != nil { + return err + } else if res == nil { + return nil + } + + return encodeOutput([]*action.CASBackendItem{res}, casBackendListTableOutput) + }, + } + + cmd.Flags().StringVar(&backendID, "id", "", "CAS Backend ID") + err := cmd.MarkFlagRequired("id") + cobra.CheckErr(err) + + cmd.Flags().StringVar(&accessKeyID, "access-key-id", "", "AWS Access Key ID") + cmd.Flags().StringVar(&secretAccessKey, "secret-access-key", "", "AWS Secret Access Key") + cmd.Flags().StringVar(®ion, "region", "", "AWS region for the bucket") + + return cmd +} diff --git a/app/controlplane/cmd/main.go b/app/controlplane/cmd/main.go index fb87a9eca..dc0d6f3b5 100644 --- a/app/controlplane/cmd/main.go +++ b/app/controlplane/cmd/main.go @@ -28,9 +28,6 @@ import ( "github.com/chainloop-dev/chainloop/app/controlplane/internal/server" "github.com/chainloop-dev/chainloop/app/controlplane/plugins" "github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1" - backends "github.com/chainloop-dev/chainloop/internal/blobmanager" - "github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob" - "github.com/chainloop-dev/chainloop/internal/blobmanager/oci" "github.com/chainloop-dev/chainloop/internal/credentials" "github.com/chainloop-dev/chainloop/internal/credentials/manager" "github.com/chainloop-dev/chainloop/internal/servicelogger" @@ -171,16 +168,6 @@ func maskArgs(keyvals []interface{}) { } } -func loadCASBackendProviders(creader credentials.Reader) backends.Providers { - // Initialize CAS backend providers - ociProvider := oci.NewBackendProvider(creader) - azureBlobProvider := azureblob.NewBackendProvider(creader) - return backends.Providers{ - ociProvider.ID(): ociProvider, - azureBlobProvider.ID(): azureBlobProvider, - } -} - func initSentry(c *conf.Bootstrap, logger log.Logger) (cleanupFunc func(), err error) { cleanupFunc = func() { sentry.Flush(2 * time.Second) diff --git a/app/controlplane/cmd/wire.go b/app/controlplane/cmd/wire.go index 114a9624a..45c23368f 100644 --- a/app/controlplane/cmd/wire.go +++ b/app/controlplane/cmd/wire.go @@ -28,6 +28,7 @@ import ( "github.com/chainloop-dev/chainloop/app/controlplane/internal/server" "github.com/chainloop-dev/chainloop/app/controlplane/internal/service" "github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1" + "github.com/chainloop-dev/chainloop/internal/blobmanager/loader" "github.com/chainloop-dev/chainloop/internal/credentials" "github.com/go-kratos/kratos/v2/log" "github.com/google/wire" @@ -40,7 +41,7 @@ func wireApp(*conf.Bootstrap, credentials.ReaderWriter, log.Logger, sdk.Availabl server.ProviderSet, data.ProviderSet, biz.ProviderSet, - loadCASBackendProviders, + loader.LoadProviders, service.ProviderSet, wire.Bind(new(biz.CASClient), new(*biz.CASClientUseCase)), serviceOpts, diff --git a/app/controlplane/cmd/wire_gen.go b/app/controlplane/cmd/wire_gen.go index e3152f654..e337e7a6a 100644 --- a/app/controlplane/cmd/wire_gen.go +++ b/app/controlplane/cmd/wire_gen.go @@ -14,6 +14,7 @@ import ( "github.com/chainloop-dev/chainloop/app/controlplane/internal/server" "github.com/chainloop-dev/chainloop/app/controlplane/internal/service" "github.com/chainloop-dev/chainloop/app/controlplane/plugins/sdk/v1" + "github.com/chainloop-dev/chainloop/internal/blobmanager/loader" "github.com/chainloop-dev/chainloop/internal/credentials" "github.com/go-kratos/kratos/v2/log" ) @@ -31,7 +32,7 @@ func wireApp(bootstrap *conf.Bootstrap, readerWriter credentials.ReaderWriter, l membershipUseCase := biz.NewMembershipUseCase(membershipRepo, logger) organizationRepo := data.NewOrganizationRepo(dataData, logger) casBackendRepo := data.NewCASBackendRepo(dataData, logger) - providers := loadCASBackendProviders(readerWriter) + providers := loader.LoadProviders(readerWriter) casBackendUseCase := biz.NewCASBackendUseCase(casBackendRepo, readerWriter, providers, logger) integrationRepo := data.NewIntegrationRepo(dataData, logger) integrationAttachmentRepo := data.NewIntegrationAttachmentRepo(dataData, logger) diff --git a/app/controlplane/internal/biz/casbackend.go b/app/controlplane/internal/biz/casbackend.go index 95a5b6d7b..e5401cfea 100644 --- a/app/controlplane/internal/biz/casbackend.go +++ b/app/controlplane/internal/biz/casbackend.go @@ -26,6 +26,7 @@ import ( backend "github.com/chainloop-dev/chainloop/internal/blobmanager" "github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob" "github.com/chainloop-dev/chainloop/internal/blobmanager/oci" + "github.com/chainloop-dev/chainloop/internal/blobmanager/s3" "github.com/chainloop-dev/chainloop/internal/credentials" "github.com/chainloop-dev/chainloop/internal/servicelogger" "github.com/go-kratos/kratos/v2/log" @@ -470,7 +471,7 @@ func (uc *CASBackendUseCase) PerformValidation(ctx context.Context, id string) ( // Implements https://pkg.go.dev/entgo.io/ent/schema/field#EnumValues func (CASBackendProvider) Values() (kinds []string) { - for _, s := range []CASBackendProvider{azureblob.ProviderID, oci.ProviderID, CASBackendInline} { + for _, s := range []CASBackendProvider{azureblob.ProviderID, oci.ProviderID, CASBackendInline, s3.ProviderID} { kinds = append(kinds, string(s)) } diff --git a/app/controlplane/internal/data/ent/casbackend/casbackend.go b/app/controlplane/internal/data/ent/casbackend/casbackend.go index b3e97a60e..d9812d882 100644 --- a/app/controlplane/internal/data/ent/casbackend/casbackend.go +++ b/app/controlplane/internal/data/ent/casbackend/casbackend.go @@ -115,7 +115,7 @@ var ( // ProviderValidator is a validator for the "provider" field enum values. It is called by the builders before save. func ProviderValidator(pr biz.CASBackendProvider) error { switch pr { - case "AzureBlob", "OCI", "INLINE": + case "AzureBlob", "OCI", "INLINE", "AWS-S3": return nil default: return fmt.Errorf("casbackend: invalid enum value for provider field: %q", pr) diff --git a/app/controlplane/internal/data/ent/migrate/schema.go b/app/controlplane/internal/data/ent/migrate/schema.go index a6ab7fe6e..90209dc55 100644 --- a/app/controlplane/internal/data/ent/migrate/schema.go +++ b/app/controlplane/internal/data/ent/migrate/schema.go @@ -12,7 +12,7 @@ var ( CasBackendsColumns = []*schema.Column{ {Name: "id", Type: field.TypeUUID, Unique: true}, {Name: "location", Type: field.TypeString}, - {Name: "provider", Type: field.TypeEnum, Enums: []string{"AzureBlob", "OCI", "INLINE"}}, + {Name: "provider", Type: field.TypeEnum, Enums: []string{"AzureBlob", "OCI", "INLINE", "AWS-S3"}}, {Name: "description", Type: field.TypeString, Nullable: true}, {Name: "secret_name", Type: field.TypeString}, {Name: "created_at", Type: field.TypeTime, Default: "CURRENT_TIMESTAMP"}, diff --git a/go.mod b/go.mod index 3a4efea8c..e30f27fcf 100644 --- a/go.mod +++ b/go.mod @@ -89,6 +89,7 @@ require ( github.com/cockroachdb/apd/v3 v3.2.0 // indirect github.com/cpuguy83/dockercfg v0.3.1 // indirect github.com/cyphar/filepath-securejoin v0.2.4 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/fatih/color v1.15.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect @@ -101,8 +102,12 @@ require ( github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/yamux v0.1.1 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect + github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/kylelemons/godebug v1.1.0 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 // indirect github.com/oklog/run v1.1.0 // indirect @@ -110,6 +115,7 @@ require ( github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pkg/xattr v0.4.9 // indirect + github.com/rs/xid v1.5.0 // indirect github.com/sergi/go-diff v1.3.1 // indirect github.com/skeema/knownhosts v1.2.0 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect @@ -132,6 +138,7 @@ require ( github.com/agext/levenshtein v1.2.3 // indirect github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aws/aws-sdk-go v1.45.25 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.13.11 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.41 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.35 // indirect @@ -221,6 +228,7 @@ require ( github.com/mattn/go-runewidth v0.0.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/pkcs11 v1.1.1 // indirect + github.com/minio/minio-go/v7 v7.0.63 github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect diff --git a/go.sum b/go.sum index 1be9babd6..da0c2c77f 100644 --- a/go.sum +++ b/go.sum @@ -169,8 +169,8 @@ github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3d github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.44.318 h1:Yl66rpbQHFUbxe9JBKLcvOvRivhVgP6+zH0b9KzARX8= -github.com/aws/aws-sdk-go v1.44.318/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.45.25 h1:c4fLlh5sLdK2DCRTY1z0hyuJZU4ygxX8m1FswL6/nF4= +github.com/aws/aws-sdk-go v1.45.25/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.21.0 h1:gMT0IW+03wtYJhRqTVYn0wLzwdnK9sRMcxmtfGzRdJc= github.com/aws/aws-sdk-go-v2 v1.21.0/go.mod h1:/RfNgGmRxI+iFOB1OeJUyxiU+9s88k3pfHvDagGEp0M= @@ -770,6 +770,7 @@ github.com/jhump/protoreflect v1.12.0/go.mod h1:JytZfP5d0r8pVNLZvai7U/MCuTWITgrI github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/jmhodges/clock v1.2.0 h1:eq4kys+NI0PLngzaHEe7AmPT90XMGIEySD1JfV1PDIs= github.com/jmhodges/clock v1.2.0/go.mod h1:qKjhA7x7u/lQpPB1XAqX1b1lCI/w3/fNuYpI/ZjLynI= @@ -805,8 +806,9 @@ github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= -github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= -github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -887,8 +889,8 @@ github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.55 h1:ZXqUO/8cgfHzI+08h/zGuTTFpISSA32BZmBE3FCLJas= -github.com/minio/minio-go/v7 v7.0.55/go.mod h1:NUDy4A4oXPq1l2yK6LTSvCEzAMeIcoz9lcj5dbzSrRE= +github.com/minio/minio-go/v7 v7.0.63 h1:GbZ2oCvaUdgT5640WJOpyDhhDxvknAJU2/T3yurwcbQ= +github.com/minio/minio-go/v7 v7.0.63/go.mod h1:Q6X7Qjb7WMhvG65qKf4gUgA5XaiSox74kR1uAEjxRS4= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= @@ -1435,6 +1437,7 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= @@ -1583,6 +1586,7 @@ golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= diff --git a/internal/blobmanager/azureblob/backend.go b/internal/blobmanager/azureblob/backend.go index 4044a677f..d2f09503a 100644 --- a/internal/blobmanager/azureblob/backend.go +++ b/internal/blobmanager/azureblob/backend.go @@ -84,7 +84,8 @@ func resourceName(digest string) string { // Exists check that the artifact is already present in the repository func (b *Backend) Exists(ctx context.Context, digest string) (bool, error) { _, err := b.Describe(ctx, digest) - if err != nil && errors.As(err, &backend.ErrNotFound{}) { + notFoundErr := &backend.ErrNotFound{} + if err != nil && errors.As(err, ¬FoundErr) { return false, nil } diff --git a/internal/blobmanager/azureblob/provider.go b/internal/blobmanager/azureblob/provider.go index 7fcb1be42..9c7a706f8 100644 --- a/internal/blobmanager/azureblob/provider.go +++ b/internal/blobmanager/azureblob/provider.go @@ -110,28 +110,26 @@ type Credentials struct { ClientSecret string } -var ErrValidation = errors.New("credentials validation error") - // Validate that the APICreds has all its properties set func (c *Credentials) Validate() error { if c.StorageAccountName == "" { - return fmt.Errorf("%w: missing storage account name", ErrValidation) + return fmt.Errorf("%w: missing storage account name", backend.ErrValidation) } if c.TenantID == "" { - return fmt.Errorf("%w: missing tenant ID", ErrValidation) + return fmt.Errorf("%w: missing tenant ID", backend.ErrValidation) } if c.ClientID == "" { - return fmt.Errorf("%w: missing client ID", ErrValidation) + return fmt.Errorf("%w: missing client ID", backend.ErrValidation) } if c.ClientSecret == "" { - return fmt.Errorf("%w: missing client secret", ErrValidation) + return fmt.Errorf("%w: missing client secret", backend.ErrValidation) } if c.Container == "" { - return fmt.Errorf("%w: missing container", ErrValidation) + return fmt.Errorf("%w: missing container", backend.ErrValidation) } return nil diff --git a/internal/blobmanager/backend.go b/internal/blobmanager/backend.go index eaeea1b8d..37652c009 100644 --- a/internal/blobmanager/backend.go +++ b/internal/blobmanager/backend.go @@ -17,6 +17,7 @@ package backend import ( "context" + "errors" "io" "net/http" "strings" @@ -51,6 +52,8 @@ type Downloader interface { Download(ctx context.Context, w io.Writer, digest string) error } +var ErrValidation = errors.New("credentials validation error") + // Provider is an interface that allows to create a backend from a secret type Provider interface { // Provider identifier diff --git a/internal/blobmanager/loader/loader.go b/internal/blobmanager/loader/loader.go new file mode 100644 index 000000000..d1dc6cb9c --- /dev/null +++ b/internal/blobmanager/loader/loader.go @@ -0,0 +1,37 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package loader + +import ( + backends "github.com/chainloop-dev/chainloop/internal/blobmanager" + "github.com/chainloop-dev/chainloop/internal/blobmanager/azureblob" + "github.com/chainloop-dev/chainloop/internal/blobmanager/oci" + "github.com/chainloop-dev/chainloop/internal/blobmanager/s3" + "github.com/chainloop-dev/chainloop/internal/credentials" +) + +func LoadProviders(creader credentials.Reader) backends.Providers { + // Initialize CAS backend providers + ociProvider := oci.NewBackendProvider(creader) + azureBlobProvider := azureblob.NewBackendProvider(creader) + s3Provider := s3.NewBackendProvider(creader) + + return backends.Providers{ + ociProvider.ID(): ociProvider, + azureBlobProvider.ID(): azureBlobProvider, + s3Provider.ID(): s3Provider, + } +} diff --git a/internal/blobmanager/s3/backend.go b/internal/blobmanager/s3/backend.go new file mode 100644 index 000000000..6aabf8413 --- /dev/null +++ b/internal/blobmanager/s3/backend.go @@ -0,0 +1,238 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "context" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "io" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/s3" + "github.com/aws/aws-sdk-go/service/s3/s3manager" + pb "github.com/chainloop-dev/chainloop/app/artifact-cas/api/cas/v1" + backend "github.com/chainloop-dev/chainloop/internal/blobmanager" +) + +const ( + annotationNameAuthor = "Author" + annotationNameFilename = "Filename" +) + +type Backend struct { + client *s3.S3 + bucket string +} + +var _ backend.UploaderDownloader = (*Backend)(nil) + +type ConnOpt func(*aws.Config) + +// Optional endpoint configuration +func WithEndpoint(endpoint string) ConnOpt { + return func(cfg *aws.Config) { + cfg.Endpoint = aws.String(endpoint) + } +} + +func WithForcedS3PathStyle(force bool) ConnOpt { + return func(cfg *aws.Config) { + cfg.S3ForcePathStyle = aws.Bool(force) + } +} + +func NewBackend(creds *Credentials, connOpts ...ConnOpt) (*Backend, error) { + if creds == nil { + return nil, errors.New("credentials cannot be nil") + } + + if err := creds.Validate(); err != nil { + return nil, fmt.Errorf("invalid credentials: %w", err) + } + + c := credentials.NewStaticCredentials(creds.AccessKeyID, creds.SecretAccessKey, "") + // Configure AWS session + cfg := &aws.Config{Credentials: c, Region: aws.String(creds.Region)} + for _, opt := range connOpts { + opt(cfg) + } + + session, err := session.NewSession(cfg) + if err != nil { + return nil, fmt.Errorf("failed to create AWS session: %w", err) + } + + return &Backend{ + client: s3.New(session), + bucket: creds.BucketName, + }, nil +} + +// Exists check that the artifact is already present in the repository +func (b *Backend) Exists(ctx context.Context, digest string) (bool, error) { + _, err := b.Describe(ctx, digest) + notFoundErr := &backend.ErrNotFound{} + if err != nil && errors.As(err, ¬FoundErr) { + return false, nil + } + + return err == nil, err +} + +func (b *Backend) Upload(ctx context.Context, r io.Reader, resource *pb.CASResource) error { + uploader := s3manager.NewUploaderWithClient(b.client) + + _, err := uploader.UploadWithContext(ctx, &s3manager.UploadInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(resourceName(resource.Digest)), + Body: r, + // Check that the object is uploaded correctly + ChecksumSHA256: aws.String(hexSha256ToBinaryB64(resource.Digest)), + Metadata: map[string]*string{ + annotationNameAuthor: aws.String(backend.AuthorAnnotation), + annotationNameFilename: aws.String(resource.FileName), + }, + }) + + return err +} + +func (b *Backend) Describe(ctx context.Context, digest string) (*pb.CASResource, error) { + // and read the object back + validate integrity + resp, err := b.client.HeadObjectWithContext(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(resourceName(digest)), + ChecksumMode: aws.String("ENABLED"), + }) + + // check error is aws error + var awsErr awserr.Error + if err != nil { + if errors.As(err, &awsErr) && awsErr.Code() == "NotFound" { + return nil, &backend.ErrNotFound{} + } + + return nil, fmt.Errorf("failed to read from bucket: %w", err) + } + + // Check integrity of the remote object + if resp.ChecksumSHA256 != nil && *resp.ChecksumSHA256 != hexSha256ToBinaryB64(digest) { + return nil, fmt.Errorf("failed to validate integrity of object, got=%s, want=%s", *resp.ChecksumSHA256, hexSha256ToBinaryB64(digest)) + } + + // Check asset author is Chainloop that way we can ignore files uploaded by other tools + // note: this is not a security mechanism, an additional check will be put in place for tamper check + author, ok := resp.Metadata[annotationNameAuthor] + if !ok || *author != backend.AuthorAnnotation { + return nil, errors.New("asset not uploaded by Chainloop") + } + + fileName, ok := resp.Metadata[annotationNameFilename] + if !ok { + return nil, fmt.Errorf("couldn't find file metadata") + } + + return &pb.CASResource{ + FileName: *fileName, + Size: *resp.ContentLength, + Digest: digest, + }, nil +} + +func (b *Backend) Download(ctx context.Context, w io.Writer, digest string) error { + exists, err := b.Exists(ctx, digest) + if err != nil { + return err + } else if !exists { + return backend.NewErrNotFound("artifact") + } + + downloader := s3manager.NewDownloaderWithClient(b.client) + // force sequential downloads so we can wrap the writer and ignore the offset + // Important! Do not change this value, otherwise the fakeWriterAt will not work + downloader.Concurrency = 1 + output := fakeWriterAt{w} + + _, err = downloader.DownloadWithContext(ctx, output, &s3.GetObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(resourceName(digest)), + }) + + return err +} + +// CheckWritePermissions performs an actual write to the repository to check that the credentials +func (b *Backend) CheckWritePermissions(ctx context.Context) error { + testObject := "healthcheck" + + input := &s3.PutObjectInput{ + Body: aws.ReadSeekCloser(strings.NewReader("healthcheckdata")), + Bucket: aws.String(b.bucket), + Key: aws.String(testObject), + } + + // Write to the bucket + if _, err := b.client.PutObjectWithContext(ctx, input); err != nil { + return fmt.Errorf("failed to write to bucket: %w", err) + } + + // and read the object back + _, err := b.client.GetObjectWithContext(ctx, &s3.GetObjectInput{ + Bucket: aws.String(b.bucket), + Key: aws.String(testObject), + }) + + if err != nil { + return fmt.Errorf("failed to read from bucket: %w", err) + } + + return nil +} + +// decode the hex string into a []byte slice. +// base64 encode the result +func hexSha256ToBinaryB64(hexString string) string { + // Decode the hex string into a []byte slice. + decoded, err := hex.DecodeString(hexString) + if err != nil { + return "" + } + + return base64.StdEncoding.EncodeToString(decoded) +} + +func resourceName(digest string) string { + return fmt.Sprintf("sha256:%s", digest) +} + +// fakeWriterAt is a wrapper around io.Writer that ignores the offset +// we have this wrapper as a compatibility bridge between the backend.Downloader and io.WriterAt +// This is ok since we force sequential downloads with concurrency=1 +type fakeWriterAt struct { + w io.Writer +} + +func (fw fakeWriterAt) WriteAt(p []byte, _ int64) (n int, err error) { + // ignore 'offset' because we forced sequential downloads + return fw.w.Write(p) +} diff --git a/internal/blobmanager/s3/backend_test.go b/internal/blobmanager/s3/backend_test.go new file mode 100644 index 000000000..90fdcd37b --- /dev/null +++ b/internal/blobmanager/s3/backend_test.go @@ -0,0 +1,284 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "os" + "testing" + "time" + + pb "github.com/chainloop-dev/chainloop/app/artifact-cas/api/cas/v1" + backend "github.com/chainloop-dev/chainloop/internal/blobmanager" + "github.com/docker/go-connections/nat" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" +) + +func (s *testSuite) TestHexSha256ToBinaryB64() { + testCases := []struct { + name string + hexSha string + expected string + }{ + { + name: "valid sha", + hexSha: "aabbccddeeff", + expected: "qrvM3e7/", + }, + { + name: "invalid sha", + hexSha: "aabbccddeeffgg", + expected: "", + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + actual := hexSha256ToBinaryB64(tc.hexSha) + s.Equal(tc.expected, actual) + }) + } +} + +func (s *testSuite) TestResourceName() { + testCases := []struct { + name string + sha string + expected string + }{ + { + name: "valid sha", + sha: "aabbccddeeff", + expected: "sha256:aabbccddeeff", + }, + } + + for _, tc := range testCases { + s.Run(tc.name, func() { + actual := resourceName(tc.sha) + s.Equal(tc.expected, actual) + }) + } +} + +func (s *testSuite) TestWritePermissions() { + s.T().Run("invalid credentials", func(t *testing.T) { + s.Error(s.invalidBackend.CheckWritePermissions(context.Background())) + }) + + s.T().Run("valid credentials", func(t *testing.T) { + s.NoError(s.backend.CheckWritePermissions(context.Background())) + }) +} + +func (s *testSuite) TestExists() { + s.T().Run("doesn't exist", func(t *testing.T) { + found, err := s.backend.Exists(context.Background(), "aabbccddeeff") + s.NoError(err) + s.False(found) + }) + + s.T().Run("found", func(t *testing.T) { + found, err := s.backend.Exists(context.Background(), s.ownedObjectDigest) + s.NoError(err) + s.True(found) + }) + + s.T().Run("exists but not uploaded by chainloop", func(t *testing.T) { + found, err := s.backend.Exists(context.Background(), s.externalObjectDigest) + s.ErrorContains(err, "not uploaded by Chainloop") + s.False(found) + }) +} + +func (s *testSuite) TestDescribe() { + s.T().Run("doesn't exist", func(t *testing.T) { + artifact, err := s.backend.Describe(context.Background(), "aabbccddeeff") + s.Error(err) + notFoundErr := &backend.ErrNotFound{} + s.ErrorAs(err, ¬FoundErr) + s.Nil(artifact) + }) + + s.T().Run("found", func(t *testing.T) { + artifact, err := s.backend.Describe(context.Background(), s.ownedObjectDigest) + s.NoError(err) + s.Equal("test.txt", artifact.FileName) + s.Equal(s.ownedObjectDigest, artifact.Digest) + s.Equal(int64(4), artifact.Size) + }) +} + +func (s *testSuite) TestDownload() { + s.T().Run("exist but not uploaded by Chainloop", func(t *testing.T) { + buf := bytes.NewBuffer(nil) + err := s.backend.Download(context.Background(), buf, s.externalObjectDigest) + s.ErrorContains(err, "asset not uploaded by Chainloop") + s.Empty(buf) + }) + + s.T().Run("doesn't exist", func(t *testing.T) { + buf := bytes.NewBuffer(nil) + err := s.backend.Download(context.Background(), buf, "deadbeef") + s.ErrorContains(err, "artifact not found") + s.Empty(buf) + }) + + s.T().Run("exists", func(t *testing.T) { + buf := bytes.NewBuffer(nil) + err := s.backend.Download(context.Background(), buf, s.ownedObjectDigest) + s.NoError(err) + s.Equal("test", buf.String()) + }) + + s.T().Run("it's been tampered", func(t *testing.T) { + buf := bytes.NewBuffer(nil) + err := s.backend.Download(context.Background(), buf, s.tamperedObjectDigest) + s.ErrorContains(err, "failed to validate integrity of object") + }) +} + +type testSuite struct { + suite.Suite + minio *minioInstance + backend, invalidBackend *Backend + ownedObjectDigest string + externalObjectDigest string + tamperedObjectDigest string +} + +func TestS3Backend(t *testing.T) { + suite.Run(t, new(testSuite)) +} + +func (s *testSuite) SetupSuite() { + if os.Getenv("SKIP_INTEGRATION") == "true" { + s.T().Skip() + } +} + +// Run before each test +const testBucket = "test-bucket" + +func (s *testSuite) SetupTest() { + s.minio = newMinioInstance(s.T()) + + // Create backend + backend, err := NewBackend(&Credentials{ + AccessKeyID: "root", + SecretAccessKey: "test-password", + Region: "us-east-1", + BucketName: testBucket, + }, WithEndpoint(fmt.Sprintf("http://%s", s.minio.ConnectionString(s.T()))), WithForcedS3PathStyle(true)) + require.NoError(s.T(), err) + s.backend = backend + + invalidBackend, err := NewBackend(&Credentials{ + AccessKeyID: "root", + SecretAccessKey: "wrong-password", + Region: "us-east-1", + BucketName: testBucket, + }, WithEndpoint(fmt.Sprintf("http://%s", s.minio.ConnectionString(s.T()))), WithForcedS3PathStyle(true)) + require.NoError(s.T(), err) + s.invalidBackend = invalidBackend + + // create bucket + minioClient, err := minio.New(s.minio.ConnectionString(s.T()), &minio.Options{ + Creds: credentials.NewStaticV4("root", "test-password", ""), Secure: false, + }) + require.NoError(s.T(), err) + require.NoError(s.T(), minioClient.MakeBucket(context.TODO(), testBucket, minio.MakeBucketOptions{})) + + // upload a valid artifact + buf := bytes.NewBuffer([]byte("test")) + s.ownedObjectDigest = fmt.Sprintf("%x", sha256.Sum256(buf.Bytes())) + // calculate sha256 of the content in the buffer + err = s.backend.Upload(context.Background(), buf, &pb.CASResource{Digest: s.ownedObjectDigest, FileName: "test.txt"}) + require.NoError(s.T(), err) + + // Copy an existing object but reference it from somewhere else + s.tamperedObjectDigest = "b5bb9d8014a0f9b1d61e21e796d78dccdf1352f23cd32812f4850b878ae4944c" + _, err = minioClient.CopyObject(context.Background(), minio.CopyDestOptions{ + Bucket: testBucket, Object: fmt.Sprintf("sha256:%s", s.tamperedObjectDigest), + }, minio.CopySrcOptions{ + Bucket: testBucket, Object: fmt.Sprintf("sha256:%s", s.ownedObjectDigest), + }) + require.NoError(s.T(), err) + + // upload another one but by the client directly + reader := bytes.NewReader([]byte("hello world")) + s.externalObjectDigest = "external-deadbeef" + _, err = minioClient.PutObject(context.Background(), testBucket, fmt.Sprintf("sha256:%s", s.externalObjectDigest), reader, reader.Size(), + minio.PutObjectOptions{}) + require.NoError(s.T(), err) +} + +func (s *testSuite) TearDownTest() { + if s.minio == nil { + return + } + + assert.NoError(s.T(), s.minio.instance.Terminate(context.Background())) +} + +func newMinioInstance(t *testing.T) *minioInstance { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + + port, err := nat.NewPort("", "9000") + require.NoError(t, err) + + req := testcontainers.ContainerRequest{ + Image: "minio/minio:RELEASE.2023-09-04T19-57-37Z", + ExposedPorts: []string{port.Port()}, + Env: map[string]string{ + "MINIO_ROOT_USER": "root", + "MINIO_ROOT_PASSWORD": "test-password", + }, + Cmd: []string{"server", "/data"}, + WaitingFor: wait.ForListeningPort(port).WithStartupTimeout(5 * time.Minute), + } + + instance, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: req, + Started: true, + }) + require.NoError(t, err) + + return &minioInstance{instance} +} + +func (c *minioInstance) ConnectionString(t *testing.T) string { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + p, err := c.instance.MappedPort(ctx, "9000") + assert.NoError(t, err) + + return fmt.Sprintf("0.0.0.0:%d", p.Int()) +} + +type minioInstance struct { + instance testcontainers.Container +} diff --git a/internal/blobmanager/s3/provider.go b/internal/blobmanager/s3/provider.go new file mode 100644 index 000000000..1c4952ea3 --- /dev/null +++ b/internal/blobmanager/s3/provider.go @@ -0,0 +1,120 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "context" + "encoding/json" + "fmt" + + backend "github.com/chainloop-dev/chainloop/internal/blobmanager" + "github.com/chainloop-dev/chainloop/internal/credentials" +) + +type BackendProvider struct { + cReader credentials.Reader +} + +var _ backend.Provider = (*BackendProvider)(nil) + +func NewBackendProvider(cReader credentials.Reader) *BackendProvider { + return &BackendProvider{cReader: cReader} +} + +const ProviderID = "AWS-S3" + +func (p *BackendProvider) ID() string { + return ProviderID +} + +func (p *BackendProvider) FromCredentials(ctx context.Context, secretName string) (backend.UploaderDownloader, error) { + creds := &Credentials{} + if err := p.cReader.ReadCredentials(ctx, secretName, creds); err != nil { + return nil, err + } + + if err := creds.Validate(); err != nil { + return nil, fmt.Errorf("invalid credentials retrieved from storage: %w", err) + } + + return NewBackend(creds) +} + +func (p *BackendProvider) ValidateAndExtractCredentials(location string, credsJSON []byte) (any, error) { + creds, err := extractCreds(location, credsJSON) + if err != nil { + return nil, fmt.Errorf("extracting credentials: %w", err) + } + + // Validate that the credentials are valid against the storage account + b, err := NewBackend(creds) + if err != nil { + return nil, fmt.Errorf("creating backend: %w", err) + } + + if err := b.CheckWritePermissions(context.TODO()); err != nil { + return nil, fmt.Errorf("checking write permissions: %w", err) + } + + return creds, nil +} + +func extractCreds(bucketName string, credsJSON []byte) (*Credentials, error) { + var creds *Credentials + if err := json.Unmarshal(credsJSON, &creds); err != nil { + return nil, fmt.Errorf("unmarshaling credentials: %w", err) + } + + creds.BucketName = bucketName + + if err := creds.Validate(); err != nil { + return nil, fmt.Errorf("invalid credentials: %w", err) + } + + return creds, nil +} + +type Credentials struct { + // AWS Access Key ID + AccessKeyID string + // AWS Secret Access Key + SecretAccessKey string + // Bucket name + BucketName string + // Region ID, i.e us-east-1 + Region string +} + +// Validate that the APICreds has all its properties set +func (c *Credentials) Validate() error { + if c.AccessKeyID == "" { + return fmt.Errorf("%w: missing accessKeyID", backend.ErrValidation) + } + + if c.SecretAccessKey == "" { + return fmt.Errorf("%w: missing secretAccessKey", backend.ErrValidation) + } + + if c.BucketName == "" { + return fmt.Errorf("%w: missing bucket name", backend.ErrValidation) + } + + if c.Region == "" { + return fmt.Errorf("%w: missing region", backend.ErrValidation) + } + + return nil +} diff --git a/internal/blobmanager/s3/provider_test.go b/internal/blobmanager/s3/provider_test.go new file mode 100644 index 000000000..eec3a9582 --- /dev/null +++ b/internal/blobmanager/s3/provider_test.go @@ -0,0 +1,168 @@ +// +// Copyright 2023 The Chainloop Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package s3 + +import ( + "context" + "testing" + + "github.com/chainloop-dev/chainloop/internal/credentials/mocks" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestValidate(t *testing.T) { + testCases := []struct { + name string + creds *Credentials + wantErr bool + }{ + { + name: "valid credentials", + creds: &Credentials{ + AccessKeyID: "test", + SecretAccessKey: "test", + Region: "test", + BucketName: "test", + }, + }, + { + name: "missing access key id", + creds: &Credentials{ + SecretAccessKey: "test", + Region: "test", + BucketName: "test", + }, + wantErr: true, + }, + { + name: "missing secret access key", + creds: &Credentials{ + AccessKeyID: "test", + Region: "test", + BucketName: "test", + }, + wantErr: true, + }, + { + name: "missing region", + creds: &Credentials{ + AccessKeyID: "test", + SecretAccessKey: "test", + BucketName: "test", + }, + wantErr: true, + }, + { + name: "missing bucket name", + creds: &Credentials{ + AccessKeyID: "test", + SecretAccessKey: "test", + Region: "test", + }, + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.creds.Validate() + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + } + }) + } +} + +func TestFromCredentials(t *testing.T) { + ctx := context.Background() + assert := assert.New(t) + r := mocks.NewReader(t) + const bucket, keyID, keySecret, region = "my-bucket", "key-id", "key-secret", "region-1" + + r.On("ReadCredentials", ctx, "secretName", mock.AnythingOfType("*s3.Credentials")).Return(nil).Run( + func(args mock.Arguments) { + credentials := args.Get(2).(*Credentials) + credentials.BucketName = bucket + credentials.Region = region + credentials.SecretAccessKey = keySecret + credentials.AccessKeyID = keyID + }) + + _, err := NewBackendProvider(r).FromCredentials(ctx, "secretName") + assert.NoError(err) +} + +func TestExtractCreds(t *testing.T) { + tetCases := []struct { + name string + bucketName string + credsJSON []byte + wantErr bool + }{ + { + name: "valid credentials", + bucketName: "mybucket", + credsJSON: []byte(`{ + "AccessKeyID": "keyID", + "SecretAccessKey": "keySecret", + "Region": "region-1" + }`), + }, + { + name: "invalid location, missing bucket", + bucketName: "", + wantErr: true, + credsJSON: []byte(`{ + "AccessKeyID": "test", + "SecretAccessKey": "keySecret", + "Region": "test" + }`), + }, + { + name: "invalid credentials, missing secret", + bucketName: "account/container", + credsJSON: []byte(`{ + "AccessKeyID": "test", + "Region": "region-1" + }`), + wantErr: true, + }, + } + + for _, tc := range tetCases { + t.Run(tc.name, func(t *testing.T) { + creds, err := extractCreds(tc.bucketName, tc.credsJSON) + if tc.wantErr { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Equal(t, &Credentials{ + Region: "region-1", + SecretAccessKey: "keySecret", + AccessKeyID: "keyID", + BucketName: tc.bucketName, + }, creds) + } + }) + } +} + +func TestProviderID(t *testing.T) { + assert.Equal(t, "AWS-S3", NewBackendProvider(nil).ID()) +}