diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d207b18 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +*.go text eol=lf diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..bda9352 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,7 @@ +--- +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" \ No newline at end of file diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..f97af8a --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,118 @@ +name: Build +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + build-all-clis: + name: Build Only (No Release) + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@v5 + + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: 'go.mod' + + - name: Alioss CLI Build for Linux + env: + GOOS: linux + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building Alioss CLI for Linux" + go build -o "alioss-cli-linux-amd64" ./alioss + sha1sum "alioss-cli-linux-amd64" + + - name: Alioss CLI Build for Windows + env: + GOOS: windows + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building Alioss CLI for Windows" + go build -o "alioss-cli-windows-amd64.exe" ./alioss + sha1sum "alioss-cli-windows-amd64.exe" + + - name: Azurebs CLI Build for Linux + env: + GOOS: linux + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building Azurebs CLI for Linux" + go build -o "azurebs-cli-linux-amd64" ./azurebs + sha1sum "azurebs-cli-linux-amd64" + + - name: Azurebs CLI Build for Windows + env: + GOOS: windows + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building Azurebs CLI for Windows" + go build -o "azurebs-cli-windows-amd64.exe" ./azurebs + sha1sum "azurebs-cli-windows-amd64.exe" + + - name: Dav CLI Build for Linux + env: + GOOS: linux + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building Dav CLI for Linux" + go build -o "dav-cli-linux-amd64" ./dav/main + sha1sum "dav-cli-linux-amd64" + + - name: Dav CLI Build for Windows + env: + GOOS: windows + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building Dav CLI for Windows" + go build -o "dav-cli-windows-amd64.exe" ./dav/main + sha1sum "dav-cli-windows-amd64.exe" + + - name: GCS CLI Build for Linux + env: + GOOS: linux + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building Gcs CLI for Linux" + go build -o "gcs-cli-linux-amd64" ./gcs + sha1sum "gcs-cli-linux-amd64" + + - name: GCS CLI Build for Windows + env: + GOOS: windows + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building Gcs CLI for Windows" + go build -o "gcs-cli-windows-amd64.exe" ./gcs + sha1sum "gcs-cli-windows-amd64.exe" + + - name: S3 CLI Build for Linux + env: + GOOS: linux + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building S3 CLI for Linux" + go build -o "s3-cli-linux-amd64" ./s3 + sha1sum "s3-cli-linux-amd64" + + - name: S3 CLI Build for Windows + env: + GOOS: windows + GOARCH: amd64 + CGO_ENABLED: 0 + run: | + echo "Building S3 CLI for Windows" + go build -o "s3-cli-windows-amd64.exe" ./s3 + sha1sum "s3-cli-windows-amd64.exe" \ No newline at end of file diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml new file mode 100644 index 0000000..5d5e365 --- /dev/null +++ b/.github/workflows/unit-test.yml @@ -0,0 +1,58 @@ +name : Unit Test + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +jobs: + tests: + name: Run Unit Tests + runs-on: [ubuntu-latest] + steps: + - uses: actions/checkout@v5 + - name: Set up Go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + # TODO: Re-enable linting after fixing existing issues on alioss + # - name: Lint code + # uses: golangci/golangci-lint-action@v8 + - name: Cache Go modules + uses: actions/cache@v4 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: alioss unit tests + run: | + export CGO_ENABLED=0 + go version + go run github.com/onsi/ginkgo/v2/ginkgo --skip-package=integration ./alioss/... + + - name: azurebs unit tests + run: | + export CGO_ENABLED=0 + go version + go run github.com/onsi/ginkgo/v2/ginkgo --skip-package=integration ./azurebs/... + + - name: dav unit tests + run: | + export CGO_ENABLED=0 + go version + go test -v ./dav/... + + - name: gcs unit tests + run: | + export CGO_ENABLED=0 + go version + go run github.com/onsi/ginkgo/v2/ginkgo --skip-package=integration ./gcs/... + + - name: s3 unit tests + run: | + export CGO_ENABLED=0 + go version + go run github.com/onsi/ginkgo/v2/ginkgo --skip-package=integration ./s3/... \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8e1cc9e --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib +*.out +*.bin + +# Go coverage files +*.coverprofile +*.cov + +# Output folders +/bin/ +/build/ +/dist/ + +# Test binaries +*.test + +# Dependency / tooling caches +/vendor/ +/coverage/ +/tmp/ +/.cache/ + +# IDE/editor +.vscode/ +.idea/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..fa73478 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,12 @@ +version: "2" + +linters: + default: standard + + settings: + errcheck: + check-blank: true # assignment to blank identifier: `_ := someFunc()`. + +formatters: + enable: + - goimports diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..a162275 --- /dev/null +++ b/NOTICE @@ -0,0 +1,10 @@ +Copyright (c) 2022 Cloud Foundry Contributors. All Rights Reserved. + +Other copyright notices for portions of this software are listed in the LICENSE file. + +This product is licensed to you under the Apache License, Version 2.0 (the "License"). +You may not use this product except in compliance with the License. + +This product may include a number of subcomponents with separate copyright notices +and license terms. Your use of these subcomponents is subject to the terms and +conditions of the subcomponent's license, as noted in the LICENSE file. diff --git a/README.md b/README.md new file mode 100644 index 0000000..87e6f9e --- /dev/null +++ b/README.md @@ -0,0 +1,39 @@ +# Storage CLI +This repository consolidates five independent blob-storage CLIs, one per provider, into a single codebase. Each provider has its own dedicated directory (azurebs/, s3/, gcs/, alioss/, dav/), containing an independent main package and implementation. The tools are intentionally maintained as separate binaries, preserving each provider’s native SDK, command-line flags, and operational semantics. Each CLI exposes similar high-level operations (e.g., put, get, delete). + +Key points + +- Each provider builds independently. + +- Client setup, config, and options are contained within the provider’s folder. + +- All tools support the same core commands (such as put, get, and delete) for a familiar workflow, while each provider defines its own flags, parameters, and execution flow that align with its native SDK and terminology. + +- Central issue tracking, shared CI, and aligned release process without merging implementations. + + +## Providers +- [Alioss](./alioss/README.md) +- [Azurebs](./azurebs/README.md) +- [Dav](./dav/README.md) +- [Gcs](./gcs/README.md) +- [S3](./s3/README.md) + + +## Build +Use following command to build it locally + +```shell +go build -o / /main.go +``` +e.g. `go build -o alioss/alioss-cli alioss/main.go` + + +## Notes +These commit IDs represent the last migration checkpoint from each provider's original repository, marking the final commit that was copied during the consolidation process. + +- alioss -> c303a62679ff467ba5012cc1a7ecfb7b6be47ea0 +- azurebs -> 18667d2a0b5237c38d053238906b4500cfb82ce8 +- dav -> c64e57857539d0173d46e79093c2e998ec71ab63 +- gcs -> d4ab2040f37415a559942feb7e264c6b28950f77 +- s3 -> 7ac9468ba8567eaf79828f30007c5a44066ef50f \ No newline at end of file diff --git a/alioss/README.md b/alioss/README.md new file mode 100644 index 0000000..18c17fe --- /dev/null +++ b/alioss/README.md @@ -0,0 +1,60 @@ +# Ali Storage CLI + +The Ali Storage CLI is for uploading, fetching and deleting content to and from an Ali OSS. +It is highly inspired by the https://github.com/cloudfoundry/bosh-s3cli. + +## Usage + +Given a JSON config file (`config.json`)... + +``` json +{ + "access_key_id": " (required)", + "access_key_secret": " (required)", + "endpoint": " (required)", + "bucket_name": " (required)" +} +``` + +``` bash +# Command: "put" +# Upload a blob to the blobstore. +./alioss-cli -c config.json put + +# Command: "get" +# Fetch a blob from the blobstore. +# Destination file will be overwritten if exists. +./alioss-cli -c config.json get + +# Command: "delete" +# Remove a blob from the blobstore. +./alioss-cli -c config.json delete + +# Command: "exists" +# Checks if blob exists in the blobstore. +./alioss-cli -c config.json exists + +# Command: "sign" +# Create a self-signed url for a blob in the blobstore. +./alioss-cli -c config.json sign +``` + +### Using signed urls with curl +``` bash +# Uploading a blob: +curl -X PUT -T path/to/file + +# Downloading a blob: +curl -X GET +``` +## Running integration tests + +To run the integration tests: +- Export the following variables into your environment: + ``` bash + export ACCESS_KEY_ID= + export ACCESS_KEY_SECRET= + export ENDPOINT= + export BUCKET_NAME= + ``` +- go build && go test ./integration/... diff --git a/alioss/client/client.go b/alioss/client/client.go new file mode 100644 index 0000000..516680c --- /dev/null +++ b/alioss/client/client.go @@ -0,0 +1,77 @@ +package client + +import ( + "crypto/md5" + "encoding/base64" + "fmt" + "io" + "log" + "os" + "strings" +) + +type AliBlobstore struct { + storageClient StorageClient +} + +func New(storageClient StorageClient) (AliBlobstore, error) { + return AliBlobstore{storageClient: storageClient}, nil +} + +func (client *AliBlobstore) Put(sourceFilePath string, destinationObject string) error { + sourceFileMD5, err := client.getMD5(sourceFilePath) + if err != nil { + return err + } + + err = client.storageClient.Upload(sourceFilePath, sourceFileMD5, destinationObject) + if err != nil { + return fmt.Errorf("upload failure: %w", err) + } + + log.Println("Successfully uploaded file") + return nil +} + +func (client *AliBlobstore) Get(sourceObject string, destinationFilePath string) error { + return client.storageClient.Download(sourceObject, destinationFilePath) +} + +func (client *AliBlobstore) Delete(object string) error { + return client.storageClient.Delete(object) +} + +func (client *AliBlobstore) Exists(object string) (bool, error) { + return client.storageClient.Exists(object) +} + +func (client *AliBlobstore) Sign(object string, action string, expiredInSec int64) (string, error) { + action = strings.ToUpper(action) + switch action { + case "PUT": + return client.storageClient.SignedUrlPut(object, expiredInSec) + case "GET": + return client.storageClient.SignedUrlGet(object, expiredInSec) + default: + return "", fmt.Errorf("action not implemented: %s", action) + } +} + +func (client *AliBlobstore) getMD5(filePath string) (string, error) { + file, err := os.Open(filePath) + if err != nil { + return "", err + } + + defer file.Close() + + hash := md5.New() + _, err = io.Copy(hash, file) + if err != nil { + return "", fmt.Errorf("failed to calculate md5: %w", err) + } + + md5 := base64.StdEncoding.EncodeToString(hash.Sum(nil)) + + return md5, nil +} diff --git a/alioss/client/client_suite_test.go b/alioss/client/client_suite_test.go new file mode 100644 index 0000000..79e004c --- /dev/null +++ b/alioss/client/client_suite_test.go @@ -0,0 +1,13 @@ +package client_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Client Suite") +} diff --git a/alioss/client/client_test.go b/alioss/client/client_test.go new file mode 100644 index 0000000..229b33b --- /dev/null +++ b/alioss/client/client_test.go @@ -0,0 +1,150 @@ +package client_test + +import ( + "errors" + "os" + + "github.com/cloudfoundry/storage-cli/alioss/client" + "github.com/cloudfoundry/storage-cli/alioss/client/clientfakes" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Client", func() { + + Context("Put", func() { + It("uploads a file to a blob", func() { + storageClient := clientfakes.FakeStorageClient{} + + aliBlobstore, err := client.New(&storageClient) + Expect(err).ToNot(HaveOccurred()) + + tmpFile, err := os.CreateTemp("", "azure-storage-cli-test") + + aliBlobstore.Put(tmpFile.Name(), "destination_object") + + Expect(storageClient.UploadCallCount()).To(Equal(1)) + sourceFilePath, sourceFileMD5, destination := storageClient.UploadArgsForCall(0) + + Expect(sourceFilePath).To(BeAssignableToTypeOf("source/file/path")) + Expect(sourceFileMD5).To(Equal("1B2M2Y8AsgTpgAmY7PhCfg==")) + Expect(destination).To(Equal("destination_object")) + }) + }) + + Context("Get", func() { + It("get blob downloads to a file", func() { + storageClient := clientfakes.FakeStorageClient{} + + aliBlobstore, err := client.New(&storageClient) + Expect(err).ToNot(HaveOccurred()) + + aliBlobstore.Get("source_object", "destination/file/path") + + Expect(storageClient.DownloadCallCount()).To(Equal(1)) + sourceObject, destinationFilePath := storageClient.DownloadArgsForCall(0) + + Expect(sourceObject).To(Equal("source_object")) + Expect(destinationFilePath).To(Equal("destination/file/path")) + }) + }) + + Context("Delete", func() { + It("delete blob deletes the blob", func() { + storageClient := clientfakes.FakeStorageClient{} + + aliBlobstore, err := client.New(&storageClient) + Expect(err).ToNot(HaveOccurred()) + + aliBlobstore.Delete("blob") + + Expect(storageClient.DeleteCallCount()).To(Equal(1)) + object := storageClient.DeleteArgsForCall(0) + + Expect(object).To(Equal("blob")) + }) + }) + + Context("Exists", func() { + It("returns blob.Existing on success", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.ExistsReturns(true, nil) + + aliBlobstore, _ := client.New(&storageClient) + existsState, err := aliBlobstore.Exists("blob") + Expect(existsState == true).To(BeTrue()) + Expect(err).ToNot(HaveOccurred()) + + object := storageClient.ExistsArgsForCall(0) + Expect(object).To(Equal("blob")) + }) + + It("returns blob.NotExisting for not existing blobs", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.ExistsReturns(false, nil) + + aliBlobstore, _ := client.New(&storageClient) + existsState, err := aliBlobstore.Exists("blob") + Expect(existsState == false).To(BeTrue()) + Expect(err).ToNot(HaveOccurred()) + + object := storageClient.ExistsArgsForCall(0) + Expect(object).To(Equal("blob")) + }) + + It("returns blob.ExistenceUnknown and an error in case an error occurred", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.ExistsReturns(false, errors.New("boom")) + + aliBlobstore, _ := client.New(&storageClient) + existsState, err := aliBlobstore.Exists("blob") + Expect(existsState == false).To(BeTrue()) + Expect(err).To(HaveOccurred()) + + object := storageClient.ExistsArgsForCall(0) + Expect(object).To(Equal("blob")) + }) + }) + + Context("signed url", func() { + It("returns a signed url for action 'get'", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.SignedUrlGetReturns("https://the-signed-url", nil) + + aliBlobstore, _ := client.New(&storageClient) + url, err := aliBlobstore.Sign("blob", "get", 100) + Expect(url == "https://the-signed-url").To(BeTrue()) + Expect(err).ToNot(HaveOccurred()) + + object, expiration := storageClient.SignedUrlGetArgsForCall(0) + Expect(object).To(Equal("blob")) + Expect(int(expiration)).To(Equal(100)) + }) + + It("returns a signed url for action 'put'", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.SignedUrlPutReturns("https://the-signed-url", nil) + + aliBlobstore, _ := client.New(&storageClient) + url, err := aliBlobstore.Sign("blob", "put", 100) + Expect(url == "https://the-signed-url").To(BeTrue()) + Expect(err).ToNot(HaveOccurred()) + + object, expiration := storageClient.SignedUrlPutArgsForCall(0) + Expect(object).To(Equal("blob")) + Expect(int(expiration)).To(Equal(100)) + }) + + It("fails on unknown action", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.SignedUrlGetReturns("", errors.New("boom")) + + aliBlobstore, _ := client.New(&storageClient) + url, err := aliBlobstore.Sign("blob", "unknown", 100) + Expect(url).To(Equal("")) + Expect(err).To(HaveOccurred()) + + Expect(storageClient.SignedUrlGetCallCount()).To(Equal(0)) + }) + }) +}) diff --git a/alioss/client/clientfakes/fake_storage_client.go b/alioss/client/clientfakes/fake_storage_client.go new file mode 100644 index 0000000..b963907 --- /dev/null +++ b/alioss/client/clientfakes/fake_storage_client.go @@ -0,0 +1,506 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package clientfakes + +import ( + "sync" + + "github.com/cloudfoundry/storage-cli/alioss/client" +) + +type FakeStorageClient struct { + DeleteStub func(string) error + deleteMutex sync.RWMutex + deleteArgsForCall []struct { + arg1 string + } + deleteReturns struct { + result1 error + } + deleteReturnsOnCall map[int]struct { + result1 error + } + DownloadStub func(string, string) error + downloadMutex sync.RWMutex + downloadArgsForCall []struct { + arg1 string + arg2 string + } + downloadReturns struct { + result1 error + } + downloadReturnsOnCall map[int]struct { + result1 error + } + ExistsStub func(string) (bool, error) + existsMutex sync.RWMutex + existsArgsForCall []struct { + arg1 string + } + existsReturns struct { + result1 bool + result2 error + } + existsReturnsOnCall map[int]struct { + result1 bool + result2 error + } + SignedUrlGetStub func(string, int64) (string, error) + signedUrlGetMutex sync.RWMutex + signedUrlGetArgsForCall []struct { + arg1 string + arg2 int64 + } + signedUrlGetReturns struct { + result1 string + result2 error + } + signedUrlGetReturnsOnCall map[int]struct { + result1 string + result2 error + } + SignedUrlPutStub func(string, int64) (string, error) + signedUrlPutMutex sync.RWMutex + signedUrlPutArgsForCall []struct { + arg1 string + arg2 int64 + } + signedUrlPutReturns struct { + result1 string + result2 error + } + signedUrlPutReturnsOnCall map[int]struct { + result1 string + result2 error + } + UploadStub func(string, string, string) error + uploadMutex sync.RWMutex + uploadArgsForCall []struct { + arg1 string + arg2 string + arg3 string + } + uploadReturns struct { + result1 error + } + uploadReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeStorageClient) Delete(arg1 string) error { + fake.deleteMutex.Lock() + ret, specificReturn := fake.deleteReturnsOnCall[len(fake.deleteArgsForCall)] + fake.deleteArgsForCall = append(fake.deleteArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.DeleteStub + fakeReturns := fake.deleteReturns + fake.recordInvocation("Delete", []interface{}{arg1}) + fake.deleteMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) DeleteCallCount() int { + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + return len(fake.deleteArgsForCall) +} + +func (fake *FakeStorageClient) DeleteCalls(stub func(string) error) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = stub +} + +func (fake *FakeStorageClient) DeleteArgsForCall(i int) string { + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + argsForCall := fake.deleteArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) DeleteReturns(result1 error) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = nil + fake.deleteReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) DeleteReturnsOnCall(i int, result1 error) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = nil + if fake.deleteReturnsOnCall == nil { + fake.deleteReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Download(arg1 string, arg2 string) error { + fake.downloadMutex.Lock() + ret, specificReturn := fake.downloadReturnsOnCall[len(fake.downloadArgsForCall)] + fake.downloadArgsForCall = append(fake.downloadArgsForCall, struct { + arg1 string + arg2 string + }{arg1, arg2}) + stub := fake.DownloadStub + fakeReturns := fake.downloadReturns + fake.recordInvocation("Download", []interface{}{arg1, arg2}) + fake.downloadMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) DownloadCallCount() int { + fake.downloadMutex.RLock() + defer fake.downloadMutex.RUnlock() + return len(fake.downloadArgsForCall) +} + +func (fake *FakeStorageClient) DownloadCalls(stub func(string, string) error) { + fake.downloadMutex.Lock() + defer fake.downloadMutex.Unlock() + fake.DownloadStub = stub +} + +func (fake *FakeStorageClient) DownloadArgsForCall(i int) (string, string) { + fake.downloadMutex.RLock() + defer fake.downloadMutex.RUnlock() + argsForCall := fake.downloadArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStorageClient) DownloadReturns(result1 error) { + fake.downloadMutex.Lock() + defer fake.downloadMutex.Unlock() + fake.DownloadStub = nil + fake.downloadReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) DownloadReturnsOnCall(i int, result1 error) { + fake.downloadMutex.Lock() + defer fake.downloadMutex.Unlock() + fake.DownloadStub = nil + if fake.downloadReturnsOnCall == nil { + fake.downloadReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.downloadReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Exists(arg1 string) (bool, error) { + fake.existsMutex.Lock() + ret, specificReturn := fake.existsReturnsOnCall[len(fake.existsArgsForCall)] + fake.existsArgsForCall = append(fake.existsArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.ExistsStub + fakeReturns := fake.existsReturns + fake.recordInvocation("Exists", []interface{}{arg1}) + fake.existsMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) ExistsCallCount() int { + fake.existsMutex.RLock() + defer fake.existsMutex.RUnlock() + return len(fake.existsArgsForCall) +} + +func (fake *FakeStorageClient) ExistsCalls(stub func(string) (bool, error)) { + fake.existsMutex.Lock() + defer fake.existsMutex.Unlock() + fake.ExistsStub = stub +} + +func (fake *FakeStorageClient) ExistsArgsForCall(i int) string { + fake.existsMutex.RLock() + defer fake.existsMutex.RUnlock() + argsForCall := fake.existsArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) ExistsReturns(result1 bool, result2 error) { + fake.existsMutex.Lock() + defer fake.existsMutex.Unlock() + fake.ExistsStub = nil + fake.existsReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) ExistsReturnsOnCall(i int, result1 bool, result2 error) { + fake.existsMutex.Lock() + defer fake.existsMutex.Unlock() + fake.ExistsStub = nil + if fake.existsReturnsOnCall == nil { + fake.existsReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.existsReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) SignedUrlGet(arg1 string, arg2 int64) (string, error) { + fake.signedUrlGetMutex.Lock() + ret, specificReturn := fake.signedUrlGetReturnsOnCall[len(fake.signedUrlGetArgsForCall)] + fake.signedUrlGetArgsForCall = append(fake.signedUrlGetArgsForCall, struct { + arg1 string + arg2 int64 + }{arg1, arg2}) + stub := fake.SignedUrlGetStub + fakeReturns := fake.signedUrlGetReturns + fake.recordInvocation("SignedUrlGet", []interface{}{arg1, arg2}) + fake.signedUrlGetMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) SignedUrlGetCallCount() int { + fake.signedUrlGetMutex.RLock() + defer fake.signedUrlGetMutex.RUnlock() + return len(fake.signedUrlGetArgsForCall) +} + +func (fake *FakeStorageClient) SignedUrlGetCalls(stub func(string, int64) (string, error)) { + fake.signedUrlGetMutex.Lock() + defer fake.signedUrlGetMutex.Unlock() + fake.SignedUrlGetStub = stub +} + +func (fake *FakeStorageClient) SignedUrlGetArgsForCall(i int) (string, int64) { + fake.signedUrlGetMutex.RLock() + defer fake.signedUrlGetMutex.RUnlock() + argsForCall := fake.signedUrlGetArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStorageClient) SignedUrlGetReturns(result1 string, result2 error) { + fake.signedUrlGetMutex.Lock() + defer fake.signedUrlGetMutex.Unlock() + fake.SignedUrlGetStub = nil + fake.signedUrlGetReturns = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) SignedUrlGetReturnsOnCall(i int, result1 string, result2 error) { + fake.signedUrlGetMutex.Lock() + defer fake.signedUrlGetMutex.Unlock() + fake.SignedUrlGetStub = nil + if fake.signedUrlGetReturnsOnCall == nil { + fake.signedUrlGetReturnsOnCall = make(map[int]struct { + result1 string + result2 error + }) + } + fake.signedUrlGetReturnsOnCall[i] = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) SignedUrlPut(arg1 string, arg2 int64) (string, error) { + fake.signedUrlPutMutex.Lock() + ret, specificReturn := fake.signedUrlPutReturnsOnCall[len(fake.signedUrlPutArgsForCall)] + fake.signedUrlPutArgsForCall = append(fake.signedUrlPutArgsForCall, struct { + arg1 string + arg2 int64 + }{arg1, arg2}) + stub := fake.SignedUrlPutStub + fakeReturns := fake.signedUrlPutReturns + fake.recordInvocation("SignedUrlPut", []interface{}{arg1, arg2}) + fake.signedUrlPutMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) SignedUrlPutCallCount() int { + fake.signedUrlPutMutex.RLock() + defer fake.signedUrlPutMutex.RUnlock() + return len(fake.signedUrlPutArgsForCall) +} + +func (fake *FakeStorageClient) SignedUrlPutCalls(stub func(string, int64) (string, error)) { + fake.signedUrlPutMutex.Lock() + defer fake.signedUrlPutMutex.Unlock() + fake.SignedUrlPutStub = stub +} + +func (fake *FakeStorageClient) SignedUrlPutArgsForCall(i int) (string, int64) { + fake.signedUrlPutMutex.RLock() + defer fake.signedUrlPutMutex.RUnlock() + argsForCall := fake.signedUrlPutArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStorageClient) SignedUrlPutReturns(result1 string, result2 error) { + fake.signedUrlPutMutex.Lock() + defer fake.signedUrlPutMutex.Unlock() + fake.SignedUrlPutStub = nil + fake.signedUrlPutReturns = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) SignedUrlPutReturnsOnCall(i int, result1 string, result2 error) { + fake.signedUrlPutMutex.Lock() + defer fake.signedUrlPutMutex.Unlock() + fake.SignedUrlPutStub = nil + if fake.signedUrlPutReturnsOnCall == nil { + fake.signedUrlPutReturnsOnCall = make(map[int]struct { + result1 string + result2 error + }) + } + fake.signedUrlPutReturnsOnCall[i] = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) Upload(arg1 string, arg2 string, arg3 string) error { + fake.uploadMutex.Lock() + ret, specificReturn := fake.uploadReturnsOnCall[len(fake.uploadArgsForCall)] + fake.uploadArgsForCall = append(fake.uploadArgsForCall, struct { + arg1 string + arg2 string + arg3 string + }{arg1, arg2, arg3}) + stub := fake.UploadStub + fakeReturns := fake.uploadReturns + fake.recordInvocation("Upload", []interface{}{arg1, arg2, arg3}) + fake.uploadMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) UploadCallCount() int { + fake.uploadMutex.RLock() + defer fake.uploadMutex.RUnlock() + return len(fake.uploadArgsForCall) +} + +func (fake *FakeStorageClient) UploadCalls(stub func(string, string, string) error) { + fake.uploadMutex.Lock() + defer fake.uploadMutex.Unlock() + fake.UploadStub = stub +} + +func (fake *FakeStorageClient) UploadArgsForCall(i int) (string, string, string) { + fake.uploadMutex.RLock() + defer fake.uploadMutex.RUnlock() + argsForCall := fake.uploadArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeStorageClient) UploadReturns(result1 error) { + fake.uploadMutex.Lock() + defer fake.uploadMutex.Unlock() + fake.UploadStub = nil + fake.uploadReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) UploadReturnsOnCall(i int, result1 error) { + fake.uploadMutex.Lock() + defer fake.uploadMutex.Unlock() + fake.UploadStub = nil + if fake.uploadReturnsOnCall == nil { + fake.uploadReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.uploadReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + fake.downloadMutex.RLock() + defer fake.downloadMutex.RUnlock() + fake.existsMutex.RLock() + defer fake.existsMutex.RUnlock() + fake.signedUrlGetMutex.RLock() + defer fake.signedUrlGetMutex.RUnlock() + fake.signedUrlPutMutex.RLock() + defer fake.signedUrlPutMutex.RUnlock() + fake.uploadMutex.RLock() + defer fake.uploadMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeStorageClient) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ client.StorageClient = new(FakeStorageClient) diff --git a/alioss/client/storage_client.go b/alioss/client/storage_client.go new file mode 100644 index 0000000..e4e34b0 --- /dev/null +++ b/alioss/client/storage_client.go @@ -0,0 +1,173 @@ +package client + +import ( + "fmt" + "log" + + "github.com/aliyun/aliyun-oss-go-sdk/oss" + "github.com/cloudfoundry/storage-cli/alioss/config" +) + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . StorageClient +type StorageClient interface { + Upload( + sourceFilePath string, + sourceFileMD5 string, + destinationObject string, + ) error + + Download( + sourceObject string, + destinationFilePath string, + ) error + + Delete( + object string, + ) error + + Exists( + object string, + ) (bool, error) + + SignedUrlPut( + object string, + expiredInSec int64, + ) (string, error) + + SignedUrlGet( + object string, + expiredInSec int64, + ) (string, error) +} + +type DefaultStorageClient struct { + storageConfig config.AliStorageConfig +} + +func NewStorageClient(storageConfig config.AliStorageConfig) (StorageClient, error) { + return DefaultStorageClient{storageConfig: storageConfig}, nil +} + +func (dsc DefaultStorageClient) Upload( + sourceFilePath string, + sourceFileMD5 string, + destinationObject string, +) error { + log.Println(fmt.Sprintf("Uploading %s/%s", dsc.storageConfig.BucketName, destinationObject)) + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return err + } + + return bucket.PutObjectFromFile(destinationObject, sourceFilePath, oss.ContentMD5(sourceFileMD5)) +} + +func (dsc DefaultStorageClient) Download( + sourceObject string, + destinationFilePath string, +) error { + log.Println(fmt.Sprintf("Downloading %s/%s", dsc.storageConfig.BucketName, sourceObject)) + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return err + } + + return bucket.GetObjectToFile(sourceObject, destinationFilePath) +} + +func (dsc DefaultStorageClient) Delete( + object string, +) error { + log.Println(fmt.Sprintf("Deleting %s/%s", dsc.storageConfig.BucketName, object)) + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return err + } + + return bucket.DeleteObject(object) +} + +func (dsc DefaultStorageClient) Exists(object string) (bool, error) { + log.Println(fmt.Sprintf("Checking if blob: %s/%s", dsc.storageConfig.BucketName, object)) + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return false, err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return false, err + } + + objectExists, err := bucket.IsObjectExist(object) + if err != nil { + return false, err + } + + if objectExists { + log.Printf("File '%s' exists in bucket '%s'\n", object, dsc.storageConfig.BucketName) + return true, nil + } else { + log.Printf("File '%s' does not exist in bucket '%s'\n", object, dsc.storageConfig.BucketName) + return false, nil + } +} + +func (dsc DefaultStorageClient) SignedUrlPut( + object string, + expiredInSec int64, +) (string, error) { + + log.Println(fmt.Sprintf("Getting signed PUT url for blob %s/%s", dsc.storageConfig.BucketName, object)) + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return "", err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return "", err + } + + return bucket.SignURL(object, oss.HTTPPut, expiredInSec) +} + +func (dsc DefaultStorageClient) SignedUrlGet( + object string, + expiredInSec int64, +) (string, error) { + + log.Println(fmt.Sprintf("Getting signed GET url for blob %s/%s", dsc.storageConfig.BucketName, object)) + + client, err := oss.New(dsc.storageConfig.Endpoint, dsc.storageConfig.AccessKeyID, dsc.storageConfig.AccessKeySecret) + if err != nil { + return "", err + } + + bucket, err := client.Bucket(dsc.storageConfig.BucketName) + if err != nil { + return "", err + } + + return bucket.SignURL(object, oss.HTTPGet, expiredInSec) +} diff --git a/alioss/config/config.go b/alioss/config/config.go new file mode 100644 index 0000000..72fb15b --- /dev/null +++ b/alioss/config/config.go @@ -0,0 +1,30 @@ +package config + +import ( + "encoding/json" + "io" +) + +type AliStorageConfig struct { + AccessKeyID string `json:"access_key_id"` + AccessKeySecret string `json:"access_key_secret"` + Endpoint string `json:"endpoint"` + BucketName string `json:"bucket_name"` +} + +// NewFromReader returns a new ali-storage-cli configuration struct from the contents of reader. +// reader.Read() is expected to return valid JSON +func NewFromReader(reader io.Reader) (AliStorageConfig, error) { + bytes, err := io.ReadAll(reader) + if err != nil { + return AliStorageConfig{}, err + } + config := AliStorageConfig{} + + err = json.Unmarshal(bytes, &config) + if err != nil { + return AliStorageConfig{}, err + } + + return config, nil +} diff --git a/alioss/config/config_suite_test.go b/alioss/config/config_suite_test.go new file mode 100644 index 0000000..c6e29ba --- /dev/null +++ b/alioss/config/config_suite_test.go @@ -0,0 +1,13 @@ +package config_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Config Suite") +} diff --git a/alioss/config/config_test.go b/alioss/config/config_test.go new file mode 100644 index 0000000..6212d37 --- /dev/null +++ b/alioss/config/config_test.go @@ -0,0 +1,58 @@ +package config_test + +import ( + "bytes" + "errors" + + "github.com/cloudfoundry/storage-cli/alioss/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Config", func() { + + It("contains mandatory properties", func() { + configJson := []byte(`{"access_key_id": "foo_access_key_id", + "access_key_secret": "foo_access_key_secret", + "endpoint": "foo_endpoint", + "bucket_name": "foo_bucket_name"}`) + configReader := bytes.NewReader(configJson) + + config, err := config.NewFromReader(configReader) + + Expect(err).ToNot(HaveOccurred()) + Expect(config.AccessKeyID).To(Equal("foo_access_key_id")) + Expect(config.AccessKeySecret).To(Equal("foo_access_key_secret")) + Expect(config.Endpoint).To(Equal("foo_endpoint")) + Expect(config.BucketName).To(Equal("foo_bucket_name")) + }) + + It("is empty if config cannot be parsed", func() { + configJson := []byte(`~`) + configReader := bytes.NewReader(configJson) + + config, err := config.NewFromReader(configReader) + + Expect(err.Error()).To(Equal("invalid character '~' looking for beginning of value")) + Expect(config.AccessKeyID).Should(BeEmpty()) + Expect(config.AccessKeySecret).Should(BeEmpty()) + Expect(config.Endpoint).Should(BeEmpty()) + Expect(config.BucketName).Should(BeEmpty()) + }) + + Context("when the configuration file cannot be read", func() { + It("returns an error", func() { + f := explodingReader{} + + _, err := config.NewFromReader(f) + Expect(err).To(MatchError("explosion")) + }) + }) + +}) + +type explodingReader struct{} + +func (e explodingReader) Read([]byte) (int, error) { + return 0, errors.New("explosion") +} diff --git a/alioss/integration/general_ali_test.go b/alioss/integration/general_ali_test.go new file mode 100644 index 0000000..2df26e8 --- /dev/null +++ b/alioss/integration/general_ali_test.go @@ -0,0 +1,213 @@ +package integration_test + +import ( + "bytes" + "io/ioutil" + "os" + + "github.com/cloudfoundry/storage-cli/alioss/config" + "github.com/cloudfoundry/storage-cli/alioss/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("General testing for all Ali regions", func() { + + var blobName string + var configPath string + var contentFile string + + BeforeEach(func() { + blobName = integration.GenerateRandomString() + configPath = integration.MakeConfigFile(&defaultConfig) + contentFile = integration.MakeContentFile("foo") + }) + + AfterEach(func() { + defer func() { _ = os.Remove(configPath) }() + defer func() { _ = os.Remove(contentFile) }() + }) + + Describe("Invoking `put`", func() { + It("uploads a file", func() { + defer func() { + cliSession, err := integration.RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + }() + + cliSession, err := integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + Expect(string(cliSession.Err.Contents())).To(MatchRegexp("File '" + blobName + "' exists in bucket '" + bucketName + "'")) + }) + + It("overwrites an existing file", func() { + defer func() { + cliSession, err := integration.RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + }() + + tmpLocalFile, _ := os.CreateTemp("", "ali-storage-cli-download") + tmpLocalFile.Close() + defer func() { _ = os.Remove(tmpLocalFile.Name()) }() + + contentFile = integration.MakeContentFile("initial content") + cliSession, err := integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "get", blobName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + gottenBytes, _ := os.ReadFile(tmpLocalFile.Name()) + Expect(string(gottenBytes)).To(Equal("initial content")) + + contentFile = integration.MakeContentFile("updated content") + cliSession, err = integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "get", blobName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + gottenBytes, _ = os.ReadFile(tmpLocalFile.Name()) + Expect(string(gottenBytes)).To(Equal("updated content")) + }) + + It("returns the appropriate error message", func() { + cfg := &config.AliStorageConfig{ + AccessKeyID: accessKeyID, + AccessKeySecret: accessKeySecret, + Endpoint: endpoint, + BucketName: "not-existing", + } + + configPath = integration.MakeConfigFile(cfg) + + cliSession, err := integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(1)) + + consoleOutput := bytes.NewBuffer(cliSession.Err.Contents()).String() + Expect(consoleOutput).To(ContainSubstring("upload failure")) + }) + }) + + Describe("Invoking `get`", func() { + It("downloads a file", func() { + outputFilePath := "/tmp/" + integration.GenerateRandomString() + + defer func() { + cliSession, err := integration.RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + _ = os.Remove(outputFilePath) + }() + + cliSession, err := integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "get", blobName, outputFilePath) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + fileContent, _ := ioutil.ReadFile(outputFilePath) + Expect(string(fileContent)).To(Equal("foo")) + }) + }) + + Describe("Invoking `delete`", func() { + It("deletes a file", func() { + defer func() { + cliSession, err := integration.RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + }() + + cliSession, err := integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(3)) + }) + }) + + Describe("Invoking `exists`", func() { + It("returns 0 for an existing blob", func() { + defer func() { + cliSession, err := integration.RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + }() + + cliSession, err := integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(0)) + }) + + It("returns 3 for a not existing blob", func() { + cliSession, err := integration.RunCli(cliPath, configPath, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(3)) + }) + }) + + Describe("Invoking `sign`", func() { + It("returns 0 for an existing blob", func() { + cliSession, err := integration.RunCli(cliPath, configPath, "sign", "some-blob", "get", "60s") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + getUrl := bytes.NewBuffer(cliSession.Out.Contents()).String() + Expect(getUrl).To(MatchRegexp("http://" + bucketName + "." + endpoint + "/some-blob")) + + cliSession, err = integration.RunCli(cliPath, configPath, "sign", "some-blob", "put", "60s") + Expect(err).ToNot(HaveOccurred()) + + putUrl := bytes.NewBuffer(cliSession.Out.Contents()).String() + Expect(putUrl).To(MatchRegexp("http://" + bucketName + "." + endpoint + "/some-blob")) + }) + + It("returns 3 for a not existing blob", func() { + cliSession, err := integration.RunCli(cliPath, configPath, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(3)) + }) + }) + + Describe("Invoking `-v`", func() { + It("returns the cli version", func() { + configPath := integration.MakeConfigFile(&defaultConfig) + defer func() { _ = os.Remove(configPath) }() + + cliSession, err := integration.RunCli(cliPath, configPath, "-v") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(0)) + + consoleOutput := bytes.NewBuffer(cliSession.Out.Contents()).String() + Expect(consoleOutput).To(ContainSubstring("version")) + }) + }) +}) diff --git a/alioss/integration/integration_suite_test.go b/alioss/integration/integration_suite_test.go new file mode 100644 index 0000000..a8e4fa2 --- /dev/null +++ b/alioss/integration/integration_suite_test.go @@ -0,0 +1,55 @@ +package integration_test + +import ( + "os" + "testing" + + "github.com/cloudfoundry/storage-cli/alioss/config" + "github.com/onsi/gomega/gexec" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestIntegration(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Integration Suite") +} + +var cliPath string +var accessKeyID string +var accessKeySecret string +var endpoint string +var bucketName string +var defaultConfig config.AliStorageConfig + +var _ = BeforeSuite(func() { + if len(cliPath) == 0 { + var err error + cliPath, err = gexec.Build("github.com/cloudfoundry/storage-cli/alioss") + Expect(err).ShouldNot(HaveOccurred()) + } + + accessKeyID = os.Getenv("ACCESS_KEY_ID") + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + + accessKeySecret = os.Getenv("ACCESS_KEY_SECRET") + Expect(accessKeySecret).ToNot(BeEmpty(), "ACCESS_KEY_SECRET must be set") + + endpoint = os.Getenv("ENDPOINT") + Expect(endpoint).ToNot(BeEmpty(), "ENDPOINT must be set") + + bucketName = os.Getenv("BUCKET_NAME") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + + defaultConfig = config.AliStorageConfig{ + AccessKeyID: accessKeyID, + AccessKeySecret: accessKeySecret, + Endpoint: endpoint, + BucketName: bucketName, + } +}) + +var _ = AfterSuite(func() { + gexec.CleanupBuildArtifacts() +}) diff --git a/alioss/integration/utils.go b/alioss/integration/utils.go new file mode 100644 index 0000000..c689f85 --- /dev/null +++ b/alioss/integration/utils.go @@ -0,0 +1,68 @@ +package integration + +import ( + "encoding/json" + "math/rand" + "os" + "os/exec" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + + "github.com/cloudfoundry/storage-cli/alioss/config" +) + +const alphanum = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +func GenerateRandomString(params ...int) string { + size := 25 + if len(params) == 1 { + size = params[0] + } + + randBytes := make([]byte, size) + for i := range randBytes { + randBytes[i] = alphanum[rand.Intn(len(alphanum))] + } + return string(randBytes) +} + +func MakeConfigFile(cfg *config.AliStorageConfig) string { + cfgBytes, err := json.Marshal(cfg) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + tmpFile, err := os.CreateTemp("", "azure-storage-cli-test") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = tmpFile.Write(cfgBytes) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = tmpFile.Close() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + return tmpFile.Name() +} + +func MakeContentFile(content string) string { + tmpFile, err := os.CreateTemp("", "azure-storage-test-content") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = tmpFile.Write([]byte(content)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = tmpFile.Close() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + return tmpFile.Name() +} + +func RunCli(cliPath string, configPath string, subcommand string, args ...string) (*gexec.Session, error) { + cmdArgs := []string{ + "-c", + configPath, + subcommand, + } + cmdArgs = append(cmdArgs, args...) + command := exec.Command(cliPath, cmdArgs...) + gexecSession, err := gexec.Start(command, ginkgo.GinkgoWriter, ginkgo.GinkgoWriter) + if err != nil { + return nil, err + } + gexecSession.Wait(1 * time.Minute) + return gexecSession, nil +} diff --git a/alioss/main.go b/alioss/main.go new file mode 100644 index 0000000..348de6d --- /dev/null +++ b/alioss/main.go @@ -0,0 +1,135 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + + "github.com/cloudfoundry/storage-cli/alioss/client" + "github.com/cloudfoundry/storage-cli/alioss/config" +) + +var version string + +func main() { + + configPath := flag.String("c", "", "configuration path") + showVer := flag.Bool("v", false, "version") + flag.Parse() + + if *showVer { + fmt.Printf("version %s\n", version) + os.Exit(0) + } + + configFile, err := os.Open(*configPath) + if err != nil { + log.Fatalln(err) + } + + aliConfig, err := config.NewFromReader(configFile) + if err != nil { + log.Fatalln(err) + } + + storageClient, err := client.NewStorageClient(aliConfig) + if err != nil { + log.Fatalln(err) + } + + blobstoreClient, err := client.New(storageClient) + if err != nil { + log.Fatalln(err) + } + + nonFlagArgs := flag.Args() + if len(nonFlagArgs) < 2 { + log.Fatalf("Expected at least two arguments got %d\n", len(nonFlagArgs)) + } + + cmd := nonFlagArgs[0] + + switch cmd { + case "put": + if len(nonFlagArgs) != 3 { + log.Fatalf("Put method expected 3 arguments got %d\n", len(nonFlagArgs)) + } + sourceFilePath, destination := nonFlagArgs[1], nonFlagArgs[2] + + _, err := os.Stat(sourceFilePath) + if err != nil { + log.Fatalln(err) + } + + err = blobstoreClient.Put(sourceFilePath, destination) + fatalLog(cmd, err) + + case "get": + if len(nonFlagArgs) != 3 { + log.Fatalf("Get method expected 3 arguments got %d\n", len(nonFlagArgs)) + } + source, destinationFilePath := nonFlagArgs[1], nonFlagArgs[2] + + err = blobstoreClient.Get(source, destinationFilePath) + fatalLog(cmd, err) + + case "delete": + if len(nonFlagArgs) != 2 { + log.Fatalf("Delete method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + + err = blobstoreClient.Delete(nonFlagArgs[1]) + fatalLog(cmd, err) + + case "exists": + if len(nonFlagArgs) != 2 { + log.Fatalf("Exists method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + + var exists bool + exists, err = blobstoreClient.Exists(nonFlagArgs[1]) + + // If the object exists the exit status is 0, otherwise it is 3 + // We are using `3` since `1` and `2` have special meanings + if err == nil && !exists { + os.Exit(3) + } + + case "sign": + if len(nonFlagArgs) != 4 { + log.Fatalf("Sign method expects 3 arguments got %d\n", len(nonFlagArgs)-1) + } + + object, action := nonFlagArgs[1], nonFlagArgs[2] + + if action != "get" && action != "put" { + log.Fatalf("Action not implemented: %s. Available actions are 'get' and 'put'", action) + } + + duration, err := time.ParseDuration(nonFlagArgs[3]) + if err != nil { + log.Fatalf("Expiration should be in the format of a duration i.e. 1h, 60m, 3600s. Got: %s", nonFlagArgs[3]) + } + + expiredInSec := int64(duration.Seconds()) + signedURL, err := blobstoreClient.Sign(object, action, expiredInSec) + + if err != nil { + log.Fatalf("Failed to sign request: %s", err) + } + + fmt.Println(signedURL) + os.Exit(0) + + default: + log.Fatalf("unknown command: '%s'\n", cmd) + } +} + +func fatalLog(cmd string, err error) { + if err != nil { + log.Fatalf("performing operation %s: %s\n", cmd, err) + } +} diff --git a/azurebs/README.md b/azurebs/README.md new file mode 100644 index 0000000..4b94514 --- /dev/null +++ b/azurebs/README.md @@ -0,0 +1,83 @@ +# Azure Storage CLI + +The Azure Storage CLI is for uploading, fetching and deleting content to and from an Azure blobstore. +It is highly inspired by the https://github.com/cloudfoundry/bosh-s3cli. + +## Usage + +Given a JSON config file (`config.json`)... + +``` json +{ + "account_name": " (required)", + "account_key": " (required)", + "container_name": " (required)", + "environment": " (optional, default: 'AzureCloud')", +} +``` + +``` bash +# Command: "put" +# Upload a blob to the blobstore. +./azurebs-cli -c config.json put + +# Command: "get" +# Fetch a blob from the blobstore. +# Destination file will be overwritten if exists. +./azurebs-cli -c config.json get + +# Command: "delete" +# Remove a blob from the blobstore. +./azurebs-cli -c config.json delete + +# Command: "exists" +# Checks if blob exists in the blobstore. +./azurebs-cli -c config.json exists + +# Command: "sign" +# Create a self-signed url for a blob in the blobstore. +./azurebs-cli -c config.json sign +``` + +### Using signed urls with curl + +``` bash +# Uploading a blob: +curl -X PUT -H "x-ms-blob-type: blockblob" -F 'fileX=' + +# Downloading a blob: +curl -X GET +``` + +## Running tests + +### Unit tests + +Using ginkgo: + +``` bash +go install github.com/onsi/ginkgo/v2/ginkgo +ginkgo --skip-package=integration --randomize-all --cover -v -r +``` + +Using go test: + +``` bash +go test $(go list ./... | grep -v integration) +``` + +### Integration tests + +1. Export the following variables into your environment: + + ``` bash + export ACCOUNT_NAME= + export ACCOUNT_KEY= + export CONTAINER_NAME= + ``` + +2. Run integration tests + + ```bash + go test ./integration/... + ``` diff --git a/azurebs/client/client.go b/azurebs/client/client.go new file mode 100644 index 0000000..5021e6e --- /dev/null +++ b/azurebs/client/client.go @@ -0,0 +1,119 @@ +package client + +import ( + "bytes" + "crypto/md5" + "fmt" + "io" + "log" + "os" + "strings" + "time" +) + +type AzBlobstore struct { + storageClient StorageClient +} + +func New(storageClient StorageClient) (AzBlobstore, error) { + return AzBlobstore{storageClient: storageClient}, nil +} + +func (client *AzBlobstore) Put(sourceFilePath string, dest string) error { + sourceMD5, err := client.getMD5(sourceFilePath) + if err != nil { + return err + } + + source, err := os.Open(sourceFilePath) + if err != nil { + return err + } + + defer source.Close() //nolint:errcheck + + md5, err := client.storageClient.Upload(source, dest) + if err != nil { + return fmt.Errorf("upload failure: %w", err) + } + + if !bytes.Equal(sourceMD5, md5) { + log.Println("The upload failed because of an MD5 inconsistency. Triggering blob deletion ...") + + err := client.storageClient.Delete(dest) + if err != nil { + log.Println(fmt.Errorf("blob deletion failed: %w", err)) + } + + return fmt.Errorf("the upload responded an MD5 %v does not match the source file MD5 %v", md5, sourceMD5) + } + + log.Println("Successfully uploaded file") + return nil +} + +func (client *AzBlobstore) Get(source string, dest *os.File) error { + + return client.storageClient.Download(source, dest) +} + +func (client *AzBlobstore) Delete(dest string) error { + + return client.storageClient.Delete(dest) +} + +func (client *AzBlobstore) DeleteRecursive(prefix string) error { + + return client.storageClient.DeleteRecursive(prefix) +} + +func (client *AzBlobstore) Exists(dest string) (bool, error) { + + return client.storageClient.Exists(dest) +} + +func (client *AzBlobstore) Sign(dest string, action string, expiration time.Duration) (string, error) { + action = strings.ToUpper(action) + switch action { + case "GET", "PUT": + return client.storageClient.SignedUrl(action, dest, expiration) + default: + return "", fmt.Errorf("action not implemented: %s", action) + } +} + +func (client *AzBlobstore) getMD5(filePath string) ([]byte, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, err + } + + defer file.Close() //nolint:errcheck + + hash := md5.New() + _, err = io.Copy(hash, file) + if err != nil { + return nil, fmt.Errorf("failed to calculate md5: %w", err) + } + + return hash.Sum(nil), nil +} + +func (client *AzBlobstore) List(prefix string) ([]string, error) { + return client.storageClient.List(prefix) +} + +func (client *AzBlobstore) Copy(srcBlob string, dstBlob string) error { + + return client.storageClient.Copy(srcBlob, dstBlob) +} + +func (client *AzBlobstore) Properties(dest string) error { + + return client.storageClient.Properties(dest) +} + +func (client *AzBlobstore) EnsureContainerExists() error { + + return client.storageClient.EnsureContainerExists() +} diff --git a/azurebs/client/client_suite_test.go b/azurebs/client/client_suite_test.go new file mode 100644 index 0000000..79e004c --- /dev/null +++ b/azurebs/client/client_suite_test.go @@ -0,0 +1,13 @@ +package client_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Client Suite") +} diff --git a/azurebs/client/client_test.go b/azurebs/client/client_test.go new file mode 100644 index 0000000..9d2df7b --- /dev/null +++ b/azurebs/client/client_test.go @@ -0,0 +1,218 @@ +package client_test + +import ( + "errors" + "os" + "runtime" + + "github.com/cloudfoundry/storage-cli/azurebs/client" + "github.com/cloudfoundry/storage-cli/azurebs/client/clientfakes" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Client", func() { + + Context("Put", func() { + It("uploads a file to a blob", func() { + storageClient := clientfakes.FakeStorageClient{} + + azBlobstore, err := client.New(&storageClient) + Expect(err).ToNot(HaveOccurred()) + + file, _ := os.CreateTemp("", "tmpfile") //nolint:errcheck + + azBlobstore.Put(file.Name(), "target/blob") //nolint:errcheck + + Expect(storageClient.UploadCallCount()).To(Equal(1)) + source, dest := storageClient.UploadArgsForCall(0) + + Expect(source).To(BeAssignableToTypeOf((*os.File)(nil))) + Expect(dest).To(Equal("target/blob")) + }) + + It("skips the upload if the md5 cannot be calculated from the file", func() { + storageClient := clientfakes.FakeStorageClient{} + + azBlobstore, err := client.New(&storageClient) + Expect(err).ToNot(HaveOccurred()) + + err = azBlobstore.Put("the/path", "target/blob") + + Expect(storageClient.UploadCallCount()).To(Equal(0)) + var expectedError string + if runtime.GOOS == "windows" { + expectedError = "open the/path: The system cannot find the path specified." + } else { + expectedError = "open the/path: no such file or directory" + } + Expect(err.Error()).To(Equal(expectedError)) + }) + + It("fails if the source file md5 does not match the responded md5", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.UploadReturns([]byte{1, 2, 3}, nil) + + azBlobstore, err := client.New(&storageClient) + Expect(err).ToNot(HaveOccurred()) + + file, _ := os.CreateTemp("", "tmpfile") //nolint:errcheck + + putError := azBlobstore.Put(file.Name(), "target/blob") + Expect(putError.Error()).To(Equal("the upload responded an MD5 [1 2 3] does not match the source file MD5 [212 29 140 217 143 0 178 4 233 128 9 152 236 248 66 126]")) + + Expect(storageClient.UploadCallCount()).To(Equal(1)) + source, dest := storageClient.UploadArgsForCall(0) + Expect(source).To(BeAssignableToTypeOf((*os.File)(nil))) + Expect(dest).To(Equal("target/blob")) + + Expect(storageClient.DeleteCallCount()).To(Equal(1)) + dest = storageClient.DeleteArgsForCall(0) + Expect(dest).To(Equal("target/blob")) + }) + }) + + It("get blob downloads to a file", func() { + storageClient := clientfakes.FakeStorageClient{} + + azBlobstore, err := client.New(&storageClient) + Expect(err).ToNot(HaveOccurred()) + + file, _ := os.CreateTemp("", "tmpfile") //nolint:errcheck + + azBlobstore.Get("source/blob", file) //nolint:errcheck + + Expect(storageClient.DownloadCallCount()).To(Equal(1)) + source, dest := storageClient.DownloadArgsForCall(0) + + Expect(source).To(Equal("source/blob")) + Expect(dest).To(Equal(file)) + }) + + It("delete blob deletes the blob", func() { + storageClient := clientfakes.FakeStorageClient{} + + azBlobstore, err := client.New(&storageClient) + Expect(err).ToNot(HaveOccurred()) + + azBlobstore.Delete("blob") //nolint:errcheck + + Expect(storageClient.DeleteCallCount()).To(Equal(1)) + dest := storageClient.DeleteArgsForCall(0) + + Expect(dest).To(Equal("blob")) + }) + + Context("if the blob existence is checked", func() { + It("returns blob.Existing on success", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.ExistsReturns(true, nil) + + azBlobstore, _ := client.New(&storageClient) //nolint:errcheck + existsState, err := azBlobstore.Exists("blob") + Expect(existsState == true).To(BeTrue()) + Expect(err).ToNot(HaveOccurred()) + + dest := storageClient.ExistsArgsForCall(0) + Expect(dest).To(Equal("blob")) + }) + + It("returns blob.NotExisting for not existing blobs", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.ExistsReturns(false, nil) + + azBlobstore, _ := client.New(&storageClient) //nolint:errcheck + existsState, err := azBlobstore.Exists("blob") + Expect(existsState == false).To(BeTrue()) + Expect(err).ToNot(HaveOccurred()) + + dest := storageClient.ExistsArgsForCall(0) + Expect(dest).To(Equal("blob")) + }) + + It("returns blob.ExistenceUnknown and an error in case an error occurred", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.ExistsReturns(false, errors.New("boom")) + + azBlobstore, _ := client.New(&storageClient) //nolint:errcheck + existsState, err := azBlobstore.Exists("blob") + Expect(existsState == false).To(BeTrue()) + Expect(err).To(HaveOccurred()) + + dest := storageClient.ExistsArgsForCall(0) + Expect(dest).To(Equal("blob")) + }) + }) + + Context("signed url", func() { + It("returns a signed url for action 'get'", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.SignedUrlReturns("https://the-signed-url", nil) + + azBlobstore, _ := client.New(&storageClient) //nolint:errcheck + url, err := azBlobstore.Sign("blob", "get", 100) + Expect(url == "https://the-signed-url").To(BeTrue()) + Expect(err).ToNot(HaveOccurred()) + + action, dest, expiration := storageClient.SignedUrlArgsForCall(0) + Expect(action).To(Equal("GET")) + Expect(dest).To(Equal("blob")) + Expect(int(expiration)).To(Equal(100)) + }) + + It("fails on unknown action", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.SignedUrlReturns("", errors.New("boom")) + + azBlobstore, _ := client.New(&storageClient) //nolint:errcheck + url, err := azBlobstore.Sign("blob", "unknown", 100) + Expect(url).To(Equal("")) + Expect(err).To(HaveOccurred()) + + Expect(storageClient.SignedUrlCallCount()).To(Equal(0)) + }) + }) + + Context("list", func() { + It("lists blobs in a container", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.ListReturns([]string{"blob1", "blob2"}, nil) + + azBlobstore, _ := client.New(&storageClient) //nolint:errcheck + blobs, err := azBlobstore.List("") + Expect(blobs).To(Equal([]string{"blob1", "blob2"})) + Expect(err).ToNot(HaveOccurred()) + + containerName := storageClient.ListArgsForCall(0) + Expect(containerName).To(Equal("")) + }) + + It("lists blobs with a prefix in a container", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.ListReturns([]string{"pre-blob1", "pre-blob2"}, nil) + + azBlobstore, _ := client.New(&storageClient) //nolint:errcheck + blobs, err := azBlobstore.List("pre-") + Expect(blobs).To(Equal([]string{"pre-blob1", "pre-blob2"})) + Expect(err).ToNot(HaveOccurred()) + + containerName := storageClient.ListArgsForCall(0) + Expect(containerName).To(Equal("pre-")) + }) + + It("returns an error if listing fails", func() { + storageClient := clientfakes.FakeStorageClient{} + storageClient.ListReturns(nil, errors.New("boom")) + + azBlobstore, _ := client.New(&storageClient) //nolint:errcheck + blobs, err := azBlobstore.List("container") + Expect(blobs).To(BeNil()) + Expect(err).To(HaveOccurred()) + + containerName := storageClient.ListArgsForCall(0) + Expect(containerName).To(Equal("container")) + }) + }) + +}) diff --git a/azurebs/client/clientfakes/fake_storage_client.go b/azurebs/client/clientfakes/fake_storage_client.go new file mode 100644 index 0000000..11b8530 --- /dev/null +++ b/azurebs/client/clientfakes/fake_storage_client.go @@ -0,0 +1,801 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package clientfakes + +import ( + "io" + "os" + "sync" + "time" + + "github.com/cloudfoundry/storage-cli/azurebs/client" +) + +type FakeStorageClient struct { + CopyStub func(string, string) error + copyMutex sync.RWMutex + copyArgsForCall []struct { + arg1 string + arg2 string + } + copyReturns struct { + result1 error + } + copyReturnsOnCall map[int]struct { + result1 error + } + DeleteStub func(string) error + deleteMutex sync.RWMutex + deleteArgsForCall []struct { + arg1 string + } + deleteReturns struct { + result1 error + } + deleteReturnsOnCall map[int]struct { + result1 error + } + DeleteRecursiveStub func(string) error + deleteRecursiveMutex sync.RWMutex + deleteRecursiveArgsForCall []struct { + arg1 string + } + deleteRecursiveReturns struct { + result1 error + } + deleteRecursiveReturnsOnCall map[int]struct { + result1 error + } + DownloadStub func(string, *os.File) error + downloadMutex sync.RWMutex + downloadArgsForCall []struct { + arg1 string + arg2 *os.File + } + downloadReturns struct { + result1 error + } + downloadReturnsOnCall map[int]struct { + result1 error + } + EnsureContainerExistsStub func() error + ensureContainerExistsMutex sync.RWMutex + ensureContainerExistsArgsForCall []struct { + } + ensureContainerExistsReturns struct { + result1 error + } + ensureContainerExistsReturnsOnCall map[int]struct { + result1 error + } + ExistsStub func(string) (bool, error) + existsMutex sync.RWMutex + existsArgsForCall []struct { + arg1 string + } + existsReturns struct { + result1 bool + result2 error + } + existsReturnsOnCall map[int]struct { + result1 bool + result2 error + } + ListStub func(string) ([]string, error) + listMutex sync.RWMutex + listArgsForCall []struct { + arg1 string + } + listReturns struct { + result1 []string + result2 error + } + listReturnsOnCall map[int]struct { + result1 []string + result2 error + } + PropertiesStub func(string) error + propertiesMutex sync.RWMutex + propertiesArgsForCall []struct { + arg1 string + } + propertiesReturns struct { + result1 error + } + propertiesReturnsOnCall map[int]struct { + result1 error + } + SignedUrlStub func(string, string, time.Duration) (string, error) + signedUrlMutex sync.RWMutex + signedUrlArgsForCall []struct { + arg1 string + arg2 string + arg3 time.Duration + } + signedUrlReturns struct { + result1 string + result2 error + } + signedUrlReturnsOnCall map[int]struct { + result1 string + result2 error + } + UploadStub func(io.ReadSeekCloser, string) ([]byte, error) + uploadMutex sync.RWMutex + uploadArgsForCall []struct { + arg1 io.ReadSeekCloser + arg2 string + } + uploadReturns struct { + result1 []byte + result2 error + } + uploadReturnsOnCall map[int]struct { + result1 []byte + result2 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeStorageClient) Copy(arg1 string, arg2 string) error { + fake.copyMutex.Lock() + ret, specificReturn := fake.copyReturnsOnCall[len(fake.copyArgsForCall)] + fake.copyArgsForCall = append(fake.copyArgsForCall, struct { + arg1 string + arg2 string + }{arg1, arg2}) + stub := fake.CopyStub + fakeReturns := fake.copyReturns + fake.recordInvocation("Copy", []interface{}{arg1, arg2}) + fake.copyMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) CopyCallCount() int { + fake.copyMutex.RLock() + defer fake.copyMutex.RUnlock() + return len(fake.copyArgsForCall) +} + +func (fake *FakeStorageClient) CopyCalls(stub func(string, string) error) { + fake.copyMutex.Lock() + defer fake.copyMutex.Unlock() + fake.CopyStub = stub +} + +func (fake *FakeStorageClient) CopyArgsForCall(i int) (string, string) { + fake.copyMutex.RLock() + defer fake.copyMutex.RUnlock() + argsForCall := fake.copyArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStorageClient) CopyReturns(result1 error) { + fake.copyMutex.Lock() + defer fake.copyMutex.Unlock() + fake.CopyStub = nil + fake.copyReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) CopyReturnsOnCall(i int, result1 error) { + fake.copyMutex.Lock() + defer fake.copyMutex.Unlock() + fake.CopyStub = nil + if fake.copyReturnsOnCall == nil { + fake.copyReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.copyReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Delete(arg1 string) error { + fake.deleteMutex.Lock() + ret, specificReturn := fake.deleteReturnsOnCall[len(fake.deleteArgsForCall)] + fake.deleteArgsForCall = append(fake.deleteArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.DeleteStub + fakeReturns := fake.deleteReturns + fake.recordInvocation("Delete", []interface{}{arg1}) + fake.deleteMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) DeleteCallCount() int { + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + return len(fake.deleteArgsForCall) +} + +func (fake *FakeStorageClient) DeleteCalls(stub func(string) error) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = stub +} + +func (fake *FakeStorageClient) DeleteArgsForCall(i int) string { + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + argsForCall := fake.deleteArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) DeleteReturns(result1 error) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = nil + fake.deleteReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) DeleteReturnsOnCall(i int, result1 error) { + fake.deleteMutex.Lock() + defer fake.deleteMutex.Unlock() + fake.DeleteStub = nil + if fake.deleteReturnsOnCall == nil { + fake.deleteReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) DeleteRecursive(arg1 string) error { + fake.deleteRecursiveMutex.Lock() + ret, specificReturn := fake.deleteRecursiveReturnsOnCall[len(fake.deleteRecursiveArgsForCall)] + fake.deleteRecursiveArgsForCall = append(fake.deleteRecursiveArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.DeleteRecursiveStub + fakeReturns := fake.deleteRecursiveReturns + fake.recordInvocation("DeleteRecursive", []interface{}{arg1}) + fake.deleteRecursiveMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) DeleteRecursiveCallCount() int { + fake.deleteRecursiveMutex.RLock() + defer fake.deleteRecursiveMutex.RUnlock() + return len(fake.deleteRecursiveArgsForCall) +} + +func (fake *FakeStorageClient) DeleteRecursiveCalls(stub func(string) error) { + fake.deleteRecursiveMutex.Lock() + defer fake.deleteRecursiveMutex.Unlock() + fake.DeleteRecursiveStub = stub +} + +func (fake *FakeStorageClient) DeleteRecursiveArgsForCall(i int) string { + fake.deleteRecursiveMutex.RLock() + defer fake.deleteRecursiveMutex.RUnlock() + argsForCall := fake.deleteRecursiveArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) DeleteRecursiveReturns(result1 error) { + fake.deleteRecursiveMutex.Lock() + defer fake.deleteRecursiveMutex.Unlock() + fake.DeleteRecursiveStub = nil + fake.deleteRecursiveReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) DeleteRecursiveReturnsOnCall(i int, result1 error) { + fake.deleteRecursiveMutex.Lock() + defer fake.deleteRecursiveMutex.Unlock() + fake.DeleteRecursiveStub = nil + if fake.deleteRecursiveReturnsOnCall == nil { + fake.deleteRecursiveReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.deleteRecursiveReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Download(arg1 string, arg2 *os.File) error { + fake.downloadMutex.Lock() + ret, specificReturn := fake.downloadReturnsOnCall[len(fake.downloadArgsForCall)] + fake.downloadArgsForCall = append(fake.downloadArgsForCall, struct { + arg1 string + arg2 *os.File + }{arg1, arg2}) + stub := fake.DownloadStub + fakeReturns := fake.downloadReturns + fake.recordInvocation("Download", []interface{}{arg1, arg2}) + fake.downloadMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) DownloadCallCount() int { + fake.downloadMutex.RLock() + defer fake.downloadMutex.RUnlock() + return len(fake.downloadArgsForCall) +} + +func (fake *FakeStorageClient) DownloadCalls(stub func(string, *os.File) error) { + fake.downloadMutex.Lock() + defer fake.downloadMutex.Unlock() + fake.DownloadStub = stub +} + +func (fake *FakeStorageClient) DownloadArgsForCall(i int) (string, *os.File) { + fake.downloadMutex.RLock() + defer fake.downloadMutex.RUnlock() + argsForCall := fake.downloadArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStorageClient) DownloadReturns(result1 error) { + fake.downloadMutex.Lock() + defer fake.downloadMutex.Unlock() + fake.DownloadStub = nil + fake.downloadReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) DownloadReturnsOnCall(i int, result1 error) { + fake.downloadMutex.Lock() + defer fake.downloadMutex.Unlock() + fake.DownloadStub = nil + if fake.downloadReturnsOnCall == nil { + fake.downloadReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.downloadReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) EnsureContainerExists() error { + fake.ensureContainerExistsMutex.Lock() + ret, specificReturn := fake.ensureContainerExistsReturnsOnCall[len(fake.ensureContainerExistsArgsForCall)] + fake.ensureContainerExistsArgsForCall = append(fake.ensureContainerExistsArgsForCall, struct { + }{}) + stub := fake.EnsureContainerExistsStub + fakeReturns := fake.ensureContainerExistsReturns + fake.recordInvocation("EnsureContainerExists", []interface{}{}) + fake.ensureContainerExistsMutex.Unlock() + if stub != nil { + return stub() + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) EnsureContainerExistsCallCount() int { + fake.ensureContainerExistsMutex.RLock() + defer fake.ensureContainerExistsMutex.RUnlock() + return len(fake.ensureContainerExistsArgsForCall) +} + +func (fake *FakeStorageClient) EnsureContainerExistsCalls(stub func() error) { + fake.ensureContainerExistsMutex.Lock() + defer fake.ensureContainerExistsMutex.Unlock() + fake.EnsureContainerExistsStub = stub +} + +func (fake *FakeStorageClient) EnsureContainerExistsReturns(result1 error) { + fake.ensureContainerExistsMutex.Lock() + defer fake.ensureContainerExistsMutex.Unlock() + fake.EnsureContainerExistsStub = nil + fake.ensureContainerExistsReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) EnsureContainerExistsReturnsOnCall(i int, result1 error) { + fake.ensureContainerExistsMutex.Lock() + defer fake.ensureContainerExistsMutex.Unlock() + fake.EnsureContainerExistsStub = nil + if fake.ensureContainerExistsReturnsOnCall == nil { + fake.ensureContainerExistsReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.ensureContainerExistsReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) Exists(arg1 string) (bool, error) { + fake.existsMutex.Lock() + ret, specificReturn := fake.existsReturnsOnCall[len(fake.existsArgsForCall)] + fake.existsArgsForCall = append(fake.existsArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.ExistsStub + fakeReturns := fake.existsReturns + fake.recordInvocation("Exists", []interface{}{arg1}) + fake.existsMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) ExistsCallCount() int { + fake.existsMutex.RLock() + defer fake.existsMutex.RUnlock() + return len(fake.existsArgsForCall) +} + +func (fake *FakeStorageClient) ExistsCalls(stub func(string) (bool, error)) { + fake.existsMutex.Lock() + defer fake.existsMutex.Unlock() + fake.ExistsStub = stub +} + +func (fake *FakeStorageClient) ExistsArgsForCall(i int) string { + fake.existsMutex.RLock() + defer fake.existsMutex.RUnlock() + argsForCall := fake.existsArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) ExistsReturns(result1 bool, result2 error) { + fake.existsMutex.Lock() + defer fake.existsMutex.Unlock() + fake.ExistsStub = nil + fake.existsReturns = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) ExistsReturnsOnCall(i int, result1 bool, result2 error) { + fake.existsMutex.Lock() + defer fake.existsMutex.Unlock() + fake.ExistsStub = nil + if fake.existsReturnsOnCall == nil { + fake.existsReturnsOnCall = make(map[int]struct { + result1 bool + result2 error + }) + } + fake.existsReturnsOnCall[i] = struct { + result1 bool + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) List(arg1 string) ([]string, error) { + fake.listMutex.Lock() + ret, specificReturn := fake.listReturnsOnCall[len(fake.listArgsForCall)] + fake.listArgsForCall = append(fake.listArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.ListStub + fakeReturns := fake.listReturns + fake.recordInvocation("List", []interface{}{arg1}) + fake.listMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) ListCallCount() int { + fake.listMutex.RLock() + defer fake.listMutex.RUnlock() + return len(fake.listArgsForCall) +} + +func (fake *FakeStorageClient) ListCalls(stub func(string) ([]string, error)) { + fake.listMutex.Lock() + defer fake.listMutex.Unlock() + fake.ListStub = stub +} + +func (fake *FakeStorageClient) ListArgsForCall(i int) string { + fake.listMutex.RLock() + defer fake.listMutex.RUnlock() + argsForCall := fake.listArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) ListReturns(result1 []string, result2 error) { + fake.listMutex.Lock() + defer fake.listMutex.Unlock() + fake.ListStub = nil + fake.listReturns = struct { + result1 []string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) ListReturnsOnCall(i int, result1 []string, result2 error) { + fake.listMutex.Lock() + defer fake.listMutex.Unlock() + fake.ListStub = nil + if fake.listReturnsOnCall == nil { + fake.listReturnsOnCall = make(map[int]struct { + result1 []string + result2 error + }) + } + fake.listReturnsOnCall[i] = struct { + result1 []string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) Properties(arg1 string) error { + fake.propertiesMutex.Lock() + ret, specificReturn := fake.propertiesReturnsOnCall[len(fake.propertiesArgsForCall)] + fake.propertiesArgsForCall = append(fake.propertiesArgsForCall, struct { + arg1 string + }{arg1}) + stub := fake.PropertiesStub + fakeReturns := fake.propertiesReturns + fake.recordInvocation("Properties", []interface{}{arg1}) + fake.propertiesMutex.Unlock() + if stub != nil { + return stub(arg1) + } + if specificReturn { + return ret.result1 + } + return fakeReturns.result1 +} + +func (fake *FakeStorageClient) PropertiesCallCount() int { + fake.propertiesMutex.RLock() + defer fake.propertiesMutex.RUnlock() + return len(fake.propertiesArgsForCall) +} + +func (fake *FakeStorageClient) PropertiesCalls(stub func(string) error) { + fake.propertiesMutex.Lock() + defer fake.propertiesMutex.Unlock() + fake.PropertiesStub = stub +} + +func (fake *FakeStorageClient) PropertiesArgsForCall(i int) string { + fake.propertiesMutex.RLock() + defer fake.propertiesMutex.RUnlock() + argsForCall := fake.propertiesArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeStorageClient) PropertiesReturns(result1 error) { + fake.propertiesMutex.Lock() + defer fake.propertiesMutex.Unlock() + fake.PropertiesStub = nil + fake.propertiesReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) PropertiesReturnsOnCall(i int, result1 error) { + fake.propertiesMutex.Lock() + defer fake.propertiesMutex.Unlock() + fake.PropertiesStub = nil + if fake.propertiesReturnsOnCall == nil { + fake.propertiesReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.propertiesReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeStorageClient) SignedUrl(arg1 string, arg2 string, arg3 time.Duration) (string, error) { + fake.signedUrlMutex.Lock() + ret, specificReturn := fake.signedUrlReturnsOnCall[len(fake.signedUrlArgsForCall)] + fake.signedUrlArgsForCall = append(fake.signedUrlArgsForCall, struct { + arg1 string + arg2 string + arg3 time.Duration + }{arg1, arg2, arg3}) + stub := fake.SignedUrlStub + fakeReturns := fake.signedUrlReturns + fake.recordInvocation("SignedUrl", []interface{}{arg1, arg2, arg3}) + fake.signedUrlMutex.Unlock() + if stub != nil { + return stub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) SignedUrlCallCount() int { + fake.signedUrlMutex.RLock() + defer fake.signedUrlMutex.RUnlock() + return len(fake.signedUrlArgsForCall) +} + +func (fake *FakeStorageClient) SignedUrlCalls(stub func(string, string, time.Duration) (string, error)) { + fake.signedUrlMutex.Lock() + defer fake.signedUrlMutex.Unlock() + fake.SignedUrlStub = stub +} + +func (fake *FakeStorageClient) SignedUrlArgsForCall(i int) (string, string, time.Duration) { + fake.signedUrlMutex.RLock() + defer fake.signedUrlMutex.RUnlock() + argsForCall := fake.signedUrlArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeStorageClient) SignedUrlReturns(result1 string, result2 error) { + fake.signedUrlMutex.Lock() + defer fake.signedUrlMutex.Unlock() + fake.SignedUrlStub = nil + fake.signedUrlReturns = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) SignedUrlReturnsOnCall(i int, result1 string, result2 error) { + fake.signedUrlMutex.Lock() + defer fake.signedUrlMutex.Unlock() + fake.SignedUrlStub = nil + if fake.signedUrlReturnsOnCall == nil { + fake.signedUrlReturnsOnCall = make(map[int]struct { + result1 string + result2 error + }) + } + fake.signedUrlReturnsOnCall[i] = struct { + result1 string + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) Upload(arg1 io.ReadSeekCloser, arg2 string) ([]byte, error) { + fake.uploadMutex.Lock() + ret, specificReturn := fake.uploadReturnsOnCall[len(fake.uploadArgsForCall)] + fake.uploadArgsForCall = append(fake.uploadArgsForCall, struct { + arg1 io.ReadSeekCloser + arg2 string + }{arg1, arg2}) + stub := fake.UploadStub + fakeReturns := fake.uploadReturns + fake.recordInvocation("Upload", []interface{}{arg1, arg2}) + fake.uploadMutex.Unlock() + if stub != nil { + return stub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeStorageClient) UploadCallCount() int { + fake.uploadMutex.RLock() + defer fake.uploadMutex.RUnlock() + return len(fake.uploadArgsForCall) +} + +func (fake *FakeStorageClient) UploadCalls(stub func(io.ReadSeekCloser, string) ([]byte, error)) { + fake.uploadMutex.Lock() + defer fake.uploadMutex.Unlock() + fake.UploadStub = stub +} + +func (fake *FakeStorageClient) UploadArgsForCall(i int) (io.ReadSeekCloser, string) { + fake.uploadMutex.RLock() + defer fake.uploadMutex.RUnlock() + argsForCall := fake.uploadArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeStorageClient) UploadReturns(result1 []byte, result2 error) { + fake.uploadMutex.Lock() + defer fake.uploadMutex.Unlock() + fake.UploadStub = nil + fake.uploadReturns = struct { + result1 []byte + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) UploadReturnsOnCall(i int, result1 []byte, result2 error) { + fake.uploadMutex.Lock() + defer fake.uploadMutex.Unlock() + fake.UploadStub = nil + if fake.uploadReturnsOnCall == nil { + fake.uploadReturnsOnCall = make(map[int]struct { + result1 []byte + result2 error + }) + } + fake.uploadReturnsOnCall[i] = struct { + result1 []byte + result2 error + }{result1, result2} +} + +func (fake *FakeStorageClient) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.copyMutex.RLock() + defer fake.copyMutex.RUnlock() + fake.deleteMutex.RLock() + defer fake.deleteMutex.RUnlock() + fake.deleteRecursiveMutex.RLock() + defer fake.deleteRecursiveMutex.RUnlock() + fake.downloadMutex.RLock() + defer fake.downloadMutex.RUnlock() + fake.ensureContainerExistsMutex.RLock() + defer fake.ensureContainerExistsMutex.RUnlock() + fake.existsMutex.RLock() + defer fake.existsMutex.RUnlock() + fake.listMutex.RLock() + defer fake.listMutex.RUnlock() + fake.propertiesMutex.RLock() + defer fake.propertiesMutex.RUnlock() + fake.signedUrlMutex.RLock() + defer fake.signedUrlMutex.RUnlock() + fake.uploadMutex.RLock() + defer fake.uploadMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeStorageClient) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ client.StorageClient = new(FakeStorageClient) diff --git a/azurebs/client/storage_client.go b/azurebs/client/storage_client.go new file mode 100644 index 0000000..02f9f28 --- /dev/null +++ b/azurebs/client/storage_client.go @@ -0,0 +1,428 @@ +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "os" + "strconv" + "strings" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" + + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + azBlob "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blockblob" + azContainer "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/container" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/sas" + + "github.com/cloudfoundry/storage-cli/azurebs/config" +) + +//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 . StorageClient +type StorageClient interface { + Upload( + source io.ReadSeekCloser, + dest string, + ) ([]byte, error) + + Download( + source string, + dest *os.File, + ) error + + Copy( + srcBlob string, + destBlob string, + ) error + + Delete( + dest string, + ) error + + DeleteRecursive( + dest string, + ) error + + Exists( + dest string, + ) (bool, error) + + SignedUrl( + requestType string, + dest string, + expiration time.Duration, + ) (string, error) + + List( + prefix string, + ) ([]string, error) + Properties( + dest string, + ) error + EnsureContainerExists() error +} + +type DefaultStorageClient struct { + credential *azblob.SharedKeyCredential + serviceURL string + storageConfig config.AZStorageConfig +} + +func NewStorageClient(storageConfig config.AZStorageConfig) (StorageClient, error) { + credential, err := azblob.NewSharedKeyCredential(storageConfig.AccountName, storageConfig.AccountKey) + if err != nil { + return nil, err + } + + serviceURL := fmt.Sprintf("https://%s.%s/%s", storageConfig.AccountName, storageConfig.StorageEndpoint(), storageConfig.ContainerName) + + return DefaultStorageClient{credential: credential, serviceURL: serviceURL, storageConfig: storageConfig}, nil +} + +func (dsc DefaultStorageClient) Upload( + source io.ReadSeekCloser, + dest string, +) ([]byte, error) { + blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest) + + var ctx context.Context + var cancel context.CancelFunc + + if dsc.storageConfig.Timeout != "" { + timeoutInt, err := strconv.Atoi(dsc.storageConfig.Timeout) + timeout := time.Duration(timeoutInt) * time.Second + if timeout < 1 && err == nil { + log.Printf("Invalid time \"%s\", need at least 1 second", dsc.storageConfig.Timeout) + return nil, fmt.Errorf("invalid time: %w", err) + } + if err != nil { + log.Printf("Invalid timeout format \"%s\", need \"\" e.g. 30", dsc.storageConfig.Timeout) + return nil, fmt.Errorf("invalid timeout format: %w", err) + } + log.Println(fmt.Sprintf("Uploading %s with a timeout of %s", blobURL, timeout)) //nolint:staticcheck + ctx, cancel = context.WithTimeout(context.Background(), timeout) + } else { + log.Println(fmt.Sprintf("Uploading %s with no timeout", blobURL)) //nolint:staticcheck + ctx, cancel = context.WithCancel(context.Background()) + } + defer cancel() + + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + if err != nil { + return nil, err + } + uploadResponse, err := client.Upload(ctx, source, nil) + if err != nil { + if dsc.storageConfig.Timeout != "" && errors.Is(err, context.DeadlineExceeded) { + return nil, fmt.Errorf("upload failed: timeout of %s reached while uploading %s", dsc.storageConfig.Timeout, dest) + } + return nil, fmt.Errorf("upload failure: %w", err) + } + return uploadResponse.ContentMD5, err +} + +func (dsc DefaultStorageClient) Download( + source string, + dest *os.File, +) error { + + blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, source) + + log.Println(fmt.Sprintf("Downloading %s", blobURL)) //nolint:staticcheck + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + if err != nil { + return err + } + + blobSize, err := client.DownloadFile(context.Background(), dest, nil) //nolint:ineffassign,staticcheck + if err != nil { + return err + } + info, err := dest.Stat() + if err != nil { + return err + } + if blobSize != info.Size() { + log.Printf("Truncating file according to the blob size %v", blobSize) + dest.Truncate(blobSize) //nolint:errcheck + } + + return nil +} + +func (dsc DefaultStorageClient) Copy( + srcBlob string, + destBlob string, +) error { + log.Printf("Copying blob from %s to %s", srcBlob, destBlob) + + srcURL := fmt.Sprintf("%s/%s", dsc.serviceURL, srcBlob) + destURL := fmt.Sprintf("%s/%s", dsc.serviceURL, destBlob) + + destClient, err := blockblob.NewClientWithSharedKeyCredential(destURL, dsc.credential, nil) + if err != nil { + return fmt.Errorf("failed to create destination client: %w", err) + } + + resp, err := destClient.StartCopyFromURL(context.Background(), srcURL, nil) + if err != nil { + return fmt.Errorf("failed to start copy: %w", err) + } + + copyID := *resp.CopyID + log.Printf("Copy started with CopyID: %s", copyID) + + // Wait for completion + for { + props, err := destClient.GetProperties(context.Background(), nil) + if err != nil { + return fmt.Errorf("failed to get properties: %w", err) + } + + copyStatus := *props.CopyStatus + log.Printf("Copy status: %s", copyStatus) + + switch copyStatus { + case "success": + log.Println("Copy completed successfully") + return nil + case "pending": + time.Sleep(200 * time.Millisecond) + default: + return fmt.Errorf("copy failed or aborted with status: %s", copyStatus) + } + } +} + +func (dsc DefaultStorageClient) Delete( + dest string, +) error { + + blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest) + + log.Println(fmt.Sprintf("Deleting %s", blobURL)) //nolint:staticcheck + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + if err != nil { + return err + } + + _, err = client.Delete(context.Background(), nil) + + if err == nil { + return nil + } + + if strings.Contains(err.Error(), "RESPONSE 404") { + return nil + } + + return err +} + +func (dsc DefaultStorageClient) DeleteRecursive( + prefix string, +) error { + if prefix != "" { + log.Printf("Deleting all blobs in container %s with prefix '%s'\n", dsc.storageConfig.ContainerName, prefix) + } else { + log.Printf("Deleting all blobs in container %s\n", dsc.storageConfig.ContainerName) + } + + containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, nil) + if err != nil { + return fmt.Errorf("failed to create container client: %w", err) + } + + options := &azContainer.ListBlobsFlatOptions{} + if prefix != "" { + options.Prefix = &prefix + } + + pager := containerClient.NewListBlobsFlatPager(options) + + for pager.More() { + resp, err := pager.NextPage(context.Background()) + if err != nil { + return fmt.Errorf("error retrieving page of blobs: %w", err) + } + + for _, blob := range resp.Segment.BlobItems { + blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, *blob.Name) + blobClient, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + if err != nil { + log.Printf("Failed to create blob client for %s: %v\n", *blob.Name, err) + continue + } + + _, err = blobClient.BlobClient().Delete(context.Background(), nil) + if err != nil && !strings.Contains(err.Error(), "RESPONSE 404") { + log.Printf("Failed to delete blob %s: %v\n", *blob.Name, err) + } + } + } + + return nil +} + +func (dsc DefaultStorageClient) Exists( + dest string, +) (bool, error) { + + blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest) + + log.Println(fmt.Sprintf("Checking if blob: %s exists", blobURL)) //nolint:staticcheck + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + if err != nil { + return false, err + } + + _, err = client.BlobClient().GetProperties(context.Background(), nil) + if err == nil { + log.Printf("File '%s' exists in bucket '%s'\n", dest, dsc.storageConfig.ContainerName) + return true, nil + } + if strings.Contains(err.Error(), "RESPONSE 404") { + log.Printf("File '%s' does not exist in bucket '%s'\n", dest, dsc.storageConfig.ContainerName) + return false, nil + } + + return false, err +} + +func (dsc DefaultStorageClient) SignedUrl( + requestType string, + dest string, + expiration time.Duration, +) (string, error) { + + blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest) + + log.Println(fmt.Sprintf("Getting signed url for blob %s", blobURL)) //nolint:staticcheck + client, err := azBlob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + if err != nil { + return "", err + } + + url, err := client.GetSASURL(sas.BlobPermissions{Read: true, Create: true}, time.Now().Add(expiration), nil) + if err != nil { + return "", err + } + + // There could be occasional issues with the Azure Storage Account when requests hitting + // the server are not responded to, and then BOSH hangs while expecting a reply from the server. + // That's why we implement a server-side timeout here (30 mins for GET and 45 mins for PUT) + // (see: https://learn.microsoft.com/en-us/rest/api/storageservices/setting-timeouts-for-blob-service-operations) + if requestType == "GET" { + url += "&timeout=1800" + } else { + url += "&timeout=2700" + } + + return url, err +} + +func (dsc DefaultStorageClient) List( + prefix string, +) ([]string, error) { + + if prefix != "" { + log.Println(fmt.Sprintf("Listing blobs in container %s with prefix '%s'", dsc.storageConfig.ContainerName, prefix)) //nolint:staticcheck + } else { + log.Println(fmt.Sprintf("Listing blobs in container %s", dsc.storageConfig.ContainerName)) //nolint:staticcheck + } + + client, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, nil) + if err != nil { + return nil, fmt.Errorf("failed to create container client: %w", err) + } + + options := &azContainer.ListBlobsFlatOptions{} + if prefix != "" { + options.Prefix = &prefix + } + + pager := client.NewListBlobsFlatPager(options) + var blobs []string + + for pager.More() { + resp, err := pager.NextPage(context.Background()) + if err != nil { + return nil, fmt.Errorf("error retrieving page of blobs: %w", err) + } + + for _, blob := range resp.Segment.BlobItems { + blobs = append(blobs, *blob.Name) + } + } + + return blobs, nil +} + +type BlobProperties struct { + ETag string `json:"etag,omitempty"` + LastModified time.Time `json:"last_modified,omitempty"` + ContentLength int64 `json:"content_length,omitempty"` +} + +func (dsc DefaultStorageClient) Properties( + dest string, +) error { + blobURL := fmt.Sprintf("%s/%s", dsc.serviceURL, dest) + + log.Println(fmt.Sprintf("Getting properties for blob %s", blobURL)) //nolint:staticcheck + client, err := blockblob.NewClientWithSharedKeyCredential(blobURL, dsc.credential, nil) + if err != nil { + return err + } + + resp, err := client.GetProperties(context.Background(), nil) + if err != nil { + if strings.Contains(err.Error(), "RESPONSE 404") { + fmt.Println(`{}`) + return nil + } + return fmt.Errorf("failed to get properties for blob %s: %w", dest, err) + } + + props := BlobProperties{ + ETag: strings.Trim(string(*resp.ETag), `"`), + LastModified: *resp.LastModified, + ContentLength: *resp.ContentLength, + } + + output, err := json.MarshalIndent(props, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal blob properties: %w", err) + } + + fmt.Println(string(output)) + return nil +} + +func (dsc DefaultStorageClient) EnsureContainerExists() error { + log.Printf("Ensuring container '%s' exists\n", dsc.storageConfig.ContainerName) + + containerClient, err := azContainer.NewClientWithSharedKeyCredential(dsc.serviceURL, dsc.credential, nil) + if err != nil { + return fmt.Errorf("failed to create container client: %w", err) + } + + _, err = containerClient.Create(context.Background(), nil) + if err != nil { + var respErr *azcore.ResponseError + if errors.As(err, &respErr) && respErr.ErrorCode == string(bloberror.ContainerAlreadyExists) { + log.Printf("Container '%s' already exists", dsc.storageConfig.ContainerName) + return nil + } + return fmt.Errorf("failed to create container: %w", err) + } + + log.Printf("Container '%s' created successfully", dsc.storageConfig.ContainerName) + return nil +} diff --git a/azurebs/config/config.go b/azurebs/config/config.go new file mode 100644 index 0000000..1407094 --- /dev/null +++ b/azurebs/config/config.go @@ -0,0 +1,76 @@ +package config + +import ( + "encoding/json" + "errors" + "io" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/cloud" +) + +const storage cloud.ServiceName = "storage" + +var cloudConfig cloud.Configuration + +func init() { + // Configure the cloud endpoints for the storage service + // as the SDK does not have a configuration for it + cloud.AzurePublic.Services[storage] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.windows.net", + } + cloud.AzureChina.Services[storage] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.chinacloudapi.cn", + } + cloud.AzureGovernment.Services[storage] = cloud.ServiceConfiguration{ + Endpoint: "blob.core.usgovcloudapi.net", + } +} + +type AZStorageConfig struct { + AccountName string `json:"account_name"` + AccountKey string `json:"account_key"` + ContainerName string `json:"container_name"` + Environment string `json:"environment"` + Timeout string `json:"put_timeout_in_seconds"` +} + +// NewFromReader returns a new azure-storage-cli configuration struct from the contents of reader. +// reader.Read() is expected to return valid JSON +func NewFromReader(reader io.Reader) (AZStorageConfig, error) { + bytes, err := io.ReadAll(reader) + if err != nil { + return AZStorageConfig{}, err + } + config := AZStorageConfig{} + + err = json.Unmarshal(bytes, &config) + if err != nil { + return AZStorageConfig{}, err + } + + err = config.configureCloud() + if err != nil { + return AZStorageConfig{}, err + } + + return config, nil +} + +func (c AZStorageConfig) StorageEndpoint() string { + return cloudConfig.Services[storage].Endpoint +} + +func (c *AZStorageConfig) configureCloud() error { + switch c.Environment { + case "AzureCloud", "": + c.Environment = "AzureCloud" + cloudConfig = cloud.AzurePublic + case "AzureChinaCloud": + cloudConfig = cloud.AzureChina + case "AzureUSGovernment": + cloudConfig = cloud.AzureGovernment + default: + return errors.New("unknown cloud environment: " + c.Environment) + } + return nil +} diff --git a/azurebs/config/config_suite_test.go b/azurebs/config/config_suite_test.go new file mode 100644 index 0000000..c6e29ba --- /dev/null +++ b/azurebs/config/config_suite_test.go @@ -0,0 +1,13 @@ +package config_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Config Suite") +} diff --git a/azurebs/config/config_test.go b/azurebs/config/config_test.go new file mode 100644 index 0000000..9f54c4f --- /dev/null +++ b/azurebs/config/config_test.go @@ -0,0 +1,96 @@ +package config_test + +import ( + "bytes" + "errors" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/cloudfoundry/storage-cli/azurebs/config" +) + +var _ = Describe("Config", func() { + + It("contains account-name and account-name", func() { + configJson := []byte(`{"account_name": "foo-account-name", + "account_key": "bar-account-key", + "container_name": "baz-container-name"}`) + configReader := bytes.NewReader(configJson) + + config, err := config.NewFromReader(configReader) + + Expect(err).ToNot(HaveOccurred()) + Expect(config.AccountName).To(Equal("foo-account-name")) + Expect(config.AccountKey).To(Equal("bar-account-key")) + Expect(config.ContainerName).To(Equal("baz-container-name")) + Expect(config.Environment).To(Equal("AzureCloud")) + Expect(config.StorageEndpoint()).To(Equal("blob.core.windows.net")) + }) + + It("is empty if config cannot be parsed", func() { + configJson := []byte(`~`) + configReader := bytes.NewReader(configJson) + + config, err := config.NewFromReader(configReader) + + Expect(err.Error()).To(Equal("invalid character '~' looking for beginning of value")) + Expect(config.AccountName).Should(BeEmpty()) + Expect(config.AccountKey).Should(BeEmpty()) + }) + + Context("when the configuration file cannot be read", func() { + It("returns an error", func() { + f := explodingReader{} + + _, err := config.NewFromReader(f) + Expect(err).To(MatchError("explosion")) + }) + }) + + Context("environment", func() { + When("environment is invalid", func() { + It("returns an error", func() { + configJson := []byte(`{"environment": "invalid-cloud"}`) + configReader := bytes.NewReader(configJson) + + config, err := config.NewFromReader(configReader) + + Expect(err.Error()).To(Equal("unknown cloud environment: invalid-cloud")) + Expect(config.Environment).Should(BeEmpty()) + }) + }) + + When("environment is AzureChinaCloud", func() { + It("sets the endpoint for china", func() { + configJson := []byte(`{"environment": "AzureChinaCloud"}`) + configReader := bytes.NewReader(configJson) + + config, err := config.NewFromReader(configReader) + + Expect(err).ToNot(HaveOccurred()) + Expect(config.Environment).To(Equal("AzureChinaCloud")) + Expect(config.StorageEndpoint()).To(Equal("blob.core.chinacloudapi.cn")) + }) + }) + + When("environment is AzureUSGovernment", func() { + It("sets the endpoint for usgovernment", func() { + configJson := []byte(`{"environment": "AzureUSGovernment"}`) + configReader := bytes.NewReader(configJson) + + config, err := config.NewFromReader(configReader) + + Expect(err).ToNot(HaveOccurred()) + Expect(config.Environment).To(Equal("AzureUSGovernment")) + Expect(config.StorageEndpoint()).To(Equal("blob.core.usgovcloudapi.net")) + }) + }) + }) +}) + +type explodingReader struct{} + +func (e explodingReader) Read([]byte) (int, error) { + return 0, errors.New("explosion") +} diff --git a/azurebs/integration/assertions.go b/azurebs/integration/assertions.go new file mode 100644 index 0000000..5baba9e --- /dev/null +++ b/azurebs/integration/assertions.go @@ -0,0 +1,414 @@ +package integration + +import ( + "bytes" + "os" + + "github.com/cloudfoundry/storage-cli/azurebs/config" + + . "github.com/onsi/gomega" //nolint:staticcheck +) + +func AssertPutUsesNoTimeout(cliPath string, cfg *config.AZStorageConfig) { + cfg2 := *cfg + cfg2.Timeout = "" // unset -> no timeout + configPath := MakeConfigFile(&cfg2) + defer os.Remove(configPath) //nolint:errcheck + + content := MakeContentFile("hello") + defer os.Remove(content) //nolint:errcheck + blob := GenerateRandomString() + + sess, err := RunCli(cliPath, configPath, "put", content, blob) + Expect(err).ToNot(HaveOccurred()) + Expect(sess.ExitCode()).To(BeZero()) + Expect(string(sess.Err.Contents())).To(ContainSubstring("Uploading ")) // stderr has log.Println + Expect(string(sess.Err.Contents())).To(ContainSubstring("with no timeout")) + + sess, err = RunCli(cliPath, configPath, "delete", blob) + Expect(err).ToNot(HaveOccurred()) + Expect(sess.ExitCode()).To(BeZero()) +} + +func AssertPutHonorsCustomTimeout(cliPath string, cfg *config.AZStorageConfig) { + cfg2 := *cfg + cfg2.Timeout = "3" + configPath := MakeConfigFile(&cfg2) + defer os.Remove(configPath) //nolint:errcheck + + content := MakeContentFile("ok") + defer os.Remove(content) //nolint:errcheck + blob := GenerateRandomString() + + sess, err := RunCli(cliPath, configPath, "put", content, blob) + Expect(err).ToNot(HaveOccurred()) + Expect(sess.ExitCode()).To(BeZero()) + Expect(string(sess.Err.Contents())).To(ContainSubstring("with a timeout of 3s")) + + sess, err = RunCli(cliPath, configPath, "delete", blob) + Expect(err).ToNot(HaveOccurred()) + Expect(sess.ExitCode()).To(BeZero()) +} + +func AssertPutTimesOut(cliPath string, cfg *config.AZStorageConfig) { + cfg2 := *cfg + cfg2.Timeout = "1" + configPath := MakeConfigFile(&cfg2) + defer os.Remove(configPath) //nolint:errcheck + + const mb = 1024 * 1024 + big := bytes.Repeat([]byte("x"), 25*mb) + content := MakeContentFile(string(big)) + defer os.Remove(content) //nolint:errcheck + blob := GenerateRandomString() + + sess, err := RunCli(cliPath, configPath, "put", content, blob) + Expect(err).ToNot(HaveOccurred()) + Expect(sess.ExitCode()).ToNot(BeZero()) + Expect(string(sess.Err.Contents())).To(ContainSubstring("timeout of 1 reached while uploading")) +} + +func AssertInvalidTimeoutIsError(cliPath string, cfg *config.AZStorageConfig) { + cfg2 := *cfg + cfg2.Timeout = "bananas" + configPath := MakeConfigFile(&cfg2) + defer os.Remove(configPath) //nolint:errcheck + + content := MakeContentFile("x") + defer os.Remove(content) //nolint:errcheck + blob := GenerateRandomString() + + sess, err := RunCli(cliPath, configPath, "put", content, blob) + Expect(err).ToNot(HaveOccurred()) + Expect(sess.ExitCode()).ToNot(BeZero()) + Expect(string(sess.Err.Contents())).To(ContainSubstring(`Invalid timeout format "bananas"`)) +} + +func AssertZeroTimeoutIsError(cliPath string, cfg *config.AZStorageConfig) { + cfg2 := *cfg + cfg2.Timeout = "0" + configPath := MakeConfigFile(&cfg2) + defer os.Remove(configPath) //nolint:errcheck + + content := MakeContentFile("x") + defer os.Remove(content) //nolint:errcheck + blob := GenerateRandomString() + + sess, err := RunCli(cliPath, configPath, "put", content, blob) + Expect(err).ToNot(HaveOccurred()) + Expect(sess.ExitCode()).ToNot(BeZero()) + + Expect(string(sess.Err.Contents())).To(ContainSubstring(`Invalid time "0", need at least 1 second`)) +} + +func AssertNegativeTimeoutIsError(cliPath string, cfg *config.AZStorageConfig) { + cfg2 := *cfg + cfg2.Timeout = "-1" + configPath := MakeConfigFile(&cfg2) + defer os.Remove(configPath) //nolint:errcheck + + content := MakeContentFile("y") + defer os.Remove(content) //nolint:errcheck + blob := GenerateRandomString() + + sess, err := RunCli(cliPath, configPath, "put", content, blob) + Expect(err).ToNot(HaveOccurred()) + Expect(sess.ExitCode()).ToNot(BeZero()) + + Expect(string(sess.Err.Contents())).To(ContainSubstring(`Invalid time "-1", need at least 1 second`)) +} + +func AssertSignedURLTimeouts(cliPath string, cfg *config.AZStorageConfig) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + sess, err := RunCli(cliPath, configPath, "sign", "some-blob", "get", "60s") + Expect(err).ToNot(HaveOccurred()) + url := string(sess.Out.Contents()) + Expect(url).To(ContainSubstring("timeout=1800")) + + sess, err = RunCli(cliPath, configPath, "sign", "some-blob", "put", "60s") + Expect(err).ToNot(HaveOccurred()) + url = string(sess.Out.Contents()) + Expect(url).To(ContainSubstring("timeout=2700")) +} + +func AssertEnsureBucketIdempotent(cliPath string, cfg *config.AZStorageConfig) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + s1, err := RunCli(cliPath, configPath, "ensure-bucket-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(s1.ExitCode()).To(BeZero()) + + s2, err := RunCli(cliPath, configPath, "ensure-bucket-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(s2.ExitCode()).To(BeZero()) +} + +func AssertPutGetWithSpecialNames(cliPath string, cfg *config.AZStorageConfig) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + name := "dir a/üñîçødë file.txt" + content := "weird name content" + f := MakeContentFile(content) + defer os.Remove(f) //nolint:errcheck + + s, err := RunCli(cliPath, configPath, "put", f, name) + Expect(err).ToNot(HaveOccurred()) + Expect(s.ExitCode()).To(BeZero()) + + tmp, _ := os.CreateTemp("", "dl") //nolint:errcheck + tmp.Close() //nolint:errcheck + defer os.Remove(tmp.Name()) //nolint:errcheck + + s, err = RunCli(cliPath, configPath, "get", name, tmp.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(s.ExitCode()).To(BeZero()) + + b, _ := os.ReadFile(tmp.Name()) //nolint:errcheck + Expect(string(b)).To(Equal(content)) + + s, err = RunCli(cliPath, configPath, "delete", name) + Expect(err).ToNot(HaveOccurred()) + Expect(s.ExitCode()).To(BeZero()) +} + +func AssertLifecycleWorks(cliPath string, cfg *config.AZStorageConfig) { + expectedString := GenerateRandomString() + blobName := GenerateRandomString() + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + contentFile := MakeContentFile(expectedString) + defer os.Remove(contentFile) //nolint:errcheck + + // Ensure container/bucket exists + cliSession, err := RunCli(cliPath, configPath, "ensure-bucket-exists") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = RunCli(cliPath, configPath, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + Expect(cliSession.Err.Contents()).To(MatchRegexp("File '.*' exists in bucket '.*'")) + + // Check blob properties + cliSession, err = RunCli(cliPath, configPath, "properties", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + output := string(cliSession.Out.Contents()) + Expect(output).To(MatchRegexp(`"etag":\s*".+?"`)) + Expect(output).To(MatchRegexp(`"last_modified":\s*".+?"`)) + Expect(output).To(MatchRegexp(`"content_length":\s*\d+`)) + + tmpLocalFile, err := os.CreateTemp("", "azure-storage-cli-download") + Expect(err).ToNot(HaveOccurred()) + err = tmpLocalFile.Close() + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpLocalFile.Name()) //nolint:errcheck + + cliSession, err = RunCli(cliPath, configPath, "get", blobName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + gottenBytes, err := os.ReadFile(tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(string(gottenBytes)).To(Equal(expectedString)) + + cliSession, err = RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = RunCli(cliPath, configPath, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(3)) + Expect(cliSession.Err.Contents()).To(MatchRegexp("File '.*' does not exist in bucket '.*'")) + + cliSession, err = RunCli(cliPath, configPath, "properties", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(0)) + Expect(cliSession.Out.Contents()).To(MatchRegexp("{}")) +} + +func AssertOnCliVersion(cliPath string, cfg *config.AZStorageConfig) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + cliSession, err := RunCli(cliPath, configPath, "-v") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(0)) + + consoleOutput := bytes.NewBuffer(cliSession.Out.Contents()).String() + Expect(consoleOutput).To(ContainSubstring("version")) +} + +func AssertGetNonexistentFails(cliPath string, cfg *config.AZStorageConfig) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + cliSession, err := RunCli(cliPath, configPath, "get", "non-existent-file", "/dev/null") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).ToNot(BeZero()) +} + +func AssertDeleteNonexistentWorks(cliPath string, cfg *config.AZStorageConfig) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + cliSession, err := RunCli(cliPath, configPath, "delete", "non-existent-file") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) +} + +func AssertOnSignedURLs(cliPath string, cfg *config.AZStorageConfig) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + regex := "https://" + cfg.AccountName + ".blob.*/" + cfg.ContainerName + "/some-blob.*" + + cliSession, err := RunCli(cliPath, configPath, "sign", "some-blob", "get", "60s") + Expect(err).ToNot(HaveOccurred()) + + getUrl := bytes.NewBuffer(cliSession.Out.Contents()).String() + Expect(getUrl).To(MatchRegexp(regex)) + + cliSession, err = RunCli(cliPath, configPath, "sign", "some-blob", "put", "60s") + Expect(err).ToNot(HaveOccurred()) + + putUrl := bytes.NewBuffer(cliSession.Out.Contents()).String() + Expect(putUrl).To(MatchRegexp(regex)) +} + +func AssertOnListDeleteLifecyle(cliPath string, cfg *config.AZStorageConfig) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + cli, err := RunCli(cliPath, configPath, "delete-recursive", "") + Expect(err).ToNot(HaveOccurred()) + Expect(cli.ExitCode()).To(BeZero()) + cliSession, err := RunCli(cliPath, configPath, "list") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + Expect(len(cliSession.Out.Contents())).To(BeZero()) + + CreateRandomBlobs(cliPath, cfg, 4, "") + + customPrefix := "custom-prefix-" + CreateRandomBlobs(cliPath, cfg, 4, customPrefix) + + otherPrefix := "other-prefix-" + CreateRandomBlobs(cliPath, cfg, 2, otherPrefix) + + // Assert that the blobs are listed correctly + cliSession, err = RunCli(cliPath, configPath, "list") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + Expect(len(bytes.FieldsFunc(cliSession.Out.Contents(), func(r rune) bool { return r == '\n' || r == '\r' }))).To(BeNumerically("==", 10)) + + // Assert that the all blobs with custom prefix are listed correctly + cliSession, err = RunCli(cliPath, configPath, "list", customPrefix) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + Expect(len(bytes.FieldsFunc(cliSession.Out.Contents(), func(r rune) bool { return r == '\n' || r == '\r' }))).To(BeNumerically("==", 4)) + + // Delete all blobs with custom prefix + cliSession, err = RunCli(cliPath, configPath, "delete-recursive", customPrefix) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + // Assert that the blobs with custom prefix are deleted + cliSession, err = RunCli(cliPath, configPath, "list", customPrefix) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + Expect(len(cliSession.Out.Contents())).To(BeZero()) + + // Assert that the other prefixed blobs are still listed + cliSession, err = RunCli(cliPath, configPath, "list", otherPrefix) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + Expect(len(bytes.FieldsFunc(cliSession.Out.Contents(), func(r rune) bool { return r == '\n' || r == '\r' }))).To(BeNumerically("==", 2)) + + // Delete all other blobs + cliSession, err = RunCli(cliPath, configPath, "delete-recursive", "") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + // Assert that all blobs are deleted + cliSession, err = RunCli(cliPath, configPath, "list") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + Expect(len(cliSession.Out.Contents())).To(BeZero()) +} + +func AssertOnCopy(cliPath string, cfg *config.AZStorageConfig) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + // Create a blob to copy + blobName := GenerateRandomString() + blobContent := GenerateRandomString() + contentFile := MakeContentFile(blobContent) + defer os.Remove(contentFile) //nolint:errcheck + + cliSession, err := RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + // Copy the blob to a new name + copiedBlobName := GenerateRandomString() + cliSession, err = RunCli(cliPath, configPath, "copy", blobName, copiedBlobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + // Assert that the copied blob exists + cliSession, err = RunCli(cliPath, configPath, "exists", copiedBlobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + // Compare the content of the original and copied blobs + tmpLocalFile, err := os.CreateTemp("", "download-copy") + Expect(err).ToNot(HaveOccurred()) + err = tmpLocalFile.Close() + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpLocalFile.Name()) //nolint:errcheck + cliSession, err = RunCli(cliPath, configPath, "get", blobName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + gottenBytes, err := os.ReadFile(tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(string(gottenBytes)).To(Equal(blobContent)) + + // Clean up + cliSession, err = RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + cliSession, err = RunCli(cliPath, configPath, "delete", copiedBlobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) +} + +func CreateRandomBlobs(cliPath string, cfg *config.AZStorageConfig, count int, prefix string) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + for i := 0; i < count; i++ { + blobName := GenerateRandomString() + if prefix != "" { + blobName = prefix + blobName + } + contentFile := MakeContentFile(GenerateRandomString()) + defer os.Remove(contentFile) //nolint:errcheck + + cliSession, err := RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + } +} diff --git a/azurebs/integration/general_azure_test.go b/azurebs/integration/general_azure_test.go new file mode 100644 index 0000000..bff36b7 --- /dev/null +++ b/azurebs/integration/general_azure_test.go @@ -0,0 +1,189 @@ +package integration_test + +import ( + "bytes" + "os" + + "github.com/cloudfoundry/storage-cli/azurebs/config" + "github.com/cloudfoundry/storage-cli/azurebs/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("General testing for all Azure regions", func() { + var defaultConfig config.AZStorageConfig + + BeforeEach(func() { + defaultConfig = config.AZStorageConfig{ + AccountName: os.Getenv("ACCOUNT_NAME"), + AccountKey: os.Getenv("ACCOUNT_KEY"), + ContainerName: os.Getenv("CONTAINER_NAME"), + Environment: os.Getenv("ENVIRONMENT"), + } + if defaultConfig.Environment == "" { + defaultConfig.Environment = "AzureCloud" + } + + Expect(defaultConfig.AccountName).ToNot(BeEmpty(), "ACCOUNT_NAME must be set") + Expect(defaultConfig.AccountKey).ToNot(BeEmpty(), "ACCOUNT_KEY must be set") + Expect(defaultConfig.ContainerName).ToNot(BeEmpty(), "CONTAINER_NAME must be set") + }) + + configurations := []TableEntry{ + Entry("with default config", &defaultConfig), + } + DescribeTable("Assert Put Uses No Timeout When Not Specified", + func(cfg *config.AZStorageConfig) { integration.AssertPutUsesNoTimeout(cliPath, cfg) }, + configurations, + ) + DescribeTable("Assert Put Honors Custom Timeout", + func(cfg *config.AZStorageConfig) { integration.AssertPutHonorsCustomTimeout(cliPath, cfg) }, + configurations, + ) + DescribeTable("Assert Put Times Out", + func(cfg *config.AZStorageConfig) { integration.AssertPutTimesOut(cliPath, cfg) }, + configurations, + ) + DescribeTable("Assert Invalid Timeout Error", + func(cfg *config.AZStorageConfig) { integration.AssertInvalidTimeoutIsError(cliPath, cfg) }, + configurations, + ) + DescribeTable("Assert Signed URL Timeouts", + func(cfg *config.AZStorageConfig) { integration.AssertSignedURLTimeouts(cliPath, cfg) }, + configurations, + ) + DescribeTable("Rejects zero timeout", + func(cfg *config.AZStorageConfig) { integration.AssertZeroTimeoutIsError(cliPath, cfg) }, + configurations, + ) + DescribeTable("Rejects negative timeout", + func(cfg *config.AZStorageConfig) { integration.AssertNegativeTimeoutIsError(cliPath, cfg) }, + configurations, + ) + DescribeTable("Assert Ensure Bucket Idempotent", + func(cfg *config.AZStorageConfig) { integration.AssertEnsureBucketIdempotent(cliPath, cfg) }, + configurations, + ) + DescribeTable("Assert Put Get With Special Names", + func(cfg *config.AZStorageConfig) { integration.AssertPutGetWithSpecialNames(cliPath, cfg) }, + configurations, + ) + DescribeTable("Blobstore lifecycle works", + func(cfg *config.AZStorageConfig) { integration.AssertLifecycleWorks(cliPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `get` on a non-existent-key fails", + func(cfg *config.AZStorageConfig) { integration.AssertGetNonexistentFails(cliPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `delete` on a non-existent-key does not fail", + func(cfg *config.AZStorageConfig) { integration.AssertDeleteNonexistentWorks(cliPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `sign` returns a signed URL", + func(cfg *config.AZStorageConfig) { integration.AssertOnSignedURLs(cliPath, cfg) }, + configurations, + ) + DescribeTable("Blobstore list and delete lifecycle works", + func(cfg *config.AZStorageConfig) { integration.AssertOnListDeleteLifecyle(cliPath, cfg) }, + configurations, + ) + + DescribeTable("Server-side copy works", + func(cfg *config.AZStorageConfig) { integration.AssertOnCopy(cliPath, cfg) }, + configurations, + ) + + Describe("Invoking `put`", func() { + var blobName string + var configPath string + var contentFile string + + BeforeEach(func() { + blobName = integration.GenerateRandomString() + configPath = integration.MakeConfigFile(&defaultConfig) + contentFile = integration.MakeContentFile("foo") + }) + + AfterEach(func() { + os.Remove(configPath) //nolint:errcheck + os.Remove(contentFile) //nolint:errcheck + }) + + It("uploads a file", func() { + defer func() { + cliSession, err := integration.RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + }() + + cliSession, err := integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "exists", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + Expect(string(cliSession.Err.Contents())).To(MatchRegexp("File '" + blobName + "' exists in bucket '" + defaultConfig.ContainerName + "'")) + }) + + It("overwrites an existing file", func() { + defer func() { + cliSession, err := integration.RunCli(cliPath, configPath, "delete", blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + }() + + tmpLocalFile, _ := os.CreateTemp("", "azure-storage-cli-download") //nolint:errcheck + tmpLocalFile.Close() //nolint:errcheck + os.Remove(tmpLocalFile.Name()) //nolint:errcheck + + contentFile = integration.MakeContentFile("initial content") + cliSession, err := integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "get", blobName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + gottenBytes, _ := os.ReadFile(tmpLocalFile.Name()) //nolint:errcheck + Expect(string(gottenBytes)).To(Equal("initial content")) + + contentFile = integration.MakeContentFile("updated content") + cliSession, err = integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + cliSession, err = integration.RunCli(cliPath, configPath, "get", blobName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + gottenBytes, _ = os.ReadFile(tmpLocalFile.Name()) //nolint:errcheck + Expect(string(gottenBytes)).To(Equal("updated content")) + }) + + It("returns the appropriate error message", func() { + cfg := &config.AZStorageConfig{ + AccountName: os.Getenv("ACCOUNT_NAME"), + AccountKey: os.Getenv("ACCOUNT_KEY"), + ContainerName: "not-existing", + } + + configPath = integration.MakeConfigFile(cfg) + + cliSession, err := integration.RunCli(cliPath, configPath, "put", contentFile, blobName) + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(Equal(1)) + + consoleOutput := bytes.NewBuffer(cliSession.Err.Contents()).String() + Expect(consoleOutput).To(ContainSubstring("upload failure")) + }) + }) + Describe("Invoking `-v`", func() { + It("returns the cli version", func() { + integration.AssertOnCliVersion(cliPath, &defaultConfig) + }) + }) +}) diff --git a/azurebs/integration/integration_suite_test.go b/azurebs/integration/integration_suite_test.go new file mode 100644 index 0000000..5b30b75 --- /dev/null +++ b/azurebs/integration/integration_suite_test.go @@ -0,0 +1,30 @@ +package integration_test + +import ( + "testing" + + "github.com/onsi/gomega/gexec" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestIntegration(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Integration Suite") +} + +var cliPath string +var largeContent string //nolint:unused + +var _ = BeforeSuite(func() { + if len(cliPath) == 0 { + var err error + cliPath, err = gexec.Build("github.com/cloudfoundry/storage-cli/azurebs") + Expect(err).ShouldNot(HaveOccurred()) + } +}) + +var _ = AfterSuite(func() { + gexec.CleanupBuildArtifacts() +}) diff --git a/azurebs/integration/utils.go b/azurebs/integration/utils.go new file mode 100644 index 0000000..1977221 --- /dev/null +++ b/azurebs/integration/utils.go @@ -0,0 +1,68 @@ +package integration + +import ( + "encoding/json" + "math/rand" + "os" + "os/exec" + "time" + + "github.com/onsi/ginkgo/v2" + "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + + "github.com/cloudfoundry/storage-cli/azurebs/config" +) + +const alphanum = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +func GenerateRandomString(params ...int) string { + size := 25 + if len(params) == 1 { + size = params[0] + } + + randBytes := make([]byte, size) + for i := range randBytes { + randBytes[i] = alphanum[rand.Intn(len(alphanum))] + } + return string(randBytes) +} + +func MakeConfigFile(cfg *config.AZStorageConfig) string { + cfgBytes, err := json.Marshal(cfg) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + tmpFile, err := os.CreateTemp("", "azure-storage-cli-test") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = tmpFile.Write(cfgBytes) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = tmpFile.Close() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + return tmpFile.Name() +} + +func MakeContentFile(content string) string { + tmpFile, err := os.CreateTemp("", "azure-storage-test-content") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = tmpFile.Write([]byte(content)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = tmpFile.Close() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + return tmpFile.Name() +} + +func RunCli(cliPath string, configPath string, subcommand string, args ...string) (*gexec.Session, error) { + cmdArgs := []string{ + "-c", + configPath, + subcommand, + } + cmdArgs = append(cmdArgs, args...) + command := exec.Command(cliPath, cmdArgs...) + gexecSession, err := gexec.Start(command, ginkgo.GinkgoWriter, ginkgo.GinkgoWriter) + if err != nil { + return nil, err + } + gexecSession.Wait(1 * time.Minute) + return gexecSession, nil +} diff --git a/azurebs/main.go b/azurebs/main.go new file mode 100644 index 0000000..698fc15 --- /dev/null +++ b/azurebs/main.go @@ -0,0 +1,197 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + + "github.com/cloudfoundry/storage-cli/azurebs/client" + "github.com/cloudfoundry/storage-cli/azurebs/config" +) + +var version string + +func main() { + + configPath := flag.String("c", "", "configuration path") + showVer := flag.Bool("v", false, "version") + flag.Parse() + + if *showVer { + fmt.Printf("version %s\n", version) + os.Exit(0) + } + + configFile, err := os.Open(*configPath) + if err != nil { + log.Fatalln(err) + } + + azConfig, err := config.NewFromReader(configFile) + if err != nil { + log.Fatalln(err) + } + + storageClient, err := client.NewStorageClient(azConfig) + if err != nil { + log.Fatalln(err) + } + + blobstoreClient, err := client.New(storageClient) + if err != nil { + log.Fatalln(err) + } + + nonFlagArgs := flag.Args() + cmd := nonFlagArgs[0] + + switch cmd { + case "put": + if len(nonFlagArgs) != 3 { + log.Fatalf("Put method expected 3 arguments got %d\n", len(nonFlagArgs)) + } + sourceFilePath, dst := nonFlagArgs[1], nonFlagArgs[2] + + _, err := os.Stat(sourceFilePath) + if err != nil { + log.Fatalln(err) + } + + err = blobstoreClient.Put(sourceFilePath, dst) + fatalLog(cmd, err) + + case "get": + if len(nonFlagArgs) != 3 { + log.Fatalf("Get method expected 3 arguments got %d\n", len(nonFlagArgs)) + } + src, dst := nonFlagArgs[1], nonFlagArgs[2] + + var dstFile *os.File + dstFile, err = os.Create(dst) + if err != nil { + log.Fatalln(err) + } + + defer dstFile.Close() //nolint:errcheck + + err = blobstoreClient.Get(src, dstFile) + fatalLog(cmd, err) + + case "copy": + if len(nonFlagArgs) != 3 { + log.Fatalf("Get method expected 3 arguments got %d\n", len(nonFlagArgs)) + } + + srcBlob, dstBlob := nonFlagArgs[1], nonFlagArgs[2] + + err = blobstoreClient.Copy(srcBlob, dstBlob) + fatalLog(cmd, err) + + case "delete": + if len(nonFlagArgs) != 2 { + log.Fatalf("Delete method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + + err = blobstoreClient.Delete(nonFlagArgs[1]) + fatalLog(cmd, err) + + case "delete-recursive": + var prefix string + if len(nonFlagArgs) > 2 { + log.Fatalf("delete-recursive takes at most one argument (prefix) got %d\n", len(nonFlagArgs)-1) + } else if len(nonFlagArgs) == 2 { + prefix = nonFlagArgs[1] + } else { + prefix = "" + } + err = blobstoreClient.DeleteRecursive(prefix) + fatalLog("delete-recursive", err) + + case "exists": + if len(nonFlagArgs) != 2 { + log.Fatalf("Exists method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + + var exists bool + exists, err = blobstoreClient.Exists(nonFlagArgs[1]) + + // If the object exists the exit status is 0, otherwise it is 3 + // We are using `3` since `1` and `2` have special meanings + if err == nil && !exists { + os.Exit(3) + } + + case "sign": + if len(nonFlagArgs) != 4 { + log.Fatalf("Sign method expects 3 arguments got %d\n", len(nonFlagArgs)-1) + } + + objectID, action := nonFlagArgs[1], nonFlagArgs[2] + + if action != "get" && action != "put" { + log.Fatalf("Action not implemented: %s. Available actions are 'get' and 'put'", action) + } + + expiration, err := time.ParseDuration(nonFlagArgs[3]) + if err != nil { + log.Fatalf("Expiration should be in the format of a duration i.e. 1h, 60m, 3600s. Got: %s", nonFlagArgs[3]) + } + + signedURL, err := blobstoreClient.Sign(objectID, action, expiration) + + if err != nil { + log.Fatalf("Failed to sign request: %s", err) + } + + fmt.Print(signedURL) + os.Exit(0) + + case "list": + var prefix string + + if len(nonFlagArgs) == 1 { + prefix = "" + } else if len(nonFlagArgs) == 2 { + prefix = nonFlagArgs[1] + } else { + log.Fatalf("List method expected 1 or 2 arguments, got %d\n", len(nonFlagArgs)-1) + } + + var objects []string + objects, err = blobstoreClient.List(prefix) + if err != nil { + log.Fatalf("Failed to list objects: %s", err) + } + + for _, object := range objects { + fmt.Println(object) + } + + case "properties": + if len(nonFlagArgs) != 2 { + log.Fatalf("Properties method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + + err = blobstoreClient.Properties(nonFlagArgs[1]) + fatalLog("properties", err) + + case "ensure-bucket-exists": + if len(nonFlagArgs) != 1 { + log.Fatalf("EnsureBucketExists method expected 1 arguments got %d\n", len(nonFlagArgs)) + } + + err = blobstoreClient.EnsureContainerExists() + fatalLog("ensure-bucket-exists", err) + + default: + log.Fatalf("unknown command: '%s'\n", cmd) + } +} + +func fatalLog(cmd string, err error) { + if err != nil { + log.Fatalf("performing operation %s: %s\n", cmd, err) + } +} diff --git a/dav/README.md b/dav/README.md new file mode 100644 index 0000000..8601a09 --- /dev/null +++ b/dav/README.md @@ -0,0 +1,36 @@ +# Dav Storage CLI + +A CLI utility the BOSH Agent uses for accessing the [DAV blobstore](https://bosh.io/docs/director-configure-blobstore.html). + +Inside stemcells this binary is on the PATH as `bosh-blobstore-dav`. + +### Developers + +To update dependencies, use `gvt update`. Here is a typical invocation to update the `bosh-utils` dependency: + +``` +gvt update github.com/cloudfoundry/bosh-utils +``` + +### Run tests + +You can run the unit test with `ginkgo` as follows. + +``` +ginkgo -r -race -progress -mod vendor . +``` + +# Pre-signed URLs + +The command `sign` generates a pre-signed url for a specific object, action and duration: + +`dav-cli ` + +The request will be signed using HMAC-SHA256 with a secret provided in configuration. + +The HMAC format is: +`` + +The generated URL will be of format: + +`https://blobstore.url/signed/object-id?st=HMACSignatureHash&ts=GenerationTimestamp&e=ExpirationTimestamp` diff --git a/dav/app/app.go b/dav/app/app.go new file mode 100644 index 0000000..523e446 --- /dev/null +++ b/dav/app/app.go @@ -0,0 +1,72 @@ +package app + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + + davcmd "github.com/cloudfoundry/storage-cli/dav/cmd" + davconfig "github.com/cloudfoundry/storage-cli/dav/config" +) + +type App struct { + runner davcmd.Runner +} + +func New(runner davcmd.Runner) (app App) { + app.runner = runner + return +} + +func (app App) Run(args []string) (err error) { + args = args[1:] + var configFilePath string + var printVersion bool + + flagSet := flag.NewFlagSet("davcli-args", flag.ContinueOnError) + flagSet.StringVar(&configFilePath, "c", "", "Config file path") + flagSet.BoolVar(&printVersion, "v", false, "print version info") + + err = flagSet.Parse(args) + if err != nil { + return + } + + if printVersion { + fmt.Println("davcli version [[version]]") + return + } + + if configFilePath == "" { + err = errors.New("Config file arg `-c` is missing") //nolint:staticcheck + return + } + + file, err := os.Open(configFilePath) + if err != nil { + return + } + + configBytes, err := io.ReadAll(file) + if err != nil { + return + } + + config := davconfig.Config{} + err = json.Unmarshal(configBytes, &config) + if err != nil { + return + } + + err = app.runner.SetConfig(config) + if err != nil { + err = fmt.Errorf("Invalid CA Certificate: %s", err.Error()) //nolint:staticcheck + return + } + + err = app.runner.Run(args[2:]) + return +} diff --git a/dav/app/app_suite_test.go b/dav/app/app_suite_test.go new file mode 100644 index 0000000..c352179 --- /dev/null +++ b/dav/app/app_suite_test.go @@ -0,0 +1,13 @@ +package app_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestApp(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Davcli App Suite") +} diff --git a/dav/app/app_test.go b/dav/app/app_test.go new file mode 100644 index 0000000..39696d8 --- /dev/null +++ b/dav/app/app_test.go @@ -0,0 +1,124 @@ +package app_test + +import ( + "errors" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + . "github.com/cloudfoundry/storage-cli/dav/app" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +type FakeRunner struct { + Config davconf.Config + SetConfigErr error + RunArgs []string + RunErr error +} + +func (r *FakeRunner) SetConfig(newConfig davconf.Config) (err error) { + r.Config = newConfig + return r.SetConfigErr +} + +func (r *FakeRunner) Run(cmdArgs []string) (err error) { + r.RunArgs = cmdArgs + return r.RunErr +} + +func pathToFixture(file string) string { + pwd, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + fixturePath := filepath.Join(pwd, "../test_assets", file) + + absPath, err := filepath.Abs(fixturePath) + Expect(err).ToNot(HaveOccurred()) + + return absPath +} + +var _ = Describe("App", func() { + It("reads the CA cert from config", func() { + runner := &FakeRunner{} + + app := New(runner) + err := app.Run([]string{"dav-cli", "-c", pathToFixture("dav-cli-config-with-ca.json"), "put", "localFile", "remoteFile"}) + Expect(err).ToNot(HaveOccurred()) + + expectedConfig := davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: "https://example.com/some/endpoint", + Secret: "77D47E3A0B0F590B73CF3EBD9BB6761E244F90FA6F28BB39F941B0905789863FBE2861FDFD8195ADC81B72BB5310BC18969BEBBF4656366E7ACD3F0E4186FDDA", + TLS: davconf.TLS{ + Cert: davconf.Cert{ + CA: "ca-cert", + }, + }, + } + + Expect(runner.Config).To(Equal(expectedConfig)) + Expect(runner.Config.TLS.Cert.CA).ToNot(BeNil()) + }) + + It("returns error if CA Cert is invalid", func() { + runner := &FakeRunner{ + SetConfigErr: errors.New("invalid cert"), + } + + app := New(runner) + err := app.Run([]string{"dav-cli", "-c", pathToFixture("dav-cli-config-with-ca.json"), "put", "localFile", "remoteFile"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Invalid CA Certificate: invalid cert")) + + }) + + It("runs the put command", func() { + runner := &FakeRunner{} + + app := New(runner) + err := app.Run([]string{"dav-cli", "-c", pathToFixture("dav-cli-config.json"), "put", "localFile", "remoteFile"}) + Expect(err).ToNot(HaveOccurred()) + + expectedConfig := davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: "http://example.com/some/endpoint", + Secret: "77D47E3A0B0F590B73CF3EBD9BB6761E244F90FA6F28BB39F941B0905789863FBE2861FDFD8195ADC81B72BB5310BC18969BEBBF4656366E7ACD3F0E4186FDDA", + } + + Expect(runner.Config).To(Equal(expectedConfig)) + Expect(runner.Config.TLS.Cert.CA).To(BeEmpty()) + Expect(runner.RunArgs).To(Equal([]string{"put", "localFile", "remoteFile"})) + }) + + It("returns error with no config argument", func() { + runner := &FakeRunner{} + + app := New(runner) + err := app.Run([]string{"put", "localFile", "remoteFile"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Config file arg `-c` is missing")) + }) + It("prints the version info with the -v flag", func() { + runner := &FakeRunner{} + app := New(runner) + err := app.Run([]string{"dav-cli", "-v"}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns error from the cmd runner", func() { + runner := &FakeRunner{ + RunErr: errors.New("fake-run-error"), + } + + app := New(runner) + err := app.Run([]string{"dav-cli", "-c", pathToFixture("dav-cli-config.json"), "put", "localFile", "remoteFile"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("fake-run-error")) + }) +}) diff --git a/dav/client/client.go b/dav/client/client.go new file mode 100644 index 0000000..cd43926 --- /dev/null +++ b/dav/client/client.go @@ -0,0 +1,197 @@ +package client + +import ( + "crypto/sha1" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strings" + "time" + + URLsigner "github.com/cloudfoundry/storage-cli/dav/signer" + + bosherr "github.com/cloudfoundry/bosh-utils/errors" + "github.com/cloudfoundry/bosh-utils/httpclient" + boshlog "github.com/cloudfoundry/bosh-utils/logger" + + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +type Client interface { + Get(path string) (content io.ReadCloser, err error) + Put(path string, content io.ReadCloser, contentLength int64) (err error) + Exists(path string) (err error) + Delete(path string) (err error) + Sign(objectID, action string, duration time.Duration) (string, error) +} + +func NewClient(config davconf.Config, httpClient httpclient.Client, logger boshlog.Logger) (c Client) { + if config.RetryAttempts == 0 { + config.RetryAttempts = 3 + } + + // @todo should a logger now be passed in to this client? + duration := time.Duration(0) + retryClient := httpclient.NewRetryClient( + httpClient, + config.RetryAttempts, + duration, + logger, + ) + + return client{ + config: config, + httpClient: retryClient, + } +} + +type client struct { + config davconf.Config + httpClient httpclient.Client +} + +func (c client) Get(path string) (io.ReadCloser, error) { + req, err := c.createReq("GET", path, nil) + if err != nil { + return nil, err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return nil, bosherr.WrapErrorf(err, "Getting dav blob %s", path) + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("Getting dav blob %s: Wrong response code: %d; body: %s", path, resp.StatusCode, c.readAndTruncateBody(resp)) //nolint:staticcheck + } + + return resp.Body, nil +} + +func (c client) Put(path string, content io.ReadCloser, contentLength int64) error { + req, err := c.createReq("PUT", path, content) + if err != nil { + return err + } + defer content.Close() //nolint:errcheck + + req.ContentLength = contentLength + resp, err := c.httpClient.Do(req) + if err != nil { + return bosherr.WrapErrorf(err, "Putting dav blob %s", path) + } + + if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusNoContent { + return fmt.Errorf("Putting dav blob %s: Wrong response code: %d; body: %s", path, resp.StatusCode, c.readAndTruncateBody(resp)) //nolint:staticcheck + } + + return nil +} + +func (c client) Exists(path string) error { + req, err := c.createReq("HEAD", path, nil) + if err != nil { + return err + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return bosherr.WrapErrorf(err, "Checking if dav blob %s exists", path) + } + + if resp.StatusCode == http.StatusNotFound { + err := fmt.Errorf("%s not found", path) + return bosherr.WrapErrorf(err, "Checking if dav blob %s exists", path) + } + + if resp.StatusCode != http.StatusOK { + err := fmt.Errorf("invalid status: %d", resp.StatusCode) + return bosherr.WrapErrorf(err, "Checking if dav blob %s exists", path) + } + + return nil +} + +func (c client) Delete(path string) error { + req, err := c.createReq("DELETE", path, nil) + if err != nil { + return bosherr.WrapErrorf(err, "Creating delete request for blob '%s'", path) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + return bosherr.WrapErrorf(err, "Deleting blob '%s'", path) + } + + if resp.StatusCode == http.StatusNotFound { + return nil + } + + if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices { + err := fmt.Errorf("invalid status: %d", resp.StatusCode) + return bosherr.WrapErrorf(err, "Deleting blob '%s'", path) + } + + return nil +} + +func (c client) Sign(blobID, action string, duration time.Duration) (string, error) { + signer := URLsigner.NewSigner(c.config.Secret) + signTime := time.Now() + + prefixedBlob := fmt.Sprintf("%s/%s", getBlobPrefix(blobID), blobID) + + signedURL, err := signer.GenerateSignedURL(c.config.Endpoint, prefixedBlob, action, signTime, duration) + + if err != nil { + return "", bosherr.WrapErrorf(err, "pre-signing the url") + } + + return signedURL, err +} + +func (c client) createReq(method, blobID string, body io.Reader) (*http.Request, error) { + blobURL, err := url.Parse(c.config.Endpoint) + if err != nil { + return nil, err + } + + blobPrefix := getBlobPrefix(blobID) + + newPath := path.Join(blobURL.Path, blobPrefix, blobID) + if !strings.HasPrefix(newPath, "/") { + newPath = "/" + newPath + } + + blobURL.Path = newPath + + req, err := http.NewRequest(method, blobURL.String(), body) + if err != nil { + return req, err + } + + if c.config.User != "" { + req.SetBasicAuth(c.config.User, c.config.Password) + } + return req, nil +} + +func (c client) readAndTruncateBody(resp *http.Response) string { + body := "" + if resp.Body != nil { + buf := make([]byte, 1024) + n, err := resp.Body.Read(buf) + if err == io.EOF || err == nil { + body = string(buf[0:n]) + } + } + return body +} + +func getBlobPrefix(blobID string) string { + digester := sha1.New() + digester.Write([]byte(blobID)) + return fmt.Sprintf("%02x", digester.Sum(nil)[0]) +} diff --git a/dav/client/client_suite_test.go b/dav/client/client_suite_test.go new file mode 100644 index 0000000..a904d86 --- /dev/null +++ b/dav/client/client_suite_test.go @@ -0,0 +1,13 @@ +package client_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Davcli Client Suite") +} diff --git a/dav/client/client_test.go b/dav/client/client_test.go new file mode 100644 index 0000000..c210e55 --- /dev/null +++ b/dav/client/client_test.go @@ -0,0 +1,298 @@ +package client_test + +import ( + "io" + "net/http" + "strings" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/ghttp" + + "github.com/cloudfoundry/bosh-utils/httpclient" + boshlog "github.com/cloudfoundry/bosh-utils/logger" + + . "github.com/cloudfoundry/storage-cli/dav/client" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +var _ = Describe("Client", func() { + var ( + server *ghttp.Server + config davconf.Config + client Client + logger boshlog.Logger + ) + + BeforeEach(func() { + server = ghttp.NewServer() + config.Endpoint = server.URL() + config.User = "some_user" + config.Password = "some password" + logger = boshlog.NewLogger(boshlog.LevelNone) + client = NewClient(config, httpclient.DefaultClient, logger) + }) + + disconnectingRequestHandler := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + conn, _, err := w.(http.Hijacker).Hijack() + Expect(err).NotTo(HaveOccurred()) + + conn.Close() //nolint:errcheck + }) + + Describe("Exists", func() { + It("does not return an error if file exists", func() { + server.AppendHandlers(ghttp.RespondWith(200, "")) + err := client.Exists("/somefile") + Expect(err).NotTo(HaveOccurred()) + }) + + Context("the file does not exist", func() { + BeforeEach(func() { + server.AppendHandlers( + ghttp.RespondWith(404, ""), + ghttp.RespondWith(404, ""), + ghttp.RespondWith(404, ""), + ) + }) + + It("returns an error saying blob was not found", func() { + err := client.Exists("/somefile") + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("Checking if dav blob /somefile exists: /somefile not found"))) + }) + }) + + Context("unexpected http status code returned", func() { + BeforeEach(func() { + server.AppendHandlers( + ghttp.RespondWith(601, ""), + ghttp.RespondWith(601, ""), + ghttp.RespondWith(601, ""), + ) + }) + + It("returns an error saying an unexpected error occurred", func() { + err := client.Exists("/somefile") + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("Checking if dav blob /somefile exists:"))) + }) + }) + }) + + Describe("Delete", func() { + Context("when the file does not exist", func() { + BeforeEach(func() { + server.AppendHandlers( + ghttp.RespondWith(404, ""), + ghttp.RespondWith(404, ""), + ghttp.RespondWith(404, ""), + ) + }) + + It("does not return an error if file does not exists", func() { + err := client.Delete("/somefile") + Expect(err).NotTo(HaveOccurred()) + }) + }) + + Context("when the file exists", func() { + BeforeEach(func() { + server.AppendHandlers(ghttp.RespondWith(204, "")) + }) + + It("does not return an error", func() { + err := client.Delete("/somefile") + Expect(err).ToNot(HaveOccurred()) + Expect(server.ReceivedRequests()).To(HaveLen(1)) + request := server.ReceivedRequests()[0] + Expect(request.URL.Path).To(Equal("/19/somefile")) + Expect(request.Method).To(Equal("DELETE")) + Expect(request.Header["Authorization"]).To(Equal([]string{"Basic c29tZV91c2VyOnNvbWUgcGFzc3dvcmQ="})) + Expect(request.Host).To(Equal(server.Addr())) + }) + }) + + Context("when the status code is not in the 2xx range", func() { + It("returns an error saying an unexpected error occurred when the status code is greater than 299", func() { + server.AppendHandlers( + ghttp.RespondWith(300, ""), + ghttp.RespondWith(300, ""), + ghttp.RespondWith(300, ""), + ) + + err := client.Delete("/somefile") + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(Equal("Deleting blob '/somefile': invalid status: 300"))) + }) + }) + }) + + Describe("Get", func() { + It("returns the response body from the given path", func() { + server.AppendHandlers(ghttp.RespondWith(200, "response")) + + responseBody, err := client.Get("/") + Expect(err).NotTo(HaveOccurred()) + buf := make([]byte, 1024) + n, _ := responseBody.Read(buf) //nolint:errcheck + Expect(string(buf[0:n])).To(Equal("response")) + }) + + Context("when the http request fails", func() { + BeforeEach(func() { + server.Close() + }) + + It("returns err", func() { + responseBody, err := client.Get("/") + Expect(responseBody).To(BeNil()) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Getting dav blob /")) + }) + }) + + Context("when the http response code is not 200", func() { + BeforeEach(func() { + server.AppendHandlers( + ghttp.RespondWith(300, "response"), + ghttp.RespondWith(300, "response"), + ghttp.RespondWith(300, "response"), + ) + }) + + It("returns err", func() { + responseBody, err := client.Get("/") + Expect(responseBody).To(BeNil()) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("Getting dav blob /: Wrong response code: 300"))) + Expect(server.ReceivedRequests()).To(HaveLen(3)) + }) + }) + }) + + Describe("Put", func() { + Context("When the put request succeeds", func() { + itUploadsABlob := func() { + body := io.NopCloser(strings.NewReader("content")) + err := client.Put("/", body, int64(7)) + Expect(err).NotTo(HaveOccurred()) + + Expect(server.ReceivedRequests()).To(HaveLen(1)) + req := server.ReceivedRequests()[0] + Expect(req.ContentLength).To(Equal(int64(7))) + } + + It("uploads the given content if the blob does not exist", func() { + server.AppendHandlers( + ghttp.CombineHandlers( + ghttp.RespondWith(201, ""), + ghttp.VerifyBody([]byte("content")), + ), + ) + itUploadsABlob() + }) + + It("uploads the given content if the blob exists", func() { + server.AppendHandlers( + ghttp.CombineHandlers( + ghttp.RespondWith(204, ""), + ghttp.VerifyBody([]byte("content")), + ), + ) + itUploadsABlob() + }) + + It("adds an Authorizatin header to the request", func() { + server.AppendHandlers( + ghttp.CombineHandlers( + ghttp.RespondWith(204, ""), + ghttp.VerifyBody([]byte("content")), + ), + ) + itUploadsABlob() + req := server.ReceivedRequests()[0] + Expect(req.Header.Get("Authorization")).NotTo(BeEmpty()) + }) + + Context("when neither user nor password is provided in blobstore options", func() { + BeforeEach(func() { + config.User = "" + config.Password = "" + client = NewClient(config, httpclient.DefaultClient, logger) + }) + + It("sends a request with no Basic Auth header", func() { + server.AppendHandlers( + ghttp.CombineHandlers( + ghttp.RespondWith(204, ""), + ghttp.VerifyBody([]byte("content")), + ), + ) + itUploadsABlob() + req := server.ReceivedRequests()[0] + Expect(req.Header.Get("Authorization")).To(BeEmpty()) + }) + }) + }) + + Context("when the http request fails", func() { + BeforeEach(func() { + server.AppendHandlers( + disconnectingRequestHandler, + disconnectingRequestHandler, + disconnectingRequestHandler, + ) + }) + + It("returns err", func() { + body := io.NopCloser(strings.NewReader("content")) + err := client.Put("/", body, int64(7)) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("Putting dav blob /: Put \"%s/42\": EOF", server.URL()))) + Expect(server.ReceivedRequests()).To(HaveLen(3)) + }) + }) + + Context("when the http response code is not 201 or 204", func() { + BeforeEach(func() { + server.AppendHandlers( + ghttp.RespondWith(300, "response"), + ghttp.RespondWith(300, "response"), + ghttp.RespondWith(300, "response"), + ) + }) + + It("returns err", func() { + body := io.NopCloser(strings.NewReader("content")) + err := client.Put("/", body, int64(7)) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("Putting dav blob /: Wrong response code: 300"))) + }) + }) + }) + + Describe("retryable count is configurable", func() { + BeforeEach(func() { + server.AppendHandlers( + disconnectingRequestHandler, + disconnectingRequestHandler, + disconnectingRequestHandler, + disconnectingRequestHandler, + disconnectingRequestHandler, + disconnectingRequestHandler, + disconnectingRequestHandler, + ) + config = davconf.Config{RetryAttempts: 7, Endpoint: server.URL()} + client = NewClient(config, httpclient.DefaultClient, logger) + }) + + It("tries the specified number of times", func() { + body := io.NopCloser(strings.NewReader("content")) + err := client.Put("/", body, int64(7)) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError(ContainSubstring("Putting dav blob /: Put \"%s/42\": EOF", server.URL()))) + Expect(server.ReceivedRequests()).To(HaveLen(7)) + }) + }) +}) diff --git a/dav/client/fakes/fake_client.go b/dav/client/fakes/fake_client.go new file mode 100644 index 0000000..9627637 --- /dev/null +++ b/dav/client/fakes/fake_client.go @@ -0,0 +1,37 @@ +package fakes + +import ( + "io" +) + +type FakeClient struct { + GetPath string + GetContents io.ReadCloser + GetErr error + + PutPath string + PutContents string + PutContentLength int64 + PutErr error +} + +func NewFakeClient() *FakeClient { + return &FakeClient{} +} + +func (c *FakeClient) Get(path string) (io.ReadCloser, error) { + c.GetPath = path + + return c.GetContents, c.GetErr +} + +func (c *FakeClient) Put(path string, content io.ReadCloser, contentLength int64) error { + c.PutPath = path + contentBytes := make([]byte, contentLength) + content.Read(contentBytes) //nolint:errcheck + defer content.Close() //nolint:errcheck + c.PutContents = string(contentBytes) + c.PutContentLength = contentLength + + return c.PutErr +} diff --git a/dav/cmd/cmd.go b/dav/cmd/cmd.go new file mode 100644 index 0000000..6f69763 --- /dev/null +++ b/dav/cmd/cmd.go @@ -0,0 +1,5 @@ +package cmd + +type Cmd interface { + Run(args []string) (err error) +} diff --git a/dav/cmd/cmd_suite_test.go b/dav/cmd/cmd_suite_test.go new file mode 100644 index 0000000..f960f5b --- /dev/null +++ b/dav/cmd/cmd_suite_test.go @@ -0,0 +1,13 @@ +package cmd_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestCmd(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Davcli Cmd Suite") +} diff --git a/dav/cmd/delete.go b/dav/cmd/delete.go new file mode 100644 index 0000000..f291828 --- /dev/null +++ b/dav/cmd/delete.go @@ -0,0 +1,25 @@ +package cmd + +import ( + "errors" + + davclient "github.com/cloudfoundry/storage-cli/dav/client" +) + +type DeleteCmd struct { + client davclient.Client +} + +func newDeleteCmd(client davclient.Client) (cmd DeleteCmd) { + cmd.client = client + return +} + +func (cmd DeleteCmd) Run(args []string) (err error) { + if len(args) != 1 { + err = errors.New("Incorrect usage, delete needs remote blob path") //nolint:staticcheck + return + } + err = cmd.client.Delete(args[0]) + return +} diff --git a/dav/cmd/delete_test.go b/dav/cmd/delete_test.go new file mode 100644 index 0000000..3b230ce --- /dev/null +++ b/dav/cmd/delete_test.go @@ -0,0 +1,105 @@ +package cmd_test + +import ( + "net/http" + "net/http/httptest" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + . "github.com/cloudfoundry/storage-cli/dav/cmd" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + testcmd "github.com/cloudfoundry/storage-cli/dav/cmd/testing" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +func runDelete(config davconf.Config, args []string) error { + logger := boshlog.NewLogger(boshlog.LevelNone) + factory := NewFactory(logger) + factory.SetConfig(config) //nolint:errcheck + + cmd, err := factory.Create("delete") + Expect(err).ToNot(HaveOccurred()) + + return cmd.Run(args) +} + +var _ = Describe("DeleteCmd", func() { + var ( + handler func(http.ResponseWriter, *http.Request) + requestedBlob string + ts *httptest.Server + config davconf.Config + ) + + BeforeEach(func() { + requestedBlob = "0ca907f2-dde8-4413-a304-9076c9d0978b" + + handler = func(w http.ResponseWriter, r *http.Request) { + req := testcmd.NewHTTPRequest(r) + + username, password, err := req.ExtractBasicAuth() + Expect(err).ToNot(HaveOccurred()) + Expect(req.URL.Path).To(Equal("/0d/" + requestedBlob)) + Expect(req.Method).To(Equal("DELETE")) + Expect(username).To(Equal("some user")) + Expect(password).To(Equal("some pwd")) + + w.WriteHeader(http.StatusOK) + } + }) + + AfterEach(func() { + ts.Close() + }) + + AssertDeleteBehavior := func() { + It("with valid args", func() { + err := runDelete(config, []string{requestedBlob}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("returns err with incorrect arg count", func() { + err := runDelete(davconf.Config{}, []string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Incorrect usage")) + }) + } + + Context("with http endpoint", func() { + BeforeEach(func() { + ts = httptest.NewServer(http.HandlerFunc(handler)) + config = davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: ts.URL, + } + + }) + + AssertDeleteBehavior() + }) + + Context("with https endpoint", func() { + BeforeEach(func() { + ts = httptest.NewTLSServer(http.HandlerFunc(handler)) + + rootCa, err := testcmd.ExtractRootCa(ts) + Expect(err).ToNot(HaveOccurred()) + + config = davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: ts.URL, + TLS: davconf.TLS{ + Cert: davconf.Cert{ + CA: rootCa, + }, + }, + } + }) + + AssertDeleteBehavior() + }) +}) diff --git a/dav/cmd/exists.go b/dav/cmd/exists.go new file mode 100644 index 0000000..220ccc6 --- /dev/null +++ b/dav/cmd/exists.go @@ -0,0 +1,25 @@ +package cmd + +import ( + "errors" + + davclient "github.com/cloudfoundry/storage-cli/dav/client" +) + +type ExistsCmd struct { + client davclient.Client +} + +func newExistsCmd(client davclient.Client) (cmd ExistsCmd) { + cmd.client = client + return +} + +func (cmd ExistsCmd) Run(args []string) (err error) { + if len(args) != 1 { + err = errors.New("Incorrect usage, exists needs remote blob path") //nolint:staticcheck + return + } + err = cmd.client.Exists(args[0]) + return +} diff --git a/dav/cmd/exists_test.go b/dav/cmd/exists_test.go new file mode 100644 index 0000000..e5d11d8 --- /dev/null +++ b/dav/cmd/exists_test.go @@ -0,0 +1,104 @@ +package cmd_test + +import ( + "net/http" + "net/http/httptest" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + . "github.com/cloudfoundry/storage-cli/dav/cmd" + testcmd "github.com/cloudfoundry/storage-cli/dav/cmd/testing" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +func runExists(config davconf.Config, args []string) error { + logger := boshlog.NewLogger(boshlog.LevelNone) + factory := NewFactory(logger) + factory.SetConfig(config) //nolint:errcheck + + cmd, err := factory.Create("exists") + Expect(err).ToNot(HaveOccurred()) + + return cmd.Run(args) +} + +var _ = Describe("Exists", func() { + var ( + handler func(http.ResponseWriter, *http.Request) + requestedBlob string + ts *httptest.Server + config davconf.Config + ) + + BeforeEach(func() { + requestedBlob = "0ca907f2-dde8-4413-a304-9076c9d0978b" + + handler = func(w http.ResponseWriter, r *http.Request) { + req := testcmd.NewHTTPRequest(r) + + username, password, err := req.ExtractBasicAuth() + Expect(err).ToNot(HaveOccurred()) + Expect(req.URL.Path).To(Equal("/0d/" + requestedBlob)) + Expect(req.Method).To(Equal("HEAD")) + Expect(username).To(Equal("some user")) + Expect(password).To(Equal("some pwd")) + + w.WriteHeader(200) + } + }) + + AfterEach(func() { + ts.Close() + }) + + AssertExistsBehavior := func() { + It("with valid args", func() { + err := runExists(config, []string{requestedBlob}) + Expect(err).ToNot(HaveOccurred()) + }) + + It("with incorrect arg count", func() { + err := runExists(davconf.Config{}, []string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Incorrect usage")) + }) + } + + Context("with http endpoint", func() { + BeforeEach(func() { + ts = httptest.NewServer(http.HandlerFunc(handler)) + config = davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: ts.URL, + } + + }) + + AssertExistsBehavior() + }) + + Context("with https endpoint", func() { + BeforeEach(func() { + ts = httptest.NewTLSServer(http.HandlerFunc(handler)) + + rootCa, err := testcmd.ExtractRootCa(ts) + Expect(err).ToNot(HaveOccurred()) + + config = davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: ts.URL, + TLS: davconf.TLS{ + Cert: davconf.Cert{ + CA: rootCa, + }, + }, + } + }) + + AssertExistsBehavior() + }) +}) diff --git a/dav/cmd/factory.go b/dav/cmd/factory.go new file mode 100644 index 0000000..6b68025 --- /dev/null +++ b/dav/cmd/factory.go @@ -0,0 +1,62 @@ +package cmd + +import ( + "crypto/x509" + "fmt" + + boshcrypto "github.com/cloudfoundry/bosh-utils/crypto" + boshhttpclient "github.com/cloudfoundry/bosh-utils/httpclient" + boshlog "github.com/cloudfoundry/bosh-utils/logger" + + davclient "github.com/cloudfoundry/storage-cli/dav/client" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +type Factory interface { + Create(name string) (cmd Cmd, err error) + SetConfig(config davconf.Config) (err error) +} + +func NewFactory(logger boshlog.Logger) Factory { + return &factory{ + cmds: make(map[string]Cmd), + logger: logger, + } +} + +type factory struct { + config davconf.Config //nolint:unused + cmds map[string]Cmd + logger boshlog.Logger +} + +func (f *factory) Create(name string) (cmd Cmd, err error) { + cmd, found := f.cmds[name] + if !found { + err = fmt.Errorf("Could not find command with name %s", name) //nolint:staticcheck + } + return +} + +func (f *factory) SetConfig(config davconf.Config) (err error) { + var httpClient boshhttpclient.Client + var certPool *x509.CertPool + + if len(config.TLS.Cert.CA) != 0 { + certPool, err = boshcrypto.CertPoolFromPEM([]byte(config.TLS.Cert.CA)) + } + + httpClient = boshhttpclient.CreateDefaultClient(certPool) + + client := davclient.NewClient(config, httpClient, f.logger) + + f.cmds = map[string]Cmd{ + "put": newPutCmd(client), + "get": newGetCmd(client), + "exists": newExistsCmd(client), + "delete": newDeleteCmd(client), + "sign": newSignCmd(client), + } + + return +} diff --git a/dav/cmd/factory_test.go b/dav/cmd/factory_test.go new file mode 100644 index 0000000..4caf61a --- /dev/null +++ b/dav/cmd/factory_test.go @@ -0,0 +1,111 @@ +package cmd_test + +import ( + "reflect" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + . "github.com/cloudfoundry/storage-cli/dav/cmd" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +func buildFactory() (factory Factory) { + config := davconf.Config{User: "some user"} + logger := boshlog.NewLogger(boshlog.LevelNone) + factory = NewFactory(logger) + factory.SetConfig(config) //nolint:errcheck + return +} + +var _ = Describe("Factory", func() { + Describe("Create", func() { + It("factory create a put command", func() { + factory := buildFactory() + cmd, err := factory.Create("put") + + Expect(err).ToNot(HaveOccurred()) + Expect(reflect.TypeOf(cmd)).To(Equal(reflect.TypeOf(PutCmd{}))) + }) + + It("factory create a get command", func() { + factory := buildFactory() + cmd, err := factory.Create("get") + + Expect(err).ToNot(HaveOccurred()) + Expect(reflect.TypeOf(cmd)).To(Equal(reflect.TypeOf(GetCmd{}))) + }) + + It("factory create a delete command", func() { + factory := buildFactory() + cmd, err := factory.Create("delete") + + Expect(err).ToNot(HaveOccurred()) + Expect(reflect.TypeOf(cmd)).To(Equal(reflect.TypeOf(DeleteCmd{}))) + }) + + It("factory create when cmd is unknown", func() { + factory := buildFactory() + _, err := factory.Create("some unknown cmd") + + Expect(err).To(HaveOccurred()) + }) + }) + + Describe("SetConfig", func() { + It("returns an error if CaCert is given but invalid", func() { + factory := buildFactory() + config := davconf.Config{ + TLS: davconf.TLS{ + Cert: davconf.Cert{ + CA: "--- INVALID CERTIFICATE ---", + }, + }, + } + + err := factory.SetConfig(config) + Expect(err).To(HaveOccurred()) + }) + It("does not return an error if CaCert is valid", func() { + factory := buildFactory() + cert := `-----BEGIN CERTIFICATE----- +MIICEzCCAXygAwIBAgIQMIMChMLGrR+QvmQvpwAU6zANBgkqhkiG9w0BAQsFADAS +MRAwDgYDVQQKEwdBY21lIENvMCAXDTcwMDEwMTAwMDAwMFoYDzIwODQwMTI5MTYw +MDAwWjASMRAwDgYDVQQKEwdBY21lIENvMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCB +iQKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9SjY1bIw4 +iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZBl2+XsDul +rKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQABo2gwZjAO +BgNVHQ8BAf8EBAMCAqQwEwYDVR0lBAwwCgYIKwYBBQUHAwEwDwYDVR0TAQH/BAUw +AwEB/zAuBgNVHREEJzAlggtleGFtcGxlLmNvbYcEfwAAAYcQAAAAAAAAAAAAAAAA +AAAAATANBgkqhkiG9w0BAQsFAAOBgQCEcetwO59EWk7WiJsG4x8SY+UIAA+flUI9 +tyC4lNhbcF2Idq9greZwbYCqTTTr2XiRNSMLCOjKyI7ukPoPjo16ocHj+P3vZGfs +h1fIw3cSS2OolhloGw/XM6RWPWtPAlGykKLciQrBru5NAPvCMsb/I1DAceTiotQM +fblo6RBxUQ== +-----END CERTIFICATE-----` + config := davconf.Config{ + TLS: davconf.TLS{ + Cert: davconf.Cert{ + CA: cert, + }, + }, + } + + err := factory.SetConfig(config) + Expect(err).ToNot(HaveOccurred()) + }) + It("does not return an error if CaCert is not provided", func() { + factory := buildFactory() + config := davconf.Config{ + TLS: davconf.TLS{ + Cert: davconf.Cert{ + CA: "", + }, + }, + } + + err := factory.SetConfig(config) + Expect(err).ToNot(HaveOccurred()) + }) + }) +}) diff --git a/dav/cmd/get.go b/dav/cmd/get.go new file mode 100644 index 0000000..3009585 --- /dev/null +++ b/dav/cmd/get.go @@ -0,0 +1,39 @@ +package cmd + +import ( + "errors" + "io" + "os" + + davclient "github.com/cloudfoundry/storage-cli/dav/client" +) + +type GetCmd struct { + client davclient.Client +} + +func newGetCmd(client davclient.Client) (cmd GetCmd) { + cmd.client = client + return +} + +func (cmd GetCmd) Run(args []string) (err error) { + if len(args) != 2 { + err = errors.New("Incorrect usage, get needs remote blob path and local file destination") //nolint:staticcheck + return + } + + readCloser, err := cmd.client.Get(args[0]) + if err != nil { + return + } + defer readCloser.Close() //nolint:errcheck + + targetFile, err := os.Create(args[1]) + if err != nil { + return + } + + _, err = io.Copy(targetFile, readCloser) + return +} diff --git a/dav/cmd/get_test.go b/dav/cmd/get_test.go new file mode 100644 index 0000000..c3d7008 --- /dev/null +++ b/dav/cmd/get_test.go @@ -0,0 +1,122 @@ +package cmd_test + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + . "github.com/cloudfoundry/storage-cli/dav/cmd" + testcmd "github.com/cloudfoundry/storage-cli/dav/cmd/testing" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +func runGet(config davconf.Config, args []string) error { + logger := boshlog.NewLogger(boshlog.LevelNone) + factory := NewFactory(logger) + factory.SetConfig(config) //nolint:errcheck + + cmd, err := factory.Create("get") + Expect(err).ToNot(HaveOccurred()) + + return cmd.Run(args) +} + +func getFileContent(path string) string { + file, err := os.Open(path) + Expect(err).ToNot(HaveOccurred()) + + fileBytes, err := io.ReadAll(file) + Expect(err).ToNot(HaveOccurred()) + + return string(fileBytes) +} + +var _ = Describe("GetCmd", func() { + var ( + handler func(http.ResponseWriter, *http.Request) + targetFilePath string + requestedBlob string + ts *httptest.Server + config davconf.Config + ) + + BeforeEach(func() { + requestedBlob = "0ca907f2-dde8-4413-a304-9076c9d0978b" + targetFilePath = filepath.Join(os.TempDir(), "testRunGetCommand.txt") + + handler = func(w http.ResponseWriter, r *http.Request) { + req := testcmd.NewHTTPRequest(r) + + username, password, err := req.ExtractBasicAuth() + Expect(err).ToNot(HaveOccurred()) + Expect(req.URL.Path).To(Equal("/0d/" + requestedBlob)) + Expect(req.Method).To(Equal("GET")) + Expect(username).To(Equal("some user")) + Expect(password).To(Equal("some pwd")) + + w.Write([]byte("this is your blob")) //nolint:errcheck + } + + }) + + AfterEach(func() { + os.RemoveAll(targetFilePath) //nolint:errcheck + ts.Close() + }) + + AssertGetBehavior := func() { + It("get run with valid args", func() { + err := runGet(config, []string{requestedBlob, targetFilePath}) + Expect(err).ToNot(HaveOccurred()) + Expect(getFileContent(targetFilePath)).To(Equal("this is your blob")) + }) + + It("get run with incorrect arg count", func() { + err := runGet(davconf.Config{}, []string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Incorrect usage")) + }) + } + + Context("with http endpoint", func() { + BeforeEach(func() { + ts = httptest.NewServer(http.HandlerFunc(handler)) + + config = davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: ts.URL, + } + }) + + AssertGetBehavior() + }) + + Context("with https endpoint", func() { + BeforeEach(func() { + ts = httptest.NewTLSServer(http.HandlerFunc(handler)) + + rootCa, err := testcmd.ExtractRootCa(ts) + Expect(err).ToNot(HaveOccurred()) + + config = davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: ts.URL, + TLS: davconf.TLS{ + Cert: davconf.Cert{ + CA: rootCa, + }, + }, + } + }) + + AssertGetBehavior() + }) +}) diff --git a/dav/cmd/put.go b/dav/cmd/put.go new file mode 100644 index 0000000..44f6d84 --- /dev/null +++ b/dav/cmd/put.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "errors" + "os" + + davclient "github.com/cloudfoundry/storage-cli/dav/client" +) + +type PutCmd struct { + client davclient.Client +} + +func newPutCmd(client davclient.Client) (cmd PutCmd) { + cmd.client = client + return +} + +func (cmd PutCmd) Run(args []string) error { + if len(args) != 2 { + return errors.New("Incorrect usage, put needs local file and remote blob destination") //nolint:staticcheck + } + + file, err := os.OpenFile(args[0], os.O_RDWR, os.ModeExclusive) + if err != nil { + return err + } + + fileInfo, err := file.Stat() + if err != nil { + return err + } + + return cmd.client.Put(args[1], file, fileInfo.Size()) +} diff --git a/dav/cmd/put_test.go b/dav/cmd/put_test.go new file mode 100644 index 0000000..0c234ec --- /dev/null +++ b/dav/cmd/put_test.go @@ -0,0 +1,134 @@ +package cmd_test + +import ( + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + + . "github.com/cloudfoundry/storage-cli/dav/cmd" + testcmd "github.com/cloudfoundry/storage-cli/dav/cmd/testing" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +func runPut(config davconf.Config, args []string) error { + logger := boshlog.NewLogger(boshlog.LevelNone) + factory := NewFactory(logger) + factory.SetConfig(config) //nolint:errcheck + + cmd, err := factory.Create("put") + Expect(err).ToNot(HaveOccurred()) + + return cmd.Run(args) +} + +func fileBytes(path string) []byte { + file, err := os.Open(path) + Expect(err).ToNot(HaveOccurred()) + + content, err := io.ReadAll(file) + Expect(err).ToNot(HaveOccurred()) + + return content +} + +var _ = Describe("PutCmd", func() { + Describe("Run", func() { + var ( + handler func(http.ResponseWriter, *http.Request) + config davconf.Config + ts *httptest.Server + sourceFilePath string + targetBlob string + serverWasHit bool + ) + BeforeEach(func() { + pwd, err := os.Getwd() + Expect(err).ToNot(HaveOccurred()) + + sourceFilePath = filepath.Join(pwd, "../test_assets/cat.jpg") + targetBlob = "some-other-awesome-guid" + serverWasHit = false + + handler = func(w http.ResponseWriter, r *http.Request) { + defer GinkgoRecover() + serverWasHit = true + req := testcmd.NewHTTPRequest(r) + + username, password, err := req.ExtractBasicAuth() + Expect(err).ToNot(HaveOccurred()) + Expect(req.URL.Path).To(Equal("/d1/" + targetBlob)) + Expect(req.Method).To(Equal("PUT")) + Expect(req.ContentLength).To(Equal(int64(1718186))) + Expect(username).To(Equal("some user")) + Expect(password).To(Equal("some pwd")) + + expectedBytes := fileBytes(sourceFilePath) + actualBytes, _ := io.ReadAll(r.Body) //nolint:errcheck + Expect(expectedBytes).To(Equal(actualBytes)) + + w.WriteHeader(201) + } + }) + + AfterEach(func() { + defer ts.Close() + }) + + AssertPutBehavior := func() { + It("uploads the blob with valid args", func() { + err := runPut(config, []string{sourceFilePath, targetBlob}) + Expect(err).ToNot(HaveOccurred()) + Expect(serverWasHit).To(BeTrue()) + }) + + It("returns err with incorrect arg count", func() { + err := runPut(davconf.Config{}, []string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Incorrect usage")) + }) + } + + Context("with http endpoint", func() { + BeforeEach(func() { + ts = httptest.NewServer(http.HandlerFunc(handler)) + config = davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: ts.URL, + } + + }) + + AssertPutBehavior() + }) + + Context("with https endpoint", func() { + BeforeEach(func() { + ts = httptest.NewTLSServer(http.HandlerFunc(handler)) + + rootCa, err := testcmd.ExtractRootCa(ts) + Expect(err).ToNot(HaveOccurred()) + + config = davconf.Config{ + User: "some user", + Password: "some pwd", + Endpoint: ts.URL, + TLS: davconf.TLS{ + Cert: davconf.Cert{ + CA: rootCa, + }, + }, + } + }) + + AssertPutBehavior() + }) + }) +}) diff --git a/dav/cmd/runner.go b/dav/cmd/runner.go new file mode 100644 index 0000000..0fbf423 --- /dev/null +++ b/dav/cmd/runner.go @@ -0,0 +1,40 @@ +package cmd + +import ( + "errors" + + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +type Runner interface { + SetConfig(newConfig davconf.Config) (err error) + Run(cmdArgs []string) (err error) +} + +func NewRunner(factory Factory) Runner { + return runner{ + factory: factory, + } +} + +type runner struct { + factory Factory +} + +func (r runner) Run(cmdArgs []string) (err error) { + if len(cmdArgs) == 0 { + err = errors.New("Missing command name") //nolint:staticcheck + return + } + + cmd, err := r.factory.Create(cmdArgs[0]) + if err != nil { + return + } + + return cmd.Run(cmdArgs[1:]) +} + +func (r runner) SetConfig(newConfig davconf.Config) (err error) { + return r.factory.SetConfig(newConfig) +} diff --git a/dav/cmd/runner_test.go b/dav/cmd/runner_test.go new file mode 100644 index 0000000..df65651 --- /dev/null +++ b/dav/cmd/runner_test.go @@ -0,0 +1,111 @@ +package cmd_test + +import ( + "errors" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + . "github.com/cloudfoundry/storage-cli/dav/cmd" + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +type FakeFactory struct { + CreateName string + CreateCmd *FakeCmd + CreateErr error + + Config davconf.Config + SetConfigErr error +} + +func (f *FakeFactory) Create(name string) (cmd Cmd, err error) { + f.CreateName = name + cmd = f.CreateCmd + err = f.CreateErr + return +} + +func (f *FakeFactory) SetConfig(config davconf.Config) (err error) { + f.Config = config + return f.SetConfigErr +} + +type FakeCmd struct { + RunArgs []string + RunErr error +} + +func (cmd *FakeCmd) Run(args []string) (err error) { + cmd.RunArgs = args + err = cmd.RunErr + return +} + +var _ = Describe("Runner", func() { + Describe("Run", func() { + It("run can run a command and return its error", func() { + factory := &FakeFactory{ + CreateCmd: &FakeCmd{ + RunErr: errors.New("fake-run-error"), + }, + } + cmdRunner := NewRunner(factory) + + err := cmdRunner.Run([]string{"put", "foo", "bar"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("fake-run-error")) + + Expect(factory.CreateName).To(Equal("put")) + Expect(factory.CreateCmd.RunArgs).To(Equal([]string{"foo", "bar"})) + }) + + It("run expects at least one argument", func() { + factory := &FakeFactory{ + CreateCmd: &FakeCmd{}, + } + cmdRunner := NewRunner(factory) + + err := cmdRunner.Run([]string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("Missing command name")) + }) + + It("accepts exactly one argument", func() { + factory := &FakeFactory{ + CreateCmd: &FakeCmd{}, + } + cmdRunner := NewRunner(factory) + + err := cmdRunner.Run([]string{"put"}) + Expect(err).ToNot(HaveOccurred()) + + Expect(factory.CreateName).To(Equal("put")) + Expect(factory.CreateCmd.RunArgs).To(Equal([]string{})) + }) + }) + + Describe("SetConfig", func() { + It("delegates to factory", func() { + factory := &FakeFactory{} + cmdRunner := NewRunner(factory) + conf := davconf.Config{User: "foo"} + + err := cmdRunner.SetConfig(conf) + + Expect(factory.Config).To(Equal(conf)) + Expect(err).ToNot(HaveOccurred()) + }) + It("propagates errors", func() { + setConfigErr := errors.New("some error") + factory := &FakeFactory{ + SetConfigErr: setConfigErr, + } + cmdRunner := NewRunner(factory) + conf := davconf.Config{User: "foo"} + + err := cmdRunner.SetConfig(conf) + Expect(err).To(HaveOccurred()) + }) + }) +}) diff --git a/dav/cmd/sign.go b/dav/cmd/sign.go new file mode 100644 index 0000000..27b9ac6 --- /dev/null +++ b/dav/cmd/sign.go @@ -0,0 +1,41 @@ +package cmd + +import ( + "errors" + "fmt" + "time" + + davclient "github.com/cloudfoundry/storage-cli/dav/client" +) + +type SignCmd struct { + client davclient.Client +} + +func newSignCmd(client davclient.Client) (cmd SignCmd) { + cmd.client = client + return +} + +func (cmd SignCmd) Run(args []string) (err error) { + if len(args) != 3 { + err = errors.New("incorrect usage, sign requires: ") + return + } + + objectID, action := args[0], args[1] + + expiration, err := time.ParseDuration(args[2]) + if err != nil { + err = fmt.Errorf("expiration should be a duration value eg: 45s or 1h43m. Got: %s", args[2]) + return + } + + signedURL, err := cmd.client.Sign(objectID, action, expiration) + if err != nil { + return err + } + + fmt.Print(signedURL) + return +} diff --git a/dav/cmd/sign_test.go b/dav/cmd/sign_test.go new file mode 100644 index 0000000..f49731a --- /dev/null +++ b/dav/cmd/sign_test.go @@ -0,0 +1,80 @@ +package cmd_test + +import ( + "bytes" + "io" + "os" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + . "github.com/cloudfoundry/storage-cli/dav/cmd" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + + davconf "github.com/cloudfoundry/storage-cli/dav/config" +) + +func runSign(config davconf.Config, args []string) error { + logger := boshlog.NewLogger(boshlog.LevelNone) + factory := NewFactory(logger) + factory.SetConfig(config) //nolint:errcheck + + cmd, err := factory.Create("sign") + Expect(err).ToNot(HaveOccurred()) + + return cmd.Run(args) +} + +var _ = Describe("SignCmd", func() { + var ( + objectID = "0ca907f2-dde8-4413-a304-9076c9d0978b" + config davconf.Config + ) + + It("with valid args", func() { + old := os.Stdout // keep backup of the real stdout + r, w, _ := os.Pipe() //nolint:errcheck + os.Stdout = w + + err := runSign(config, []string{objectID, "get", "15m"}) + + outC := make(chan string) + // copy the output in a separate goroutine so printing can't block indefinitely + go func() { + var buf bytes.Buffer + io.Copy(&buf, r) //nolint:errcheck + outC <- buf.String() + }() + + // back to normal state + w.Close() //nolint:errcheck + os.Stdout = old // restoring the real stdout + out := <-outC + + Expect(err).ToNot(HaveOccurred()) + Expect(out).To(HavePrefix("signed/")) + Expect(out).To(ContainSubstring(objectID)) + Expect(out).To(ContainSubstring("?e=")) + Expect(out).To(ContainSubstring("&st=")) + Expect(out).To(ContainSubstring("&ts=")) + }) + + It("returns err with incorrect arg count", func() { + err := runSign(davconf.Config{}, []string{}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("incorrect usage")) + }) + + It("returns err with non-implemented action", func() { + err := runSign(davconf.Config{}, []string{objectID, "delete", "15m"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("action not implemented")) + }) + + It("returns err with incorrect duration", func() { + err := runSign(davconf.Config{}, []string{objectID, "put", "15"}) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("expiration should be a duration value")) + }) +}) diff --git a/dav/cmd/testing/http_request.go b/dav/cmd/testing/http_request.go new file mode 100644 index 0000000..912d363 --- /dev/null +++ b/dav/cmd/testing/http_request.go @@ -0,0 +1,47 @@ +package testing + +import ( + "encoding/base64" + "errors" + "net/http" + "strings" +) + +type HTTPRequest struct { + *http.Request +} + +func NewHTTPRequest(req *http.Request) (testReq HTTPRequest) { + return HTTPRequest{req} +} + +func (req HTTPRequest) ExtractBasicAuth() (username, password string, err error) { + authHeader := req.Header["Authorization"] + if len(authHeader) != 1 { + err = errors.New("Missing basic auth header") //nolint:staticcheck + return + } + + encodedAuth := authHeader[0] + encodedAuthParts := strings.Split(encodedAuth, " ") + if len(encodedAuthParts) != 2 { + err = errors.New("Invalid basic auth header format") //nolint:staticcheck + return + } + + clearAuth, err := base64.StdEncoding.DecodeString(encodedAuthParts[1]) + if len(encodedAuthParts) != 2 { + err = errors.New("Invalid basic auth header encoding") //nolint:staticcheck + return + } + + clearAuthParts := strings.Split(string(clearAuth), ":") + if len(clearAuthParts) != 2 { + err = errors.New("Invalid basic auth header encoded username and pwd") //nolint:staticcheck + return + } + + username = clearAuthParts[0] + password = clearAuthParts[1] + return +} diff --git a/dav/cmd/testing/testing_suite_test.go b/dav/cmd/testing/testing_suite_test.go new file mode 100644 index 0000000..eb53406 --- /dev/null +++ b/dav/cmd/testing/testing_suite_test.go @@ -0,0 +1,13 @@ +package testing_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestTesting(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Davcli Testing Suite") +} diff --git a/dav/cmd/testing/tls_server.go b/dav/cmd/testing/tls_server.go new file mode 100644 index 0000000..6bdeb96 --- /dev/null +++ b/dav/cmd/testing/tls_server.go @@ -0,0 +1,31 @@ +package testing + +import ( + "bytes" + "crypto/x509" + "encoding/pem" + "net/http/httptest" +) + +func ExtractRootCa(server *httptest.Server) (rootCaStr string, err error) { + rootCa := new(bytes.Buffer) + + cert, err := x509.ParseCertificate(server.TLS.Certificates[0].Certificate[0]) + if err != nil { + panic(err.Error()) + } + // TODO: Replace above with following on Go 1.9 + //cert := server.Certificate() + + block := &pem.Block{ + Type: "CERTIFICATE", + Bytes: cert.Raw, + } + + err = pem.Encode(rootCa, block) + if err != nil { + return "", err + } + + return rootCa.String(), nil +} diff --git a/dav/config/config.go b/dav/config/config.go new file mode 100644 index 0000000..31c637b --- /dev/null +++ b/dav/config/config.go @@ -0,0 +1,18 @@ +package config + +type Config struct { + User string + Password string + Endpoint string + RetryAttempts uint + TLS TLS + Secret string +} + +type TLS struct { + Cert Cert +} + +type Cert struct { + CA string +} diff --git a/dav/config/config_suite_test.go b/dav/config/config_suite_test.go new file mode 100644 index 0000000..dd39ef7 --- /dev/null +++ b/dav/config/config_suite_test.go @@ -0,0 +1,13 @@ +package config_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Davcli Config Suite") +} diff --git a/dav/main/dav.go b/dav/main/dav.go new file mode 100644 index 0000000..6f1822a --- /dev/null +++ b/dav/main/dav.go @@ -0,0 +1,30 @@ +package main + +import ( + "fmt" + "os" + "strings" + + boshlog "github.com/cloudfoundry/bosh-utils/logger" + "github.com/cloudfoundry/storage-cli/dav/app" + "github.com/cloudfoundry/storage-cli/dav/cmd" +) + +func main() { + logger := boshlog.NewLogger(boshlog.LevelNone) + cmdFactory := cmd.NewFactory(logger) + + cmdRunner := cmd.NewRunner(cmdFactory) + + cli := app.New(cmdRunner) + + err := cli.Run(os.Args) + if err != nil { + if strings.Contains(err.Error(), "not found") { + fmt.Printf("Blob not found - %s", err.Error()) + os.Exit(3) + } + fmt.Printf("Error running app - %s", err.Error()) + os.Exit(1) + } +} diff --git a/dav/main/main_suite_test.go b/dav/main/main_suite_test.go new file mode 100644 index 0000000..92223fd --- /dev/null +++ b/dav/main/main_suite_test.go @@ -0,0 +1,13 @@ +package main_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestMain(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Davcli Main Suite") +} diff --git a/dav/signer/signer.go b/dav/signer/signer.go new file mode 100644 index 0000000..55f203e --- /dev/null +++ b/dav/signer/signer.go @@ -0,0 +1,63 @@ +package signer + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "time" +) + +type Signer interface { + GenerateSignedURL(endpoint, prefixedBlobID, verb string, timeStamp time.Time, expiresAfter time.Duration) (string, error) +} + +type signer struct { + secret string +} + +func NewSigner(secret string) Signer { + return &signer{ + secret: secret, + } +} + +func (s *signer) generateSignature(prefixedBlobID, verb string, timeStamp time.Time, expires int) string { + verb = strings.ToUpper(verb) + signature := fmt.Sprintf("%s%s%d%d", verb, prefixedBlobID, timeStamp.Unix(), expires) + hmac := hmac.New(sha256.New, []byte(s.secret)) + hmac.Write([]byte(signature)) + sigBytes := hmac.Sum(nil) + return base64.URLEncoding.WithPadding(base64.NoPadding).EncodeToString(sigBytes) +} + +func (s *signer) GenerateSignedURL(endpoint, prefixedBlobID, verb string, timeStamp time.Time, expiresAfter time.Duration) (string, error) { + verb = strings.ToUpper(verb) + if verb != "GET" && verb != "PUT" { + return "", fmt.Errorf("action not implemented: %s. Available actions are 'GET' and 'PUT'", verb) + } + + endpoint = strings.TrimSuffix(endpoint, "/") + expiresAfterSeconds := int(expiresAfter.Seconds()) + signature := s.generateSignature(prefixedBlobID, verb, timeStamp, expiresAfterSeconds) + + blobURL, err := url.Parse(endpoint) + if err != nil { + return "", err + } + blobURL.Path = path.Join(blobURL.Path, "signed", prefixedBlobID) + req, err := http.NewRequest(verb, blobURL.String(), nil) + if err != nil { + return "", err + } + q := req.URL.Query() + q.Add("st", signature) + q.Add("ts", fmt.Sprintf("%d", timeStamp.Unix())) + q.Add("e", fmt.Sprintf("%d", expiresAfterSeconds)) + req.URL.RawQuery = q.Encode() + return req.URL.String(), nil +} diff --git a/dav/signer/signer_suite_test.go b/dav/signer/signer_suite_test.go new file mode 100644 index 0000000..4e8df40 --- /dev/null +++ b/dav/signer/signer_suite_test.go @@ -0,0 +1,13 @@ +package signer_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "testing" +) + +func TestSigner(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "DavCli Signer Suite") +} diff --git a/dav/signer/signer_test.go b/dav/signer/signer_test.go new file mode 100644 index 0000000..d76264a --- /dev/null +++ b/dav/signer/signer_test.go @@ -0,0 +1,30 @@ +package signer_test + +import ( + "time" + + "github.com/cloudfoundry/storage-cli/dav/signer" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Signer", func() { + secret := "mefq0umpmwevpv034m890j34m0j0-9!fijm434j99j034mjrwjmv9m304mj90;2ef32buf32gbu2i3" + objectID := "fake-object-id" + verb := "get" + signer := signer.NewSigner(secret) + duration := time.Duration(15 * time.Minute) + timeStamp := time.Date(2019, 8, 26, 11, 11, 0, 0, time.UTC) + path := "https://api.example.com/" + + Context("HMAC Signed URL", func() { + + expected := "https://api.example.com/signed/fake-object-id?e=900&st=BxLKZK_dTSLyBis1pAjdwq4aYVrJvXX6vvLpdCClGYo&ts=1566817860" + + It("Generates a properly formed URL", func() { + actual, err := signer.GenerateSignedURL(path, objectID, verb, timeStamp, duration) + Expect(err).To(BeNil()) + Expect(actual).To(Equal(expected)) + }) + }) +}) diff --git a/dav/test_assets/cat.jpg b/dav/test_assets/cat.jpg new file mode 100644 index 0000000..995d644 Binary files /dev/null and b/dav/test_assets/cat.jpg differ diff --git a/dav/test_assets/dav-cli-config-with-ca.json b/dav/test_assets/dav-cli-config-with-ca.json new file mode 100644 index 0000000..fdec3de --- /dev/null +++ b/dav/test_assets/dav-cli-config-with-ca.json @@ -0,0 +1,11 @@ +{ + "user":"some user", + "password":"some pwd", + "endpoint":"https://example.com/some/endpoint", + "secret": "77D47E3A0B0F590B73CF3EBD9BB6761E244F90FA6F28BB39F941B0905789863FBE2861FDFD8195ADC81B72BB5310BC18969BEBBF4656366E7ACD3F0E4186FDDA", + "tls": { + "cert": { + "ca":"ca-cert" + } + } +} diff --git a/dav/test_assets/dav-cli-config.json b/dav/test_assets/dav-cli-config.json new file mode 100644 index 0000000..5defc00 --- /dev/null +++ b/dav/test_assets/dav-cli-config.json @@ -0,0 +1,6 @@ +{ + "user":"some user", + "password":"some pwd", + "endpoint":"http://example.com/some/endpoint", + "secret": "77D47E3A0B0F590B73CF3EBD9BB6761E244F90FA6F28BB39F941B0905789863FBE2861FDFD8195ADC81B72BB5310BC18969BEBBF4656366E7ACD3F0E4186FDDA" +} diff --git a/gcs/README.md b/gcs/README.md new file mode 100644 index 0000000..3e6ac17 --- /dev/null +++ b/gcs/README.md @@ -0,0 +1,88 @@ +# GCS Storage CLI +A Golang CLI for uploading, fetching and deleting content to/from [Google Cloud Storage](https://cloud.google.com/storage/). +This tool exists to work with the [bosh-cli](https://github.com/cloudfoundry/bosh-cli) and [director](https://github.com/cloudfoundry/bosh). + +This is **not** an official Google Product. + + +## Commands + +### Usage +```bash +gcs-cli --help +``` +### Upload an object +```bash +gcs-cli -c config.json put +``` +### Fetch an object +```bash +gcs-cli -c config.json get +``` +### Delete an object +```bash +gcs-cli -c config.json delete +``` +### Check if an object exists +```bash +gcs-cli -c config.json exists +``` + +### Generate a signed url for an object +If there is an encryption key present in the config, then an additional header is sent + +```bash +gcs-cli -c config.json sign +``` +Where: + - `` is GET, PUT, or DELETE + - `` is a duration string less than 7 days (e.g. "6h") + +## Configuration +The command line tool expects a JSON configuration file. Run `storage-cli-gcs --help` for details. + +### Authentication Methods (`credentials_source`) +* `static`: A [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) key will be provided via the `json_key` field. +* `none`: No credentials are provided. The client is reading from a public bucket. +* <empty>: [Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials) + will be used if they exist (either through `gcloud auth application-default login` or a [service account](https://cloud.google.com/iam/docs/understanding-service-accounts)). + If they don't exist the client will fall back to `none` behavior. + +## Running Integration Tests + +1. Ensure [gcloud](https://cloud.google.com/sdk/downloads) is installed and you have authenticated (`gcloud auth login`). + These credentials will be used by the Makefile to create/destroy Google Cloud Storage buckets for testing. +1. Set the Google Cloud project: `gcloud config set project ` +1. Generate a service account with the `Storage Admin` role for your project and set the contents as + the environment variable `GOOGLE_APPLICATION_CREDENTIALS`, for example: + ```bash + export project_id=$(gcloud config get-value project) + + export service_account_name=storage-cli-gcs-integration-tests + export service_account_email=${service_account_name}@${project_id}.iam.gserviceaccount.com + credentials_file=$(mktemp) + + gcloud config set project ${project_id} + gcloud iam service-accounts create ${service_account_name} --display-name "Integration Test Access for storage-cli-gcs " + gcloud iam service-accounts keys create ${credentials_file} --iam-account ${service_account_email} + gcloud project add-iam-policy-binding ${project_id} --member serviceAccount:${service_account_email} --role roles/storage.admin + + export GOOGLE_SERVICE_ACCOUNT="$(cat ${credentials_file})" + export GOOGLE_APPLICATION_CREDENTIALS="$(cat ${credentials_file})" + export LC_ALL=C # fix `tr` complaining about "illegal byte sequence" on OSX + ``` +1. Run the unit and fast integration tests: `make test-fast-int` +1. Clean up buckets: `make clean-gcs` + +## Development + +* A Makefile is provided that automates integration testing. Try `make help` to get started. +* [gvt](https://godoc.org/github.com/FiloSottile/gvt) is used for vendoring. + +## Contributing + +For details on how to contribute to this project - including filing bug reports and contributing code changes - please see [CONTRIBUTING.md](./CONTRIBUTING.md). + +## License + +This tool is licensed under Apache 2.0. Full license text is available in [LICENSE](LICENSE). diff --git a/gcs/client/client.go b/gcs/client/client.go new file mode 100644 index 0000000..10b6aad --- /dev/null +++ b/gcs/client/client.go @@ -0,0 +1,227 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "context" + "errors" + "fmt" + "io" + "log" + "time" + + "golang.org/x/oauth2/google" + + "cloud.google.com/go/storage" + + "github.com/cloudfoundry/storage-cli/gcs/config" +) + +// ErrInvalidROWriteOperation is returned when credentials associated with the +// client disallow an attempted write operation. +var ErrInvalidROWriteOperation = errors.New("the client operates in read only mode. Change 'credentials_source' parameter value ") + +// GCSBlobstore encapsulates interaction with the GCS blobstore +type GCSBlobstore struct { + authenticatedGCS *storage.Client + publicGCS *storage.Client + config *config.GCSCli +} + +// validateRemoteConfig determines if the configuration of the client matches +// against the remote configuration +// +// If operating in read-only mode, no mutations can be performed +// so the remote bucket location is always compatible. +func (client *GCSBlobstore) validateRemoteConfig() error { + if client.readOnly() { + return nil + } + + bucket := client.authenticatedGCS.Bucket(client.config.BucketName) + _, err := bucket.Attrs(context.Background()) + return err +} + +// getObjectHandle returns a handle to an object named src +func (client *GCSBlobstore) getObjectHandle(gcs *storage.Client, src string) *storage.ObjectHandle { + handle := gcs.Bucket(client.config.BucketName).Object(src) + if client.config.EncryptionKey != nil { + handle = handle.Key(client.config.EncryptionKey) + } + return handle +} + +// New returns a GCSBlobstore configured to operate using the given config +// +// non-nil error is returned on invalid Client or config. If the configuration +// is incompatible with the GCS bucket, a non-nil error is also returned. +func New(ctx context.Context, cfg *config.GCSCli) (*GCSBlobstore, error) { + if cfg == nil { + return nil, errors.New("expected non-nill config object") + } + + authenticatedGCS, publicGCS, err := newStorageClients(ctx, cfg) + if err != nil { + return nil, fmt.Errorf("creating storage client: %v", err) + } + + return &GCSBlobstore{authenticatedGCS: authenticatedGCS, publicGCS: publicGCS, config: cfg}, nil +} + +// Get fetches a blob from the GCS blobstore. +// Destination will be overwritten if it already exists. +func (client *GCSBlobstore) Get(src string, dest io.Writer) error { + reader, err := client.getReader(client.publicGCS, src) + + // If the public client fails, try using it as an authenticated actor + if err != nil && client.authenticatedGCS != nil { + reader, err = client.getReader(client.authenticatedGCS, src) + } + + if err != nil { + return err + } + + _, err = io.Copy(dest, reader) + return err +} + +func (client *GCSBlobstore) getReader(gcs *storage.Client, src string) (*storage.Reader, error) { + return client.getObjectHandle(gcs, src).NewReader(context.Background()) +} + +// Put uploads a blob to the GCS blobstore. +// Destination will be overwritten if it already exists. +// +// Put retries retryAttempts times +const retryAttempts = 3 + +func (client *GCSBlobstore) Put(src io.ReadSeeker, dest string) error { + if client.readOnly() { + return ErrInvalidROWriteOperation + } + + if err := client.validateRemoteConfig(); err != nil { + return err + } + + pos, err := src.Seek(0, io.SeekCurrent) + if err != nil { + return fmt.Errorf("finding buffer position: %v", err) + } + + var errs []error + for i := 0; i < retryAttempts; i++ { + err := client.putOnce(src, dest) + if err == nil { + return nil + } + + errs = append(errs, err) + log.Printf("upload failed for %s, attempt %d/%d: %v\n", dest, i+1, retryAttempts, err) + + if _, err := src.Seek(pos, io.SeekStart); err != nil { + return fmt.Errorf("restting buffer position after failed upload: %v", err) + } + } + + return fmt.Errorf("upload failed for %s after %d attempts: %v", dest, retryAttempts, errs) +} + +func (client *GCSBlobstore) putOnce(src io.ReadSeeker, dest string) error { + remoteWriter := client.getObjectHandle(client.authenticatedGCS, dest).NewWriter(context.Background()) //nolint:staticcheck + remoteWriter.ObjectAttrs.StorageClass = client.config.StorageClass //nolint:staticcheck + + if _, err := io.Copy(remoteWriter, src); err != nil { + remoteWriter.CloseWithError(err) //nolint:errcheck,staticcheck + return err + } + + return remoteWriter.Close() +} + +// Delete removes a blob from from the GCS blobstore. +// +// If the object does not exist, Delete returns a nil error. +func (client *GCSBlobstore) Delete(dest string) error { + if client.readOnly() { + return ErrInvalidROWriteOperation + } + + err := client.getObjectHandle(client.authenticatedGCS, dest).Delete(context.Background()) + if errors.Is(err, storage.ErrObjectNotExist) { + return nil + } + return err +} + +// Exists checks if a blob exists in the GCS blobstore. +func (client *GCSBlobstore) Exists(dest string) (exists bool, err error) { + if exists, err = client.exists(client.publicGCS, dest); err == nil { + return exists, nil + } + + // If the public client fails, try using it as an authenticated actor + if client.authenticatedGCS != nil { + return client.exists(client.authenticatedGCS, dest) + } + + return +} + +func (client *GCSBlobstore) exists(gcs *storage.Client, dest string) (bool, error) { + _, err := client.getObjectHandle(gcs, dest).Attrs(context.Background()) + if err == nil { + log.Printf("File '%s' exists in bucket '%s'\n", dest, client.config.BucketName) + return true, nil + } else if errors.Is(err, storage.ErrObjectNotExist) { + log.Printf("File '%s' does not exist in bucket '%s'\n", dest, client.config.BucketName) + return false, nil + } + return false, err +} + +func (client *GCSBlobstore) readOnly() bool { + return client.authenticatedGCS == nil +} + +func (client *GCSBlobstore) Sign(id string, action string, expiry time.Duration) (string, error) { + token, err := google.JWTConfigFromJSON([]byte(client.config.ServiceAccountFile), storage.ScopeFullControl) + if err != nil { + return "", err + } + options := storage.SignedURLOptions{ + Method: action, + Expires: time.Now().Add(expiry), + PrivateKey: token.PrivateKey, + GoogleAccessID: token.Email, + Scheme: storage.SigningSchemeV4, + } + + // GET/PUT to the resultant signed url must include, in addition to the below: + // 'x-goog-encryption-key' and 'x-goog-encryption-key-sha256' + willEncrypt := len(client.config.EncryptionKey) > 0 + if willEncrypt { + options.Headers = []string{ + "x-goog-encryption-algorithm: AES256", + fmt.Sprintf("x-goog-encryption-key: %s", client.config.EncryptionKeyEncoded), + fmt.Sprintf("x-goog-encryption-key-sha256: %s", client.config.EncryptionKeySha256), + } + } + return storage.SignedURL(client.config.BucketName, id, &options) +} diff --git a/gcs/client/sdk.go b/gcs/client/sdk.go new file mode 100644 index 0000000..77c2249 --- /dev/null +++ b/gcs/client/sdk.go @@ -0,0 +1,54 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package client + +import ( + "context" + "errors" + + "golang.org/x/oauth2/google" + + "google.golang.org/api/option" + + "net/http" + + "cloud.google.com/go/storage" + "github.com/cloudfoundry/storage-cli/gcs/config" +) + +const uaString = "storage-cli-gcs" + +func newStorageClients(ctx context.Context, cfg *config.GCSCli) (*storage.Client, *storage.Client, error) { + publicClient, err := storage.NewClient(ctx, option.WithUserAgent(uaString), option.WithHTTPClient(http.DefaultClient)) + var authenticatedClient *storage.Client + + switch cfg.CredentialsSource { + case config.NoneCredentialsSource: + // no-op + case config.DefaultCredentialsSource: + if tokenSource, err := google.DefaultTokenSource(ctx, storage.ScopeFullControl); err == nil { + authenticatedClient, err = storage.NewClient(ctx, option.WithUserAgent(uaString), option.WithTokenSource(tokenSource)) //nolint:ineffassign,staticcheck + } + case config.ServiceAccountFileCredentialsSource: + if token, err := google.JWTConfigFromJSON([]byte(cfg.ServiceAccountFile), storage.ScopeFullControl); err == nil { + authenticatedClient, err = storage.NewClient(ctx, option.WithUserAgent(uaString), option.WithTokenSource(token.TokenSource(ctx))) //nolint:ineffassign,staticcheck + } + default: + return nil, nil, errors.New("unknown credentials_source in configuration") + } + return authenticatedClient, publicClient, err +} diff --git a/gcs/config/config.go b/gcs/config/config.go new file mode 100644 index 0000000..29ee733 --- /dev/null +++ b/gcs/config/config.go @@ -0,0 +1,111 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "io" +) + +// GCSCli represents the configuration for the gcscli +type GCSCli struct { + // BucketName is the GCS bucket operations will use. + BucketName string `json:"bucket_name"` + // CredentialsSource is the location of a Service Account File. + // If left empty, Application Default Credentials will be used if available. + // If equal to 'none', read-only scope will be used. + // If equal to 'static', json_key will be used. + CredentialsSource string `json:"credentials_source"` + // ServiceAccountFile is the contents of a JSON Service Account File. + // Required if credentials_source is 'static', otherwise ignored. + ServiceAccountFile string `json:"json_key"` + // StorageClass is the type of storage used for objects added to the bucket + // https://cloud.google.com/storage/docs/storage-classes + StorageClass string `json:"storage_class"` + // EncryptionKey is a Customer-Supplied encryption key used to + // encrypt objects added to the bucket. + // If left empty, no explicit encryption key will be used; + // GCS transparently encrypts data using server-side encryption keys. + // https://cloud.google.com/storage/docs/encryption + EncryptionKey []byte `json:"encryption_key"` + + EncryptionKeyEncoded string + EncryptionKeySha256 string +} + +// DefaultCredentialsSource specifies that credentials should be detected. +// Application Default Credentials will be used if avaliable. +// A read-only client will be used otherwise. +const DefaultCredentialsSource = "" + +// NoneCredentialsSource specifies that credentials are explicitly empty +// and that the client should be restricted to a read-only scope. +const NoneCredentialsSource = "none" + +// ServiceAccountFileCredentialsSource specifies that a service account file +// included in json_key should be used for authentication. +const ServiceAccountFileCredentialsSource = "static" + +// ErrEmptyBucketName is returned when a bucket_name in the config is empty +var ErrEmptyBucketName = errors.New("bucket_name must be set") + +// ErrEmptyServiceAccountFile is returned when json_key in the +// config is empty when StaticCredentialsSource is explicitly requested. +var ErrEmptyServiceAccountFile = errors.New("json_key must be set") + +// ErrWrongLengthEncryptionKey is returned when a non-nil encryption_key +// in the config is not exactly 32 bytes. +var ErrWrongLengthEncryptionKey = errors.New("encryption_key not 32 bytes") + +// NewFromReader returns the new gcscli configuration struct from the +// contents of the reader. +// +// reader.Read() is expected to return valid JSON. +func NewFromReader(reader io.Reader) (GCSCli, error) { + + dec := json.NewDecoder(reader) + var c GCSCli + if err := dec.Decode(&c); err != nil { + return GCSCli{}, err + } + + if c.BucketName == "" { + return GCSCli{}, ErrEmptyBucketName + } + + if c.CredentialsSource == ServiceAccountFileCredentialsSource && + c.ServiceAccountFile == "" { + return GCSCli{}, ErrEmptyServiceAccountFile + } + + if len(c.EncryptionKey) != 32 && c.EncryptionKey != nil { + return GCSCli{}, ErrWrongLengthEncryptionKey + } + + if len(c.EncryptionKey) > 0 { + c.EncryptionKeyEncoded = base64.StdEncoding.EncodeToString(c.EncryptionKey) + + encryptionKeySha := sha256.New() + encryptionKeySha.Write(c.EncryptionKey) + c.EncryptionKeySha256 = base64.StdEncoding.EncodeToString(encryptionKeySha.Sum(nil)) + } + + return c, nil +} diff --git a/gcs/config/config_suite_test.go b/gcs/config/config_suite_test.go new file mode 100644 index 0000000..2bd95f0 --- /dev/null +++ b/gcs/config/config_suite_test.go @@ -0,0 +1,29 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "testing" +) + +func TestConfig(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Config Suite") +} diff --git a/gcs/config/config_test.go b/gcs/config/config_test.go new file mode 100644 index 0000000..2d856c8 --- /dev/null +++ b/gcs/config/config_test.go @@ -0,0 +1,149 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package config_test + +import ( + "bytes" + + . "github.com/cloudfoundry/storage-cli/gcs/config" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("BlobstoreClient configuration", func() { + Describe("when bucket is not specified", func() { + dummyJSONBytes := []byte(`{}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("returns an error", func() { + _, err := NewFromReader(dummyJSONReader) + Expect(err).To(MatchError(ErrEmptyBucketName)) + }) + }) + + Describe("when bucket is specified", func() { + dummyJSONBytes := []byte(`{"bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("uses the given bucket", func() { + c, err := NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.BucketName).To(Equal("some-bucket")) + }) + }) + + Describe("when credentials_source is specified", func() { + dummyJSONBytes := []byte(`{"credentials_source": "/tmp/foobar.json", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("uses the credentials", func() { + c, err := NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.CredentialsSource).To(Equal("/tmp/foobar.json")) + }) + }) + + Describe("when credentials_source is 'static' with json_key", func() { + dummyJSONBytes := []byte(`{"credentials_source": "static", "json_key": "{\"foo\": \"bar\"}", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("uses the credentials", func() { + c, err := NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.ServiceAccountFile).ToNot(BeEmpty()) + }) + }) + + Describe("when credentials_source is 'static' without json_key", func() { + dummyJSONBytes := []byte(`{"credentials_source": "static", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("returns an error", func() { + _, err := NewFromReader(dummyJSONReader) + Expect(err).To(Equal(ErrEmptyServiceAccountFile)) + }) + }) + + Describe("when credentials_source is not specified", func() { + dummyJSONBytes := []byte(`{"credentials_source": "", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("uses the Application Default Credentials", func() { + _, err := NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Describe("when encryption_key is specified", func() { + // encryption_key = []byte{0, 1, 2, ..., 31} as base64 + dummyJSONBytes := []byte(`{"encryption_key": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("uses the given key", func() { + c, err := NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(len(c.EncryptionKey)).To(Equal(32)) + Expect(c.EncryptionKeyEncoded).To(Equal("AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8=")) + Expect(c.EncryptionKeySha256).To(Equal("Yw3NKWbEM2aRElRIu7JbT/QSpJxzLbLIq8G4WBvXEN0=")) + }) + }) + + Describe("when encryption_key is too long", func() { + // encryption_key = []byte{0, 1, 2, ..., 31, 32} as base64 + dummyJSONBytes := []byte(`{"encryption_key": "AAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8g", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("returns an error", func() { + _, err := NewFromReader(dummyJSONReader) + Expect(err).To(Equal(ErrWrongLengthEncryptionKey)) + }) + }) + + Describe("when encryption_key is malformed", func() { + // encryption_key is not valid base64 + dummyJSONBytes := []byte(`{"encryption_key": "zzz", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("returns an error", func() { + _, err := NewFromReader(dummyJSONReader) + Expect(err).ToNot(BeNil()) + }) + }) + + Describe("when encryption_key is not specified", func() { + dummyJSONBytes := []byte(`{"bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("uses no encryption", func() { + c, err := NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.EncryptionKey).To(BeNil()) + }) + }) + + Describe("when json is invalid", func() { + dummyJSONBytes := []byte(`{"credentials_source": '`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("returns an error", func() { + _, err := NewFromReader(dummyJSONReader) + Expect(err).ToNot(BeNil()) + }) + }) + +}) diff --git a/gcs/integration/assertcontext.go b/gcs/integration/assertcontext.go new file mode 100644 index 0000000..f3fcd97 --- /dev/null +++ b/gcs/integration/assertcontext.go @@ -0,0 +1,182 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "fmt" + "os" + + "github.com/cloudfoundry/storage-cli/gcs/config" + + . "github.com/onsi/gomega" //nolint:staticcheck + "golang.org/x/net/context" +) + +// GoogleAppCredentialsEnv is the environment variable +// expected to be populated with a path to a Service Account File for testing +// Application Default Credentials +const GoogleAppCredentialsEnv = "GOOGLE_APPLICATION_CREDENTIALS" + +// ServiceAccountFileEnv is the environment variable +// expected to be populated with a Service Account File for testing +const ServiceAccountFileEnv = "GOOGLE_SERVICE_ACCOUNT" + +// ServiceAccountFileMsg is the template used when ServiceAccountFileEnv +// has not been populated +const ServiceAccountFileMsg = "environment variable %s expected to contain a valid Service Account File but was empty" + +// AssertContext contains the generated content to be used within tests. +// +// This allows Assertions to not have to worry about setup and teardown. +type AssertContext struct { + // Config is the configuration used to + Config *config.GCSCli + // ConfigPath is the path to the file containing the + // serialized content of Config. + ConfigPath string + + // GCSFileName is the name of whatever blob is generated in an + // assertion. It is the assert's responsibility to remove the blob. + GCSFileName string + // ExpectedString is the generated content used in an assertion. + ExpectedString string + // ContentFile is the path of the file containing ExpectedString + ContentFile string + + // serviceAccountFile is the contents of a Service Account File + // This is used in various contexts to authenticate + serviceAccountFile string + + // serviceAccountPath is the path to a Service Account File created + // to allow the use of Application Default Credentials + serviceAccountPath string + + // options are the AssertContextConfigOption which are used to modify + // the configuration whenever AddConfig is called. + options []AssertContextConfigOption + + // ctx is the context used by the individual test + ctx context.Context +} + +// NewAssertContext returns an AssertContext with all fields +// which can be generated filled in. +func NewAssertContext(options ...AssertContextConfigOption) AssertContext { + expectedString := GenerateRandomString() + + serviceAccountFile := os.Getenv(ServiceAccountFileEnv) + Expect(serviceAccountFile).ToNot(BeEmpty(), + fmt.Sprintf(ServiceAccountFileMsg, ServiceAccountFileEnv)) + + return AssertContext{ + ExpectedString: expectedString, + ContentFile: MakeContentFile(expectedString), + GCSFileName: GenerateRandomString(), + serviceAccountFile: serviceAccountFile, + options: options, + ctx: context.Background(), + } +} + +// AddConfig includes the config.GCSCli required for AssertContext +// +// Configuration is typically not available immediately before a test +// can be run, hence the need to add it later. +func (ctx *AssertContext) AddConfig(config *config.GCSCli) { + ctx.Config = config + for _, opt := range ctx.options { + opt(ctx) + } + ctx.ConfigPath = MakeConfigFile(ctx.Config) +} + +// AssertContextConfigOption is an option used for configuring an +// AssertContext's handling of the config. +// +// The behavior for an AssertContextAuthOption is applied when config is added +type AssertContextConfigOption func(ctx *AssertContext) + +// AsReadOnlyCredentials configures the AssertContext to be used soley for +// public, read-only operations. +func AsReadOnlyCredentials(ctx *AssertContext) { + conf := ctx.Config + Expect(conf).ToNot(BeNil(), + "cannot set read-only AssertContext without config") + + conf.CredentialsSource = config.NoneCredentialsSource + conf.ServiceAccountFile = "" +} + +// AsStaticCredentials configures the AssertContext to authenticate explicitly +// using a Service Account File +func AsStaticCredentials(ctx *AssertContext) { + conf := ctx.Config + Expect(conf).ToNot(BeNil(), + "cannot set static AssertContext without config") + + conf.ServiceAccountFile = ctx.serviceAccountFile + conf.CredentialsSource = config.ServiceAccountFileCredentialsSource +} + +// AsDefaultCredentials configures the AssertContext to authenticate using +// Application Default Credentials populated using the +// testing service account file. +func AsDefaultCredentials(ctx *AssertContext) { + conf := ctx.Config + Expect(conf).ToNot(BeNil(), + "cannot set static AssertContext without config") + + tempFile, err := os.CreateTemp("", "bosh-gcscli-service-account-file") + Expect(err).ToNot(HaveOccurred()) + defer tempFile.Close() //nolint:errcheck + + tempFile.WriteString(ctx.serviceAccountFile) //nolint:errcheck + + ctx.serviceAccountPath = tempFile.Name() + os.Setenv(GoogleAppCredentialsEnv, ctx.serviceAccountPath) //nolint:errcheck + + conf.CredentialsSource = config.DefaultCredentialsSource +} + +// Clone returns a new AssertContext configured using the provided options. +// This overwrites the previous options of the context. +// +// This is useful in assertions where initial setup must be done under one +// form of authentication and the actual assertion is done under another. +// +// Note: The returned AssertContext is a distinct AssertContext, Cleanup must +// be called to remove testing files from the filesystem. +func (ctx *AssertContext) Clone(options ...AssertContextConfigOption) AssertContext { + conf := *ctx.Config + + newContext := *ctx + newContext.options = options + newContext.AddConfig(&conf) + + return newContext +} + +// Cleanup removes artifacts generated by the AssertContext. +func (ctx *AssertContext) Cleanup() { + os.Remove(ctx.ConfigPath) //nolint:errcheck + os.Remove(ctx.ContentFile) //nolint:errcheck + + if ctx.serviceAccountPath != "" { + os.Remove(ctx.serviceAccountPath) //nolint:errcheck + } + os.Unsetenv(GoogleAppCredentialsEnv) //nolint:errcheck +} diff --git a/gcs/integration/assertions.go b/gcs/integration/assertions.go new file mode 100644 index 0000000..2fa76f7 --- /dev/null +++ b/gcs/integration/assertions.go @@ -0,0 +1,75 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "os" + + . "github.com/onsi/gomega" //nolint:staticcheck +) + +// NoLongEnv must be set in the environment +// to enable skipping long running tests +const NoLongEnv = "SKIP_LONG_TESTS" + +// NoLongMsg is the template used when BucketNoLongEnv's environment variable +// has not been populated. +const NoLongMsg = "environment variable %s filled, skipping long test" + +// AssertLifecycleWorks tests the main blobstore object lifecycle from +// creation to deletion. +// +// This is using gomega matchers, so it will fail if called outside an +// 'It' test. +func AssertLifecycleWorks(gcsCLIPath string, ctx AssertContext) { + session, err := RunGCSCLI(gcsCLIPath, ctx.ConfigPath, + "put", ctx.ContentFile, ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, + "exists", ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + Expect(session.Err.Contents()).To(MatchRegexp("File '.*' exists in bucket '.*'")) + + tmpLocalFile, err := os.CreateTemp("", "gcscli-download") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpLocalFile.Name()) //nolint:errcheck + err = tmpLocalFile.Close() + Expect(err).ToNot(HaveOccurred()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, + "get", ctx.GCSFileName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + gottenBytes, err := os.ReadFile(tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(string(gottenBytes)).To(Equal(ctx.ExpectedString)) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, + "delete", ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, + "exists", ctx.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(Equal(3)) + Expect(session.Err.Contents()).To(MatchRegexp("File '.*' does not exist in bucket '.*'")) +} diff --git a/gcs/integration/configurations.go b/gcs/integration/configurations.go new file mode 100644 index 0000000..3c997a2 --- /dev/null +++ b/gcs/integration/configurations.go @@ -0,0 +1,128 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "context" + "errors" + "fmt" + "net/http" + "os" + + "cloud.google.com/go/storage" + + "github.com/cloudfoundry/storage-cli/gcs/config" + + . "github.com/onsi/ginkgo/v2" //nolint:staticcheck + "golang.org/x/oauth2" + "golang.org/x/oauth2/google" + "golang.org/x/oauth2/jwt" + "google.golang.org/api/option" +) + +const regionalBucketEnv = "REGIONAL_BUCKET_NAME" +const multiRegionalBucketEnv = "MULTIREGIONAL_BUCKET_NAME" +const publicBucketEnv = "PUBLIC_BUCKET_NAME" + +// noBucketMsg is the template used when a BucketEnv's environment variable +// has not been populated. +const noBucketMsg = "environment variable %s expected to contain a valid Google Cloud Storage bucket but was empty" + +const getConfigErrMsg = "creating %s configs: %v" + +func readBucketEnv(env string) (string, error) { + bucket := os.Getenv(env) + if len(bucket) == 0 { + return "", fmt.Errorf(noBucketMsg, env) + } + return bucket, nil +} + +func getRegionalConfig() *config.GCSCli { + var regional string + var err error + + if regional, err = readBucketEnv(regionalBucketEnv); err != nil { + panic(fmt.Errorf(getConfigErrMsg, "base", err)) + } + + return &config.GCSCli{BucketName: regional} +} + +func getMultiRegionConfig() *config.GCSCli { + var multiRegional string + var err error + + if multiRegional, err = readBucketEnv(multiRegionalBucketEnv); err != nil { + panic(fmt.Errorf(getConfigErrMsg, "base", err)) + } + + return &config.GCSCli{BucketName: multiRegional} +} + +func getBaseConfigs() []TableEntry { + regional := getRegionalConfig() + multiRegion := getMultiRegionConfig() + + return []TableEntry{ + Entry("Regional bucket, default StorageClass", regional), + Entry("MultiRegion bucket, default StorageClass", multiRegion), + } +} + +func getPublicConfig() *config.GCSCli { + public, err := readBucketEnv(publicBucketEnv) + if err != nil { + panic(fmt.Errorf(getConfigErrMsg, "public", err)) + } + + return &config.GCSCli{ + BucketName: public, + } +} + +// newSDK builds the GCS SDK Client from a valid config.GCSCli +// TODO: Simplify and remove this. Tests should expect a single config and use it. +func newSDK(ctx context.Context, c config.GCSCli) (*storage.Client, error) { + var client *storage.Client + var err error + var opt option.ClientOption + switch c.CredentialsSource { + case config.DefaultCredentialsSource: + var tokenSource oauth2.TokenSource + tokenSource, err = google.DefaultTokenSource(ctx, storage.ScopeFullControl) + if err == nil { + opt = option.WithTokenSource(tokenSource) + } + case config.NoneCredentialsSource: + opt = option.WithHTTPClient(http.DefaultClient) + case config.ServiceAccountFileCredentialsSource: + var token *jwt.Config + token, err = google.JWTConfigFromJSON([]byte(c.ServiceAccountFile), storage.ScopeFullControl) + if err == nil { + tokenSource := token.TokenSource(ctx) + opt = option.WithTokenSource(tokenSource) + } + default: + err = errors.New("unknown credentials_source in configuration") + } + if err != nil { + return client, err + } + + return storage.NewClient(ctx, opt) +} diff --git a/gcs/integration/gcs_encryption_test.go b/gcs/integration/gcs_encryption_test.go new file mode 100644 index 0000000..c37d71e --- /dev/null +++ b/gcs/integration/gcs_encryption_test.go @@ -0,0 +1,108 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "bytes" + "crypto/sha256" + + "github.com/cloudfoundry/storage-cli/gcs/client" + "github.com/cloudfoundry/storage-cli/gcs/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// encryptionKeyBytes are used as the key in tests requiring encryption. +var encryptionKeyBytes = []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31} + +// encryptionKeyBytesHash is the has of the encryptionKeyBytes +// +// Typical usage is ensuring the encryption key is actually used by GCS. +var encryptionKeyBytesHash = sha256.Sum256(encryptionKeyBytes) //nolint:unused + +var _ = Describe("Integration", func() { + Context("general (Default Applicaton Credentials) configuration", func() { + var ( + env AssertContext + cfg *config.GCSCli + ) + BeforeEach(func() { + cfg = getMultiRegionConfig() + cfg.EncryptionKey = encryptionKeyBytes + + env = NewAssertContext(AsDefaultCredentials) + env.AddConfig(cfg) + }) + AfterEach(func() { + env.Cleanup() + }) + + // tests that a blob uploaded with a specified encryption_key can be downloaded again. + It("can perform encrypted lifecycle", func() { + AssertLifecycleWorks(gcsCLIPath, env) + }) + + // tests that uploading a blob with encryption + // results in failure to download when the key is changed. + It("fails to get with the wrong encryption_key", func() { + Expect(env.Config.EncryptionKey).ToNot(BeNil(), + "Need encryption key for test") + + session, err := RunGCSCLI(gcsCLIPath, env.ConfigPath, + "put", env.ContentFile, env.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + blobstoreClient, err := client.New(env.ctx, env.Config) + Expect(err).ToNot(HaveOccurred()) + + env.Config.EncryptionKey[0]++ + + var target bytes.Buffer + err = blobstoreClient.Get(env.GCSFileName, &target) + Expect(err).To(HaveOccurred()) + + session, err = RunGCSCLI(gcsCLIPath, env.ConfigPath, "delete", env.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + }) + + // tests that uploading a blob with encryption + // results in failure to download without encryption. + It("fails to get with no encryption_key", func() { + Expect(env.Config.EncryptionKey).ToNot(BeNil(), + "Need encryption key for test") + + session, err := RunGCSCLI(gcsCLIPath, env.ConfigPath, "put", env.ContentFile, env.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + + blobstoreClient, err := client.New(env.ctx, env.Config) + Expect(err).ToNot(HaveOccurred()) + + env.Config.EncryptionKey = nil + + var target bytes.Buffer + err = blobstoreClient.Get(env.GCSFileName, &target) + Expect(err).To(HaveOccurred()) + + session, err = RunGCSCLI(gcsCLIPath, env.ConfigPath, "delete", env.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + }) + }) +}) diff --git a/gcs/integration/gcs_general_test.go b/gcs/integration/gcs_general_test.go new file mode 100644 index 0000000..d0ce8da --- /dev/null +++ b/gcs/integration/gcs_general_test.go @@ -0,0 +1,151 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "crypto/rand" + "fmt" + "io" + "os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/cloudfoundry/storage-cli/gcs/client" + "github.com/cloudfoundry/storage-cli/gcs/config" +) + +// randReadSeeker is a ReadSeeker which returns random content and +// non-nil error for every operation. +// +// crypto/rand is used to ensure any compression +// applied to the reader's output doesn't effect the work we intend to do. +type randReadSeeker struct { + reader io.Reader +} + +func newrandReadSeeker(maxSize int64) randReadSeeker { + limited := io.LimitReader(rand.Reader, maxSize) + return randReadSeeker{limited} +} + +func (rrs *randReadSeeker) Read(p []byte) (n int, err error) { + return rrs.reader.Read(p) +} + +func (rrs *randReadSeeker) Seek(offset int64, whenc int) (n int64, err error) { + return offset, nil +} + +// badReadSeeker is a ReadSeeker which returns a non-nil error +// for every operation. +type badReadSeeker struct{} + +var badReadSeekerErr = io.ErrUnexpectedEOF + +func (brs *badReadSeeker) Read(p []byte) (n int, err error) { + return 0, badReadSeekerErr +} + +func (brs *badReadSeeker) Seek(offset int64, whenc int) (n int64, err error) { + return 0, badReadSeekerErr +} + +var _ = Describe("Integration", func() { + Context("general (Default Applicaton Credentials) configuration", func() { + var env AssertContext + BeforeEach(func() { + env = NewAssertContext(AsDefaultCredentials) + }) + AfterEach(func() { + env.Cleanup() + }) + + configurations := getBaseConfigs() + + DescribeTable("Blobstore lifecycle works", + func(config *config.GCSCli) { + env.AddConfig(config) + AssertLifecycleWorks(gcsCLIPath, env) + }, + configurations) + + DescribeTable("Delete silently ignores that the file doesn't exist", + func(config *config.GCSCli) { + env.AddConfig(config) + + session, err := RunGCSCLI(gcsCLIPath, env.ConfigPath, + "delete", env.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + }, + configurations) + + Context("with a regional bucket", func() { + var cfg *config.GCSCli + BeforeEach(func() { + cfg = getRegionalConfig() + env.AddConfig(cfg) + }) + AfterEach(func() { + env.Cleanup() + }) + + It("can perform large file upload (multi-part)", func() { + if os.Getenv(NoLongEnv) != "" { + Skip(fmt.Sprintf(NoLongMsg, NoLongEnv)) + } + + const twoGB = 1024 * 1024 * 1024 * 2 + limited := newrandReadSeeker(twoGB) + + blobstoreClient, err := client.New(env.ctx, env.Config) + Expect(err).ToNot(HaveOccurred()) + + err = blobstoreClient.Put(&limited, env.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + + blobstoreClient.Delete(env.GCSFileName) //nolint:errcheck + Expect(err).ToNot(HaveOccurred()) + }) + }) + + DescribeTable("Invalid Put should fail", + func(config *config.GCSCli) { + env.AddConfig(config) + + blobstoreClient, err := client.New(env.ctx, env.Config) + Expect(err).ToNot(HaveOccurred()) + + err = blobstoreClient.Put(&badReadSeeker{}, env.GCSFileName) + Expect(err).To(HaveOccurred()) + }, + configurations) + + DescribeTable("Invalid Get should fail", + func(config *config.GCSCli) { + env.AddConfig(config) + + session, err := RunGCSCLI(gcsCLIPath, env.ConfigPath, + "get", env.GCSFileName, "/dev/null") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(session.Err.Contents()).To(ContainSubstring("object doesn't exist")) + }, + configurations) + }) +}) diff --git a/gcs/integration/gcs_public_test.go b/gcs/integration/gcs_public_test.go new file mode 100644 index 0000000..3a07fb6 --- /dev/null +++ b/gcs/integration/gcs_public_test.go @@ -0,0 +1,114 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "context" + "fmt" + "os" + + "cloud.google.com/go/storage" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/cloudfoundry/storage-cli/gcs/client" + "github.com/cloudfoundry/storage-cli/gcs/config" +) + +var _ = Describe("GCS Public Bucket", func() { + Context("with read-only configuration", func() { + var ( + setupEnv AssertContext + publicEnv AssertContext + cfg *config.GCSCli + ) + + BeforeEach(func() { + cfg = getPublicConfig() + + setupEnv = NewAssertContext(AsDefaultCredentials) + setupEnv.AddConfig(cfg) + Expect(setupEnv.Config.CredentialsSource).ToNot(Equal(config.NoneCredentialsSource), "Cannot use 'none' credentials to setup") + + publicEnv = setupEnv.Clone(AsReadOnlyCredentials) + }) + AfterEach(func() { + setupEnv.Cleanup() + publicEnv.Cleanup() + }) + + Describe("with a public file", func() { + BeforeEach(func() { + // Place a file in the bucket + RunGCSCLI(gcsCLIPath, setupEnv.ConfigPath, "put", setupEnv.ContentFile, setupEnv.GCSFileName) //nolint:errcheck + + // Make the file public + rwClient, err := newSDK(setupEnv.ctx, *setupEnv.Config) + Expect(err).ToNot(HaveOccurred()) + bucket := rwClient.Bucket(setupEnv.Config.BucketName) + obj := bucket.Object(setupEnv.GCSFileName) + Expect(obj.ACL().Set(context.Background(), storage.AllUsers, storage.RoleReader)).To(Succeed()) + }) + AfterEach(func() { + RunGCSCLI(gcsCLIPath, setupEnv.ConfigPath, "delete", setupEnv.GCSFileName) //nolint:errcheck + publicEnv.Cleanup() + }) + + It("can check if it exists", func() { + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, "exists", setupEnv.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero()) + }) + + It("can get", func() { + tmpLocalFile, err := os.CreateTemp("", "gcscli-download") + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpLocalFile.Name()) //nolint:errcheck + Expect(tmpLocalFile.Close()).To(Succeed()) + + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, "get", setupEnv.GCSFileName, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(BeZero(), fmt.Sprintf("unexpected '%s'", session.Err.Contents())) + + gottenBytes, err := os.ReadFile(tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(string(gottenBytes)).To(Equal(setupEnv.ExpectedString)) + }) + }) + + It("fails to get a missing file", func() { + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, "get", setupEnv.GCSFileName, "/dev/null") + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(session.Err.Contents()).To(ContainSubstring("object doesn't exist")) + }) + + It("fails to put", func() { + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, "put", publicEnv.ContentFile, publicEnv.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(session.Err.Contents()).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) + }) + + It("fails to delete", func() { + session, err := RunGCSCLI(gcsCLIPath, publicEnv.ConfigPath, "delete", publicEnv.GCSFileName) + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).ToNot(BeZero()) + Expect(session.Err.Contents()).To(ContainSubstring(client.ErrInvalidROWriteOperation.Error())) + }) + }) +}) diff --git a/gcs/integration/gcs_static_test.go b/gcs/integration/gcs_static_test.go new file mode 100644 index 0000000..fff4f29 --- /dev/null +++ b/gcs/integration/gcs_static_test.go @@ -0,0 +1,128 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "encoding/base64" + "net/http" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/cloudfoundry/storage-cli/gcs/config" +) + +var _ = Describe("Integration", func() { + Context("static credentials configuration with a regional bucket", func() { + var ( + ctx AssertContext + cfg *config.GCSCli + ) + BeforeEach(func() { + cfg = getRegionalConfig() + ctx = NewAssertContext(AsStaticCredentials) + ctx.AddConfig(cfg) + }) + AfterEach(func() { + ctx.Cleanup() + }) + + It("can perform blobstore lifecycle", func() { + AssertLifecycleWorks(gcsCLIPath, ctx) + }) + + It("validates the action is valid", func() { + session, err := RunGCSCLI(gcsCLIPath, ctx.ConfigPath, "sign", ctx.GCSFileName, "not-valid", "1h") + Expect(err).NotTo(HaveOccurred()) + Expect(session.ExitCode()).ToNot(Equal(0)) + }) + + It("can generate a signed url for a given object and action", func() { + session, err := RunGCSCLI(gcsCLIPath, ctx.ConfigPath, "sign", ctx.GCSFileName, "put", "1h") + + Expect(err).ToNot(HaveOccurred()) + Expect(session.ExitCode()).To(Equal(0)) + url := string(session.Out.Contents()) + Expect(url).To(MatchRegexp("https://")) + + body := strings.NewReader(`bar`) + req, err := http.NewRequest("PUT", url, body) + Expect(err).ToNot(HaveOccurred()) + + resp, err := http.DefaultClient.Do(req) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(200)) + defer resp.Body.Close() //nolint:errcheck + }) + + Context("encryption key is set", func() { + var key string + + BeforeEach(func() { + // even though the config file holds a base64 encodeded key, + // config at this point needs it to be decoded + // openssl rand 32 | base64 + key = "PG+tLm6vjBZXpU6S5Oiv/rpkA4KLioQRTXU3AfVzyHc=" + data, err := base64.StdEncoding.DecodeString(key) + Expect(err).NotTo(HaveOccurred()) + + newcfg := ctx.Config + newcfg.EncryptionKey = data + ctx.AddConfig(newcfg) + }) + + It("can generate a signed url for encrypting later", func() { + // echo -n key | base64 -D | shasum -a 256 | cut -f1 -d' ' | tr -d '\n' | xxd -r -p | base64 + hash := "bQOB9Mp048LRjpIoKm2njgQgiC3FRO2gn/+x6Vlfa4E=" + + session, err := RunGCSCLI(gcsCLIPath, ctx.ConfigPath, "sign", ctx.GCSFileName, "PUT", "1h") + Expect(err).ToNot(HaveOccurred()) + signedPutUrl := string(session.Out.Contents()) + Expect(signedPutUrl).ToNot(BeNil()) + + session, err = RunGCSCLI(gcsCLIPath, ctx.ConfigPath, "sign", ctx.GCSFileName, "GET", "1h") + Expect(err).ToNot(HaveOccurred()) + signedGetUrl := string(session.Out.Contents()) + Expect(signedGetUrl).ToNot(BeNil()) + + stuff := strings.NewReader(`stuff`) //nolint:errcheck + putReq, _ := http.NewRequest("PUT", signedPutUrl, stuff) //nolint:errcheck + getReq, _ := http.NewRequest("GET", signedGetUrl, nil) //nolint:errcheck + + headers := map[string][]string{ + "x-goog-encryption-algorithm": []string{"AES256"}, + "x-goog-encryption-key": []string{key}, + "x-goog-encryption-key-sha256": []string{hash}, + } + + putReq.Header = headers + getReq.Header = headers + + resp, err := http.DefaultClient.Do(putReq) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(200)) + resp.Body.Close() //nolint:errcheck + + resp, err = http.DefaultClient.Do(getReq) + Expect(err).ToNot(HaveOccurred()) + Expect(resp.StatusCode).To(Equal(200)) + resp.Body.Close() //nolint:errcheck + }) + }) + }) +}) diff --git a/gcs/integration/integration_suite_test.go b/gcs/integration/integration_suite_test.go new file mode 100644 index 0000000..d18d685 --- /dev/null +++ b/gcs/integration/integration_suite_test.go @@ -0,0 +1,43 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" + + "testing" +) + +var gcsCLIPath string + +func TestIntegration(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Integration Suite") +} + +var _ = BeforeSuite(func() { + // Integration test against the CLI means we need the binary. + var err error + gcsCLIPath, err = gexec.Build("github.com/cloudfoundry/storage-cli/gcs") + Expect(err).ShouldNot(HaveOccurred()) +}) + +var _ = AfterSuite(func() { + gexec.CleanupBuildArtifacts() +}) diff --git a/gcs/integration/utils.go b/gcs/integration/utils.go new file mode 100644 index 0000000..400db50 --- /dev/null +++ b/gcs/integration/utils.go @@ -0,0 +1,93 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package integration + +import ( + "encoding/json" + "math/rand" + "os" + "os/exec" + "time" + + "github.com/cloudfoundry/storage-cli/gcs/config" + "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" //nolint:staticcheck + "github.com/onsi/gomega/gexec" +) + +const alphanum = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +// GenerateRandomString generates a random string of desired length (default: 25) +func GenerateRandomString(params ...int) string { + size := 25 + if len(params) == 1 { + size = params[0] + } + + randBytes := make([]byte, size) + for i := range randBytes { + randBytes[i] = alphanum[rand.Intn(len(alphanum))] + } + return string(randBytes) +} + +// MakeConfigFile creates a config file from a GCSCli config struct +func MakeConfigFile(cfg *config.GCSCli) string { + cfgBytes, err := json.Marshal(cfg) + Expect(err).ToNot(HaveOccurred()) + tmpFile, err := os.CreateTemp("", "gcscli-test") + Expect(err).ToNot(HaveOccurred()) + _, err = tmpFile.Write(cfgBytes) + Expect(err).ToNot(HaveOccurred()) + err = tmpFile.Close() + Expect(err).ToNot(HaveOccurred()) + return tmpFile.Name() +} + +// MakeContentFile creates a temporary file with content to upload to GCS +func MakeContentFile(content string) string { + tmpFile, err := os.CreateTemp("", "gcscli-test-content") + Expect(err).ToNot(HaveOccurred()) + _, err = tmpFile.Write([]byte(content)) + Expect(err).ToNot(HaveOccurred()) + err = tmpFile.Close() + Expect(err).ToNot(HaveOccurred()) + return tmpFile.Name() +} + +// RunGCSCLI run the gcscli and outputs the session +// after waiting for it to finish +func RunGCSCLI(gcsCLIPath, configPath, subcommand string, + args ...string) (*gexec.Session, error) { + + cmdArgs := []string{ + "-c", + configPath, + subcommand, + } + cmdArgs = append(cmdArgs, args...) + + command := exec.Command(gcsCLIPath, cmdArgs...) + gexecSession, err := gexec.Start(command, + ginkgo.GinkgoWriter, ginkgo.GinkgoWriter) + if err != nil { + return nil, err + } + gexecSession.Wait(1 * time.Minute) + + return gexecSession, nil +} diff --git a/gcs/main.go b/gcs/main.go new file mode 100644 index 0000000..1cf4ad2 --- /dev/null +++ b/gcs/main.go @@ -0,0 +1,220 @@ +/* + * Copyright 2017 Google Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "flag" + "fmt" + "log" + "net/http" + "os" + "strings" + "time" + + "golang.org/x/net/context" + + "github.com/cloudfoundry/storage-cli/gcs/client" + "github.com/cloudfoundry/storage-cli/gcs/config" +) + +var version = "dev" + +// usageExample provides examples of how to use the CLI. +const usageExample = ` +# Usage +storage-cli-gcs --help + +# Upload a blob to the GCS blobstore. +storage-cli-gcs -c config.json put + +# Fetch a blob from the GCS blobstore. +# Destination file will be overwritten if exists. +storage-cli-gcs -c config.json get + +# Remove a blob from the GCS blobstore. +storage-cli-gcs -c config.json delete + +# Checks if blob exists in the GCS blobstore. +storage-cli-gcs -c config.json exists + +# Generate a signed url for an object +# if an encryption key is present in config, the appropriate header will be sent +# users of the signed url must include encryption headers in request +# Where: +# - is GET, PUT, or DELETE +# - is a duration string less than 7 days (e.g. "6h") +# eg storage-cli-gcs -c config.json sign blobid PUT 24h +storage-cli-gcs -c config.json sign ` + +var ( + showVer = flag.Bool("v", false, "Print CLI version") + shortHelp = flag.Bool("h", false, "Print this help text") + longHelp = flag.Bool("help", false, "Print this help text") + configPath = flag.String("c", "", + `path to a JSON file with the following contents: + { + "bucket_name": "name of Google Cloud Storage bucket (required)", + "credentials_source": "Optional, defaults to Application Default Credentials or none) + (can be 'static' for a service account specified in json_key), + (can be 'none' for explicitly no credentials)" + "json_key": "JSON Service Account File + (optional, required for 'static' credentials)", + "storage_class": "storage class for objects + (optional, defaults to bucket settings)", + "encryption_key": "Base64 encoded 32 byte Customer-Supplied + encryption key used to encrypt objects + (optional, defaults to GCS controlled key)" + } + + storage_class is one of MULTI_REGIONAL, REGIONAL, NEARLINE, or COLDLINE. + For more information on characteristics and location compatibility: + https://cloud.google.com/storage/docs/storage-classes + + For more information on Customer-Supplied encryption keys: + https://cloud.google.com/storage/docs/encryption +`) +) + +func main() { + flag.Parse() + + if *showVer { + fmt.Printf("version %s\n", version) + os.Exit(0) + } + + if *shortHelp || *longHelp || len(flag.Args()) == 0 { + flag.Usage() + fmt.Println(usageExample) + os.Exit(0) + } + + if *configPath == "" { + log.Fatalf("no config file provided\nSee -help for usage\n") + } + + configFile, err := os.Open(*configPath) + if err != nil { + log.Fatalf("opening config %s: %v\n", *configPath, err) + } + + gcsConfig, err := config.NewFromReader(configFile) + if err != nil { + log.Fatalf("reading config %s: %v\n", *configPath, err) + } + + ctx := context.Background() + blobstoreClient, err := client.New(ctx, &gcsConfig) + if err != nil { + log.Fatalf("creating gcs client: %v\n", err) + } + + nonFlagArgs := flag.Args() + if len(nonFlagArgs) < 2 { + log.Fatalf("Expected at least two arguments got %d\n", len(nonFlagArgs)) + } + + cmd := nonFlagArgs[0] + + switch cmd { + case "put": + if len(nonFlagArgs) != 3 { + log.Fatalf("put method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + src, dst := nonFlagArgs[1], nonFlagArgs[2] + + var sourceFile *os.File + sourceFile, err = os.Open(src) + if err != nil { + log.Fatalln(err) + } + + defer sourceFile.Close() //nolint:errcheck + err = blobstoreClient.Put(sourceFile, dst) + fmt.Println(err) + case "get": + if len(nonFlagArgs) != 3 { + log.Fatalf("get method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + src, dst := nonFlagArgs[1], nonFlagArgs[2] + + var dstFile *os.File + dstFile, err = os.Create(dst) + if err != nil { + log.Fatalln(err) + } + + defer dstFile.Close() //nolint:errcheck + err = blobstoreClient.Get(src, dstFile) + case "delete": + if len(nonFlagArgs) != 2 { + log.Fatalf("delete method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + + err = blobstoreClient.Delete(nonFlagArgs[1]) + case "exists": + if len(nonFlagArgs) != 2 { + log.Fatalf("exists method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + + var exists bool + exists, err = blobstoreClient.Exists(nonFlagArgs[1]) + + // If the object exists the exit status is 0, otherwise it is 3 + // We are using `3` since `1` and `2` have special meanings + if err == nil && !exists { + os.Exit(3) + } + case "sign": + if len(nonFlagArgs) != 4 { + log.Fatalf("sign method expected 3 arguments got %d\n", len(nonFlagArgs)) + } + + id, action, expiry := nonFlagArgs[1], nonFlagArgs[2], nonFlagArgs[3] + + action = strings.ToUpper(action) + err = validateAction(action) + if err != nil { + log.Fatal(err) + } + + var expiryDuration time.Duration + expiryDuration, err = time.ParseDuration(expiry) + if err != nil { + log.Fatalf("Invalid expiry duration: %v", err) + } + url := "" + url, err = blobstoreClient.Sign(id, action, expiryDuration) + if err == nil { + os.Stdout.WriteString(url) //nolint:errcheck + } + + default: + log.Fatalf("unknown command: '%s'\n", cmd) + } + + if err != nil { + log.Fatalf("performing operation %s: %s\n", cmd, err) + } +} + +func validateAction(action string) error { + if action != http.MethodGet && action != http.MethodPut && action != http.MethodDelete { + return fmt.Errorf("invalid signing action: %s must be GET, PUT, or DELETE", action) + } + return nil +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..ff1d936 --- /dev/null +++ b/go.mod @@ -0,0 +1,103 @@ +module github.com/cloudfoundry/storage-cli + +go 1.24.1 + +require ( + cloud.google.com/go/storage v1.57.1 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 + github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible + github.com/aws/aws-sdk-go-v2 v1.39.5 + github.com/aws/aws-sdk-go-v2/config v1.31.16 + github.com/aws/aws-sdk-go-v2/credentials v1.18.20 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 + github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 + github.com/aws/smithy-go v1.23.1 + github.com/cloudfoundry/bosh-utils v0.0.560 + github.com/maxbrunsfeld/counterfeiter/v6 v6.12.0 + github.com/onsi/ginkgo v1.16.5 + github.com/onsi/ginkgo/v2 v2.27.2 + github.com/onsi/gomega v1.38.2 + golang.org/x/net v0.46.0 + golang.org/x/oauth2 v0.32.0 + google.golang.org/api v0.254.0 +) + +require ( + cel.dev/expr v0.24.0 // indirect + cloud.google.com/go v0.121.6 // indirect + cloud.google.com/go/auth v0.17.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.9.0 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + code.cloudfoundry.org/tlsconfig v0.34.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 // indirect + github.com/bmatcuk/doublestar v1.3.4 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/charlievieth/fs v0.0.3 // indirect + github.com/cloudfoundry/go-socks5 v0.0.0-20250423223041-4ad5fea42851 // indirect + github.com/cloudfoundry/socks5-proxy v0.2.158 // indirect + github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.2 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-task/slim-sprig/v3 v3.0.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect + github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.15.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect + github.com/nxadm/tail v1.4.11 // indirect + github.com/pivotal-cf/paraphernalia v0.0.0-20180203224945-a64ae2051c20 // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.36.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect + go.opentelemetry.io/otel v1.37.0 // indirect + go.opentelemetry.io/otel/metric v1.37.0 // indirect + go.opentelemetry.io/otel/sdk v1.37.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.37.0 // indirect + go.opentelemetry.io/otel/trace v1.37.0 // indirect + go.yaml.in/yaml/v3 v3.0.4 // indirect + golang.org/x/crypto v0.43.0 // indirect + golang.org/x/mod v0.28.0 // indirect + golang.org/x/sync v0.17.0 // indirect + golang.org/x/sys v0.37.0 // indirect + golang.org/x/text v0.30.0 // indirect + golang.org/x/time v0.14.0 // indirect + golang.org/x/tools v0.37.0 // indirect + google.golang.org/genproto v0.0.0-20250603155806-513f23925822 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect + google.golang.org/grpc v1.76.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c347d72 --- /dev/null +++ b/go.sum @@ -0,0 +1,341 @@ +cel.dev/expr v0.24.0 h1:56OvJKSH3hDGL0ml5uSxZmz3/3Pq4tJ+fb1unVLAFcY= +cel.dev/expr v0.24.0/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.121.6 h1:waZiuajrI28iAf40cWgycWNgaXPO06dupuS+sgibK6c= +cloud.google.com/go v0.121.6/go.mod h1:coChdst4Ea5vUpiALcYKXEpR1S9ZgXbhEzzMcMR66vI= +cloud.google.com/go/auth v0.17.0 h1:74yCm7hCj2rUyyAocqnFzsAYXgJhrG26XCFimrc/Kz4= +cloud.google.com/go/auth v0.17.0/go.mod h1:6wv/t5/6rOPAX4fJiRjKkJCvswLwdet7G8+UGXt7nCQ= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.9.0 h1:pDUj4QMoPejqq20dK0Pg2N4yG9zIkYGdBtwLoEkH9Zs= +cloud.google.com/go/compute/metadata v0.9.0/go.mod h1:E0bWwX5wTnLPedCKqk3pJmVgCBSM6qQI1yTBdEb3C10= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/storage v1.57.1 h1:gzao6odNJ7dR3XXYvAgPK+Iw4fVPPznEPPyNjbaVkq8= +cloud.google.com/go/storage v1.57.1/go.mod h1:329cwlpzALLgJuu8beyJ/uvQznDHpa2U5lGjWednkzg= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +code.cloudfoundry.org/clock v1.0.0 h1:kFXWQM4bxYvdBw2X8BbBeXwQNgfoWv1vqAk2ZZyBN2o= +code.cloudfoundry.org/clock v1.0.0/go.mod h1:QD9Lzhd/ux6eNQVUDVRJX/RKTigpewimNYBi7ivZKY8= +code.cloudfoundry.org/tlsconfig v0.34.0 h1:Rvo+p1v1W19kuqeoNmugi2RxsWyooz8O0ePxzYBB/js= +code.cloudfoundry.org/tlsconfig v0.34.0/go.mod h1:fUnhzvxS9xNrqzmlenDp59inXZRduL4VhLDJqzwNmy0= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1 h1:5YTBM8QDVIBN3sxBil89WfdAAqDZbyJTgh688DSxX5w= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.19.1/go.mod h1:YD5h/ldMsG0XiIw7PdyNhLxaM317eFh5yNLccNfGdyw= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0 h1:KpMC6LFL7mqpExyMC9jVOYRiVhLmamjeZfRsUpB7l4s= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.13.0/go.mod h1:J7MUC/wtRpfGVbQ5sIItY5/FuVWmvzlY21WAOfQnq/I= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2 h1:9iefClla7iYpfYWdzPCRDozdmndjTm8DXdpCzPajMgA= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.2/go.mod h1:XtLgD3ZD34DAaVIIAyG3objl5DynM3CQ/vMcbBNJZGI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1 h1:/Zt+cDPnpC3OVDm/JKLOs7M2DKmLRIIp3XIx9pHHiig= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.1/go.mod h1:Ng3urmn6dYe8gnbCMoHHVl5APYz2txho3koEkV2o2HA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3 h1:ZJJNFaQ86GVKQ9ehwqyAFE6pIfyicpuJ8IkVaPBc6/4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.3/go.mod h1:URuDvhmATVKqHBH9/0nOiNKk0+YcwfQ3WkK5PqHKxc8= +github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0 h1:XkkQbfMyuH2jTSjQjSoihryI8GINRcs4xp8lNawg0FI= +github.com/AzureAD/microsoft-authentication-library-for-go v1.5.0/go.mod h1:HKpQxkWaGLJ+D/5H8QRpyQXA1eKjxkFlOMwck5+33Jk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0 h1:UQUsRi8WTzhZntp5313l+CHIAT95ojUI2lpP/ExlZa4= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.29.0/go.mod h1:Cz6ft6Dkn3Et6l2v2a9/RpN7epQ1GtDlO6lj8bEcOvw= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0 h1:owcC2UnmsZycprQ5RfRgjydWhuoxg71LUfyiQdijZuM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.53.0/go.mod h1:ZPpqegjbE99EPKsu3iUWV22A04wzGPcAY/ziSIQEEgs= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0 h1:4LP6hvB4I5ouTbGgWtixJhgED6xdf67twf9PoY96Tbg= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.53.0/go.mod h1:jUZ5LYlw40WMd07qxcQJD5M40aUxrfwqQX1g7zxYnrQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0 h1:Ron4zCA/yk6U7WOBXhTJcDpsUBG9npumK6xw2auFltQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.53.0/go.mod h1:cSgYe11MCNYunTnRXrKiR/tHc0eoKjICUuWpNZoVCOo= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible h1:8psS8a+wKfiLt1iVDX79F7Y6wUM49Lcha2FMXt4UM8g= +github.com/aliyun/aliyun-oss-go-sdk v3.0.2+incompatible/go.mod h1:T/Aws4fEfogEE9v+HPhhw+CntffsBHJ8nXQCwKr0/g8= +github.com/aws/aws-sdk-go-v2 v1.39.5 h1:e/SXuia3rkFtapghJROrydtQpfQaaUgd1cUvyO1mp2w= +github.com/aws/aws-sdk-go-v2 v1.39.5/go.mod h1:yWSxrnioGUZ4WVv9TgMrNUeLV3PFESn/v+6T/Su8gnM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2 h1:t9yYsydLYNBk9cJ73rgPhPWqOh/52fcWDQB5b1JsKSY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.2/go.mod h1:IusfVNTmiSN3t4rhxWFaBAqn+mcNdwKtPcV16eYdgko= +github.com/aws/aws-sdk-go-v2/config v1.31.16 h1:E4Tz+tJiPc7kGnXwIfCyUj6xHJNpENlY11oKpRTgsjc= +github.com/aws/aws-sdk-go-v2/config v1.31.16/go.mod h1:2S9hBElpCyGMifv14WxQ7EfPumgoeCPZUpuPX8VtW34= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20 h1:KFndAnHd9NUuzikHjQ8D5CfFVO+bgELkmcGY8yAw98Q= +github.com/aws/aws-sdk-go-v2/credentials v1.18.20/go.mod h1:9mCi28a+fmBHSQ0UM79omkz6JtN+PEsvLrnG36uoUv0= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12 h1:VO3FIM2TDbm0kqp6sFNR0PbioXJb/HzCDW6NtIZpIWE= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.12/go.mod h1:6C39gB8kg82tx3r72muZSrNhHia9rjGkX7ORaS2GKNE= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2 h1:9/HxDeIgA7DcKK6e6ZaP5PQiXugYbNERx3Z5u30mN+k= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.20.2/go.mod h1:3N1RoxKNcVHmbOKVMMw8pvMs5TUhGYPQP/aq1zmAWqo= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12 h1:p/9flfXdoAnwJnuW9xHEAFY22R3A6skYkW19JFF9F+8= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.12/go.mod h1:ZTLHakoVCTtW8AaLGSwJ3LXqHD9uQKnOcv1TrpO6u2k= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12 h1:2lTWFvRcnWFFLzHWmtddu5MTchc5Oj2OOey++99tPZ0= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.12/go.mod h1:hI92pK+ho8HVcWMHKHrK3Uml4pfG7wvL86FzO0LVtQQ= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4 h1:WKuaxf++XKWlHWu9ECbMlha8WOEGm0OUEZqm4K/Gcfk= +github.com/aws/aws-sdk-go-v2/internal/ini v1.8.4/go.mod h1:ZWy7j6v1vWGmPReu0iSGvRiise4YI5SkR3OHKTZ6Wuc= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12 h1:itu4KHu8JK/N6NcLIISlf3LL1LccMqruLUXZ9y7yBZw= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.12/go.mod h1:i+6vTU3xziikTY3vcox23X8pPGW5X3wVgd1VZ7ha+x8= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2 h1:xtuxji5CS0JknaXoACOunXOYOQzgfTvGAc9s2QdCJA4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.2/go.mod h1:zxwi0DIR0rcRcgdbl7E2MSOvxDyyXGBlScvBkARFaLQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3 h1:NEe7FaViguRQEm8zl8Ay/kC/QRsMtWUiCGZajQIsLdc= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.3/go.mod h1:JLuCKu5VfiLBBBl/5IzZILU7rxS0koQpHzMOCzycOJU= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12 h1:MM8imH7NZ0ovIVX7D2RxfMDv7Jt9OiUXkcQ+GqywA7M= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.12/go.mod h1:gf4OGwdNkbEsb7elw2Sy76odfhwNktWII3WgvQgQQ6w= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12 h1:R3uW0iKl8rgNEXNjVGliW/oMEh9fO/LlUEV8RvIFr1I= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.12/go.mod h1:XEttbEr5yqsw8ebi7vlDoGJJjMXRez4/s9pibpJyL5s= +github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1 h1:Dq82AV+Qxpno/fG162eAhnD8d48t9S+GZCfz7yv1VeA= +github.com/aws/aws-sdk-go-v2/service/s3 v1.89.1/go.mod h1:MbKLznDKpf7PnSonNRUVYZzfP0CeLkRIUexeblgKcU4= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0 h1:xHXvxst78wBpJFgDW07xllOx0IAzbryrSdM4nMVQ4Dw= +github.com/aws/aws-sdk-go-v2/service/sso v1.30.0/go.mod h1:/e8m+AO6HNPPqMyfKRtzZ9+mBF5/x1Wk8QiDva4m07I= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4 h1:tBw2Qhf0kj4ZwtsVpDiVRU3zKLvjvjgIjHMKirxXg8M= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.4/go.mod h1:Deq4B7sRM6Awq/xyOBlxBdgW8/Z926KYNNaGMW2lrkA= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0 h1:C+BRMnasSYFcgDw8o9H5hzehKzXyAb9GY5v/8bP9DUY= +github.com/aws/aws-sdk-go-v2/service/sts v1.39.0/go.mod h1:4EjU+4mIx6+JqKQkruye+CaigV7alL3thVPfDd9VlMs= +github.com/aws/smithy-go v1.23.1 h1:sLvcH6dfAFwGkHLZ7dGiYF7aK6mg4CgKA/iDKjLDt9M= +github.com/aws/smithy-go v1.23.1/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= +github.com/bmatcuk/doublestar v1.3.4 h1:gPypJ5xD31uhX6Tf54sDPUOBXTqKH4c9aPY66CyQrS0= +github.com/bmatcuk/doublestar v1.3.4/go.mod h1:wiQtGV+rzVYxB7WIlirSN++5HPtPlXEo9MEoZQC/PmE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charlievieth/fs v0.0.3 h1:3lZQXTj4PbE81CVPwALSn+JoyCNXkZgORHN6h2XHGlg= +github.com/charlievieth/fs v0.0.3/go.mod h1:hD4sRzto1Hw8zCua76tNVKZxaeZZr1RiKftjAJQRLLo= +github.com/cloudfoundry/bosh-utils v0.0.560 h1:4vsa4dbw0gDe0/5AnFSYPrt2L+lrCxKwbJDs3hhdw6M= +github.com/cloudfoundry/bosh-utils v0.0.560/go.mod h1:LNoUi8A9pr1KHF4RjwMKx4eBF6d/jhgFxZY32QVPQGY= +github.com/cloudfoundry/go-socks5 v0.0.0-20250423223041-4ad5fea42851 h1:oy59UYcspoP44ggE8DM3kjxl1+sTFd802bbZlBBhBMk= +github.com/cloudfoundry/go-socks5 v0.0.0-20250423223041-4ad5fea42851/go.mod h1:72EEm1oq5oXqGfu9XGtaRPWEcAFYd/P10cMNln0QhA8= +github.com/cloudfoundry/socks5-proxy v0.2.158 h1:R+7NlxmzCiTMAyZqNt77G/DgdEINcEjz+cnMAu+UHm8= +github.com/cloudfoundry/socks5-proxy v0.2.158/go.mod h1:RFuO7DkORi74ijYHjGNWiW2OSNxmklFWcxp22KdbO7Y= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443 h1:aQ3y1lwWyqYPiWZThqv1aFbZMiM9vblcSArJRf2Irls= +github.com/cncf/xds/go v0.0.0-20250501225837-2ac532fd4443/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-jose/go-jose/v4 v4.1.2 h1:TK/7NqRQZfgAh+Td8AlsrvtPoUyiHh0LqVvokh+1vHI= +github.com/go-jose/go-jose/v4 v4.1.2/go.mod h1:22cg9HWM1pOlnRiY+9cQYJ9XHmya1bYW8OeDM6Ku6Oo= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= +github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= +github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= +github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5 h1:xhMrHhTJ6zxu3gA4enFM9MLn9AY7613teCdFnlUVbSQ= +github.com/google/pprof v0.0.0-20250630185457-6e76a2b096b5/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo= +github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.0 h1:aOeI7xAOVdK+R6xbVsZuU9HmCZYmQVmZgPf9xJUd2Sg= +github.com/maxbrunsfeld/counterfeiter/v6 v6.12.0/go.mod h1:0hZWbtfeCYUQeAQdPLUzETiBhUSns7O6LDj9vH88xKA= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= +github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/nxadm/tail v1.4.11 h1:8feyoE3OzPrcshW5/MJ4sGESc5cqmGkGCWlco4l0bqY= +github.com/nxadm/tail v1.4.11/go.mod h1:OTaG3NK980DZzxbRq6lEuzgU+mug70nY11sMd4JXXHc= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= +github.com/pivotal-cf/paraphernalia v0.0.0-20180203224945-a64ae2051c20 h1:DR5eMfe2+6GzLkVyWytdtgUxgbPiOfvKDuqityTV3y8= +github.com/pivotal-cf/paraphernalia v0.0.0-20180203224945-a64ae2051c20/go.mod h1:Y3IqE20LKprEpLkXb7gXinJf4vvDdQe/BS8E4kL/dgE= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8= +github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= +github.com/square/certstrap v1.3.0 h1:N9P0ZRA+DjT8pq5fGDj0z3FjafRKnBDypP0QHpMlaAk= +github.com/square/certstrap v1.3.0/go.mod h1:wGZo9eE1B7WX2GKBn0htJ+B3OuRl2UsdCFySNooy9hU= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0 h1:F7q2tNlCaHY9nMKHR6XH9/qkp8FktLnIcy6jJNyOCQw= +go.opentelemetry.io/contrib/detectors/gcp v1.36.0/go.mod h1:IbBN8uAIIx734PTonTPxAxnjc2pQTxWNkwfstZ+6H2k= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0 h1:q4XOmH/0opmeuJtPsbFNivyl7bCt7yRBbeEm2sC/XtQ= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.61.0/go.mod h1:snMWehoOh2wsEwnvvwtDyFCxVeDAODenXHtn5vzrKjo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 h1:F7Jx+6hwnZ41NSFTO5q4LYDtJRXBf2PD0rNBkeB/lus= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0/go.mod h1:UHB22Z8QsdRDrnAtX4PntOl36ajSxcdUMt1sF7Y6E7Q= +go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ= +go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0 h1:rixTyDGXFxRy1xzhKrotaHy3/KXdPhlWARrCgK+eqUY= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.36.0/go.mod h1:dowW6UsM9MKbJq5JTz2AMVp3/5iW5I/TStsk8S+CfHw= +go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE= +go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E= +go.opentelemetry.io/otel/sdk v1.37.0 h1:ItB0QUqnjesGRvNcmAcU0LyvkVyGJ2xftD29bWdDvKI= +go.opentelemetry.io/otel/sdk v1.37.0/go.mod h1:VredYzxUvuo2q3WRcDnKDjbdvmO0sCzOvVAiY+yUkAg= +go.opentelemetry.io/otel/sdk/metric v1.37.0 h1:90lI228XrB9jCMuSdA0673aubgRobVZFhbjxHHspCPc= +go.opentelemetry.io/otel/sdk/metric v1.37.0/go.mod h1:cNen4ZWfiD37l5NhS+Keb5RXVWZWpRE+9WyVCpbo5ps= +go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4= +go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0= +go.step.sm/crypto v0.70.0 h1:Q9Ft7N637mucyZcHZd1+0VVQJVwDCKqcb9CYcYi7cds= +go.step.sm/crypto v0.70.0/go.mod h1:pzfUhS5/ue7ev64PLlEgXvhx1opwbhFCjkvlhsxVds0= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04= +golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U= +golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.46.0 h1:giFlY12I07fugqwPuWJi68oOnpfqFnJIJzaIIm2JVV4= +golang.org/x/net v0.46.0/go.mod h1:Q9BGdFy1y4nkUwiLvT5qtyhAnEHgnQ/zd8PfU6nc210= +golang.org/x/oauth2 v0.32.0 h1:jsCblLleRMDrxMN29H3z/k1KliIvpLgCkE6R8FXXNgY= +golang.org/x/oauth2 v0.32.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= +golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= +golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.36.0 h1:zMPR+aF8gfksFprF/Nc/rd1wRS1EI6nDBGyWAvDzx2Q= +golang.org/x/term v0.36.0/go.mod h1:Qu394IJq6V6dCBRgwqshf3mPF85AqzYEzofzRdZkWss= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.30.0 h1:yznKA/E9zq54KzlzBEAWn1NXSQ8DIp/NYMy88xJjl4k= +golang.org/x/text v0.30.0/go.mod h1:yDdHFIX9t+tORqspjENWgzaCVXgk0yYnYuSZ8UzzBVM= +golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= +golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE= +golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/api v0.254.0 h1:jl3XrGj7lRjnlUvZAbAdhINTLbsg5dbjmR90+pTQvt4= +google.golang.org/api v0.254.0/go.mod h1:5BkSURm3D9kAqjGvBNgf0EcbX6Rnrf6UArKkwBzAyqQ= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822 h1:rHWScKit0gvAPuOnu87KpaYtjK5zBMLcULh7gxkCXu4= +google.golang.org/genproto v0.0.0-20250603155806-513f23925822/go.mod h1:HubltRL7rMh0LfnQPkMH4NPDFEWp0jw3vixw7jEM53s= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c h1:AtEkQdl5b6zsybXcbz00j1LwNodDuH6hVifIaNqk7NQ= +google.golang.org/genproto/googleapis/api v0.0.0-20250818200422-3122310a409c/go.mod h1:ea2MjsO70ssTfCjiwHgI0ZFqcw45Ksuk2ckf9G468GA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 h1:M1rk8KBnUsBDg1oPGHNCxG4vc1f49epmTO7xscSajMk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.76.0 h1:UnVkv1+uMLYXoIz6o7chp59WfQUYA2ex/BXQ9rHZu7A= +google.golang.org/grpc v1.76.0/go.mod h1:Ju12QI8M6iQJtbcsV+awF5a4hfJMLi4X0JLo94ULZ6c= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/s3/README.md b/s3/README.md new file mode 100644 index 0000000..59e822e --- /dev/null +++ b/s3/README.md @@ -0,0 +1,101 @@ +## S3 CLI + +A CLI for uploading, fetching and deleting content to/from an S3-compatible +blobstore. + +Continuous integration: + +Releases can be found in `https://s3.amazonaws.com/bosh-s3cli-artifacts`. The Linux binaries follow the regex `s3cli-(\d+\.\d+\.\d+)-linux-amd64` and the windows binaries `s3cli-(\d+\.\d+\.\d+)-windows-amd64`. + +## Usage + +Given a JSON config file (`config.json`)... + +``` json +{ + "bucket_name": " (required)", + + "credentials_source": " [static|env_or_profile|none]", + "access_key_id": " (required if credentials_source = 'static')", + "secret_access_key": " (required if credentials_source = 'static')", + + "region": " (optional - default: 'us-east-1')", + "host": " (optional)", + "port": (optional), + + "ssl_verify_peer": (optional), + "use_ssl": (optional), + "signature_version": " (optional)", + "server_side_encryption": " (optional)", + "sse_kms_key_id": " (optional)", + "multipart_upload": (optional - default: true) +} +``` + +``` bash +# Usage +s3-cli --help + +# Command: "put" +# Upload a blob to an S3-compatible blobstore. +s3-cli -c config.json put + +# Command: "get" +# Fetch a blob from an S3-compatible blobstore. +# Destination file will be overwritten if exists. +s3-cli -c config.json get + +# Command: "delete" +# Remove a blob from an S3-compatible blobstore. +s3-cli -c config.json delete + +# Command: "exists" +# Checks if blob exists in an S3-compatible blobstore. +s3-cli -c config.json exists + +# Command: "sign" +# Create a self-signed url for an object +s3-cli -c config.json sign +``` + +## Contributing + +Follow these steps to make a contribution to the project: + +- Fork this repository +- Create a feature branch based upon the `main` branch (*pull requests must be made against this branch*) + ``` bash + git checkout -b feature-name origin/main + ``` +- Run tests to check your development environment setup + ``` bash + scripts/ginkgo -r -race --skip-package=integration ./ + ``` +- Make your changes (*be sure to add/update tests*) +- Run tests to check your changes + ``` bash + scripts/ginkgo -r -race --skip-package=integration ./ + ``` +- Push changes to your fork + ``` bash + git add . + git commit -m "Commit message" + git push origin feature-name + ``` +- Create a GitHub pull request, selecting `main` as the target branch + +## Running integration tests + +To run the integration tests, export the following variables into your environment: + +``` +export access_key_id=YOUR_AWS_ACCESS_KEY +export focus_regex="GENERAL AWS|AWS V2 REGION|AWS V4 REGION|AWS US-EAST-1" +export region_name=us-east-1 +export s3_endpoint_host=https://s3.amazonaws.com +export secret_access_key=YOUR_SECRET_ACCESS_KEY +export stack_name=s3cli-iam +export bucket_name=s3cli-pipeline +``` + +Run `ci/tasks/setup-aws-infrastructure.sh` and `ci/tasks/teardown-infrastructure.sh` before and after the `run-integration-*` tests in `ci/tasks`. diff --git a/s3/client/aws_s3_blobstore.go b/s3/client/aws_s3_blobstore.go new file mode 100644 index 0000000..b9a71b7 --- /dev/null +++ b/s3/client/aws_s3_blobstore.go @@ -0,0 +1,193 @@ +package client + +import ( + "errors" + "fmt" + "io" + "log" + "strings" + "time" + + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + "github.com/aws/smithy-go" + + "github.com/cloudfoundry/storage-cli/s3/config" +) + +var errorInvalidCredentialsSourceValue = errors.New("the client operates in read only mode. Change 'credentials_source' parameter value ") +var oneTB = int64(1000 * 1024 * 1024 * 1024) + +// awsS3Client encapsulates AWS S3 blobstore interactions +type awsS3Client struct { + s3Client *s3.Client + s3cliConfig *config.S3Cli +} + +// Get fetches a blob, destination will be overwritten if exists +func (b *awsS3Client) Get(src string, dest io.WriterAt) error { + downloader := manager.NewDownloader(b.s3Client) + + _, err := downloader.Download(context.TODO(), dest, &s3.GetObjectInput{ + Bucket: aws.String(b.s3cliConfig.BucketName), + Key: b.key(src), + }) + + if err != nil { + return err + } + + return nil +} + +// Put uploads a blob +func (b *awsS3Client) Put(src io.ReadSeeker, dest string) error { + cfg := b.s3cliConfig + if cfg.CredentialsSource == config.NoneCredentialsSource { + return errorInvalidCredentialsSourceValue + } + + uploader := manager.NewUploader(b.s3Client, func(u *manager.Uploader) { + u.LeavePartsOnError = false + + if !cfg.MultipartUpload { + // disable multipart uploads by way of large PartSize configuration + u.PartSize = oneTB + } + }) + uploadInput := &s3.PutObjectInput{ + Body: src, + Bucket: aws.String(cfg.BucketName), + Key: b.key(dest), + } + if cfg.ServerSideEncryption != "" { + uploadInput.ServerSideEncryption = types.ServerSideEncryption(cfg.ServerSideEncryption) + } + if cfg.SSEKMSKeyID != "" { + uploadInput.SSEKMSKeyId = aws.String(cfg.SSEKMSKeyID) + } + + retry := 0 + maxRetries := 3 + for { + putResult, err := uploader.Upload(context.TODO(), uploadInput) + if err != nil { + if _, ok := err.(manager.MultiUploadFailure); ok { + if retry == maxRetries { + log.Println("Upload retry limit exceeded:", err.Error()) + return fmt.Errorf("upload retry limit exceeded: %s", err.Error()) + } + retry++ + time.Sleep(time.Second * time.Duration(retry)) + continue + } + log.Println("Upload failed:", err.Error()) + return fmt.Errorf("upload failure: %s", err.Error()) + } + + log.Println("Successfully uploaded file to", putResult.Location) + return nil + } +} + +// Delete removes a blob - no error is returned if the object does not exist +func (b *awsS3Client) Delete(dest string) error { + if b.s3cliConfig.CredentialsSource == config.NoneCredentialsSource { + return errorInvalidCredentialsSourceValue + } + + deleteParams := &s3.DeleteObjectInput{ + Bucket: aws.String(b.s3cliConfig.BucketName), + Key: b.key(dest), + } + + _, err := b.s3Client.DeleteObject(context.TODO(), deleteParams) + + if err == nil { + return nil + } + + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NotFound" { + return nil + } + return err +} + +// Exists checks if blob exists +func (b *awsS3Client) Exists(dest string) (bool, error) { + existsParams := &s3.HeadObjectInput{ + Bucket: aws.String(b.s3cliConfig.BucketName), + Key: b.key(dest), + } + + _, err := b.s3Client.HeadObject(context.TODO(), existsParams) + + if err == nil { + log.Printf("File '%s' exists in bucket '%s'\n", dest, b.s3cliConfig.BucketName) + return true, nil + } + + var apiErr smithy.APIError + if errors.As(err, &apiErr) && apiErr.ErrorCode() == "NotFound" { + log.Printf("File '%s' does not exist in bucket '%s'\n", dest, b.s3cliConfig.BucketName) + return false, nil + } + return false, err +} + +// Sign creates a presigned URL +func (b *awsS3Client) Sign(objectID string, action string, expiration time.Duration) (string, error) { + action = strings.ToUpper(action) + switch action { + case "GET": + return b.getSigned(objectID, expiration) + case "PUT": + return b.putSigned(objectID, expiration) + default: + return "", fmt.Errorf("action not implemented: %s", action) + } +} + +func (b *awsS3Client) key(srcOrDest string) *string { + formattedKey := aws.String(srcOrDest) + if len(b.s3cliConfig.FolderName) != 0 { + formattedKey = aws.String(fmt.Sprintf("%s/%s", b.s3cliConfig.FolderName, srcOrDest)) + } + + return formattedKey +} + +func (b *awsS3Client) getSigned(objectID string, expiration time.Duration) (string, error) { + presignClient := s3.NewPresignClient(b.s3Client) + signParams := &s3.GetObjectInput{ + Bucket: aws.String(b.s3cliConfig.BucketName), + Key: b.key(objectID), + } + + req, err := presignClient.PresignGetObject(context.TODO(), signParams, s3.WithPresignExpires(expiration)) + if err != nil { + return "", err + } + + return req.URL, nil +} + +func (b *awsS3Client) putSigned(objectID string, expiration time.Duration) (string, error) { + presignClient := s3.NewPresignClient(b.s3Client) + signParams := &s3.PutObjectInput{ + Bucket: aws.String(b.s3cliConfig.BucketName), + Key: b.key(objectID), + } + + req, err := presignClient.PresignPutObject(context.TODO(), signParams, s3.WithPresignExpires(expiration)) + if err != nil { + return "", err + } + + return req.URL, nil +} diff --git a/s3/client/client.go b/s3/client/client.go new file mode 100644 index 0000000..1e92c83 --- /dev/null +++ b/s3/client/client.go @@ -0,0 +1,62 @@ +package client + +import ( + "io" + "time" + + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudfoundry/storage-cli/s3/config" +) + +type S3CompatibleClient interface { + Get(src string, dest io.WriterAt) error + Put(src io.ReadSeeker, dest string) error + Delete(dest string) error + Exists(dest string) (bool, error) + Sign(objectID string, action string, expiration time.Duration) (string, error) +} + +// New returns an S3CompatibleClient +func New(s3Client *s3.Client, s3cliConfig *config.S3Cli) S3CompatibleClient { + return &s3CompatibleClient{ + s3cliConfig: s3cliConfig, + openstackSwiftBlobstore: &openstackSwiftS3Client{ + s3cliConfig: s3cliConfig, + }, + awsS3BlobstoreClient: &awsS3Client{ + s3Client: s3Client, + s3cliConfig: s3cliConfig, + }, + } +} + +type s3CompatibleClient struct { + s3cliConfig *config.S3Cli + awsS3BlobstoreClient *awsS3Client + openstackSwiftBlobstore *openstackSwiftS3Client +} + +func (c *s3CompatibleClient) Get(src string, dest io.WriterAt) error { + return c.awsS3BlobstoreClient.Get(src, dest) +} + +func (c *s3CompatibleClient) Put(src io.ReadSeeker, dest string) error { + return c.awsS3BlobstoreClient.Put(src, dest) +} + +func (c *s3CompatibleClient) Delete(dest string) error { + return c.awsS3BlobstoreClient.Delete(dest) +} + +func (c *s3CompatibleClient) Exists(dest string) (bool, error) { + return c.awsS3BlobstoreClient.Exists(dest) +} + +func (c *s3CompatibleClient) Sign(objectID string, action string, expiration time.Duration) (string, error) { + if c.s3cliConfig.SwiftAuthAccount != "" { + return c.openstackSwiftBlobstore.Sign(objectID, action, expiration) + } + + return c.awsS3BlobstoreClient.Sign(objectID, action, expiration) +} diff --git a/s3/client/client_suite_test.go b/s3/client/client_suite_test.go new file mode 100644 index 0000000..79e004c --- /dev/null +++ b/s3/client/client_suite_test.go @@ -0,0 +1,13 @@ +package client_test + +import ( + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Client Suite") +} diff --git a/s3/client/client_test.go b/s3/client/client_test.go new file mode 100644 index 0000000..37e463e --- /dev/null +++ b/s3/client/client_test.go @@ -0,0 +1,156 @@ +package client_test + +import ( + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + + "github.com/cloudfoundry/storage-cli/s3/client" + "github.com/cloudfoundry/storage-cli/s3/config" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("S3CompatibleClient", func() { + var blobstoreClient client.S3CompatibleClient + var s3Config *config.S3Cli + + Describe("Sign()", func() { + var objectId = "test-object-id" + var expiration = time.Duration(100) * time.Second + var action string + var urlRegexp string + + Context("when SwiftAuthAccount is empty", func() { + BeforeEach(func() { + s3Config = &config.S3Cli{ + AccessKeyID: "id", + SecretAccessKey: "key", + BucketName: "some-bucket", + Host: "host-name", + } + awsCfg := aws.Config{ + Region: "us-west-2", + Credentials: credentials.NewStaticCredentialsProvider( + s3Config.AccessKeyID, + s3Config.SecretAccessKey, + "", + ), + } + + s3Client := s3.NewFromConfig(awsCfg) + + blobstoreClient = client.New(s3Client, s3Config) + + urlRegexp = `https://some-bucket.s3.us-west-2.amazonaws.com/test-object-id` + + `\?X-Amz-Algorithm=AWS4-HMAC-SHA256` + + `&X-Amz-Credential=id%2F([0-9]+)%2Fus-west-2%2Fs3%2Faws4_request` + + `&X-Amz-Date=([0-9]+)T([0-9]+)Z` + + `&X-Amz-Expires=100` + + `&X-Amz-SignedHeaders=host` + + `&x-id=[A-Za-z]+` + + `&X-Amz-Signature=([a-f0-9]+)` + }) + + Context("when the action is GET", func() { + BeforeEach(func() { + action = "GET" + }) + + It("returns a signed URL", func() { + url, err := blobstoreClient.Sign(objectId, action, expiration) + Expect(err).NotTo(HaveOccurred()) + + Expect(url).To(MatchRegexp(urlRegexp)) + }) + }) + + Context("when the action is PUT", func() { + BeforeEach(func() { + action = "PUT" + }) + + It("returns a signed URL", func() { + url, err := blobstoreClient.Sign(objectId, action, expiration) + Expect(err).NotTo(HaveOccurred()) + + Expect(url).To(MatchRegexp(urlRegexp)) + }) + }) + + Context("when the action is neither GET nor PUT", func() { + BeforeEach(func() { + action = "UNSUPPORTED_ACTION" + }) + + It("returns an error", func() { + _, err := blobstoreClient.Sign(objectId, action, expiration) + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Context("when SwiftAuthAccount is NOT empty", func() { + BeforeEach(func() { + s3Config = &config.S3Cli{ + AccessKeyID: "id", + SecretAccessKey: "key", + BucketName: "some-bucket", + Host: "host-name", + SwiftAuthAccount: "swift_account", + SwiftTempURLKey: "temp_key", + } + + s3Client, err := client.NewAwsS3Client(s3Config) + Expect(err).NotTo(HaveOccurred()) + + blobstoreClient = client.New(s3Client, s3Config) + + urlRegexp = + "https://host-name/v1/swift_account/some-bucket/test-object-id" + + `\?temp_url_sig=([a-f0-9]+)` + + `&temp_url_expires=([0-9]+)` + }) + + Context("when the action is GET", func() { + BeforeEach(func() { + action = "GET" + }) + + It("returns a signed URL", func() { + url, err := blobstoreClient.Sign(objectId, action, expiration) + Expect(err).NotTo(HaveOccurred()) + + Expect(url).To(MatchRegexp(urlRegexp)) + }) + }) + + Context("when the action is PUT", func() { + BeforeEach(func() { + action = "PUT" + }) + + It("returns a signed URL", func() { + url, err := blobstoreClient.Sign(objectId, action, expiration) + Expect(err).NotTo(HaveOccurred()) + + Expect(url).To(MatchRegexp(urlRegexp)) + }) + }) + + Context("when the action is neither GET nor PUT", func() { + BeforeEach(func() { + action = "UNSUPPORTED_ACTION" + }) + + It("returns an error", func() { + _, err := blobstoreClient.Sign(objectId, action, expiration) + Expect(err).To(HaveOccurred()) + }) + }) + }) + }) +}) diff --git a/s3/client/openstack_swift_client.go b/s3/client/openstack_swift_client.go new file mode 100644 index 0000000..6d50cc0 --- /dev/null +++ b/s3/client/openstack_swift_client.go @@ -0,0 +1,43 @@ +package client + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "fmt" + "strconv" + "strings" + "time" + + "github.com/cloudfoundry/storage-cli/s3/config" +) + +// awsS3Client encapsulates Openstack Swift specific bloblsstore interactions +type openstackSwiftS3Client struct { + s3cliConfig *config.S3Cli +} + +func (c *openstackSwiftS3Client) Sign(objectID string, action string, expiration time.Duration) (string, error) { + action = strings.ToUpper(action) + switch action { + case "GET", "PUT": + return c.signedURL(action, objectID, expiration) + default: + return "", fmt.Errorf("action not implemented: %s", action) + } +} + +func (c *openstackSwiftS3Client) signedURL(action string, objectID string, expiration time.Duration) (string, error) { + path := fmt.Sprintf("/v1/%s/%s/%s", c.s3cliConfig.SwiftAuthAccount, c.s3cliConfig.BucketName, objectID) + + expires := time.Now().Add(expiration).Unix() + hmacBody := action + "\n" + strconv.FormatInt(expires, 10) + "\n" + path + + h := hmac.New(sha256.New, []byte(c.s3cliConfig.SwiftTempURLKey)) + h.Write([]byte(hmacBody)) + signature := hex.EncodeToString(h.Sum(nil)) + + url := fmt.Sprintf("https://%s%s?temp_url_sig=%s&temp_url_expires=%d", c.s3cliConfig.Host, path, signature, expires) + + return url, nil +} diff --git a/s3/client/sdk.go b/s3/client/sdk.go new file mode 100644 index 0000000..c362db5 --- /dev/null +++ b/s3/client/sdk.go @@ -0,0 +1,77 @@ +package client + +import ( + "net/http" + "strings" + + "context" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/sts" + boshhttp "github.com/cloudfoundry/bosh-utils/httpclient" + + s3cli_config "github.com/cloudfoundry/storage-cli/s3/config" +) + +func NewAwsS3Client(c *s3cli_config.S3Cli) (*s3.Client, error) { + var httpClient *http.Client + + if c.SSLVerifyPeer { + httpClient = boshhttp.CreateDefaultClient(nil) + } else { + httpClient = boshhttp.CreateDefaultClientInsecureSkipVerify() + } + + options := []func(*config.LoadOptions) error{ + config.WithHTTPClient(httpClient), + } + + if c.UseRegion() { + options = append(options, config.WithRegion(c.Region)) + } else { + options = append(options, config.WithRegion(s3cli_config.EmptyRegion)) + } + + if c.CredentialsSource == s3cli_config.StaticCredentialsSource { + options = append(options, config.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(c.AccessKeyID, c.SecretAccessKey, ""), + )) + } + + if c.CredentialsSource == s3cli_config.NoneCredentialsSource { + options = append(options, config.WithCredentialsProvider(aws.AnonymousCredentials{})) + } + + awsConfig, err := config.LoadDefaultConfig(context.TODO(), options...) + if err != nil { + return nil, err + } + + if c.AssumeRoleArn != "" { + stsClient := sts.NewFromConfig(awsConfig) + provider := stscreds.NewAssumeRoleProvider(stsClient, c.AssumeRoleArn) + awsConfig.Credentials = aws.NewCredentialsCache(provider) + } + + s3Client := s3.NewFromConfig(awsConfig, func(o *s3.Options) { + o.UsePathStyle = !c.HostStyle + if c.S3Endpoint() != "" { + endpoint := c.S3Endpoint() + // AWS SDK v2 requires full URI with protocol + if !strings.HasPrefix(endpoint, "http://") && !strings.HasPrefix(endpoint, "https://") { + if c.UseSSL { + endpoint = "https://" + endpoint + } else { + endpoint = "http://" + endpoint + } + } + o.BaseEndpoint = aws.String(endpoint) + } + }) + + return s3Client, nil +} diff --git a/s3/config/config.go b/s3/config/config.go new file mode 100644 index 0000000..6d6292e --- /dev/null +++ b/s3/config/config.go @@ -0,0 +1,210 @@ +package config + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "strings" +) + +// The S3Cli represents configuration for the s3cli +type S3Cli struct { + AccessKeyID string `json:"access_key_id"` + SecretAccessKey string `json:"secret_access_key"` + BucketName string `json:"bucket_name"` + FolderName string `json:"folder_name"` + CredentialsSource string `json:"credentials_source"` + Host string `json:"host"` + Port int `json:"port"` // 0 means no custom port + Region string `json:"region"` + SSLVerifyPeer bool `json:"ssl_verify_peer"` + UseSSL bool `json:"use_ssl"` + SignatureVersion int `json:"signature_version,string"` + ServerSideEncryption string `json:"server_side_encryption"` + SSEKMSKeyID string `json:"sse_kms_key_id"` + AssumeRoleArn string `json:"assume_role_arn"` + MultipartUpload bool `json:"multipart_upload"` + UseV2SigningMethod bool + HostStyle bool `json:"host_style"` + SwiftAuthAccount string `json:"swift_auth_account"` + SwiftTempURLKey string `json:"swift_temp_url_key"` +} + +// EmptyRegion is required to allow us to use the AWS SDK against S3 compatible blobstores which do not have +// the concept of a region +const EmptyRegion = " " + +const ( + defaultRegion = "us-east-1" //nolint:unused +) + +// StaticCredentialsSource specifies that credentials will be supplied using access_key_id and secret_access_key +const StaticCredentialsSource = "static" + +// NoneCredentialsSource specifies that credentials will be empty. The blobstore client operates in read only mode. +const NoneCredentialsSource = "none" + +const credentialsSourceEnvOrProfile = "env_or_profile" + +// Nothing was provided in configuration +const noCredentialsSourceProvided = "" + +var errorStaticCredentialsMissing = errors.New("access_key_id and secret_access_key must be provided") + +type errorStaticCredentialsPresent struct { + credentialsSource string +} + +func (e errorStaticCredentialsPresent) Error() string { + return fmt.Sprintf("can't use access_key_id and secret_access_key with %s credentials_source", e.credentialsSource) +} + +func newStaticCredentialsPresentError(desiredSource string) error { + return errorStaticCredentialsPresent{credentialsSource: desiredSource} +} + +// NewFromReader returns a new s3cli configuration struct from the contents of reader. +// reader.Read() is expected to return valid JSON +func NewFromReader(reader io.Reader) (S3Cli, error) { + bytes, err := io.ReadAll(reader) + if err != nil { + return S3Cli{}, err + } + + c := S3Cli{ + SSLVerifyPeer: true, + UseSSL: true, + MultipartUpload: true, + } + + err = json.Unmarshal(bytes, &c) + if err != nil { + return S3Cli{}, err + } + + if c.BucketName == "" { + return S3Cli{}, errors.New("bucket_name must be set") + } + + switch c.CredentialsSource { + case StaticCredentialsSource: + if c.AccessKeyID == "" || c.SecretAccessKey == "" { + return S3Cli{}, errorStaticCredentialsMissing + } + case credentialsSourceEnvOrProfile: + if c.AccessKeyID != "" || c.SecretAccessKey != "" { + return S3Cli{}, newStaticCredentialsPresentError(credentialsSourceEnvOrProfile) + } + case NoneCredentialsSource: + if c.AccessKeyID != "" || c.SecretAccessKey != "" { + return S3Cli{}, newStaticCredentialsPresentError(NoneCredentialsSource) + } + + case noCredentialsSourceProvided: + if c.SecretAccessKey != "" && c.AccessKeyID != "" { + c.CredentialsSource = StaticCredentialsSource + } else if c.SecretAccessKey == "" && c.AccessKeyID == "" { + c.CredentialsSource = NoneCredentialsSource + } else { + return S3Cli{}, errorStaticCredentialsMissing + } + default: + return S3Cli{}, fmt.Errorf("invalid credentials_source: %s", c.CredentialsSource) + } + + switch Provider(c.Host) { + case "aws": + c.configureAWS() + case "alicloud": + c.configureAlicloud() + case "google": + c.configureGoogle() + default: + c.configureDefault() + } + + return c, nil +} + +// Provider returns one of (aws, alicloud, google) based on a host name. +// Returns "" if a known provider cannot be detected. +func Provider(host string) string { + for provider, regex := range providerRegex { + if regex.MatchString(host) { + return provider + } + } + + return "" +} + +func (c *S3Cli) configureAWS() { + c.MultipartUpload = true + + if c.Region == "" { + c.Region = AWSHostToRegion(c.Host) + } + + switch c.SignatureVersion { + case 2: + c.UseV2SigningMethod = true + case 4: + c.UseV2SigningMethod = false + default: + c.UseV2SigningMethod = false + } +} + +func (c *S3Cli) configureAlicloud() { + c.MultipartUpload = true + c.configureDefaultSigningMethod() + c.HostStyle = true + + c.Host = strings.Split(c.Host, ":")[0] + if c.Region == "" { + c.Region = AlicloudHostToRegion(c.Host) + } +} + +func (c *S3Cli) configureGoogle() { + c.MultipartUpload = false + c.configureDefaultSigningMethod() +} + +func (c *S3Cli) configureDefault() { + c.configureDefaultSigningMethod() +} + +func (c *S3Cli) configureDefaultSigningMethod() { + switch c.SignatureVersion { + case 2: + c.UseV2SigningMethod = true + case 4: + c.UseV2SigningMethod = false + default: + c.UseV2SigningMethod = true + } +} + +// UseRegion signals to users of the S3Cli whether to use Region information +func (c *S3Cli) UseRegion() bool { + return c.Region != "" +} + +// S3Endpoint returns the S3 URI to use if custom host information has been provided +func (c *S3Cli) S3Endpoint() string { + if c.Host == "" { + return "" + } + if c.Port == 80 && !c.UseSSL { + return c.Host + } + if c.Port == 443 && c.UseSSL { + return c.Host + } + if c.Port != 0 { + return fmt.Sprintf("%s:%d", c.Host, c.Port) + } + return c.Host +} diff --git a/s3/config/config_suite_test.go b/s3/config/config_suite_test.go new file mode 100644 index 0000000..63543a8 --- /dev/null +++ b/s3/config/config_suite_test.go @@ -0,0 +1,13 @@ +package config_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "testing" +) + +func TestClient(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Config Suite") +} diff --git a/s3/config/config_test.go b/s3/config/config_test.go new file mode 100644 index 0000000..b3ecdf9 --- /dev/null +++ b/s3/config/config_test.go @@ -0,0 +1,593 @@ +package config_test + +import ( + "bytes" + "errors" + + "github.com/cloudfoundry/storage-cli/s3/config" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("BlobstoreClient configuration", func() { + Describe("empty region configuration", func() { + It("allows for the S3 SDK to be configured with empty region information", func() { + Expect(config.EmptyRegion).To(Equal(" ")) + }) + }) + + DescribeTable("Provider", + func(host, provider string) { + Expect(config.Provider(host)).To(Equal(provider)) + }, + Entry("aws 1", "s3.amazonaws.com", "aws"), + Entry("aws 2", "s3.external-1.amazonaws.com", "aws"), + Entry("aws 3", "s3.some-region.amazonaws.com", "aws"), + Entry("alicloud 1", "oss-r-s-1-internal.aliyuncs.com", "alicloud"), + Entry("alicloud 2", "oss-r-s-internal.aliyuncs.com", "alicloud"), + Entry("alicloud 3", "oss-r-s-1.aliyuncs.com", "alicloud"), + Entry("alicloud 4", "oss-r-s.aliyuncs.com", "alicloud"), + Entry("google 1", "storage.googleapis.com", "google"), + ) + + Describe("building a configuration", func() { + Describe("checking that either host or region has been set", func() { + + Context("when AWS endpoint has been set but not region", func() { + + It("sets the AWS region based on the hostname", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "s3.amazonaws.com"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.UseRegion()).To(BeTrue(), "Expected UseRegion to be true") + Expect(c.Host).To(Equal("s3.amazonaws.com")) + Expect(c.Region).To(Equal("us-east-1")) + }) + }) + + Context("when non-AWS endpoint has been set but not region", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "some-host"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("reports that region should not be used for SDK configuration", func() { + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.UseRegion()).To(BeFalse()) + Expect(c.Host).To(Equal("some-host")) + Expect(c.Region).To(Equal("")) + }) + }) + + Context("when region has been set but not host", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "region": "some-region"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("reports that region should be used for SDK configuration", func() { + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.UseRegion()).To(BeTrue()) + Expect(c.Host).To(Equal("")) + Expect(c.Region).To(Equal("some-region")) + }) + }) + + Context("when non-AWS host and region have been set", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "some-host", "region": "some-region"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("sets region and endpoint to user-specified values", func() { + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.UseRegion()).To(BeTrue()) + Expect(c.Host).To(Equal("some-host")) + Expect(c.Region).To(Equal("some-region")) + }) + }) + + Context("when AWS host and region have been set", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "s3.amazonaws.com", "region": "us-west-1"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("does not override the user-specified region based on the hostname", func() { + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.UseRegion()).To(BeTrue()) + Expect(c.Host).To(Equal("s3.amazonaws.com")) + Expect(c.Region).To(Equal("us-west-1")) + }) + }) + + Context("when neither host and region have been set", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + It("defaults region to us-east-1", func() { + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.Host).To(Equal("")) + Expect(c.Region).To(Equal("us-east-1")) + }) + }) + + Context("when MultipartUpload have been set", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "some-host", "region": "some-region", "multipart_upload": false}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + It("sets MultipartUpload to user-specified values", func() { + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.MultipartUpload).To(BeFalse()) + }) + }) + + Context("when MultipartUpload have not been set", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "some-host", "region": "some-region"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + It("default MultipartUpload to true", func() { + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.MultipartUpload).To(BeTrue()) + }) + }) + + Context("when HostStyle has been set", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "some-host", "region": "some-region", "host_style": true}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + It("sets HostStyle to user-specified value", func() { + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.HostStyle).To(BeTrue()) + }) + }) + }) + + Describe("when bucket is not specified", func() { + emptyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key"}`) + emptyJSONReader := bytes.NewReader(emptyJSONBytes) + + It("returns an error", func() { + _, err := config.NewFromReader(emptyJSONReader) + Expect(err).To(MatchError("bucket_name must be set")) + }) + }) + + Describe("when bucket is specified", func() { + emptyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket"}`) + emptyJSONReader := bytes.NewReader(emptyJSONBytes) + + It("uses the given bucket", func() { + c, err := config.NewFromReader(emptyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.BucketName).To(Equal("some-bucket")) + }) + }) + + Describe("when folder is specified", func() { + emptyJSONBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "folder_name": "some-folder/other-folder" + }`) + emptyJSONReader := bytes.NewReader(emptyJSONBytes) + + It("uses the given folder", func() { + c, err := config.NewFromReader(emptyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.FolderName).To(Equal("some-folder/other-folder")) + }) + }) + + Describe("Default SSL options", func() { + emptyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket"}`) + emptyJSONReader := bytes.NewReader(emptyJSONBytes) + + It("defaults to use SSL and peer verification", func() { + c, err := config.NewFromReader(emptyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.UseSSL).To(BeTrue()) + Expect(c.SSLVerifyPeer).To(BeTrue()) + }) + }) + + Describe("configuring signing method", func() { + + It("uses v4 signing when there is no host defined", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.UseV2SigningMethod).To(BeFalse()) + }) + + It("uses v4 signing when the hostname maps to a known Amazon region", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "host": "s3-external-1.amazonaws.com" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.UseV2SigningMethod).To(BeFalse()) + }) + + It("uses v4 signing when the hostname maps to a known Amazon china region", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "host": "s3.cn-north-1.amazonaws.com.cn" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.UseV2SigningMethod).To(BeFalse()) + }) + + It("uses v4 signing when both the hostname and the region map to a known Amazon region", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "host": "s3-external-1.amazonaws.com", + "region": "eu-central-1" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.UseV2SigningMethod).To(BeFalse()) + }) + + It("uses v2 signing when the hostname is a non-Amazon endpoint", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "host": "s3-compatible.com" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.UseV2SigningMethod).To(BeTrue()) + }) + + It("uses override signing value when signing_version is overriden", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "host": "s3-external-1.amazonaws.com", + "signature_version": "2" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.UseV2SigningMethod).To(BeTrue()) + }) + }) + + Describe("configing force path style", func() { + It("when Alibaba Cloud provider", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "host": "oss-some-region.aliyuncs.com" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.HostStyle).To(BeTrue()) + }) + + It("when AWS provider", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "host": "s3.amazonaws.com" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.HostStyle).To(BeFalse()) + }) + + It("when Google provider", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "host": "storage.googleapis.com" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.HostStyle).To(BeFalse()) + }) + + It("when Default provider", func() { + configBytes := []byte(`{ + "access_key_id": "id", + "secret_access_key": "key", + "bucket_name": "some-bucket", + "host": "storage.googleapis.com" + }`) + + configReader := bytes.NewReader(configBytes) + s3CliConfig, err := config.NewFromReader(configReader) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CliConfig.HostStyle).To(BeFalse()) + }) + }) + + Context("when the configuration file cannot be read", func() { + It("returns an error", func() { + f := explodingReader{} + + _, err := config.NewFromReader(f) + Expect(err).To(MatchError("explosion")) + }) + }) + + Context("when the configuration file is invalid JSON", func() { + It("returns an error", func() { + invalidJSONBytes := []byte(`invalid-json`) + invalidJSONReader := bytes.NewReader(invalidJSONBytes) + + _, err := config.NewFromReader(invalidJSONReader) + Expect(err).To(HaveOccurred()) + }) + }) + }) + + Describe("returning the S3 endpoint", func() { + Context("when port is provided", func() { + It("returns a URI in the form `host:port`", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "use_ssl": false, "host": "some-host-name", "port": 443}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("some-host-name:443")) + }) + It("returns a URI in the form `host` when protocol and port match", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "use_ssl": true, "host": "some-host-name", "port": 443}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("some-host-name")) + + dummyJSONBytes = []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "use_ssl": false, "host": "some-host-name", "port": 80}`) + dummyJSONReader = bytes.NewReader(dummyJSONBytes) + + c, err = config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("some-host-name")) + }) + It("returns a empty string URI if `host` is empty", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "", "port": 443}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("")) + }) + }) + + Context("when port is not provided", func() { + It("returns a URI in the form `host` only", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "some-host-name"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("some-host-name")) + }) + It("returns a empty string URI if `host` is empty", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": ""}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("")) + }) + }) + }) + + Describe("validating credentials", func() { + Describe("when credentials source is not specified", func() { + Context("when a secret key and access key are provided", func() { + It("defaults to static credentials", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.CredentialsSource).To(Equal("static")) + }) + }) + + Context("when either the secret key or access key are missing", func() { + It("raises an error", func() { + dummyJSONBytes := []byte(`{"secret_access_key": "key", "bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + _, err := config.NewFromReader(dummyJSONReader) + Expect(err).To(MatchError("access_key_id and secret_access_key must be provided")) + }) + }) + + Context("when neither an access key or secret key are provided", func() { + It("defaults credentials source to anonymous", func() { + dummyJSONBytes := []byte(`{"bucket_name": "some-bucket"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.CredentialsSource).To(Equal("none")) + }) + }) + + Describe("when credentials source is invalid", func() { + It("returns an error", func() { + dummyJSONBytes := []byte(`{"bucket_name": "some-bucket", "credentials_source": "magical_unicorns"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + _, err := config.NewFromReader(dummyJSONReader) + Expect(err).To(MatchError("invalid credentials_source: magical_unicorns")) + }) + }) + + }) + + Context("when credential source is `static`", func() { + It("validates that access key and secret key are set", func() { + dummyJSONBytes := []byte(`{"bucket_name": "some-bucket", "access_key_id": "some_id"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + _, err := config.NewFromReader(dummyJSONReader) + Expect(err).To(MatchError("access_key_id and secret_access_key must be provided")) + + dummyJSONBytes = []byte(`{"bucket_name": "some-bucket", "access_key_id": "some_id", "secret_access_key": "some_secret"}`) + dummyJSONReader = bytes.NewReader(dummyJSONBytes) + _, err = config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + }) + }) + + Context("when credentials source is `env_or_profile`", func() { + It("validates that access key and secret key are not set", func() { + dummyJSONBytes := []byte(`{"bucket_name": "some-bucket", "credentials_source": "env_or_profile"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + _, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + + dummyJSONBytes = []byte(`{"bucket_name": "some-bucket", "credentials_source": "env_or_profile", "access_key_id": "some_id"}`) + dummyJSONReader = bytes.NewReader(dummyJSONBytes) + _, err = config.NewFromReader(dummyJSONReader) + Expect(err).To(MatchError("can't use access_key_id and secret_access_key with env_or_profile credentials_source")) + + dummyJSONBytes = []byte(`{"bucket_name": "some-bucket", "credentials_source": "env_or_profile", "access_key_id": "some_id", "secret_access_key": "some_secret"}`) + dummyJSONReader = bytes.NewReader(dummyJSONBytes) + _, err = config.NewFromReader(dummyJSONReader) + Expect(err).To(MatchError("can't use access_key_id and secret_access_key with env_or_profile credentials_source")) + }) + }) + + Context("when the credentials source is `none`", func() { + It("validates that access key and secret key are not set", func() { + dummyJSONBytes := []byte(`{"bucket_name": "some-bucket", "credentials_source": "none", "access_key_id": "some_id"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + _, err := config.NewFromReader(dummyJSONReader) + Expect(err).To(MatchError("can't use access_key_id and secret_access_key with none credentials_source")) + }) + }) + }) + + Describe("returning the alibaba cloud region", func() { + Context("when host is provided", func() { + It("returns a region id in the public `host`", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "oss-some-region.aliyuncs.com"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.Region).To(Equal("some-region")) + }) + It("returns a region id in the private `host`", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "oss-some-region-internal.aliyuncs.com"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.Region).To(Equal("some-region")) + }) + It("returns a empty string if `host` is empty", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": ""}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("")) + }) + }) + }) + + Describe("returning the alibaba cloud endpoint", func() { + Context("when port is provided", func() { + It("returns a URI in the form `host:port`", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "use_ssl": false, "host": "oss-some-region.aliyuncs.com", "port": 443}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("oss-some-region.aliyuncs.com:443")) + Expect(c.Host).To(Equal("oss-some-region.aliyuncs.com")) + }) + It("returns a empty string URI if `host` is empty", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "", "port": 443}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("")) + Expect(c.Host).To(Equal("")) + }) + }) + + Context("when port is not provided", func() { + It("returns a URI in the form `host` only", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "oss-some-region.aliyuncs.com"}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("oss-some-region.aliyuncs.com")) + Expect(c.Host).To(Equal("oss-some-region.aliyuncs.com")) + }) + It("returns a empty string URI if `host` is empty", func() { + dummyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": ""}`) + dummyJSONReader := bytes.NewReader(dummyJSONBytes) + + c, err := config.NewFromReader(dummyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.S3Endpoint()).To(Equal("")) + }) + }) + }) + + Describe("checking the alibaba cloud MultipartUpload", func() { + emptyJSONBytes := []byte(`{"access_key_id": "id", "secret_access_key": "key", "bucket_name": "some-bucket", "host": "oss-some-region.aliyuncs.com"}`) + emptyJSONReader := bytes.NewReader(emptyJSONBytes) + + It("defaults to support multipart uploading", func() { + c, err := config.NewFromReader(emptyJSONReader) + Expect(err).ToNot(HaveOccurred()) + Expect(c.MultipartUpload).To(BeTrue()) + }) + }) +}) + +type explodingReader struct{} + +func (e explodingReader) Read([]byte) (int, error) { + return 0, errors.New("explosion") +} diff --git a/s3/config/endpoints.go b/s3/config/endpoints.go new file mode 100644 index 0000000..13a996a --- /dev/null +++ b/s3/config/endpoints.go @@ -0,0 +1,35 @@ +package config + +import ( + "regexp" +) + +var ( + providerRegex = map[string]*regexp.Regexp{ + "aws": regexp.MustCompile(`(^$|s3[-.]?(.*)\.amazonaws\.com(\.cn)?$)`), + "alicloud": regexp.MustCompile(`^oss-([a-z]+-[a-z]+(-[1-9])?)(-internal)?.aliyuncs.com$`), + "google": regexp.MustCompile(`^storage.googleapis.com$`), + } +) + +func AWSHostToRegion(host string) string { + regexMatches := providerRegex["aws"].FindStringSubmatch(host) + + region := "us-east-1" + + if len(regexMatches) == 4 && regexMatches[2] != "" && regexMatches[2] != "external-1" { + region = regexMatches[2] + } + + return region +} + +func AlicloudHostToRegion(host string) string { + regexMatches := providerRegex["alicloud"].FindStringSubmatch(host) + + if len(regexMatches) == 4 { + return regexMatches[1] + } + + return "" +} diff --git a/s3/config/endpoints_test.go b/s3/config/endpoints_test.go new file mode 100644 index 0000000..bb07ca8 --- /dev/null +++ b/s3/config/endpoints_test.go @@ -0,0 +1,35 @@ +package config_test + +import ( + "github.com/cloudfoundry/storage-cli/s3/config" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Endpoints", func() { + DescribeTable("AWSHostToRegion", + func(host, region string) { + Expect(config.AWSHostToRegion(host)).To(Equal(region)) + }, + Entry("us-east-1", "this-should-default", "us-east-1"), + Entry("us-east-1", "s3.amazonaws.com", "us-east-1"), + Entry("us-east-1", "s3-external-1.amazonaws.com", "us-east-1"), + Entry("us-east-2", "s3.us-east-2.amazonaws.com", "us-east-2"), + Entry("us-east-2", "s3-us-east-2.amazonaws.com", "us-east-2"), + Entry("cn-north-1", "s3.cn-north-1.amazonaws.com.cn", "cn-north-1"), + Entry("whatever-region", "s3.whatever-region.amazonaws.com", "whatever-region"), + Entry("some-region", "s3-some-region.amazonaws.com", "some-region"), + ) + + DescribeTable("AlicloudHostToRegion", + func(host, region string) { + Expect(config.AlicloudHostToRegion(host)).To(Equal(region)) + }, + Entry("with internal and number", "oss-country-zone-9-internal.aliyuncs.com", "country-zone-9"), + Entry("with internal and no number", "oss-sichuan-chengdu-internal.aliyuncs.com", "sichuan-chengdu"), + Entry("without internal and number", "oss-one-two-1.aliyuncs.com", "one-two-1"), + Entry("without internal and no number", "oss-country-zone.aliyuncs.com", "country-zone"), + Entry("not alicloud", "s3-us-east-2.amazonaws.com", ""), + ) +}) diff --git a/s3/integration/assertions.go b/s3/integration/assertions.go new file mode 100644 index 0000000..c2951f8 --- /dev/null +++ b/s3/integration/assertions.go @@ -0,0 +1,228 @@ +package integration + +import ( + "context" + "fmt" + "log" + "os" + "strings" + "time" + + "github.com/cloudfoundry/storage-cli/s3/client" + "github.com/cloudfoundry/storage-cli/s3/config" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + . "github.com/onsi/gomega" //nolint:staticcheck +) + +// AssertLifecycleWorks tests the main blobstore object lifecycle from creation to deletion +func AssertLifecycleWorks(s3CLIPath string, cfg *config.S3Cli) { + expectedString := GenerateRandomString() + s3Filename := GenerateRandomString() + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + contentFile := MakeContentFile(expectedString) + defer os.Remove(contentFile) //nolint:errcheck + + s3CLISession, err := RunS3CLI(s3CLIPath, configPath, "put", contentFile, s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(BeZero()) + + if len(cfg.FolderName) != 0 { + folderName := cfg.FolderName + cfg.FolderName = "" + noFolderConfigPath := MakeConfigFile(cfg) + defer os.Remove(noFolderConfigPath) //nolint:errcheck + + s3CLISession, err := + RunS3CLI(s3CLIPath, noFolderConfigPath, "exists", fmt.Sprintf("%s/%s", folderName, s3Filename)) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(BeZero()) + } + + s3CLISession, err = RunS3CLI(s3CLIPath, configPath, "exists", s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(BeZero()) + Expect(s3CLISession.Err.Contents()).To(MatchRegexp("File '.*' exists in bucket '.*'")) + + tmpLocalFile, err := os.CreateTemp("", "s3cli-download") + Expect(err).ToNot(HaveOccurred()) + err = tmpLocalFile.Close() + Expect(err).ToNot(HaveOccurred()) + defer os.Remove(tmpLocalFile.Name()) //nolint:errcheck + + s3CLISession, err = RunS3CLI(s3CLIPath, configPath, "get", s3Filename, tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(BeZero()) + + gottenBytes, err := os.ReadFile(tmpLocalFile.Name()) + Expect(err).ToNot(HaveOccurred()) + Expect(string(gottenBytes)).To(Equal(expectedString)) + + s3CLISession, err = RunS3CLI(s3CLIPath, configPath, "delete", s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(BeZero()) + + s3CLISession, err = RunS3CLI(s3CLIPath, configPath, "exists", s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(Equal(3)) + Expect(s3CLISession.Err.Contents()).To(MatchRegexp("File '.*' does not exist in bucket '.*'")) +} + +func AssertOnPutFailures(s3CLIPath string, cfg *config.S3Cli, content, errorMessage string) { + s3Filename := GenerateRandomString() + sourceContent := strings.NewReader(content) + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + configFile, err := os.Open(configPath) + Expect(err).ToNot(HaveOccurred()) + + s3Config, err := config.NewFromReader(configFile) + Expect(err).ToNot(HaveOccurred()) + + s3Client, err := CreateS3ClientWithFailureInjection(&s3Config) + if err != nil { + log.Fatalln(err) + } + blobstoreClient := client.New(s3Client, &s3Config) + + err = blobstoreClient.Put(sourceContent, s3Filename) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(errorMessage)) +} + +// AssertPutOptionsApplied asserts that `s3cli put` uploads files with the requested encryption options +func AssertPutOptionsApplied(s3CLIPath string, cfg *config.S3Cli) { + expectedString := GenerateRandomString() + s3Filename := GenerateRandomString() + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + contentFile := MakeContentFile(expectedString) + defer os.Remove(contentFile) //nolint:errcheck + + configFile, err := os.Open(configPath) + Expect(err).ToNot(HaveOccurred()) + + s3CLISession, err := RunS3CLI(s3CLIPath, configPath, "put", contentFile, s3Filename) //nolint:ineffassign,staticcheck + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(BeZero()) + + s3Config, err := config.NewFromReader(configFile) + Expect(err).ToNot(HaveOccurred()) + + s3Client, err := client.NewAwsS3Client(&s3Config) + Expect(err).ToNot(HaveOccurred()) + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + resp, err := s3Client.HeadObject(ctx, &s3.HeadObjectInput{ + Bucket: aws.String(cfg.BucketName), + Key: aws.String(s3Filename), + }) + Expect(err).ToNot(HaveOccurred()) + + if cfg.ServerSideEncryption == "" { + Expect(resp.ServerSideEncryption).To(Or(BeNil(), HaveValue(Equal(types.ServerSideEncryptionAes256)))) + } else { + Expect(string(resp.ServerSideEncryption)).To(Equal(cfg.ServerSideEncryption)) + } +} + +// AssertGetNonexistentFails asserts that `s3cli get` on a non-existent object will fail +func AssertGetNonexistentFails(s3CLIPath string, cfg *config.S3Cli) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + s3CLISession, err := RunS3CLI(s3CLIPath, configPath, "get", "non-existent-file", "/dev/null") + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).ToNot(BeZero()) + Expect(s3CLISession.Err.Contents()).To(ContainSubstring("NoSuchKey")) +} + +// AssertDeleteNonexistentWorks asserts that `s3cli delete` on a non-existent +// object exits with status 0 (tests idempotency) +func AssertDeleteNonexistentWorks(s3CLIPath string, cfg *config.S3Cli) { + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + s3CLISession, err := RunS3CLI(s3CLIPath, configPath, "delete", "non-existent-file") + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(BeZero()) +} + +func AssertOnMultipartUploads(s3CLIPath string, cfg *config.S3Cli, content string) { + s3Filename := GenerateRandomString() + sourceContent := strings.NewReader(content) + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + configFile, err := os.Open(configPath) + Expect(err).ToNot(HaveOccurred()) + + s3Config, err := config.NewFromReader(configFile) + Expect(err).ToNot(HaveOccurred()) + + // Create S3 client with tracing middleware + calls := []string{} + s3Client, err := CreateTracingS3Client(&s3Config, &calls) + if err != nil { + log.Fatalln(err) + } + + blobstoreClient := client.New(s3Client, &s3Config) + + err = blobstoreClient.Put(sourceContent, s3Filename) + Expect(err).ToNot(HaveOccurred()) + + switch cfg.Host { + case "storage.googleapis.com": + Expect(calls).To(Equal([]string{"PutObject"})) + default: + Expect(calls).To(Equal([]string{"CreateMultipart", "UploadPart", "UploadPart", "CompleteMultipart"})) + } +} + +// AssertOnSignedURLs asserts on using signed URLs for upload and download +func AssertOnSignedURLs(s3CLIPath string, cfg *config.S3Cli) { + s3Filename := GenerateRandomString() + + configPath := MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + configFile, err := os.Open(configPath) + Expect(err).ToNot(HaveOccurred()) + + s3Config, err := config.NewFromReader(configFile) + Expect(err).ToNot(HaveOccurred()) + + // Create S3 client with tracing middleware (though signing operations don't need tracing for this test) + calls := []string{} + s3Client, err := CreateTracingS3Client(&s3Config, &calls) + if err != nil { + log.Fatalln(err) + } + + blobstoreClient := client.New(s3Client, &s3Config) + + regex := `(?m)((([A-Za-z]{3,9}:(?:\/\/?)?)(?:[-;:&=\+\$,\w]+@)?[A-Za-z0-9.-]+(:[0-9]+)?|(?:www.|[-;:&=\+\$,\w]+@)[A-Za-z0-9.-]+)((?:\/[\+~%\/.\w-_]*)?\??(?:[-\+=&;%@.\w_]*)#?(?:[\w]*))?)` + + // get + url, err := blobstoreClient.Sign(s3Filename, "get", 1*time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(url).To(MatchRegexp(regex)) + + // put + url, err = blobstoreClient.Sign(s3Filename, "put", 1*time.Minute) + Expect(err).ToNot(HaveOccurred()) + Expect(url).To(MatchRegexp(regex)) +} diff --git a/s3/integration/aws_assume_role_test.go b/s3/integration/aws_assume_role_test.go new file mode 100644 index 0000000..0b43f2b --- /dev/null +++ b/s3/integration/aws_assume_role_test.go @@ -0,0 +1,61 @@ +package integration_test + +import ( + "os" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing AWS assume role ", func() { + Context("with AWS ASSUME ROLE configurations", func() { + It("get file from assumed role", func() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + + assumeRoleArn := os.Getenv("ASSUME_ROLE_ARN") + Expect(assumeRoleArn).ToNot(BeEmpty(), "ASSUME_ROLE_ARN must be set") + + bucketName := "bosh-s3cli-assume-role-integration-test" + region := "us-east-1" + + nonAssumedRoleCfg := &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + UseSSL: true, + } + + assumedRoleCfg := &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + AssumeRoleArn: assumeRoleArn, + UseSSL: true, + } + s3Filename := "test-file" + + notAssumeRoleConfigPath := integration.MakeConfigFile(nonAssumedRoleCfg) + defer os.Remove(notAssumeRoleConfigPath) //nolint:errcheck + + s3CLISession, err := integration.RunS3CLI(s3CLIPath, notAssumeRoleConfigPath, "exists", s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).ToNot(BeZero()) + + assumeRoleConfigPath := integration.MakeConfigFile(assumedRoleCfg) + defer os.Remove(assumeRoleConfigPath) //nolint:errcheck + + s3CLISession, err = integration.RunS3CLI(s3CLIPath, assumeRoleConfigPath, "exists", s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(BeZero()) + }) + }) +}) diff --git a/s3/integration/aws_iam_role_test.go b/s3/integration/aws_iam_role_test.go new file mode 100644 index 0000000..f73a359 --- /dev/null +++ b/s3/integration/aws_iam_role_test.go @@ -0,0 +1,60 @@ +package integration_test + +import ( + "os" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing inside an AWS compute resource with an IAM role", func() { + Context("with AWS STANDARD IAM ROLE (env_or_profile creds) configurations", func() { + bucketName := os.Getenv("BUCKET_NAME") + region := os.Getenv("REGION") + s3Host := os.Getenv("S3_HOST") + + BeforeEach(func() { + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + Expect(region).ToNot(BeEmpty(), "REGION must be set") + Expect(s3Host).ToNot(BeEmpty(), "S3_HOST must be set") + }) + + configurations := []TableEntry{ + Entry("with minimal config", &config.S3Cli{ + CredentialsSource: "env_or_profile", + BucketName: bucketName, + }), + Entry("with region and without host, signature version 4", &config.S3Cli{ + SignatureVersion: 4, + CredentialsSource: "env_or_profile", + BucketName: bucketName, + Region: region, + }), + Entry("with maximal config, signature version 4", &config.S3Cli{ + SignatureVersion: 4, + CredentialsSource: "env_or_profile", + BucketName: bucketName, + Host: s3Host, + Port: 443, + UseSSL: true, + SSLVerifyPeer: true, + Region: region, + }), + } + DescribeTable("Blobstore lifecycle works", + func(cfg *config.S3Cli) { integration.AssertLifecycleWorks(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli get` on a non-existent-key fails", + func(cfg *config.S3Cli) { integration.AssertGetNonexistentFails(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli delete` on a non-existent-key does not fail", + func(cfg *config.S3Cli) { integration.AssertDeleteNonexistentWorks(s3CLIPath, cfg) }, + configurations, + ) + }) +}) diff --git a/s3/integration/aws_isolated_region_test.go b/s3/integration/aws_isolated_region_test.go new file mode 100644 index 0000000..276fe23 --- /dev/null +++ b/s3/integration/aws_isolated_region_test.go @@ -0,0 +1,56 @@ +package integration_test + +import ( + "os" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing in any AWS region isolated from the US standard regions (i.e., cn-north-1)", func() { + Context("with AWS ISOLATED REGION (static creds) configurations", func() { + It("fails with a config that specifies a valid region but invalid host", func() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + + bucketName := os.Getenv("BUCKET_NAME") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + + region := os.Getenv("REGION") + Expect(region).ToNot(BeEmpty(), "REGION must be set") + + cfg := &config.S3Cli{ + SignatureVersion: 4, + CredentialsSource: "static", + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + Host: "s3-external-1.amazonaws.com", + } + s3Filename := integration.GenerateRandomString() + + configPath := integration.MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + contentFile := integration.MakeContentFile("test") + defer os.Remove(contentFile) //nolint:errcheck + + s3CLISession, err := integration.RunS3CLI(s3CLIPath, configPath, "put", contentFile, s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).ToNot(BeZero()) + Expect(s3CLISession.Err.Contents()).To(ContainSubstring("AuthorizationHeaderMalformed")) + + s3CLISession, err = integration.RunS3CLI(s3CLIPath, configPath, "delete", s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).ToNot(BeZero()) + Expect(s3CLISession.Err.Contents()).To(ContainSubstring("AuthorizationHeaderMalformed")) + }) + }) +}) diff --git a/s3/integration/aws_public_read_only_test.go b/s3/integration/aws_public_read_only_test.go new file mode 100644 index 0000000..0461a69 --- /dev/null +++ b/s3/integration/aws_public_read_only_test.go @@ -0,0 +1,76 @@ +package integration_test + +import ( + "context" + "os" + "strings" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing gets against a public AWS S3 bucket", func() { + Context("with PUBLIC READ ONLY (no creds) configuration", func() { + It("can successfully get a publicly readable file", func() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + + bucketName := os.Getenv("BUCKET_NAME") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + + region := os.Getenv("REGION") + Expect(region).ToNot(BeEmpty(), "REGION must be set") + + s3Filename := integration.GenerateRandomString() + s3FileContents := integration.GenerateRandomString() + + awsConfig, err := awsconfig.LoadDefaultConfig(context.TODO(), + awsconfig.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyID, secretAccessKey, "")), + awsconfig.WithRegion(region), + ) + Expect(err).ToNot(HaveOccurred()) + s3Client := s3.NewFromConfig(awsConfig) + + _, err = s3Client.PutObject(context.TODO(), &s3.PutObjectInput{ + Body: strings.NewReader(s3FileContents), + Bucket: &bucketName, + Key: &s3Filename, + ACL: types.ObjectCannedACLPublicRead, + }) + Expect(err).ToNot(HaveOccurred()) + + cfg := &config.S3Cli{ + BucketName: bucketName, + Region: region, + } + + configPath := integration.MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + s3CLISession, err := integration.RunS3CLI(s3CLIPath, configPath, "get", s3Filename, "public-file") + Expect(err).ToNot(HaveOccurred()) + + defer os.Remove("public-file") //nolint:errcheck + Expect(s3CLISession.ExitCode()).To(BeZero()) + + gottenBytes, err := os.ReadFile("public-file") + Expect(err).ToNot(HaveOccurred()) + Expect(string(gottenBytes)).To(Equal(s3FileContents)) + + s3CLISession, err = integration.RunS3CLI(s3CLIPath, configPath, "exists", s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).To(BeZero()) + Expect(s3CLISession.Err.Contents()).To(MatchRegexp("File '.*' exists in bucket '.*'")) + }) + }) +}) diff --git a/s3/integration/aws_us_east_test.go b/s3/integration/aws_us_east_test.go new file mode 100644 index 0000000..0824e58 --- /dev/null +++ b/s3/integration/aws_us_east_test.go @@ -0,0 +1,64 @@ +package integration_test + +import ( + "os" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing only in us-east-1", func() { + Context("with AWS US-EAST-1 (static creds) configurations", func() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + bucketName := os.Getenv("BUCKET_NAME") + + BeforeEach(func() { + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + }) + + configurations := []TableEntry{ + Entry("with minimal config", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + }), + Entry("with signature version 2", &config.S3Cli{ + SignatureVersion: 2, + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + }), + Entry("with alternate host", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: "s3-external-1.amazonaws.com", + }), + Entry("with alternate host and region", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: "s3-external-1.amazonaws.com", + Region: "us-east-1", + }), + } + DescribeTable("Blobstore lifecycle works", + func(cfg *config.S3Cli) { integration.AssertLifecycleWorks(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli get` on a non-existent-key fails", + func(cfg *config.S3Cli) { integration.AssertGetNonexistentFails(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli delete` on a non-existent-key does not fail", + func(cfg *config.S3Cli) { integration.AssertDeleteNonexistentWorks(s3CLIPath, cfg) }, + configurations, + ) + }) +}) diff --git a/s3/integration/aws_v2_region_test.go b/s3/integration/aws_v2_region_test.go new file mode 100644 index 0000000..a3ae2ae --- /dev/null +++ b/s3/integration/aws_v2_region_test.go @@ -0,0 +1,60 @@ +package integration_test + +import ( + "os" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing in any AWS region that supports v2 signature version", func() { + Context("with AWS V2 REGION (static creds) configurations", func() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + bucketName := os.Getenv("BUCKET_NAME") + region := os.Getenv("REGION") + s3Host := os.Getenv("S3_HOST") + + BeforeEach(func() { + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + Expect(region).ToNot(BeEmpty(), "REGION must be set") + Expect(s3Host).ToNot(BeEmpty(), "S3_HOST must be set") + }) + + configurations := []TableEntry{ + Entry("with host and without region, signature version 2", &config.S3Cli{ + SignatureVersion: 2, + CredentialsSource: "static", + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: s3Host, + }), + Entry("with region and without host, signature version 2", &config.S3Cli{ + SignatureVersion: 2, + CredentialsSource: "static", + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + }), + } + DescribeTable("Blobstore lifecycle works", + func(cfg *config.S3Cli) { integration.AssertLifecycleWorks(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli get` on a non-existent-key fails", + func(cfg *config.S3Cli) { integration.AssertGetNonexistentFails(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli delete` on a non-existent-key does not fail", + func(cfg *config.S3Cli) { integration.AssertDeleteNonexistentWorks(s3CLIPath, cfg) }, + configurations, + ) + }) +}) diff --git a/s3/integration/aws_v4_only_region_test.go b/s3/integration/aws_v4_only_region_test.go new file mode 100644 index 0000000..8508598 --- /dev/null +++ b/s3/integration/aws_v4_only_region_test.go @@ -0,0 +1,52 @@ +package integration_test + +import ( + "os" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing in any AWS region that only supports v4 signature version", func() { + Context("with AWS V4 ONLY REGION (static creds) configurations", func() { + It("fails with a config that specifies signature version 2", func() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + + bucketName := os.Getenv("BUCKET_NAME") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + + region := os.Getenv("REGION") + Expect(region).ToNot(BeEmpty(), "REGION must be set") + + cfg := &config.S3Cli{ + SignatureVersion: 2, + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + } + s3Filename := integration.GenerateRandomString() + + configPath := integration.MakeConfigFile(cfg) + defer os.Remove(configPath) //nolint:errcheck + + contentFile := integration.MakeContentFile("test") + defer os.Remove(contentFile) //nolint:errcheck + + s3CLISession, err := integration.RunS3CLI(s3CLIPath, configPath, "put", contentFile, s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).ToNot(BeZero()) + + s3CLISession, err = integration.RunS3CLI(s3CLIPath, configPath, "delete", s3Filename) + Expect(err).ToNot(HaveOccurred()) + Expect(s3CLISession.ExitCode()).ToNot(BeZero()) + }) + }) +}) diff --git a/s3/integration/aws_v4_region_test.go b/s3/integration/aws_v4_region_test.go new file mode 100644 index 0000000..b7c3467 --- /dev/null +++ b/s3/integration/aws_v4_region_test.go @@ -0,0 +1,72 @@ +package integration_test + +import ( + "os" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing in any AWS region that supports v4 signature version", func() { + Context("with AWS V4 REGION (static creds) configurations", func() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + bucketName := os.Getenv("BUCKET_NAME") + region := os.Getenv("REGION") + s3Host := os.Getenv("S3_HOST") + + BeforeEach(func() { + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + Expect(region).ToNot(BeEmpty(), "REGION must be set") + Expect(s3Host).ToNot(BeEmpty(), "S3_HOST must be set") + }) + + configurations := []TableEntry{ + Entry("with region and without host", &config.S3Cli{ + SignatureVersion: 4, + CredentialsSource: "static", + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + }), + Entry("with host and without region", &config.S3Cli{ + SignatureVersion: 4, + CredentialsSource: "static", + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: s3Host, + }), + Entry("with maximal config", &config.S3Cli{ + SignatureVersion: 4, + CredentialsSource: "static", + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: s3Host, + Port: 443, + UseSSL: true, + SSLVerifyPeer: true, + Region: region, + }), + } + DescribeTable("Blobstore lifecycle works", + func(cfg *config.S3Cli) { integration.AssertLifecycleWorks(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli get` on a non-existent-key fails", + func(cfg *config.S3Cli) { integration.AssertGetNonexistentFails(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli delete` on a non-existent-key does not fail", + func(cfg *config.S3Cli) { integration.AssertDeleteNonexistentWorks(s3CLIPath, cfg) }, + configurations, + ) + }) +}) diff --git a/s3/integration/general_aws_test.go b/s3/integration/general_aws_test.go new file mode 100644 index 0000000..cf94fa1 --- /dev/null +++ b/s3/integration/general_aws_test.go @@ -0,0 +1,120 @@ +package integration_test + +import ( + "os" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("General testing for all AWS regions", func() { + Context("with GENERAL AWS (static creds) configurations", func() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + bucketName := os.Getenv("BUCKET_NAME") + region := os.Getenv("REGION") + s3Host := os.Getenv("S3_HOST") + + BeforeEach(func() { + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + Expect(region).ToNot(BeEmpty(), "REGION must be set") + Expect(s3Host).ToNot(BeEmpty(), "S3_HOST must be set") + }) + + configurations := []TableEntry{ + Entry("with region and without host", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + }), + Entry("with host and without region", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: s3Host, + }), + Entry("with folder", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + FolderName: "test-folder/a-folder", + Region: region, + }), + Entry("with host style enabled", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + HostStyle: true, + }), + } + DescribeTable("Blobstore lifecycle works", + func(cfg *config.S3Cli) { integration.AssertLifecycleWorks(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli get` on a non-existent-key fails", + func(cfg *config.S3Cli) { integration.AssertGetNonexistentFails(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli delete` on a non-existent-key does not fail", + func(cfg *config.S3Cli) { integration.AssertDeleteNonexistentWorks(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli sign` returns a signed URL", + func(cfg *config.S3Cli) { integration.AssertOnSignedURLs(s3CLIPath, cfg) }, + configurations, + ) + + configurations = []TableEntry{ + Entry("with encryption", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + ServerSideEncryption: "AES256", + }), + Entry("without encryption", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + }), + } + DescribeTable("Invoking `s3cli put` uploads with options", + func(cfg *config.S3Cli) { integration.AssertPutOptionsApplied(s3CLIPath, cfg) }, + configurations, + ) + + Describe("Invoking `s3cli put` with arbitrary upload failures", func() { + It("returns the appropriate error message", func() { + cfg := &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: "http://localhost", + } + msg := "upload failure" + integration.AssertOnPutFailures(s3CLIPath, cfg, largeContent, msg) + }) + }) + + Describe("Invoking `s3cli put` with multipart upload failures", func() { + It("returns the appropriate error message", func() { + cfg := &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Region: region, + } + msg := "upload retry limit exceeded" + integration.AssertOnPutFailures(s3CLIPath, cfg, largeContent, msg) + }) + }) + }) +}) diff --git a/s3/integration/integration_suite_test.go b/s3/integration/integration_suite_test.go new file mode 100644 index 0000000..a3494e2 --- /dev/null +++ b/s3/integration/integration_suite_test.go @@ -0,0 +1,37 @@ +package integration_test + +import ( + "os" + "testing" + + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" +) + +func TestIntegration(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Integration Suite") +} + +var s3CLIPath string +var largeContent string + +var _ = BeforeSuite(func() { + // Running the IAM tests within an AWS Lambda environment + // require a pre-compiled binary + s3CLIPath = os.Getenv("S3_CLI_PATH") + largeContent = integration.GenerateRandomString(1024 * 1024 * 6) + + if len(s3CLIPath) == 0 { + var err error + s3CLIPath, err = gexec.Build("github.com/cloudfoundry/bosh-s3cli") + Expect(err).ShouldNot(HaveOccurred()) + } +}) + +var _ = AfterSuite(func() { + gexec.CleanupBuildArtifacts() +}) diff --git a/s3/integration/middlewares.go b/s3/integration/middlewares.go new file mode 100644 index 0000000..5066ea2 --- /dev/null +++ b/s3/integration/middlewares.go @@ -0,0 +1,257 @@ +package integration + +import ( + "context" + "net/http" + "sync/atomic" + + "github.com/aws/aws-sdk-go-v2/aws" + awsconfig "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/credentials/stscreds" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/sts" + "github.com/aws/smithy-go/middleware" + smithyhttp "github.com/aws/smithy-go/transport/http" + boshhttp "github.com/cloudfoundry/bosh-utils/httpclient" + "github.com/cloudfoundry/storage-cli/s3/config" +) + +// CreateUploadPartTracker creates an Initialize middleware that tracks upload parts +func CreateUploadPartTracker() middleware.InitializeMiddleware { + var partCounter int64 + + return middleware.InitializeMiddlewareFunc("UploadPartTracker", func( + ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler, + ) ( + out middleware.InitializeOutput, metadata middleware.Metadata, err error, + ) { + // Type switch to check if the input is s3.UploadPartInput + injectFailure := false + switch in.Parameters.(type) { + case *s3.UploadPartInput: + // Increment counter and mark even-numbered parts for failure + count := atomic.AddInt64(&partCounter, 1) + if count%2 == 0 { + injectFailure = true + } + } + + // Store the injectFailure flag in the context for this request + ctx = context.WithValue(ctx, failureInjectionKey{}, injectFailure) + + // Continue to next middleware + return next.HandleInitialize(ctx, in) + }) +} + +// Context key for failure injection flag +type failureInjectionKey struct{} + +// CreateSHACorruptionMiddleware creates a Finalize middleware that corrupts headers +func CreateSHACorruptionMiddleware() middleware.FinalizeMiddleware { + return middleware.FinalizeMiddlewareFunc("SHACorruptionMiddleware", func( + ctx context.Context, in middleware.FinalizeInput, next middleware.FinalizeHandler, + ) (middleware.FinalizeOutput, middleware.Metadata, error) { + // Check if we should inject a failure based on context value + if inject, ok := ctx.Value(failureInjectionKey{}).(bool); ok && inject { + if req, ok := in.Request.(*smithyhttp.Request); ok { + // Corrupt the SHA256 header to cause upload failure + req.Header.Set("X-Amz-Content-Sha256", "000") + } + } + return next.HandleFinalize(ctx, in) + }) +} + +// CreateS3ClientWithFailureInjection creates an S3 client with failure injection middleware +func CreateS3ClientWithFailureInjection(s3Config *config.S3Cli) (*s3.Client, error) { + // Create HTTP client based on SSL verification settings + var httpClient *http.Client + if s3Config.SSLVerifyPeer { + httpClient = boshhttp.CreateDefaultClient(nil) + } else { + httpClient = boshhttp.CreateDefaultClientInsecureSkipVerify() + } + + // Set up AWS config options + options := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithHTTPClient(httpClient), + } + + if s3Config.UseRegion() { + options = append(options, awsconfig.WithRegion(s3Config.Region)) + } else { + options = append(options, awsconfig.WithRegion(config.EmptyRegion)) + } + + if s3Config.CredentialsSource == config.StaticCredentialsSource { + options = append(options, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(s3Config.AccessKeyID, s3Config.SecretAccessKey, ""), + )) + } + + if s3Config.CredentialsSource == config.NoneCredentialsSource { + options = append(options, awsconfig.WithCredentialsProvider(aws.AnonymousCredentials{})) + } + + // Load AWS config + awsConfig, err := awsconfig.LoadDefaultConfig(context.TODO(), options...) + if err != nil { + return nil, err + } + + // Handle STS assume role if configured + if s3Config.AssumeRoleArn != "" { + stsClient := sts.NewFromConfig(awsConfig) + provider := stscreds.NewAssumeRoleProvider(stsClient, s3Config.AssumeRoleArn) + awsConfig.Credentials = aws.NewCredentialsCache(provider) + } + + // Create failure injection middlewares + trackingMiddleware := CreateUploadPartTracker() + corruptionMiddleware := CreateSHACorruptionMiddleware() + + // Create S3 client with custom middleware and options + s3Client := s3.NewFromConfig(awsConfig, func(o *s3.Options) { + o.UsePathStyle = !s3Config.HostStyle + if s3Config.S3Endpoint() != "" { + o.BaseEndpoint = aws.String(s3Config.S3Endpoint()) + } + + // Add the failure injection middlewares + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + // Add initialize middleware to track UploadPart operations + if err := stack.Initialize.Add(trackingMiddleware, middleware.Before); err != nil { + return err + } + // Add finalize middleware to corrupt headers after signing + return stack.Finalize.Add(corruptionMiddleware, middleware.After) + }) + }) + + return s3Client, nil +} + +// S3TracingMiddleware captures S3 operation names for testing +type S3TracingMiddleware struct { + calls *[]string +} + +// CreateS3TracingMiddleware creates a middleware that tracks S3 operation calls +func CreateS3TracingMiddleware(calls *[]string) *S3TracingMiddleware { + return &S3TracingMiddleware{calls: calls} +} + +// ID returns the middleware identifier +func (m *S3TracingMiddleware) ID() string { + return "S3TracingMiddleware" +} + +// HandleInitialize implements the InitializeMiddleware interface +func (m *S3TracingMiddleware) HandleInitialize( + ctx context.Context, in middleware.InitializeInput, next middleware.InitializeHandler, +) (middleware.InitializeOutput, middleware.Metadata, error) { + // Extract operation name from the middleware metadata + if operationName := middleware.GetStackValue(ctx, "operation"); operationName != nil { + if opName, ok := operationName.(string); ok { + *m.calls = append(*m.calls, opName) + } + } + + // Try to determine operation from the input type as fallback + switch in.Parameters.(type) { + case *s3.CreateMultipartUploadInput: + *m.calls = append(*m.calls, "CreateMultipart") + case *s3.UploadPartInput: + *m.calls = append(*m.calls, "UploadPart") + case *s3.CompleteMultipartUploadInput: + *m.calls = append(*m.calls, "CompleteMultipart") + case *s3.PutObjectInput: + *m.calls = append(*m.calls, "PutObject") + case *s3.GetObjectInput: + *m.calls = append(*m.calls, "GetObject") + case *s3.DeleteObjectInput: + *m.calls = append(*m.calls, "DeleteObject") + case *s3.HeadObjectInput: + *m.calls = append(*m.calls, "HeadObject") + } + + return next.HandleInitialize(ctx, in) +} + +// CreateS3ClientWithTracing creates a new S3 client with tracing middleware +func CreateS3ClientWithTracing(baseClient *s3.Client, tracingMiddleware *S3TracingMiddleware) *s3.Client { + // Create a wrapper that captures calls and delegates to the base client + // Since AWS SDK v2 makes it difficult to extract config from existing clients, + // we'll use a different approach: modify the traceS3 function to work differently + + // For the tracing functionality, we'll need to intercept at a higher level + // The current implementation will track operations through the middleware + // that inspects the input parameters + + return baseClient +} + +// CreateTracingS3Client creates an S3 client with tracing middleware from config +func CreateTracingS3Client(s3Config *config.S3Cli, calls *[]string) (*s3.Client, error) { + // Create HTTP client based on SSL verification settings + var httpClient *http.Client + if s3Config.SSLVerifyPeer { + httpClient = boshhttp.CreateDefaultClient(nil) + } else { + httpClient = boshhttp.CreateDefaultClientInsecureSkipVerify() + } + + // Set up AWS config options + options := []func(*awsconfig.LoadOptions) error{ + awsconfig.WithHTTPClient(httpClient), + } + + if s3Config.UseRegion() { + options = append(options, awsconfig.WithRegion(s3Config.Region)) + } else { + options = append(options, awsconfig.WithRegion(config.EmptyRegion)) + } + + if s3Config.CredentialsSource == config.StaticCredentialsSource { + options = append(options, awsconfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(s3Config.AccessKeyID, s3Config.SecretAccessKey, ""), + )) + } + + if s3Config.CredentialsSource == config.NoneCredentialsSource { + options = append(options, awsconfig.WithCredentialsProvider(aws.AnonymousCredentials{})) + } + + // Load AWS config + awsConfig, err := awsconfig.LoadDefaultConfig(context.TODO(), options...) + if err != nil { + return nil, err + } + + // Handle STS assume role if configured + if s3Config.AssumeRoleArn != "" { + stsClient := sts.NewFromConfig(awsConfig) + provider := stscreds.NewAssumeRoleProvider(stsClient, s3Config.AssumeRoleArn) + awsConfig.Credentials = aws.NewCredentialsCache(provider) + } + + // Create tracing middleware + tracingMiddleware := CreateS3TracingMiddleware(calls) + + // Create S3 client with tracing middleware + s3Client := s3.NewFromConfig(awsConfig, func(o *s3.Options) { + o.UsePathStyle = !s3Config.HostStyle + if s3Config.S3Endpoint() != "" { + o.BaseEndpoint = aws.String(s3Config.S3Endpoint()) + } + + // Add the tracing middleware + o.APIOptions = append(o.APIOptions, func(stack *middleware.Stack) error { + return stack.Initialize.Add(tracingMiddleware, middleware.Before) + }) + }) + + return s3Client, nil +} diff --git a/s3/integration/s3_compatible_test.go b/s3/integration/s3_compatible_test.go new file mode 100644 index 0000000..5eabf01 --- /dev/null +++ b/s3/integration/s3_compatible_test.go @@ -0,0 +1,92 @@ +package integration_test + +import ( + "os" + "strconv" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing in any non-AWS, S3 compatible storage service", func() { + Context("with S3 COMPATIBLE (static creds) configurations", func() { + accessKeyID := os.Getenv("ACCESS_KEY_ID") + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + bucketName := os.Getenv("BUCKET_NAME") + s3Host := os.Getenv("S3_HOST") + s3PortString := os.Getenv("S3_PORT") + s3Port, err := strconv.Atoi(s3PortString) + + BeforeEach(func() { + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + Expect(s3Host).ToNot(BeEmpty(), "S3_HOST must be set") + Expect(s3PortString).ToNot(BeEmpty(), "S3_PORT must be set") + Expect(err).ToNot(HaveOccurred()) + }) + + configurations := []TableEntry{ + Entry("with the minimal configuration", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: s3Host, + MultipartUpload: true, + }), + Entry("with region specified", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: s3Host, + Region: "invalid-region", + MultipartUpload: true, + }), + Entry("with use_ssl set to false", &config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: s3Host, + UseSSL: false, + MultipartUpload: true, + }), + Entry("with the maximal configuration", &config.S3Cli{ + SignatureVersion: 2, + CredentialsSource: "static", + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: s3Host, + Port: s3Port, + UseSSL: true, + SSLVerifyPeer: true, + Region: "invalid-region", + MultipartUpload: true, + }), + } + + DescribeTable("Blobstore lifecycle works", + func(cfg *config.S3Cli) { integration.AssertLifecycleWorks(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli get` on a non-existent-key fails", + func(cfg *config.S3Cli) { integration.AssertGetNonexistentFails(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli delete` on a non-existent-key does not fail", + func(cfg *config.S3Cli) { integration.AssertDeleteNonexistentWorks(s3CLIPath, cfg) }, + configurations, + ) + DescribeTable("Invoking `s3cli put` handling of mulitpart uploads", + func(cfg *config.S3Cli) { integration.AssertOnMultipartUploads(s3CLIPath, cfg, largeContent) }, + configurations, + ) + DescribeTable("Invoking `s3cli sign` returns a signed URL", + func(cfg *config.S3Cli) { integration.AssertOnSignedURLs(s3CLIPath, cfg) }, + configurations, + ) + }) +}) diff --git a/s3/integration/swift_signed_url_test.go b/s3/integration/swift_signed_url_test.go new file mode 100644 index 0000000..c0c2805 --- /dev/null +++ b/s3/integration/swift_signed_url_test.go @@ -0,0 +1,71 @@ +package integration_test + +import ( + "bytes" + "os" + + "github.com/cloudfoundry/storage-cli/s3/config" + "github.com/cloudfoundry/storage-cli/s3/integration" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Testing for working signed URLs all Swift/OpenStack regions", func() { + Context("with GENERAL OpenStack/Swift (static creds) configurations", func() { + var configPath string + var contentFile string + var defaultConfig config.S3Cli + + accessKeyID := os.Getenv("ACCESS_KEY_ID") + secretAccessKey := os.Getenv("SECRET_ACCESS_KEY") + bucketName := os.Getenv("BUCKET_NAME") + region := os.Getenv("REGION") + swiftHost := os.Getenv("SWIFT_HOST") + swiftTempURLKey := os.Getenv("SWIFT_TEMPURL_KEY") + swiftAuthAccount := os.Getenv("SWIFT_AUTH_ACCOUNT") + s3CLIPath := os.Getenv("S3_CLI_PATH") + + BeforeEach(func() { + if os.Getenv("SWIFT_AUTH_ACCOUNT") == "" { + Skip("Skipping because swift blobstore isn't available") + } + Expect(accessKeyID).ToNot(BeEmpty(), "ACCESS_KEY_ID must be set") + Expect(secretAccessKey).ToNot(BeEmpty(), "SECRET_ACCESS_KEY must be set") + Expect(bucketName).ToNot(BeEmpty(), "BUCKET_NAME must be set") + Expect(region).ToNot(BeEmpty(), "REGION must be set") + Expect(swiftTempURLKey).ToNot(BeEmpty(), "SWIFT_TEMPURL_KEY must be set") + Expect(swiftAuthAccount).ToNot(BeEmpty(), "SWIFT_AUTH_ACCOUNT must be set") + defaultConfig = config.S3Cli{ + AccessKeyID: accessKeyID, + SecretAccessKey: secretAccessKey, + BucketName: bucketName, + Host: swiftHost, + SwiftAuthAccount: swiftAuthAccount, + } + configPath = integration.MakeConfigFile(&defaultConfig) + contentFile = integration.MakeContentFile("foo") + }) + + AfterEach(func() { + defer os.Remove(configPath) //nolint:errcheck + defer os.Remove(contentFile) //nolint:errcheck + }) + + Describe("Invoking `sign`", func() { + It("returns 0 for an existing blob", func() { + cliSession, err := integration.RunS3CLI(s3CLIPath, configPath, "sign", "some-blob", "get", "60s") + Expect(err).ToNot(HaveOccurred()) + Expect(cliSession.ExitCode()).To(BeZero()) + + getUrl := bytes.NewBuffer(cliSession.Out.Contents()).String() + Expect(getUrl).To(MatchRegexp("https://" + swiftHost + ".*?" + "/some-blob")) + cliSession, err = integration.RunS3CLI(s3CLIPath, configPath, "sign", "some-blob", "put", "60s") + Expect(err).ToNot(HaveOccurred()) + + putUrl := bytes.NewBuffer(cliSession.Out.Contents()).String() + Expect(putUrl).To(MatchRegexp("https://" + swiftHost + ".*?" + "/some-blob")) + }) + }) + }) +}) diff --git a/s3/integration/utils.go b/s3/integration/utils.go new file mode 100644 index 0000000..ba2faf2 --- /dev/null +++ b/s3/integration/utils.go @@ -0,0 +1,72 @@ +package integration + +import ( + "encoding/json" + "math/rand" + "os" + "os/exec" + "time" + + "github.com/cloudfoundry/storage-cli/s3/config" + + . "github.com/onsi/ginkgo/v2" //nolint:staticcheck + "github.com/onsi/gomega" + "github.com/onsi/gomega/gexec" +) + +const alphaNum = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +// GenerateRandomString generates a random string of desired length (default: 25) +func GenerateRandomString(params ...int) string { + size := 25 + if len(params) == 1 { + size = params[0] + } + + randBytes := make([]byte, size) + for i := range randBytes { + randBytes[i] = alphaNum[rand.Intn(len(alphaNum))] + } + return string(randBytes) +} + +// MakeConfigFile creates a config file from a S3Cli config struct +func MakeConfigFile(cfg *config.S3Cli) string { + cfgBytes, err := json.Marshal(cfg) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + tmpFile, err := os.CreateTemp("", "s3cli-test") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = tmpFile.Write(cfgBytes) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = tmpFile.Close() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + return tmpFile.Name() +} + +// MakeContentFile creates a temporary file with content to upload to S3 +func MakeContentFile(content string) string { + tmpFile, err := os.CreateTemp("", "s3cli-test-content") + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + _, err = tmpFile.Write([]byte(content)) + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + err = tmpFile.Close() + gomega.Expect(err).ToNot(gomega.HaveOccurred()) + return tmpFile.Name() +} + +// RunS3CLI runs the s3cli and outputs the session after waiting for it to finish +func RunS3CLI(s3CLIPath string, configPath string, subcommand string, args ...string) (*gexec.Session, error) { + cmdArgs := []string{ + "-c", + configPath, + subcommand, + } + cmdArgs = append(cmdArgs, args...) + command := exec.Command(s3CLIPath, cmdArgs...) + session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter) + if err != nil { + return nil, err + } + session.Wait(1 * time.Minute) + return session, nil +} diff --git a/s3/main.go b/s3/main.go new file mode 100644 index 0000000..1c0c62b --- /dev/null +++ b/s3/main.go @@ -0,0 +1,130 @@ +package main + +import ( + "flag" + "fmt" + "log" + "os" + "time" + + "github.com/cloudfoundry/storage-cli/s3/client" + "github.com/cloudfoundry/storage-cli/s3/config" +) + +var version string + +func main() { + configPath := flag.String("c", "", "configuration path") + showVer := flag.Bool("v", false, "version") + flag.Parse() + + if *showVer { + fmt.Printf("version %s\n", version) + os.Exit(0) + } + + configFile, err := os.Open(*configPath) + if err != nil { + log.Fatalln(err) + } + + s3Config, err := config.NewFromReader(configFile) + if err != nil { + log.Fatalln(err) + } + + s3Client, err := client.NewAwsS3Client(&s3Config) + if err != nil { + log.Fatalln(err) + } + + blobstoreClient := client.New(s3Client, &s3Config) + + nonFlagArgs := flag.Args() + if len(nonFlagArgs) < 2 { + log.Fatalf("Expected at least two arguments got %d\n", len(nonFlagArgs)) + } + + cmd := nonFlagArgs[0] + + switch cmd { + case "put": + if len(nonFlagArgs) != 3 { + log.Fatalf("Put method expected 3 arguments got %d\n", len(nonFlagArgs)) + } + src, dst := nonFlagArgs[1], nonFlagArgs[2] + + var sourceFile *os.File + sourceFile, err = os.Open(src) + if err != nil { + log.Fatalln(err) + } + + defer sourceFile.Close() //nolint:errcheck + err = blobstoreClient.Put(sourceFile, dst) + case "get": + if len(nonFlagArgs) != 3 { + log.Fatalf("Get method expected 3 arguments got %d\n", len(nonFlagArgs)) + } + src, dst := nonFlagArgs[1], nonFlagArgs[2] + + var dstFile *os.File + dstFile, err = os.Create(dst) + if err != nil { + log.Fatalln(err) + } + + defer dstFile.Close() //nolint:errcheck + err = blobstoreClient.Get(src, dstFile) + case "delete": + if len(nonFlagArgs) != 2 { + log.Fatalf("Delete method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + + err = blobstoreClient.Delete(nonFlagArgs[1]) + case "exists": + if len(nonFlagArgs) != 2 { + log.Fatalf("Exists method expected 2 arguments got %d\n", len(nonFlagArgs)) + } + + var exists bool + exists, err = blobstoreClient.Exists(nonFlagArgs[1]) + + // If the object exists the exit status is 0, otherwise it is 3 + // We are using `3` since `1` and `2` have special meanings + if err == nil && !exists { + os.Exit(3) + } + case "sign": + if len(nonFlagArgs) != 4 { + log.Fatalf("Sign method expects 3 arguments got %d\n", len(nonFlagArgs)-1) + } + + objectID, action := nonFlagArgs[1], nonFlagArgs[2] + + if action != "get" && action != "put" { + log.Fatalf("Action not implemented: %s. Available actions are 'get' and 'put'", action) + } + + expiration, err := time.ParseDuration(nonFlagArgs[3]) + if err != nil { + log.Fatalf("Expiration should be in the format of a duration i.e. 1h, 60m, 3600s. Got: %s", nonFlagArgs[3]) + } + + signedURL, err := blobstoreClient.Sign(objectID, action, expiration) + + if err != nil { + log.Fatalf("Failed to sign request: %s", err) + os.Exit(1) + } + + fmt.Print(signedURL) + os.Exit(0) + default: + log.Fatalf("unknown command: '%s'\n", cmd) + } + + if err != nil { + log.Fatalf("performing operation %s: %s\n", cmd, err) + } +} diff --git a/tool.go b/tool.go new file mode 100644 index 0000000..e6fa2b8 --- /dev/null +++ b/tool.go @@ -0,0 +1,11 @@ +//go:build tools +// +build tools + +package tools + +import ( + _ "github.com/maxbrunsfeld/counterfeiter/v6" +) + +// This file imports packages that are used when running go generate, or used +// during the development process but not otherwise depended on by built code. diff --git a/tools/tools.go b/tools/tools.go new file mode 100644 index 0000000..f206e8f --- /dev/null +++ b/tools/tools.go @@ -0,0 +1,11 @@ +//go:build tools +// +build tools + +package tools + +import ( + _ "github.com/onsi/ginkgo/v2" +) + +// This file imports packages that are used when running go generate, or used +// during the development process but not otherwise depended on by built code.