fix(cache-cli): allow custom S3 endpoint to be used

.github/workflows/test.yml

@@ -7,14 +7,16 @@ jobs:
       - name: Install Go
         uses: actions/setup-go@v2
         with:
-          go-version: 1.20.x
+          go-version: 1.22.x
       - name: Check out repository code
         uses: actions/checkout@v2
       - name: Install gotestsum
         run: go install gotest.tools/gotestsum@latest
       - name: Run tests
         env:
           SEMAPHORE_CACHE_S3_URL: "http://127.0.0.1:9000"
+          SEMAPHORE_CACHE_S3_KEY: minioadmin
+          SEMAPHORE_CACHE_S3_SECRET: minioadmin
           SEMAPHORE_TOOLBOX_METRICS_ENABLED: "true"
         run: |
           New-Item C:\minio -ItemType Directory > $null

.semaphore/semaphore.yml

@@ -15,7 +15,7 @@ blocks:
       jobs:
         - name: Build cache CLI
           commands:
-            - sem-version go 1.20
+            - sem-version go 1.22
             - "export GOPATH=~/go"
             - "export PATH=/home/semaphore/go/bin:$PATH"
             - checkout
@@ -32,7 +32,7 @@ blocks:
             - artifact push workflow bin/windows/cache.exe -d bin/windows/cache.exe
         - name: Build sem-context CLI
           commands:
-            - sem-version go 1.20
+            - sem-version go 1.22
             - "export GOPATH=~/go"
             - "export PATH=/home/semaphore/go/bin:$PATH"
             - checkout
@@ -319,14 +319,14 @@ blocks:
           commands:
             - bash tests/sem_service/$TEST
 
-  - name: "Bats: xcode14"
+  - name: "Bats: xcode15"
     dependencies:
       - "Build local CLIs"
     task:
       agent:
         machine:
           type: a1-standard-4
-          os_image: macos-xcode14
+          os_image: macos-xcode15
 
       prologue:
         commands:
@@ -569,7 +569,7 @@ blocks:
           - source tests/sftp_server/start_on_linux.sh
           - sudo apt-get install -y python3.8-dev
           - sem-version python 3.8
-          - sem-version go 1.13
+          - sem-version go 1.22
           - sem-version php 7.3.23
       jobs:
         - name: "Cache tests"
@@ -634,7 +634,7 @@ blocks:
           value: "on"
       prologue:
         commands:
-          - sem-version go 1.20
+          - sem-version go 1.22
           - checkout && cd cache-cli
       jobs:
         - name: Lint

cache-cli/Dockerfile.dev

@@ -1,4 +1,4 @@
-FROM golang:1.20
+FROM golang:1.22
 
 RUN go install gotest.tools/gotestsum@latest
 RUN mkdir /root/.ssh

cache-cli/docker-compose.yml

@@ -16,6 +16,8 @@ services:
       - .:/app
     environment:
       SEMAPHORE_CACHE_S3_URL: "http://s3:9000"
+      SEMAPHORE_CACHE_S3_KEY: minioadmin
+      SEMAPHORE_CACHE_S3_SECRET: minioadmin
       STORAGE_EMULATOR_HOST: "http://gcs:4443"
       SEMAPHORE_TOOLBOX_METRICS_ENABLED: "true"
   gcs:
@@ -27,7 +29,7 @@ services:
       - 4443:4443
     command: -backend memory -scheme http -port 4443 -public-host gcs:4443 -external-url http://gcs:4443
   s3:
-    image: quay.io/minio/minio:RELEASE.2021-09-15T04-54-25Z
+    image: quay.io/minio/minio:RELEASE.2024-09-22T00-33-43Z
     container_name: 's3'
     ports:
       - 9000:9000

cache-cli/go.mod

@@ -1,6 +1,6 @@
 module github.com/semaphoreci/toolbox/cache-cli
 
-go 1.20
+go 1.22
 
 require (
 	cloud.google.com/go/storage v1.30.1
@@ -9,7 +9,6 @@ require (
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.24
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67
 	github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1
-	github.com/aws/smithy-go v1.13.5
 	github.com/klauspost/pgzip v1.2.6
 	github.com/pkg/sftp v1.13.5
 	github.com/sirupsen/logrus v1.9.2
@@ -37,6 +36,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect
+	github.com/aws/smithy-go v1.13.5 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
 	github.com/golang/protobuf v1.5.3 // indirect

cache-cli/go.sum

@@ -8,6 +8,7 @@ cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2Aawl
 cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k=
 cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0=
 cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM=
+cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo=
 cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM=
 cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -88,6 +89,7 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw=
+github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk=
 github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -168,6 +170,7 @@ golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8=
+golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=

cache-cli/pkg/storage/s3.go

@@ -91,31 +91,41 @@ func createDefaultS3Storage(s3Bucket, project string, storageConfig StorageConfi
 }
 
 func createS3StorageUsingEndpoint(s3Bucket, project, s3Url string, storageConfig StorageConfig) (*S3Storage, error) {
-	resolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
-		return aws.Endpoint{
-			URL: s3Url,
-		}, nil
-	})
+	options := []func(*awsConfig.LoadOptions) error{
+		awsConfig.WithRegion("auto"),
+		awsConfig.WithEndpointResolverWithOptions(
+			aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) {
+				return aws.Endpoint{
+					URL: s3Url,
+				}, nil
+			}),
+		),
+	}
 
-	creds := credentials.NewStaticCredentialsProvider("minioadmin", "minioadmin", "")
-	cfg, err := awsConfig.LoadDefaultConfig(context.TODO(),
-		awsConfig.WithCredentialsProvider(creds),
-		awsConfig.WithEndpointResolver(resolver),
-	)
+	// If a key/secret pair is passed, we use them.
+	// Otherwise, we just rely on the default configuration methods
+	// used by LoadDefaultConfig(), for example,
+	// AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID environment variables.
+	s3Key := os.Getenv("SEMAPHORE_CACHE_S3_KEY")
+	s3Secret := os.Getenv("SEMAPHORE_CACHE_S3_SECRET")
+	if s3Key != "" && s3Secret != "" {
+		options = append(options, awsConfig.WithCredentialsProvider(
+			credentials.NewStaticCredentialsProvider(s3Key, s3Secret, ""),
+		))
+	}
 
+	cfg, err := awsConfig.LoadDefaultConfig(context.TODO(), options...)
 	if err != nil {
 		return nil, err
 	}
 
-	svc := s3.NewFromConfig(cfg, func(o *s3.Options) {
-		o.UsePathStyle = true
-	})
-
 	return &S3Storage{
-		Client:        svc,
 		Bucket:        s3Bucket,
 		Project:       project,
 		StorageConfig: storageConfig,
+		Client: s3.NewFromConfig(cfg, func(o *s3.Options) {
+			o.UsePathStyle = true
+		}),
 	}, nil
 }