diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fac82c9e..9e1e4406 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -7,7 +7,7 @@ jobs: - name: Install Go uses: actions/setup-go@v2 with: - go-version: 1.20.x + go-version: 1.22.x - name: Check out repository code uses: actions/checkout@v2 - name: Install gotestsum @@ -15,6 +15,8 @@ jobs: - name: Run tests env: SEMAPHORE_CACHE_S3_URL: "http://127.0.0.1:9000" + SEMAPHORE_CACHE_S3_KEY: minioadmin + SEMAPHORE_CACHE_S3_SECRET: minioadmin SEMAPHORE_TOOLBOX_METRICS_ENABLED: "true" run: | New-Item C:\minio -ItemType Directory > $null diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 907aa8f3..526c01e5 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -15,7 +15,7 @@ blocks: jobs: - name: Build cache CLI commands: - - sem-version go 1.20 + - sem-version go 1.22 - "export GOPATH=~/go" - "export PATH=/home/semaphore/go/bin:$PATH" - checkout @@ -32,7 +32,7 @@ blocks: - artifact push workflow bin/windows/cache.exe -d bin/windows/cache.exe - name: Build sem-context CLI commands: - - sem-version go 1.20 + - sem-version go 1.22 - "export GOPATH=~/go" - "export PATH=/home/semaphore/go/bin:$PATH" - checkout @@ -319,14 +319,14 @@ blocks: commands: - bash tests/sem_service/$TEST - - name: "Bats: xcode14" + - name: "Bats: xcode15" dependencies: - "Build local CLIs" task: agent: machine: type: a1-standard-4 - os_image: macos-xcode14 + os_image: macos-xcode15 prologue: commands: @@ -569,7 +569,7 @@ blocks: - source tests/sftp_server/start_on_linux.sh - sudo apt-get install -y python3.8-dev - sem-version python 3.8 - - sem-version go 1.13 + - sem-version go 1.22 - sem-version php 7.3.23 jobs: - name: "Cache tests" @@ -634,7 +634,7 @@ blocks: value: "on" prologue: commands: - - sem-version go 1.20 + - sem-version go 1.22 - checkout && cd cache-cli jobs: - name: Lint diff --git a/cache-cli/Dockerfile.dev b/cache-cli/Dockerfile.dev index 6a56452d..f00082cb 100644 --- a/cache-cli/Dockerfile.dev +++ b/cache-cli/Dockerfile.dev @@ -1,4 +1,4 @@ -FROM golang:1.20 +FROM golang:1.22 RUN go install gotest.tools/gotestsum@latest RUN mkdir /root/.ssh diff --git a/cache-cli/docker-compose.yml b/cache-cli/docker-compose.yml index c957d99f..2f5a44c5 100644 --- a/cache-cli/docker-compose.yml +++ b/cache-cli/docker-compose.yml @@ -16,6 +16,8 @@ services: - .:/app environment: SEMAPHORE_CACHE_S3_URL: "http://s3:9000" + SEMAPHORE_CACHE_S3_KEY: minioadmin + SEMAPHORE_CACHE_S3_SECRET: minioadmin STORAGE_EMULATOR_HOST: "http://gcs:4443" SEMAPHORE_TOOLBOX_METRICS_ENABLED: "true" gcs: @@ -27,7 +29,7 @@ services: - 4443:4443 command: -backend memory -scheme http -port 4443 -public-host gcs:4443 -external-url http://gcs:4443 s3: - image: quay.io/minio/minio:RELEASE.2021-09-15T04-54-25Z + image: quay.io/minio/minio:RELEASE.2024-09-22T00-33-43Z container_name: 's3' ports: - 9000:9000 diff --git a/cache-cli/go.mod b/cache-cli/go.mod index c5dbff9d..7bfd5dcc 100644 --- a/cache-cli/go.mod +++ b/cache-cli/go.mod @@ -1,6 +1,6 @@ module github.com/semaphoreci/toolbox/cache-cli -go 1.20 +go 1.22 require ( cloud.google.com/go/storage v1.30.1 @@ -9,7 +9,6 @@ require ( github.com/aws/aws-sdk-go-v2/credentials v1.13.24 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.67 github.com/aws/aws-sdk-go-v2/service/s3 v1.33.1 - github.com/aws/smithy-go v1.13.5 github.com/klauspost/pgzip v1.2.6 github.com/pkg/sftp v1.13.5 github.com/sirupsen/logrus v1.9.2 @@ -37,6 +36,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.12.10 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.10 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.19.0 // indirect + github.com/aws/smithy-go v1.13.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.5.3 // indirect diff --git a/cache-cli/go.sum b/cache-cli/go.sum index 92f9af68..6ba96abd 100644 --- a/cache-cli/go.sum +++ b/cache-cli/go.sum @@ -8,6 +8,7 @@ cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2Aawl cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= +cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -88,6 +89,7 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -168,6 +170,7 @@ golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.14.0 h1:LGK9IlZ8T9jvdy6cTdfKUCltatMFOehAQo9SRC46UQ8= +golang.org/x/term v0.14.0/go.mod h1:TySc+nGkYR6qt8km8wUhuFRTVSMIX3XPR58y2lC8vww= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/cache-cli/pkg/storage/s3.go b/cache-cli/pkg/storage/s3.go index 47adf797..af2a37b0 100644 --- a/cache-cli/pkg/storage/s3.go +++ b/cache-cli/pkg/storage/s3.go @@ -91,31 +91,41 @@ func createDefaultS3Storage(s3Bucket, project string, storageConfig StorageConfi } func createS3StorageUsingEndpoint(s3Bucket, project, s3Url string, storageConfig StorageConfig) (*S3Storage, error) { - resolver := aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { - return aws.Endpoint{ - URL: s3Url, - }, nil - }) + options := []func(*awsConfig.LoadOptions) error{ + awsConfig.WithRegion("auto"), + awsConfig.WithEndpointResolverWithOptions( + aws.EndpointResolverWithOptionsFunc(func(service, region string, options ...interface{}) (aws.Endpoint, error) { + return aws.Endpoint{ + URL: s3Url, + }, nil + }), + ), + } - creds := credentials.NewStaticCredentialsProvider("minioadmin", "minioadmin", "") - cfg, err := awsConfig.LoadDefaultConfig(context.TODO(), - awsConfig.WithCredentialsProvider(creds), - awsConfig.WithEndpointResolver(resolver), - ) + // If a key/secret pair is passed, we use them. + // Otherwise, we just rely on the default configuration methods + // used by LoadDefaultConfig(), for example, + // AWS_SECRET_ACCESS_KEY and AWS_ACCESS_KEY_ID environment variables. + s3Key := os.Getenv("SEMAPHORE_CACHE_S3_KEY") + s3Secret := os.Getenv("SEMAPHORE_CACHE_S3_SECRET") + if s3Key != "" && s3Secret != "" { + options = append(options, awsConfig.WithCredentialsProvider( + credentials.NewStaticCredentialsProvider(s3Key, s3Secret, ""), + )) + } + cfg, err := awsConfig.LoadDefaultConfig(context.TODO(), options...) if err != nil { return nil, err } - svc := s3.NewFromConfig(cfg, func(o *s3.Options) { - o.UsePathStyle = true - }) - return &S3Storage{ - Client: svc, Bucket: s3Bucket, Project: project, StorageConfig: storageConfig, + Client: s3.NewFromConfig(cfg, func(o *s3.Options) { + o.UsePathStyle = true + }), }, nil }