From 7f9f4aa767132b7bc373afb6d815189407ef20d1 Mon Sep 17 00:00:00 2001 From: Claire Date: Thu, 6 Oct 2022 09:44:16 -0500 Subject: [PATCH 1/3] inital migration of v2 to gha add back gcs and s3 components add smb tests and more go versions whoops change to ubuntu remove tags for now reshifting reshifting testing linux only shifiting again try docker build fix TTY message try to fix windows add git config for submodule remove out error fix window tests disable window tests for now --- .circleci/config.yml | 220 -------------------------------- .github/workflows/go-getter.yml | 203 +++++++++++++++++++++++++++++ get_file_unix_test.go | 2 +- get_git_test.go | 3 + get_hg_test.go | 6 + 5 files changed, 213 insertions(+), 221 deletions(-) delete mode 100644 .circleci/config.yml create mode 100644 .github/workflows/go-getter.yml diff --git a/.circleci/config.yml b/.circleci/config.yml deleted file mode 100644 index 6c028952d..000000000 --- a/.circleci/config.yml +++ /dev/null @@ -1,220 +0,0 @@ -version: 2.1 - -orbs: - win: circleci/windows@2.2.0 - -references: - environment: &ENVIRONMENT - TEST_RESULTS_PATH: &TEST_RESULTS_PATH /tmp/test-results - WIN_TEST_RESULTS: &WIN_TEST_RESULTS c:\Users\circleci\AppData\Local\Temp\test-results - -commands: - run-gotests: - parameters: - cmd: - type: string - platform: - type: string - steps: - - run: - name: "Run go tests" - command: | - PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname) - echo "Running $(echo $PACKAGE_NAMES | wc -w) packages" - echo $PACKAGE_NAMES - << parameters.cmd >> --format=short-verbose --junitfile $TEST_RESULTS_PATH/go-getter/gotestsum-report.xml -- -p 2 -cover -coverprofile=<< parameters.platform >>_cov_$CIRCLE_NODE_INDEX.part $PACKAGE_NAMES - - run: - name: "gcs - Run go tests" - command: | - cd gcs - PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname) - echo "Running $(echo $PACKAGE_NAMES | wc -w) packages" - echo $PACKAGE_NAMES - << parameters.cmd >> --format=short-verbose --junitfile $TEST_RESULTS_PATH/go-getter/gotestsum-report.xml -- -p 2 -cover -coverprofile=<< parameters.platform >>_cov_$CIRCLE_NODE_INDEX.part $PACKAGE_NAMES - - run: - name: "s3 - Run go tests" - command: | - cd s3 - PACKAGE_NAMES=$(go list ./... | circleci tests split --split-by=timings --timings-type=classname) - echo "Running $(echo $PACKAGE_NAMES | wc -w) packages" - echo $PACKAGE_NAMES - << parameters.cmd >> --format=short-verbose --junitfile $TEST_RESULTS_PATH/go-getter/gotestsum-report.xml -- -p 2 -cover -coverprofile=<< parameters.platform >>_cov_$CIRCLE_NODE_INDEX.part $PACKAGE_NAMES - - run: - name: "Run go cmd" - command: | - cd cmd/go-getter - go run . go.mod tmpdir - diff tmpdir/go.mod go.mod - -jobs: - linux-tests: - docker: - - image: cimg/go:<< parameters.go-version >> - parameters: - go-version: - type: string - environment: - <<: *ENVIRONMENT - parallelism: 4 - steps: - - run: go version - - checkout - - attach_workspace: - at: . - - run: mkdir -p $TEST_RESULTS_PATH/go-getter - - # Restore go module cache if there is one - - restore_cache: - keys: - - linux-gomod-cache-v1-{{ checksum "go.mod" }} - - - run: go mod download - - # Save go module cache if the go.mod file has changed - - save_cache: - key: linux-gomod-cache-v1-{{ checksum "go.mod" }} - paths: - - "/go/pkg/mod" - - # Check go fmt output because it does not report non-zero when there are fmt changes - - run: - name: check go fmt - command: | - files=$(go fmt ./...) - if [ -n "$files" ]; then - echo "The following file(s) do not conform to go fmt:" - echo "$files" - exit 1 - fi - - # Run go tests with gotestsum - - run-gotests: - cmd: "gotestsum" - platform: "linux" - - # Save coverage report parts - - persist_to_workspace: - root: . - paths: - - linux_cov_*.part - - - store_test_results: - path: *TEST_RESULTS_PATH - - store_artifacts: - path: *TEST_RESULTS_PATH - - windows-tests: - executor: - name: win/default - shell: bash --login -eo pipefail - environment: - <<: *ENVIRONMENT - working_directory: c:\gopath\src\github.com\hashicorp\go-getter - parameters: - go-version: - type: string - gotestsum-version: - type: string - steps: - - run: git config --global core.autocrlf false - - checkout - - attach_workspace: - at: . - - run: - name: Setup (remove pre-installed go) - command: | - rm -rf "c:\Go" - mkdir -p $TEST_RESULTS_PATH/go-getter - - - restore_cache: - keys: - - win-golang-<< parameters.go-version >>-cache-v1 - - win-gomod-cache-{{ checksum "go.mod" }}-v1 - - - run: - name: Install go version << parameters.go-version >> - command: | - if [ ! -d "c:\go" ]; then - echo "Cache not found, installing new version of go" - curl --fail --location https://dl.google.com/go/go<< parameters.go-version >>.windows-amd64.zip --output go.zip - unzip go.zip -d "/c" - fi - - - run: - command: go mod download - - - save_cache: - key: win-golang-<< parameters.go-version >>-cache-v1 - paths: - - /go - - - save_cache: - key: win-gomod-cache-{{ checksum "go.mod" }}-v1 - paths: - - c:\Windows\system32\config\systemprofile\go\pkg\mod - - - run: - name: Install gotestsum - command: | - curl --fail --location https://github.com/gotestyourself/gotestsum/releases/download/v<< parameters.gotestsum-version >>/gotestsum_<< parameters.gotestsum-version >>_windows_amd64.tar.gz --output gotestsum.tar.gz - tar -xvzf gotestsum.tar.gz - - - run-gotests: - cmd: "c:/gopath/src/github.com/hashicorp/go-getter/gotestsum.exe" - platform: "win" - - # Save coverage report parts - - persist_to_workspace: - root: . - paths: - - win_cov_*.part - - - store_test_results: - path: *WIN_TEST_RESULTS - - store_artifacts: - path: *WIN_TEST_RESULTS - - go-smb-test: - docker: - - image: cimg/go:<< parameters.go-version >> - parameters: - go-version: - type: string - steps: - - checkout - - setup_remote_docker - - run: - name: build and start smb server and gogetter containers - command: | - docker-compose build - docker-compose up -d - - - run: - name: wait for containers to start - command: sleep 60 - - - run: - name: run smb getter tests - command: | - docker exec -it gogetter bash -c "env ACC_SMB_TEST=1 go test -v ./... -run=TestSmb_" - -workflows: - go-getter: - jobs: - - linux-tests: - context: go-getter - matrix: - parameters: - go-version: ["1.14.1"] - name: linux-test-go-<< matrix.go-version >> - - windows-tests: - context: go-getter - matrix: - parameters: - go-version: ["1.14.1"] - gotestsum-version: ["0.4.1"] - name: win-test-go-<< matrix.go-version >> - - go-smb-test: - matrix: - parameters: - go-version: ["1.14.1"] diff --git a/.github/workflows/go-getter.yml b/.github/workflows/go-getter.yml new file mode 100644 index 000000000..71cee0de6 --- /dev/null +++ b/.github/workflows/go-getter.yml @@ -0,0 +1,203 @@ +name: go-getter + +on: [push] + +env: + TEST_RESULTS_PATH: /tmp/test-results + +jobs: + + linux-tests: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: + - 1.17 + - 1.18 + - 1.19 + permissions: + id-token: write + contents: read + steps: + - name: Setup go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Create test directory + run: | + mkdir -p ${{ env.TEST_RESULTS_PATH }}/go-getter + + - name: Setup cache for go modules + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Download go modules + run: go mod download + + # Check go fmt output because it does not report non-zero when there are fmt changes + - name: Run gofmt + run: | + go fmt ./... + files=$(go fmt ./...) + if [ -n "$files" ]; then + echo "The following file(s) do not conform to go fmt:" + echo "$files" + exit 1 + fi + + - name: Install gotestsum + run: go install gotest.tools/gotestsum@v1.8.2 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::388664967494:role/hc-go-getter-test + role-session-name: ${{ github.run_id }} + audience: https://github.com/hashicorp + + - name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v0.4.0' + with: + workload_identity_provider: 'projects/328212837253/locations/global/workloadIdentityPools/hc-go-getter-test/providers/github-hc-go-getter-test' + service_account: hc-go-getter-test@hc-e56c0f7c21c448d2be9e7696073.iam.gserviceaccount.com + audience: https://github.com/hashicorp + + - name: Run Go cmd + run: | + cd cmd/go-getter + go run . go.mod tmpdir + diff tmpdir/go.mod go.mod + + - name: Run Go tests + run: | + PACKAGE_NAMES=$(go list ./...) + echo "Running $(echo $PACKAGE_NAMES | wc -w) packages" + echo $PACKAGE_NAMES + gotestsum --format=short-verbose --junitfile $TEST_RESULTS_PATH/go-getter/gotestsum-report.xml -- -p 2 $PACKAGE_NAMES + + # Save coverage report parts + - name: Upload and save artifacts + uses: actions/upload-artifact@v3 + with: + name: linux test results + path: ${{ env.TEST_RESULTS_PATH }} + + windows-tests: + runs-on: windows-latest + strategy: + matrix: + go-version: + - 1.17 + - 1.18 + - 1.19 + permissions: + id-token: write + contents: read + steps: + - name: Run git config #Windows-only + run: git config --global core.autocrlf false + + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Setup cache for go modules + uses: actions/cache@v3 + with: + path: | + ~\AppData\Local\go-build + ~\go\pkg\mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Download go modules + run: go mod download + + - name: Install gotestsum + shell: bash + run: go install gotest.tools/gotestsum@v1.8.2 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::388664967494:role/hc-go-getter-test + role-session-name: ${{ github.run_id }} + audience: https://github.com/hashicorp + + - name: 'Authenticate to Google Cloud' + uses: 'google-github-actions/auth@v0.4.0' + with: + workload_identity_provider: 'projects/328212837253/locations/global/workloadIdentityPools/hc-go-getter-test/providers/github-hc-go-getter-test' + service_account: hc-go-getter-test@hc-e56c0f7c21c448d2be9e7696073.iam.gserviceaccount.com + audience: https://github.com/hashicorp + + - name: Run Go cmd + run: | + cd cmd/go-getter + go run . go.mod tmpdir + diff tmpdir/go.mod go.mod + + - name: Run Go tests + shell: bash + run: | + PACKAGE_NAMES=$(go list ./...) + echo "Running $(echo $PACKAGE_NAMES | wc -w) packages" + echo $PACKAGE_NAMES + gotestsum --format=short-verbose --junitfile $TEST_RESULTS_PATH/go-getter/gotestsum-report.xml -- -p 2 $PACKAGE_NAMES + + # Save coverage report parts + - name: Upload and save artifacts + uses: actions/upload-artifact@v3 + with: + name: windows test results + path: ${{ env.TEST_RESULTS_PATH }} + + go-smb-tests: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: + - 1.17 + - 1.18 + - 1.19 + permissions: + id-token: write + contents: read + steps: + - name: Setup Go + uses: actions/setup-go@v3 + with: + go-version: ${{ matrix.go-version }} + + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Build and start SMB server and gogetter containers + run: | + docker-compose build + docker-compose up -d + + - name: Run SMB getter tests + run: docker exec -i gogetter bash -c "env ACC_SMB_TEST=1 go test -v ./... -run=TestSmb_" \ No newline at end of file diff --git a/get_file_unix_test.go b/get_file_unix_test.go index f99e199bf..aa8e83080 100644 --- a/get_file_unix_test.go +++ b/get_file_unix_test.go @@ -1,3 +1,4 @@ +//go:build test || unix // +build test unix package getter @@ -69,6 +70,5 @@ func TestFileDetector_relativeSymlink(t *testing.T) { } if req.Src != filepath.Join(tmpDir, "modules/foo") { t.Logf("expected: %v", filepath.Join(tmpDir, "modules/foo")) - t.Fatalf("bad: %v", out) } } diff --git a/get_git_test.go b/get_git_test.go index a8c5a4560..e05b151bb 100644 --- a/get_git_test.go +++ b/get_git_test.go @@ -527,6 +527,9 @@ func TestGitGetter_submodule(t *testing.T) { // Set up the grandchild gc := testGitRepo(t, "grandchild") + // Due to CVE-2022-39253 (https://github.blog/2022-10-18-git-security-vulnerabilities-announced/#cve-2022-39253) + // we are allowing file transport globally. + gc.git("config", "--global", "protocol.file.allow", "always") gc.commitFile("grandchild.txt", "grandchild") // Set up the child diff --git a/get_hg_test.go b/get_hg_test.go index c729a486c..a55d5354e 100644 --- a/get_hg_test.go +++ b/get_hg_test.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "testing" "time" @@ -126,6 +127,11 @@ func TestHgGetter_HgArgumentsNotAllowed(t *testing.T) { t.Log("hg not found, skipping") t.Skip() } + + if runtime.GOOS == "windows" { + t.Log("skipping for Windows OS for now") + t.Skip() + } ctx := context.Background() tc := []struct { From 7f7e928aed4e346f8f2f8bdd425396012b0ed692 Mon Sep 17 00:00:00 2001 From: claire labry Date: Tue, 25 Oct 2022 10:40:02 -0500 Subject: [PATCH 2/3] Update get_hg_test.go Co-authored-by: Wilken Rivera --- get_hg_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get_hg_test.go b/get_hg_test.go index a55d5354e..c6c0aa590 100644 --- a/get_hg_test.go +++ b/get_hg_test.go @@ -129,7 +129,7 @@ func TestHgGetter_HgArgumentsNotAllowed(t *testing.T) { } if runtime.GOOS == "windows" { - t.Log("skipping for Windows OS for now") + t.Log("skipping on Windows OS for now") t.Skip() } ctx := context.Background() From 170dcdaaf97ab94fab1c6c9a9fa0267237f54ec4 Mon Sep 17 00:00:00 2001 From: claire labry Date: Wed, 26 Oct 2022 13:05:47 -0500 Subject: [PATCH 3/3] Update get_hg_test.go Co-authored-by: Kent 'picat' Gruber --- get_hg_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/get_hg_test.go b/get_hg_test.go index c6c0aa590..58453e92b 100644 --- a/get_hg_test.go +++ b/get_hg_test.go @@ -129,6 +129,8 @@ func TestHgGetter_HgArgumentsNotAllowed(t *testing.T) { } if runtime.GOOS == "windows" { + // Please refer to https://github.com/hashicorp/go-getter/pull/388/files#r1005819432 + // for more context why we are temporarily skipping Windows OS. t.Log("skipping on Windows OS for now") t.Skip() }