From 2c94dcab3d929db9de92424c5243931d3ea6ef58 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Mon, 8 Apr 2024 16:14:10 -0400 Subject: [PATCH 1/9] make binny taskfile parseable on Windows When running tests on Windows, core utils like grep may not be present, or may not behave the same way. Therefore, implement inline shell logic in python and invoke it from the task file. Signed-off-by: Will Murphy --- Taskfile.yaml | 2 +- scripts/list_units.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) create mode 100644 scripts/list_units.py diff --git a/Taskfile.yaml b/Taskfile.yaml index 7ee936c..bc144e2 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -122,7 +122,7 @@ tasks: desc: Run all unit tests vars: TEST_PKGS: - sh: "go list ./... | grep -v {{ .OWNER }}/{{ .PROJECT }}/test | tr '\n' ' '" + sh: "python ./scripts/list_units.py anchore binny" # unit test coverage threshold (in % coverage) COVERAGE_THRESHOLD: 45 diff --git a/scripts/list_units.py b/scripts/list_units.py new file mode 100644 index 0000000..b93c403 --- /dev/null +++ b/scripts/list_units.py @@ -0,0 +1,18 @@ +import subprocess +import sys + +def go_list_exclude_pattern(owner, project): + exclude_pattern = f"{owner}/{project}/test" + + result = subprocess.run(["go", "list", "./..."], stdout=subprocess.PIPE, text=True, check=True) + + filtered_lines = [line for line in result.stdout.splitlines() if exclude_pattern not in line] + + joined_output = ' '.join(filtered_lines) + + return joined_output + +owner = sys.argv[1] +project = sys.argv[2] +output = go_list_exclude_pattern(owner, project) +print(output) From df14c0e36e96c58e106b8a2005dbaedf34d473d5 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Mon, 8 Apr 2024 20:52:54 +0000 Subject: [PATCH 2/9] enable unit tests to run on Windows Signed-off-by: Will Murphy --- Taskfile.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index bc144e2..740e867 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -9,6 +9,7 @@ vars: SNAPSHOT_DIR: snapshot CHANGELOG: CHANGELOG.md NEXT_VERSION: VERSION + MAKEDIR_P: 'python -c "import sys; import os; os.makedirs(sys.argv[1], exist_ok=True)"' tasks: @@ -127,7 +128,7 @@ tasks: # unit test coverage threshold (in % coverage) COVERAGE_THRESHOLD: 45 cmds: - - cmd: "mkdir -p {{ .TMP_DIR }}" + - cmd: "{{ .MAKEDIR_P }} {{ .TMP_DIR }}" silent: true - "go test -coverprofile {{ .TMP_DIR }}/unit-coverage-details.txt {{ .TEST_PKGS }}" - cmd: ".github/scripts/coverage.py {{ .COVERAGE_THRESHOLD }} {{ .TMP_DIR }}/unit-coverage-details.txt" From 13182bb2a0ce6cf28107f0e156051353e6360473 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Mon, 8 Apr 2024 20:53:20 +0000 Subject: [PATCH 3/9] skip or fix some unit tests on Windows Signed-off-by: Will Murphy --- store_test.go | 4 ++++ tool/goinstall/installer_test.go | 3 +++ 2 files changed, 7 insertions(+) diff --git a/store_test.go b/store_test.go index 739c5c3..bdd518a 100644 --- a/store_test.go +++ b/store_test.go @@ -3,6 +3,7 @@ package binny import ( "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -104,6 +105,9 @@ func TestStore_GetByName(t *testing.T) { } func TestStore_Entries(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test fixtures have different sha256 digests on Windows due to linbreak conversion") + } tests := []struct { name string storeRoot string diff --git a/tool/goinstall/installer_test.go b/tool/goinstall/installer_test.go index 78c1453..689423b 100644 --- a/tool/goinstall/installer_test.go +++ b/tool/goinstall/installer_test.go @@ -2,6 +2,8 @@ package goinstall import ( "fmt" + "os" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -108,6 +110,7 @@ func TestInstaller_InstallTo(t *testing.T) { i.goInstallRunner = tt.fields.goInstallRunner got, err := i.InstallTo(tt.args.version, tt.args.destDir) + got = strings.ReplaceAll(got, string(os.PathSeparator), "/") if !tt.wantErr(t, err, fmt.Sprintf("InstallTo(%v, %v)", tt.args.version, tt.args.destDir)) { return } From 8fb320ee528131426476b2287270261a3f7ea9f1 Mon Sep 17 00:00:00 2001 From: Keith Zantow Date: Tue, 9 Apr 2024 12:21:14 -0400 Subject: [PATCH 4/9] chore: windows compatibility due to open file failures Signed-off-by: Keith Zantow --- Makefile | 6 +++--- store.go | 20 ++++++++++++-------- store_test.go | 1 + tool/githubrelease/installer.go | 4 ++-- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index 3cae59c..166afbf 100644 --- a/Makefile +++ b/Makefile @@ -10,9 +10,9 @@ TASK = $(TOOL_DIR)/task ## Bootstrapping targets ################################# $(BINNY): - @mkdir -p $(TOOL_DIR) - @# we don't have a release of binny yet, so build off of the current branch - @#curl -sSfL https://raw.githubusercontent.com/$(OWNER)/$(PROJECT)/main/install.sh | sh -s -- -b $(TOOL_DIR) + @-mkdir $(TOOL_DIR) +# we don't have a release of binny yet, so build off of the current branch +# curl -sSfL https://raw.githubusercontent.com/$(OWNER)/$(PROJECT)/main/install.sh | sh -s -- -b $(TOOL_DIR) go build -o $(TOOL_DIR)/$(PROJECT) ./cmd/$(PROJECT) .PHONY: task diff --git a/store.go b/store.go index 41fd48e..c441428 100644 --- a/store.go +++ b/store.go @@ -133,15 +133,9 @@ func (s *Store) AddTool(toolName string, resolvedVersion, pathOutsideRoot string } } - file, err := os.Open(pathOutsideRoot) + digests, err := getDigestsForFile(pathOutsideRoot) if err != nil { - return fmt.Errorf("failed to open temp copy of binary %q: %w", pathOutsideRoot, err) - } - defer file.Close() - - digests, err := getDigestsForReader(file) - if err != nil { - return nil + return err } sha256Hash, ok := digests[internal.SHA256Algorithm] @@ -338,6 +332,16 @@ func xxh64File(path string) (string, error) { return fmt.Sprintf("%x", hash.Sum(nil)), nil } +func getDigestsForFile(filePath string) (map[string]string, error) { + file, err := os.Open(filePath) + if err != nil { + return nil, fmt.Errorf("failed to open temp copy of binary %q: %w", filePath, err) + } + defer file.Close() + + return getDigestsForReader(file) +} + func getDigestsForReader(r io.Reader) (map[string]string, error) { sha256Hash := sha256.New() xxhHash := xxhash.New64() diff --git a/store_test.go b/store_test.go index bdd518a..f87435b 100644 --- a/store_test.go +++ b/store_test.go @@ -223,6 +223,7 @@ func TestStore_AddTool(t *testing.T) { createFile := func(path, contents string) { fh, err := os.Create(path) + defer fh.Close() require.NoError(t, err) _, err = fh.WriteString(contents) require.NoError(t, err) diff --git a/tool/githubrelease/installer.go b/tool/githubrelease/installer.go index 2b24cdc..5ea682e 100644 --- a/tool/githubrelease/installer.go +++ b/tool/githubrelease/installer.go @@ -133,13 +133,13 @@ func (i Installer) InstallTo(version, destDir string) (string, error) { } func downloadAndExtractAsset(lgr logger.Logger, asset ghAsset, checksumAsset *ghAsset, destDir string) (string, error) { - assetPath := path.Join(destDir, asset.Name) + assetPath := filepath.Join(destDir, asset.Name) checksum := asset.Checksum if checksumAsset != nil && checksum == "" { lgr.WithFields("asset", checksumAsset.Name).Trace("downloading checksum manifest") - checksumsPath := path.Join(destDir, checksumsFilename) + checksumsPath := filepath.Join(destDir, checksumsFilename) if err := internal.DownloadFile(lgr, checksumAsset.URL, checksumsPath, ""); err != nil { return "", fmt.Errorf("unable to download checksum asset %q: %w", checksumAsset.Name, err) From 21257d6c68f9ad385dfc550baada2ba07a2bdb43 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Fri, 12 Apr 2024 19:30:20 +0000 Subject: [PATCH 5/9] Fix and skip more unit tests For now, simply do not support hosted shell script installer on Windows. Signed-off-by: Will Murphy --- tool/githubrelease/installer_test.go | 4 +++- tool/hostedshell/installer.go | 5 +++++ tool/hostedshell/installer_test.go | 5 +++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/tool/githubrelease/installer_test.go b/tool/githubrelease/installer_test.go index b4e96ca..862b69b 100644 --- a/tool/githubrelease/installer_test.go +++ b/tool/githubrelease/installer_test.go @@ -290,7 +290,9 @@ func Test_findBinaryAssetInDir(t *testing.T) { got, err := findBinaryAssetInDir(tt.destDir) tt.wantErr(t, err) - assert.Equal(t, tt.want, got) + want := strings.ReplaceAll(tt.want, "/", string(os.PathSeparator)) + + assert.Equal(t, want, got) }) } } diff --git a/tool/hostedshell/installer.go b/tool/hostedshell/installer.go index 399478b..14abc2a 100644 --- a/tool/hostedshell/installer.go +++ b/tool/hostedshell/installer.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "strings" "text/template" @@ -113,6 +114,10 @@ func templateFlags(args string, version, destination string) (string, error) { } func runScript(scriptPath, argStr string) error { + if runtime.GOOS == "windows" { + return fmt.Errorf("script based installers are not supported on %s", runtime.GOOS) + } + userArgs, err := shlex.Split(argStr) if err != nil { return fmt.Errorf("failed to parse args: %v", err) diff --git a/tool/hostedshell/installer_test.go b/tool/hostedshell/installer_test.go index c5df877..1593738 100644 --- a/tool/hostedshell/installer_test.go +++ b/tool/hostedshell/installer_test.go @@ -6,6 +6,7 @@ import ( "net/http/httptest" "os" "path/filepath" + "runtime" "testing" "github.com/stretchr/testify/assert" @@ -13,6 +14,10 @@ import ( ) func TestInstaller_InstallTo(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("script based installer is not supported on windows") + } + type fields struct { config InstallerParameters scriptRunner func(scriptPath string, argStr string) error From 0e0d1234d80bf20820400b2d25e16b5326e338fb Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Fri, 12 Apr 2024 15:41:38 -0400 Subject: [PATCH 6/9] prevent CRLF normalization of test fixtures There are text files under tool/testdata/store that are used as binary fixtures, including taking SHA256 of the file during unit tests. Prevent git from applying CRLF autocorrection to these files by removing the "text" attribute from them so that their SHA256 will be stable between Windows and other operating systems. Signed-off-by: Will Murphy --- .gitattributes | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..7e86101 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,4 @@ +# Prevent CRLF correction from changing digest of +# text files that are committed as stand-ins for binaries +# in text fixtures: +/tool/testdata/store/** -text From 880d8dbf55166fdb6296f259cec780c3f02d5526 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Fri, 12 Apr 2024 19:49:56 +0000 Subject: [PATCH 7/9] Enable Windows unit tests This is a stepping stone towards actual Windows support. Eventually, all validations should be run, and release artifacts should be built for Windows. Signed-off-by: Will Murphy --- .github/workflows/validations.yaml | 11 +++++++++++ Taskfile.yaml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index f4bc4d3..5dfaf12 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -27,3 +27,14 @@ jobs: - name: Run all validations run: make pr-validations + + name: "Windows units" + runs-on: windows-2022 + steps: + - uses: actions/checkout@v3 + + - name: install make + run: "choco install make" + + - name: run units + run: "make unit" diff --git a/Taskfile.yaml b/Taskfile.yaml index 740e867..5962cf4 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -131,7 +131,7 @@ tasks: - cmd: "{{ .MAKEDIR_P }} {{ .TMP_DIR }}" silent: true - "go test -coverprofile {{ .TMP_DIR }}/unit-coverage-details.txt {{ .TEST_PKGS }}" - - cmd: ".github/scripts/coverage.py {{ .COVERAGE_THRESHOLD }} {{ .TMP_DIR }}/unit-coverage-details.txt" + - cmd: '{{if eq OS "windows"}}python {{end}}.github/scripts/coverage.py {{ .COVERAGE_THRESHOLD }} {{ .TMP_DIR }}/unit-coverage-details.txt' silent: true ## Build-related targets ################################# From 9e5eb8976427d1067f4515c78ecf94598d473874 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Fri, 12 Apr 2024 19:55:46 +0000 Subject: [PATCH 8/9] Fix workflow actions yaml syntax Signed-off-by: Will Murphy --- .github/workflows/validations.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/validations.yaml b/.github/workflows/validations.yaml index 5dfaf12..24e1767 100644 --- a/.github/workflows/validations.yaml +++ b/.github/workflows/validations.yaml @@ -28,6 +28,7 @@ jobs: - name: Run all validations run: make pr-validations + WindowsValidations: name: "Windows units" runs-on: windows-2022 steps: From b0f98e9b9f19c0ed7f83f933d1a6f640c7734ea8 Mon Sep 17 00:00:00 2001 From: Will Murphy Date: Fri, 12 Apr 2024 20:00:56 +0000 Subject: [PATCH 9/9] set Windows-specific coverage threshold There are a few paths that we know are not supported on Windows, so a lower coverage threshold is acceptable. Signed-off-by: Will Murphy --- Taskfile.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Taskfile.yaml b/Taskfile.yaml index 5962cf4..e2cd832 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -126,7 +126,7 @@ tasks: sh: "python ./scripts/list_units.py anchore binny" # unit test coverage threshold (in % coverage) - COVERAGE_THRESHOLD: 45 + COVERAGE_THRESHOLD: '{{if eq OS "windows"}} 43 {{else}} 45 {{end}}' cmds: - cmd: "{{ .MAKEDIR_P }} {{ .TMP_DIR }}" silent: true