From 0b95e949f591b8284ad035dc3b3e6d346602f88a Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Wed, 19 Jun 2019 11:22:42 -0700 Subject: [PATCH 1/5] e2e: better method of adding SDK repo replace directive to go.mod --- hack/lib/test_lib.sh | 25 +++++++++++++++++++++ hack/tests/e2e-ansible.sh | 21 ++---------------- hack/tests/e2e-helm.sh | 21 ++---------------- test/e2e/memcached_test.go | 45 ++++++++++++++------------------------ 4 files changed, 46 insertions(+), 66 deletions(-) diff --git a/hack/lib/test_lib.sh b/hack/lib/test_lib.sh index a9c7e35a565..195727d8d46 100644 --- a/hack/lib/test_lib.sh +++ b/hack/lib/test_lib.sh @@ -41,3 +41,28 @@ function trap_add() { fatal "unable to add to trap ${trap_add_name}" done } + +# add_go_mod_replace adds a "replace" directive from $1 to $2 with an +# optional version version $3 to the current working directory's go.mod file. +function add_go_mod_replace() { + local from_path="$1" + local to_path="$2" + local version="${3:-}" + + if [[ ! -e go.mod ]]; then + echo "go.mod file not found in $(pwd)" + fi + + # Use the local operator-sdk directory as the repo. To make the go toolchain + # happy, the directory needs a `go.mod` file that specifies the module name, + # so we need this temporary hack until we update the SDK repo itself to use + # Go modules. + echo "module ${from_path}" > "${to_path}/go.mod" + trap_add "rm ${to_path}/go.mod" EXIT + # Do not use "go mod edit" so formatting stays the same. + local replace="replace ${from_path} => ${to_path}" + if [[ -n "$version" ]]; then + replace="$replace $version" + fi + echo "$replace" >> go.mod +} diff --git a/hack/tests/e2e-ansible.sh b/hack/tests/e2e-ansible.sh index a34bf32ba30..8c7dfc2c510 100755 --- a/hack/tests/e2e-ansible.sh +++ b/hack/tests/e2e-ansible.sh @@ -137,25 +137,8 @@ then exit 1 fi -# Remove any "replace" and "require" lines for the SDK repo before vendoring -# in case this is a release PR and the tag doesn't exist yet. This must be -# done without using "go mod edit", which first parses go.mod and will error -# if it doesn't find a tag/version/package. -# TODO: remove SDK repo references if PR/branch is not from the main SDK repo. -SDK_REPO="github.com/operator-framework/operator-sdk" -sed -E -i 's|^.*'"$SDK_REPO"'.*$||g' go.mod - -# Run `go build ./...` to pull down the deps specified by the scaffolded -# `go.mod` file and verify dependencies build correctly. -go build ./... - -# Use the local operator-sdk directory as the repo. To make the go toolchain -# happy, the directory needs a `go.mod` file that specifies the module name, -# so we need this temporary hack until we update the SDK repo itself to use -# Go modules. -echo "module ${SDK_REPO}" > "${ROOTDIR}/go.mod" -trap_add "rm ${ROOTDIR}/go.mod" EXIT -go mod edit -replace="${SDK_REPO}=$ROOTDIR" +add_go_mod_replace "github.com/operator-framework/operator-sdk" "$ROOTDIR" +# Build the project to resolve dependency versions in the modfile. go build ./... operator-sdk build "$DEST_IMAGE" diff --git a/hack/tests/e2e-helm.sh b/hack/tests/e2e-helm.sh index 9b4dacaf468..5d56935f9ce 100755 --- a/hack/tests/e2e-helm.sh +++ b/hack/tests/e2e-helm.sh @@ -146,25 +146,8 @@ then exit 1 fi -# Remove any "replace" and "require" lines for the SDK repo before vendoring -# in case this is a release PR and the tag doesn't exist yet. This must be -# done without using "go mod edit", which first parses go.mod and will error -# if it doesn't find a tag/version/package. -# TODO: remove SDK repo references if PR/branch is not from the main SDK repo. -SDK_REPO="github.com/operator-framework/operator-sdk" -sed -E -i 's|^.*'"$SDK_REPO"'.*$||g' go.mod - -# Run `go build ./...` to pull down the deps specified by the scaffolded -# `go.mod` file and verify dependencies build correctly. -go build ./... - -# Use the local operator-sdk directory as the repo. To make the go toolchain -# happy, the directory needs a `go.mod` file that specifies the module name, -# so we need this temporary hack until we update the SDK repo itself to use -# Go modules. -echo "module ${SDK_REPO}" > "${ROOTDIR}/go.mod" -trap_add "rm ${ROOTDIR}/go.mod" EXIT -go mod edit -replace="${SDK_REPO}=$ROOTDIR" +add_go_mod_replace "github.com/operator-framework/operator-sdk" "$ROOTDIR" +# Build the project to resolve dependency versions in the modfile. go build ./... operator-sdk build "$DEST_IMAGE" diff --git a/test/e2e/memcached_test.go b/test/e2e/memcached_test.go index 67066d22ffe..7188579a6e1 100644 --- a/test/e2e/memcached_test.go +++ b/test/e2e/memcached_test.go @@ -17,14 +17,12 @@ package e2e import ( "bytes" "context" - "errors" "fmt" "io" "io/ioutil" "os" "os/exec" "path/filepath" - "regexp" "strings" "testing" "time" @@ -36,10 +34,10 @@ import ( "github.com/operator-framework/operator-sdk/pkg/test/e2eutil" "github.com/ghodss/yaml" + "github.com/pkg/errors" dto "github.com/prometheus/client_model/go" "github.com/prometheus/common/expfmt" "github.com/prometheus/prometheus/util/promlint" - "github.com/rogpeppe/go-internal/modfile" v1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -130,20 +128,9 @@ func TestMemcached(t *testing.T) { } }() } - modBytes, err := ioutil.ReadFile("go.mod") + modBytes, err := insertGoModReplace(t, sdkRepo, replace.repo, replace.ref) if err != nil { - t.Fatalf("Failed to read go.mod: %v", err) - } - // Remove SDK repo dependency lines so we can parse the modfile. - sdkRe := regexp.MustCompile(`.*github\.com/operator-framework/operator-sdk.*`) - modBytes = sdkRe.ReplaceAll(modBytes, nil) - modBytes, err = insertGoModReplace(t, modBytes, sdkRepo, replace.repo, replace.ref) - if err != nil { - t.Fatalf("Failed to insert replace: %v", err) - } - err = ioutil.WriteFile("go.mod", modBytes, fileutil.DefaultFileMode) - if err != nil { - t.Fatalf("Failed to write updated go.mod: %v", err) + t.Fatalf("Failed to insert go.mod replace: %v", err) } t.Logf("go.mod: %v", string(modBytes)) } @@ -322,21 +309,23 @@ func getGoModReplace(t *testing.T, localSDKPath string) goModReplace { } } -func insertGoModReplace(t *testing.T, modBytes []byte, repo, path, sha string) ([]byte, error) { - modFile, err := modfile.Parse("go.mod", modBytes, nil) +func insertGoModReplace(t *testing.T, repo, path, sha string) ([]byte, error) { + modBytes, err := ioutil.ReadFile("go.mod") if err != nil { - return nil, fmt.Errorf("failed to parse go.mod: %v", err) + return nil, errors.Wrap(err, "failed to read go.mod") } - if err = modFile.AddReplace(repo, "", path, sha); err != nil { - s := "" - if sha != "" { - s = " " + sha - } - return nil, fmt.Errorf(`failed to add "replace %s => %s%s: %v"`, repo, path, s, err) + sdkReplace := fmt.Sprintf("replace %s => %s", repo, path) + if sha != "" { + sdkReplace = fmt.Sprintf("%s %s", sdkReplace, sha) + } + modBytes = append(modBytes, []byte("\n"+sdkReplace)...) + err = ioutil.WriteFile("go.mod", modBytes, fileutil.DefaultFileMode) + if err != nil { + return nil, errors.Wrap(err, "failed to write go.mod before replacing SDK repo") } - modFile.Cleanup() - if modBytes, err = modFile.Format(); err != nil { - return nil, fmt.Errorf("failed to format go.mod: %v", err) + cmdOut, err := exec.Command("go", "build", "./...").CombinedOutput() + if err != nil { + return nil, fmt.Errorf("\"go build ./...\" failed before replacing SDK repo: %v\nCommand Output:\n%v", err, string(cmdOut)) } return modBytes, nil } From 61137014d158fdc2b2a222bdc58da614f4110779 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Sat, 22 Jun 2019 13:28:09 -0400 Subject: [PATCH 2/5] check to_path for existence --- hack/lib/test_lib.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/hack/lib/test_lib.sh b/hack/lib/test_lib.sh index 195727d8d46..2bec98ef1c0 100644 --- a/hack/lib/test_lib.sh +++ b/hack/lib/test_lib.sh @@ -49,6 +49,9 @@ function add_go_mod_replace() { local to_path="$2" local version="${3:-}" + if [[ ! -d "$to_path" ]]; then + echo "$to_path does not exist" + fi if [[ ! -e go.mod ]]; then echo "go.mod file not found in $(pwd)" fi From 3dc8e42e5fc74767582b4142da2360955e1cea93 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Wed, 26 Jun 2019 11:09:42 -0700 Subject: [PATCH 3/5] check if {sdk repo}/go.mod exists before overwriting in e2e --- hack/lib/test_lib.sh | 9 ++++----- test/e2e/memcached_test.go | 18 ++++++++++-------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/hack/lib/test_lib.sh b/hack/lib/test_lib.sh index 2bec98ef1c0..aa7fd547740 100644 --- a/hack/lib/test_lib.sh +++ b/hack/lib/test_lib.sh @@ -49,9 +49,6 @@ function add_go_mod_replace() { local to_path="$2" local version="${3:-}" - if [[ ! -d "$to_path" ]]; then - echo "$to_path does not exist" - fi if [[ ! -e go.mod ]]; then echo "go.mod file not found in $(pwd)" fi @@ -60,8 +57,10 @@ function add_go_mod_replace() { # happy, the directory needs a `go.mod` file that specifies the module name, # so we need this temporary hack until we update the SDK repo itself to use # Go modules. - echo "module ${from_path}" > "${to_path}/go.mod" - trap_add "rm ${to_path}/go.mod" EXIT + if [[ ! -e "${to_path}/go.mod" ]]; then + echo "module ${from_path}" > "${to_path}/go.mod" + trap_add "rm ${to_path}/go.mod" EXIT + fi # Do not use "go mod edit" so formatting stays the same. local replace="replace ${from_path} => ${to_path}" if [[ -n "$version" ]]; then diff --git a/test/e2e/memcached_test.go b/test/e2e/memcached_test.go index 7188579a6e1..184471dba31 100644 --- a/test/e2e/memcached_test.go +++ b/test/e2e/memcached_test.go @@ -118,15 +118,17 @@ func TestMemcached(t *testing.T) { // memcached-operator's go.mod, which allows go to recognize // the local SDK repo as a module. sdkModPath := filepath.Join(filepath.FromSlash(replace.repo), "go.mod") - err = ioutil.WriteFile(sdkModPath, []byte("module "+sdkRepo), fileutil.DefaultFileMode) - if err != nil { - t.Fatalf("Failed to write main repo go.mod file: %v", err) - } - defer func() { - if err = os.RemoveAll(sdkModPath); err != nil { - t.Fatalf("Failed to remove %s: %v", sdkModPath, err) + if _, err = os.Stat(sdkModPath); err != nil && os.IsNotExist(err) { + err = ioutil.WriteFile(sdkModPath, []byte("module "+sdkRepo), fileutil.DefaultFileMode) + if err != nil { + t.Fatalf("Failed to write main repo go.mod file: %v", err) } - }() + defer func() { + if err = os.RemoveAll(sdkModPath); err != nil { + t.Fatalf("Failed to remove %s: %v", sdkModPath, err) + } + }() + } } modBytes, err := insertGoModReplace(t, sdkRepo, replace.repo, replace.ref) if err != nil { From 77fe3296adbaa8162bdd7bd33d61772a482ad140 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Wed, 26 Jun 2019 11:11:42 -0700 Subject: [PATCH 4/5] remove extra go build ./... --- test/e2e/memcached_test.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/test/e2e/memcached_test.go b/test/e2e/memcached_test.go index 184471dba31..a79386624d4 100644 --- a/test/e2e/memcached_test.go +++ b/test/e2e/memcached_test.go @@ -325,10 +325,6 @@ func insertGoModReplace(t *testing.T, repo, path, sha string) ([]byte, error) { if err != nil { return nil, errors.Wrap(err, "failed to write go.mod before replacing SDK repo") } - cmdOut, err := exec.Command("go", "build", "./...").CombinedOutput() - if err != nil { - return nil, fmt.Errorf("\"go build ./...\" failed before replacing SDK repo: %v\nCommand Output:\n%v", err, string(cmdOut)) - } return modBytes, nil } From 793b2ec32d948e3a0ea775739a0c619835cd5308 Mon Sep 17 00:00:00 2001 From: Eric Stroczynski Date: Wed, 26 Jun 2019 11:30:48 -0700 Subject: [PATCH 5/5] add a few checks and comment updates --- hack/lib/test_lib.sh | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/hack/lib/test_lib.sh b/hack/lib/test_lib.sh index aa7fd547740..1eba3ec545f 100644 --- a/hack/lib/test_lib.sh +++ b/hack/lib/test_lib.sh @@ -45,19 +45,26 @@ function trap_add() { # add_go_mod_replace adds a "replace" directive from $1 to $2 with an # optional version version $3 to the current working directory's go.mod file. function add_go_mod_replace() { - local from_path="$1" - local to_path="$2" + local from_path="${1:?first path in replace statement is required}" + local to_path="${2:?second path in replace statement is required}" local version="${3:-}" + if [[ ! -d "$to_path" && -z "$version" ]]; then + echo "second replace path $to_path requires a version be set because it is not a directory" + exit 1 + fi if [[ ! -e go.mod ]]; then echo "go.mod file not found in $(pwd)" + exit 1 fi - # Use the local operator-sdk directory as the repo. To make the go toolchain - # happy, the directory needs a `go.mod` file that specifies the module name, - # so we need this temporary hack until we update the SDK repo itself to use - # Go modules. - if [[ ! -e "${to_path}/go.mod" ]]; then + # If $to_path is a directory, it needs a `go.mod` file that specifies the + # module name to make the go toolchain happy. + # + # TODO: remove the below if statement once + # https://github.com/operator-framework/operator-sdk/pull/1566 is merged, + # which updates the SDK to use go modules. + if [[ -d "${to_path}" && ! -e "${to_path}/go.mod" ]]; then echo "module ${from_path}" > "${to_path}/go.mod" trap_add "rm ${to_path}/go.mod" EXIT fi