Issue #673, #647 Discern if bundle embedded, specify binary for tests…

…, and other updates

- Pass arguments to make integration via flags to go test (that underlies make integration).
- Let user specify which `crc` binary they want to use with integration tests.
- Make tests run with embedded and non-embedded bundle.
- Other additions: relevant docs, clean-up in a feature.

Makefile

@@ -122,11 +122,19 @@ clean: clean_docs
 
 
 .PHONY: integration ## Run integration tests
-integration: GODOG_OPTS = --godog.tags=$(GOOS)
 integration:
-	@$(call check_defined, BUNDLE_LOCATION, "'make integration' requires BUNDLE_LOCATION to contain the full path to a bundle file")
-	@$(call check_defined, PULL_SECRET_FILE, "'make integration' requires PULL_SECRET_FILE to point to a file with the pull secret to use")
-	@go test --timeout=90m $(REPOPATH)/test/integration -v --tags=integration $(GODOG_OPTS) $(BUNDLE_LOCATION) $(PULL_SECRET_FILE)
+GODOG_OPTS = --godog.tags=$(GOOS)
+ifndef PULL_SECRET_FILE
+	PULL_SECRET_FILE = --pull-secret-file=$(HOME)/Downloads/crc-pull-secret
+endif
+ifndef BUNDLE_LOCATION
+	BUNDLE_LOCATION = --bundle-location=$(HOME)/Downloads/crc_libvirt_$(BUNDLE_VERSION).$(BUNDLE_EXTENSION)
+endif
+ifndef CRC_BINARY
+	CRC_BINARY = --crc-binary=$(GOPATH)/bin
+endif
+integration:
+	@go test --timeout=90m $(REPOPATH)/test/integration -v $(PULL_SECRET_FILE) $(BUNDLE_LOCATION) $(CRC_BINARY) --bundle-version=$(BUNDLE_VERSION) --tags=integration $(GODOG_OPTS)
 
 .PHONY: fmt
 fmt:

centos_ci.sh

@@ -130,7 +130,9 @@ function run_tests() {
   set +e
   # In Jenkins slave we have pull secret file in the $HOME/payload/crc_pull_secret
   # this is copied over using https://github.com/minishift/minishift-ci-jobs/blob/master/minishift-ci-index.yaml#L99
-  make integration BUNDLE_LOCATION=$HOME/Downloads/$BUNDLE PULL_SECRET_FILE=$HOME/payload/crc_pull_secret
+  export PULL_SECRET_FILE=--pull-secret-file=$HOME/payload/crc_pull_secret
+  export BUNDLE_LOCATION=--bundle-location=$HOME/Downloads/$BUNDLE 
+  make integration 
   if [[ $? -ne 0 ]]; then
     upload_logs $1
     exit 1

developing.adoc

@@ -60,14 +60,16 @@ Clicumber allows running commands in a persistent shell instance (`bash`, `tcsh`
 [[how-to-run-integration-tests]]
 === How to run
 
-To start integration tests, run:
+First, one needs to set the following flags in `Makefile`, under `integration` target:
 
+- `--pull-secret-file`: absolute path to your OpenShift pull secret.
+- `--bundle-location`: if bundle is embedded, this flag should be set to `--bundle-location=embedded` or not passed at all; if bundle is not embedded, then absolute path to the bundle should be passed.
+- `--crc-binary`: if `crc` binary resides in `$GOPATH/bin`, then this flag needs not be passed; otherwise absolute path to the `crc` binary should be passed.
+
+To start integrationt tests, run:
 ```bash
-$ make integration BUNDLE_LOCATION=<bundle location> PULL_SECRET_FILE=<pull secret path>
+$ make integration
 ```
-where `<bundle location>` is either the bundle's URL or its path in the filesystem,
-and `<pull secret path>` is the path to a file containing your OpenShift pull secret.
-The paths must be absolute paths.
 
 ===== How to run only a subset of all integration tests
 

test/integration/crcsuite/crcsuite.go

@@ -25,20 +25,25 @@ import (
 	"io/ioutil"
 	"net/http"
 	"os"
+	"os/user"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"time"
 
 	clicumber "github.com/code-ready/clicumber/testsuite"
-	"github.com/code-ready/crc/pkg/crc/constants"
 	"github.com/code-ready/crc/pkg/crc/oc"
 )
 
 var (
 	CRCHome        string
-	bundleURL      string
+	CRCBinary      string
+	bundleEmbedded bool
 	bundleName     string
+	bundleURL      string
+	bundleVersion  string
 	pullSecretFile string
+	goPath         string
 )
 
 // FeatureContext defines godog.Suite steps for the test suite.
@@ -75,11 +80,59 @@ func FeatureContext(s *godog.Suite) {
 		DeleteFileFromCRCHome)
 
 	s.BeforeSuite(func() {
-		// set CRC home var
-		CRCHome = SetCRCHome()
+
+		usr, _ := user.Current()
+		CRCHome = filepath.Join(usr.HomeDir, ".crc")
+
+		// init CRCBinary if no location provided by user
+		if CRCBinary == "" {
+			fmt.Println("Expecting the CRC binary to be in $HOME/go/bin.")
+			usr, _ := user.Current()
+			CRCBinary = filepath.Join(usr.HomeDir, "go", "bin")
+		}
+
+		// put CRC binary location on top of PATH
+		path := os.Getenv("PATH")
+		newPath := fmt.Sprintf("%s%s%s", CRCBinary, os.PathListSeparator, path)
+		err := os.Setenv("PATH", newPath)
+		if err != nil {
+			fmt.Println("Could not put CRC location on top of PATH")
+			os.Exit(1)
+		}
+
+		if bundleURL == "embedded" {
+			fmt.Println("Expecting the bundle to be embedded in the CRC binary.")
+			bundleEmbedded = true
+			if bundleVersion == "" {
+				fmt.Println("User must specify --bundle-version if bundle is embedded")
+				os.Exit(1)
+			}
+			// assume default hypervisor
+			var hypervisor string
+			switch platform := runtime.GOOS; platform {
+			case "darwin":
+				hypervisor = "hyperkit"
+			case "linux":
+				hypervisor = "libvirt"
+			case "windows":
+				hypervisor = "hyperv"
+			default:
+				fmt.Printf("Unsupported OS: %s", platform)
+				os.Exit(1)
+			}
+			bundleName = fmt.Sprintf("crc_%s_%s.crcbundle", hypervisor, bundleVersion)
+		} else {
+			bundleEmbedded = false
+			_, bundleName = filepath.Split(bundleURL)
+		}
+
+		if pullSecretFile == "" {
+			fmt.Println("User must specify the pull secret file via --pull-secret-file flag.")
+			os.Exit(1)
+		}
 
 		// remove $HOME/.crc
-		err := RemoveCRCHome()
+		err = RemoveCRCHome()
 		if err != nil {
 			fmt.Println(err)
 		}
@@ -89,28 +142,29 @@ func FeatureContext(s *godog.Suite) {
 	s.AfterSuite(func() {
 		err := DeleteCRC()
 		if err != nil {
-			fmt.Println(err)
+			fmt.Printf("Could not delete CRC VM: %s.", err)
 		}
 	})
 
 	s.BeforeFeature(func(this *gherkin.Feature) {
 
-		if _, err := os.Stat(bundleName); os.IsNotExist(err) {
-			// Obtain the bundle to current dir
-			fmt.Println("Obtaining bundle...")
-			bundle, err := DownloadBundle(bundleURL, ".")
-			if err != nil {
-				fmt.Printf("Failed to obtain CRC bundle, %v\n", err)
+		if bundleEmbedded == false {
+			if _, err := os.Stat(bundleName); os.IsNotExist(err) {
+				// Obtain the bundle to current dir
+				fmt.Println("Obtaining bundle...")
+				bundle, err := DownloadBundle(bundleURL, ".")
+				if err != nil {
+					fmt.Printf("Failed to obtain CRC bundle, %v\n", err)
+					os.Exit(1)
+				}
+				fmt.Println("Using bundle:", bundle)
+			} else if err != nil {
+				fmt.Printf("Unexpected error obtaining the bundle %v.\n", bundleName)
 				os.Exit(1)
+			} else {
+				fmt.Println("Using existing bundle:", bundleName)
 			}
-			fmt.Println("Using bundle:", bundle)
-		} else if err != nil {
-			fmt.Printf("Unknown error obtaining the bundle %v.\n", bundleName)
-			os.Exit(1)
-		} else {
-			fmt.Println("Using existing bundle:", bundleName)
 		}
-
 	})
 }
 
@@ -287,7 +341,7 @@ func StartCRCWithDefaultBundleAndDefaultHypervisorSucceedsOrFails(expected strin
 	var cmd string
 	var extraBundleArgs string
 
-	if !constants.BundleEmbedded() {
+	if bundleEmbedded == false {
 		extraBundleArgs = fmt.Sprintf("-b %s", bundleName)
 	}
 	cmd = fmt.Sprintf("crc start -p '%s' %s --log-level debug", pullSecretFile, extraBundleArgs)
@@ -301,7 +355,7 @@ func StartCRCWithDefaultBundleAndHypervisorSucceedsOrFails(hypervisor string, ex
 	var cmd string
 	var extraBundleArgs string
 
-	if !constants.BundleEmbedded() {
+	if bundleEmbedded == false {
 		extraBundleArgs = fmt.Sprintf("-b %s", bundleName)
 	}
 	cmd = fmt.Sprintf("crc start -d %s -p '%s' %s --log-level debug", hypervisor, pullSecretFile, extraBundleArgs)
@@ -312,9 +366,14 @@ func StartCRCWithDefaultBundleAndHypervisorSucceedsOrFails(hypervisor string, ex
 
 func StartCRCWithDefaultBundleAndNameServerSucceedsOrFails(nameserver string, expected string) error {
 
+	var extraBundleArgs string
+	if bundleEmbedded == false {
+		extraBundleArgs = fmt.Sprintf("-b %s", bundleName)
+	}
+
 	var cmd string
 
-	cmd = fmt.Sprintf("crc start -n %s -p '%s' --log-level debug", nameserver, pullSecretFile)
+	cmd = fmt.Sprintf("crc start -n %s -p '%s' %s --log-level debug", nameserver, pullSecretFile, extraBundleArgs)
 	err := clicumber.ExecuteCommandSucceedsOrFails(cmd, expected)
 
 	return err
@@ -323,7 +382,12 @@ func StartCRCWithDefaultBundleAndNameServerSucceedsOrFails(nameserver string, ex
 func SetConfigPropertyToValueSucceedsOrFails(property string, value string, expected string) error {
 
 	if value == "current bundle" {
-		value = bundleName
+
+		if bundleEmbedded {
+			value = filepath.Join(CRCHome, bundleName)
+		} else {
+			value = bundleName
+		}
 	}
 
 	cmd := "crc config set " + property + " " + value

test/integration/crcsuite/prepare.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"os/user"
 	"path/filepath"
 
 	"github.com/code-ready/crc/pkg/download"
@@ -57,22 +56,10 @@ func DownloadBundle(bundleLocation string, bundleDestination string) (string, er
 	return filename, nil
 }
 
-// Parse GODOG flags (in feature files)
 func ParseFlags() {
 
-	flag.Parse()
-	if flag.NArg() < 2 {
-		fmt.Printf("Invalid number of arguments, the paths to the bundle file and to the pull secret file are required\n")
-		os.Exit(1)
-	}
-	bundleURL = flag.Args()[0]
-	_, bundleName = filepath.Split(bundleURL)
-	pullSecretFile = flag.Args()[1]
-}
-
-// Set CRCHome var to ~/.crc
-func SetCRCHome() string {
-	usr, _ := user.Current()
-	crcHome := filepath.Join(usr.HomeDir, ".crc")
-	return crcHome
+	flag.StringVar(&bundleURL, "bundle-location", "embedded", "Path to the bundle to be used in tests")
+	flag.StringVar(&pullSecretFile, "pull-secret-file", "", "Path to the file containing pull secret")
+	flag.StringVar(&CRCBinary, "crc-binary", "", "Path to the CRC binary to be tested")
+	flag.StringVar(&bundleVersion, "bundle-version", "", "Version of the bundle used in tests")
 }

test/integration/features/story_registry.feature

@@ -41,3 +41,13 @@ Feature: Local image to image-registry to deployment
         When executing "oc logs -f dc/hello" succeeds
         Then stdout should contain "Hello, it works!"
 
+    Scenario: Clean up
+        Given executing "sudo podman images" succeeds
+        When stdout contains "localhost/hello"
+        Then executing "sudo podman image rm localhost/hello:test" succeeds
+        And executing "oc delete project testproj-img" succeeds
+        When executing "crc stop -f" succeeds
+        Then stdout should match "(.*)[Ss]topped the OpenShift cluster"
+        And executing "crc delete -f" succeeds
+        Then stdout should contain "Deleted the OpenShift cluster"
+

test/integration/integration_test.go

@@ -55,9 +55,9 @@ func getFeatureContext(s *godog.Suite) {
 }
 
 func parseFlags() {
-	// get flag values for clicumber testsuite
-	testsuite.ParseFlags()
 
-	// here you can get additional flag values if needed, for example:
+	// NOTE:
+	// testsuite.ParseFlags() needs to be last: it calls flag.Parse()
 	crcsuite.ParseFlags()
+	testsuite.ParseFlags()
 }