Organize the tests in a proper way (#1612)

redhat-developer · May 9, 2019 · 1af22f3 · 1af22f3
1 parent 857dbec
commit 1af22f3
Show file tree

Hide file tree

Showing 82 changed files with 2,529 additions and 4,251 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -28,19 +28,19 @@ jobs:
 
     - <<: *base-test
       stage: test
-      name: "main,login,component e2e tests"
+      name: "generic,login,component integration tests"
       script:
         - ./scripts/oc-cluster.sh
         - make bin
         - sudo cp odo /usr/bin
         - oc login -u developer
-        - travis_wait make test-main-e2e
+        - travis_wait make test-generic
         - travis_wait make test-odo-login-e2e
         - travis_wait make test-cmp-e2e
 
     - <<: *base-test
       stage: test
-      name: "java e2e tests"
+      name: "java e2e integration tests"
       script:
         - ./scripts/oc-cluster.sh
         - make bin
@@ -50,14 +50,15 @@ jobs:
 
     - <<: *base-test
       stage: test
-      name: "source,json e2e tests"
+      name: "source,json and config integration tests"
       script:
         - ./scripts/oc-cluster.sh
         - make bin
         - sudo cp odo /usr/bin
         - oc login -u developer
         - travis_wait make test-source-e2e
         - travis_wait make test-json-format-output
+        - travis_wait make test-odo-config
 
 # END Tests that are also in CircleCI
 
@@ -70,7 +71,7 @@ jobs:
         - make bin
         - sudo cp odo /usr/bin
         - oc login -u developer
-        - make test-service-e2e
+        - travis_wait make test-service-e2e
         - make test-link-e2e
         - travis_wait make test-cmp-sub-e2e
 
@@ -82,7 +83,7 @@ jobs:
         - make bin
         - sudo cp odo /usr/bin
         - oc login -u developer
-        - make test-watch-e2e TIMEOUT=1200s
+        - make test-watch-e2e
 
     # test installation script on linux
     # - stage: test

diff --git a/Makefile b/Makefile
@@ -5,6 +5,7 @@ COMMON_FLAGS := -X $(PROJECT)/pkg/odo/cli/version.GITCOMMIT=$(GITCOMMIT)
 BUILD_FLAGS := -ldflags="-w $(COMMON_FLAGS)"
 DEBUG_BUILD_FLAGS := -ldflags="$(COMMON_FLAGS)"
 FILES := odo dist
+TIMEOUT ?= 1800s
 
 # Slow spec threshold for ginkgo tests. After this time (in second), ginkgo marks test as slow
 SLOW_SPEC_THRESHOLD := 120
@@ -92,109 +93,70 @@ configure-installer-tests-cluster:
 test:
 	go test -race $(PKGS)
 
-# Run main e2e tests
-.PHONY: test-main-e2e
-test-main-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoe2e" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoe2e" -ginkgo.v
-endif
+# Run generic integration tests
+.PHONY: test-generic
+test-generic:
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odo generic" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run json outout tests
 .PHONY: test-json-format-output
 test-json-format-output:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odojsonoutput" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odojsonoutput" -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odojsonoutput" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run component e2e tests
 .PHONY: test-cmp-e2e
 test-cmp-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoCmpE2e" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoCmpE2e" -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odoCmpE2e" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run component subcommands e2e tests
 .PHONY: test-cmp-sub-e2e
 test-cmp-sub-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoCmpSubE2e" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoCmpSubE2e" -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odoCmpSubE2e" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run java e2e tests
 .PHONY: test-java-e2e
 test-java-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoJavaE2e" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoJavaE2e" -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odoJavaE2e" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run source e2e tests
 .PHONY: test-source-e2e
 test-source-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoSourceE2e" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoSourceE2e" -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odoSourceE2e" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run service catalog e2e tests
 .PHONY: test-service-e2e
 test-service-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoServiceE2e" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoServiceE2e" -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odoServiceE2e" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run link e2e tests
 .PHONY: test-link-e2e
 test-link-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoLinkE2e" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoLinkE2e" -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odoLinkE2e" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run link e2e tests
 .PHONY: test-watch-e2e
 test-watch-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoWatchE2e" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoWatchE2e" -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odoWatchE2e" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run login e2e tests
 .PHONY: test-odo-login-e2e
 test-odo-login-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoLoginE2e" -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e --ginkgo.focus="odoLoginE2e" -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odoLoginE2e" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
+
+# Run config tests
+.PHONY: test-odo-config
+test-odo-config:
+	go test -v github.com/openshift/odo/tests/integration --ginkgo.focus="odo config test" -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run all e2e tests
 .PHONY: test-e2e
 test-e2e:
-ifdef TIMEOUT
-	go test -v github.com/openshift/odo/tests/e2e -ginkgo.v -timeout $(TIMEOUT)
-else
-	go test -v github.com/openshift/odo/tests/e2e -ginkgo.v
-endif
+	go test -v github.com/openshift/odo/tests/integration -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # Run e2e test scenarios
 .PHONY: test-e2e-scenarios
 test-e2e-scenarios:
-	go test -v github.com/openshift/odo/tests/e2escenarios -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v
+	go test -v github.com/openshift/odo/tests/e2escenarios -ginkgo.slowSpecThreshold=$(SLOW_SPEC_THRESHOLD) -ginkgo.v -timeout $(TIMEOUT)
 
 # create deb and rpm packages using fpm in ./dist/pkgs/
 # run make cross before this!

diff --git a/docs/development.adoc b/docs/development.adoc
@@ -146,7 +146,7 @@ NOTE: You can use environment variable `CUSTOM_HOMEDIR` to specify a custom home
 
 === Integration tests
 
-Integration tests, otherwise known as end-2-end (e2e) tests are used within `odo`. All tests can be found in the `tests/e2e` directory and can be called using functions within `makefile`.
+Integration tests are used within `odo`. All tests can be found in the `tests/` directory and can be called using functions within `makefile`. Also test directory comprises `e2e` scenario and a clean `test template` for reference.
 
 .Prerequisites:
 
@@ -160,17 +160,17 @@ $ MINISHIFT_ENABLE_EXPERIMENTAL=y minishift start --extra-clusterup-flags "--ena
 
 .Procedure:
 
-To deploy an e2e test:
+To deploy an integration test:
 
 
-* For the entire suite use:
+* For the entire integration suite use:
 ----
 $ make test-e2e
 ----
 
-* For the main tests use:
+* For the generic tests use:
 ----
-$ make test-main-e2e
+$ make test-generic
 ----
 
 * For the component tests use:
@@ -183,6 +183,11 @@ $ make test-cmp-e2e
 $ make test-service-e2e
 ----
 
+* For the e2e scenario test use:
+----
+$ make test-e2e-scenarios
+----
+
 You can run a subset of tests with ginkgo by using focused specs mechanism https://onsi.github.io/ginkgo/#focused-specs
 
 === Race conditions

diff --git a/glide.lock b/glide.lock
diff --git a/glide.yaml b/glide.yaml
@@ -55,3 +55,5 @@ import:
   - pkg/oc/cli/login
 - package: github.com/mattn/go-colorable
   version: efa589957cd060542a26d2dd7832fd6a6c6c3ade
+- package: github.com/Benjamintf1/unmarshalledmatchers
+  version: bb1c1f34eaeac620307d94a2371c514fbaf6ec7f
diff --git a/scripts/generate-coverage.sh b/scripts/generate-coverage.sh
@@ -6,7 +6,7 @@
 set -e
 echo "" > coverage.txt
 go test -i -race ./cmd/odo
-for d in $(go list ./... | grep -v vendor | grep -v tests/e2e | grep -v testingutil); do
+for d in $(go list ./... | grep -v vendor | grep -v tests | grep -v testingutil); do
     # For watch related tests, race check causes issue so disabling them here as race is already tested in other tests when used with `-coverprofile=profile.out`
     if [ "$d" = "github.com/openshift/odo/pkg/component" ]; then
         go test -coverprofile=profile.out -covermode=atomic $d

diff --git a/scripts/openshiftci-presubmit-e2e.sh b/scripts/openshiftci-presubmit-e2e.sh
@@ -11,11 +11,12 @@ make configure-installer-tests-cluster
 make bin
 export PATH="$PATH:$(pwd)"
 export CUSTOM_HOMEDIR="/tmp/artifacts"
-make test-main-e2e
+make test-generic
 make test-odo-login-e2e
 make test-json-format-output
 make test-java-e2e
 make test-source-e2e
 make test-cmp-e2e
 make test-cmp-sub-e2e
-
+make test-odo-config
+make test-watch-e2e
diff --git a/tests/e2e/README.md b/tests/e2e/README.md