Merge pull request #1385 from ypu/systemtest

Add system test with ginkgo

Makefile

@@ -31,7 +31,7 @@ BASHINSTALLDIR=${PREFIX}/share/bash-completion/completions
 OCIUMOUNTINSTALLDIR=$(PREFIX)/share/oci-umount/oci-umount.d
 
 SELINUXOPT ?= $(shell test -x /usr/sbin/selinuxenabled && selinuxenabled && echo -Z)
-PACKAGES ?= $(shell $(GO) list -tags "${BUILDTAGS}" ./... | grep -v github.com/containers/libpod/vendor | grep -v e2e)
+PACKAGES ?= $(shell $(GO) list -tags "${BUILDTAGS}" ./... | grep -v github.com/containers/libpod/vendor | grep -v e2e | grep -v system )
 
 COMMIT_NO ?= $(shell git rev-parse HEAD 2> /dev/null || true)
 GIT_COMMIT ?= $(if $(shell git status --porcelain --untracked-files=no),"${COMMIT_NO}-dirty","${COMMIT_NO}")
@@ -104,6 +104,9 @@ test/copyimg/copyimg: .gopathok $(wildcard test/copyimg/*.go)
 test/checkseccomp/checkseccomp: .gopathok $(wildcard test/checkseccomp/*.go)
 	$(GO) build -ldflags '$(LDFLAGS)' -tags "$(BUILDTAGS) containers_image_ostree_stub" -o $@ $(PROJECT)/test/checkseccomp
 
+test/goecho/goecho: .gopathok $(wildcard test/goecho/*.go)
+	$(GO) build -ldflags '$(LDFLAGS)' -o $@ $(PROJECT)/test/goecho
+
 podman: .gopathok $(PODMAN_VARLINK_DEPENDENCIES)
 	$(GO) build -i -ldflags '$(LDFLAGS_PODMAN)' -tags "$(BUILDTAGS)" -o bin/$@ $(PROJECT)/cmd/podman
 
@@ -130,6 +133,7 @@ clean:
 		test/bin2img/bin2img \
 		test/checkseccomp/checkseccomp \
 		test/copyimg/copyimg \
+		test/goecho/goecho \
 		test/testdata/redis-image \
 		cmd/podman/varlink/iopodman.go \
 		libpod/container_ffjson.go \
@@ -166,14 +170,20 @@ shell: libpodimage
 testunit: libpodimage
 	${CONTAINER_RUNTIME} run -e STORAGE_OPTIONS="--storage-driver=vfs" -e TESTFLAGS -e CGROUP_MANAGER=cgroupfs -e TRAVIS -t --privileged --rm -v ${CURDIR}:/go/src/${PROJECT} ${LIBPOD_IMAGE} make localunit
 
-localunit: varlink_generate
+localunit: test/goecho/goecho varlink_generate
 	$(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES)
 
 ginkgo:
 	ginkgo -v -tags "$(BUILDTAGS)" -cover -flakeAttempts 3 -progress -trace -noColor test/e2e/.
 
 localintegration: varlink_generate test-binaries clientintegration ginkgo
 
+localsystem: .install.ginkgo .install.gomega
+	ginkgo -v -noColor test/system/
+
+system.test-binary: .install.ginkgo .install.gomega
+	$(GO) test -c ./test/system
+
 clientintegration:
 	$(MAKE) -C contrib/python/podman integration
 	$(MAKE) -C contrib/python/pypodman integration
@@ -183,7 +193,7 @@ vagrant-check:
 
 binaries: varlink_generate easyjson_generate podman
 
-test-binaries: test/bin2img/bin2img test/copyimg/copyimg test/checkseccomp/checkseccomp
+test-binaries: test/bin2img/bin2img test/copyimg/copyimg test/checkseccomp/checkseccomp test/goecho/goecho
 
 MANPAGES_MD ?= $(wildcard docs/*.md pkg/*/docs/*.md)
 MANPAGES ?= $(MANPAGES_MD:%.md=%)

test/README.md

@@ -1,8 +1,33 @@
 ![PODMAN logo](../logo/podman-logo-source.svg)
-# Integration Tests
+# Test utils
+Test utils provide common functions and structs for testing. It includes two structs:
+* `PodmanTest`: Handle the *podman* command and other global resources like temporary
+directory. It provides basic methods, like checking podman image and pod status. Test
+suites should create their owner test *struct* as a composite of `PodmanTest`, and their
+owner PodmanMakeOptions().
+
+* `PodmanSession`: Store execution session data and related *methods*. Such like get command
+output and so on. It can be used directly in the test suite, only embed it to your owner
+session struct if you need expend it.
+
+## Unittest for test/utils
+To ensure neither *tests* nor *utils* break, There are unit-tests for each *functions* and
+*structs* in `test/utils`. When you adding functions or structs to this *package*, please
+update both unit-tests for it and this documentation.
+
+### Run unit test for test/utils
+Run unit test for test/utils.
+
+```
+make localunit
+```
+
+## Structure of the test utils and test suites
+The test *utils* package is at the same level of test suites. Each test suites also have their
+owner common functions and structs stored in `libpod_suite_test.go`.
 
-Our primary means of performing integration testing for libpod is with the
-[Ginkgo](https://github.com/onsi/ginkgo) BDD testing framework. This allows
+# Ginkgo test framework
+[Ginkgo](https://github.com/onsi/ginkgo) is a BDD testing framework. This allows
 us to use native Golang to perform our tests and there is a strong affiliation
 between Ginkgo and the Go test framework.
 
@@ -32,8 +57,16 @@ The gomega sources can be simply installed with the command:
 GOPATH=~/go go get github.com/onsi/gomega/...
 ```
 
-### Running the integration tests
+# Integration Tests
+Test suite for integration test for podman command line. It has its own structs:
+* `PodmanTestIntegration`: Integration test *struct* as a composite of `PodmanTest`. It
+set up the global options for *podman* command to ignore the environment influence from
+different test system.
+
+* `PodmanSessionIntegration`: This *struct* has it own *methods* for checking command
+output with given format JSON by using *structs* defined in inspect package.
 
+## Running the integration tests
 You can run the entire suite of integration tests with the following command:
 
 ```
@@ -67,3 +100,21 @@ make shell
 ```
 
 This will run a container and give you a shell and you can follow the instructions above.
+
+# System test
+System tests are used for testing the *podman* CLI in the context of a complete system. It
+requires that *podman*, all dependencies, and configurations are in place.  The intention of
+system testing is to match as closely as possible with real-world user/developer use-cases
+and environments. The orchestration of the environments and tests is left to external
+tooling.
+
+* `PodmanTestSystem`: System test *struct* as a composite of `PodmanTest`. It will not add any
+options to the command by default. When you run system test, you can set GLOBALOPTIONS,
+PODMAN_SUBCMD_OPTIONS or PODMAN_BINARY in ENV to run the test suite for different test matrices.
+
+## Run system test
+You can run the test with following command:
+
+```
+make localsystem
+```

test/e2e/attach_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -12,15 +13,15 @@ var _ = Describe("Podman attach", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 	})
 

test/e2e/checkpoint_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/containers/libpod/pkg/criu"
+	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -13,15 +14,15 @@ var _ = Describe("Podman checkpoint", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 		if !criu.CheckForCriu() {
 			Skip("CRIU is missing or too old.")

test/e2e/commit_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -12,15 +13,15 @@ var _ = Describe("Podman commit", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 	})
 

test/e2e/create_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -12,15 +13,15 @@ var _ = Describe("Podman create", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 	})
 

test/e2e/diff_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"sort"
 
+	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -13,15 +14,15 @@ var _ = Describe("Podman diff", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 	})
 

test/e2e/exec_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -12,15 +13,15 @@ var _ = Describe("Podman exec", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 	})
 

test/e2e/export_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
+	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -13,15 +14,15 @@ var _ = Describe("Podman export", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 	})
 

test/e2e/history_test.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -12,15 +13,15 @@ var _ = Describe("Podman history", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 	})
 

test/e2e/images_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"sort"
 
+	. "github.com/containers/libpod/test/utils"
 	"github.com/docker/go-units"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -14,15 +15,15 @@ var _ = Describe("Podman images", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 	})
 

test/e2e/import_test.go

@@ -5,6 +5,7 @@ import (
 	"os"
 	"path/filepath"
 
+	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
@@ -13,15 +14,15 @@ var _ = Describe("Podman import", func() {
 	var (
 		tempdir    string
 		err        error
-		podmanTest PodmanTest
+		podmanTest *PodmanTestIntegration
 	)
 
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
-		podmanTest = PodmanCreate(tempdir)
+		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.RestoreAllArtifacts()
 	})