tests: run conformance test suite in Travis

Run conformance tests in after_script section of Travis with
non-blocking failures.

Follow up to #1003

Signed-off-by: Lokesh Mandvekar <lsm5@fedoraproject.org>

.travis.yml

@@ -26,6 +26,7 @@ matrix:
   # If the latest unstable development version of go fails, that's OK.
   allow_failures:
     - go: tip
+    - env: TEST_GROUP=conformance
 
   # Don't hold on the tip tests to finish.  Mark tests green if the
   # stable versions pass.
@@ -81,9 +82,11 @@ script:
     - tmp=`mktemp -d`; mkdir $tmp/root $tmp/runroot; sudo PATH="$PATH" ./buildah.test -test.v -root $tmp/root -runroot $tmp/runroot -storage-driver vfs -signature-policy `pwd`/tests/policy.json -registries-conf `pwd`/tests/registries.conf
     - cd tests; travis_wait 60 sudo PATH="$PATH" ./test_runner.sh
     - cd ..
-    - echo "Run container-diff tests in Fedora container..."
+after_script:
+    - echo "Run conformance tests in Fedora container..."
     - echo "Travis/host environment:"
     - env
+    - env TEST_GROUP=conformance
     - echo
     - echo "Running tests in SPC using ./hack/run_build.sh"
     - "./hack/run_build.sh"

hack/spc_build.sh

@@ -21,11 +21,10 @@ dnf -y update --best --allowerasing
 dnf -y install dnf-plugins-core
 dnf -y copr enable lsm5/container-diff
 dnf -y install autoconf automake btrfs-progs-devel \
-   buildah \
    container-diff device-mapper-devel golang go-md2man \
    git glibc-static gpgme-devel hostname iproute \
    iputils libassuan-devel libseccomp-static make \
-   moby-engine openssl
+   moby-engine podman
 
 # build buildah binary in fedora and run tests
 echo "Cleanup buildah repo and build again in fedora..."
@@ -37,7 +36,7 @@ make GOPATH=$(pwd)/_build:$(pwd) all TAGS="seccomp containers_image_ostree_stub"
 GOPATH=$(pwd)/_build:$(pwd) go test -c -tags "seccomp `./btrfs_tag.sh` `./btrfs_installed_tag.sh` `./libdm_tag.sh` `./ostree_tag.sh` `./selinux_tag.sh`" ./cmd/buildah
 tmp=$(mktemp -d); mkdir $tmp/root $tmp/runroot; PATH="$PATH" ./buildah.test -test.v -root $tmp/root -runroot $tmp/runroot -storage-driver vfs -signature-policy $(pwd)/tests/policy.json -registries-conf $(pwd)/tests/registries.conf
 
-echo "docker login to local registry..."
+echo "Logging in to local registry..."
 echo testpassword | docker login localhost:5000 --username testuser --password-stdin
 
 echo "docker build dockerfile..."
@@ -58,3 +57,7 @@ docker pull localhost:5000/buildah-test-image
 echo "Running container-diff..."
 container-diff diff --type=rpm daemon://localhost:5000/buildah-test-image daemon://docker-test-image
 
+echo "Running conformance tests..."
+GOPATH=$(pwd) go test -v ./tests/conformance
+
+echo "Finished conformance tests. Exiting..."