Auto merge of #374 - aneeshusa:test-upgradeability-on-travis, r=edunham

Also test Salt config changes as an upgrade

The current testing always starts from scratch.
However, in practice we do not rebuild all our machines from scratch
on every change, but instead usually incrementally upgrade them.

Run the tests both from scratch, which is used whenever we stand up
a new builder, and as an upgrade from the previous configuration, which
is the more common case.

Additionally, the Salt installation and Salt grains.items and
state.show_highstate commands produce output that is useful for
debugging, but is normally not necessary to view.
Add special markers to the build log that fold these sections on Travis.
This is especially useful with the longer build logs.

Note that this doubles the number of builds run on Travis.

r? @larsbergstrom @edunham

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="35" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/servo/saltfs/374)
<!-- Reviewable:end -->

.travis.yml

@@ -3,26 +3,62 @@
 language: cpp
 
 # NOTE: Make sure the matrix covers all node types in top.sls
+# NOTE: Make sure each node type is tested both from scratch and as an upgrade
 # sudo: required and dist: trusty enable usage of Trusty
 matrix:
   include:
-    - env: SALT_NODE_ID=servo-linux-cross1
+    # Salt from scratch
+    - env:
+        - SALT_NODE_ID=servo-linux-cross1
+        - SALT_FROM_SCRATCH=true
       os: linux
       sudo: required
       dist: trusty
-    - env: SALT_NODE_ID=servo-mac1
+    - env:
+        - SALT_NODE_ID=servo-mac1
+        - SALT_FROM_SCRATCH=true
       os: osx
-    - env: SALT_NODE_ID=servo-linux1
+    - env:
+        - SALT_NODE_ID=servo-linux1
+        - SALT_FROM_SCRATCH=true
       os: linux
       sudo: required
       dist: trusty
-    - env: SALT_NODE_ID=servo-master1
+    - env:
+        - SALT_NODE_ID=servo-master1
+        - SALT_FROM_SCRATCH=true
       os: linux
       sudo: required
       dist: trusty
       language: python
       python: 3.5
-    - env: SALT_NODE_ID=test # Not a Salt node, runs test suite instead
+    # Salt from previous configuration
+    - env:
+        - SALT_NODE_ID=servo-linux-cross1
+        - SALT_FROM_SCRATCH=false
+      os: linux
+      sudo: required
+      dist: trusty
+    - env:
+        - SALT_NODE_ID=servo-mac1
+        - SALT_FROM_SCRATCH=false
+      os: osx
+    - env:
+        - SALT_NODE_ID=servo-linux1
+        - SALT_FROM_SCRATCH=false
+      os: linux
+      sudo: required
+      dist: trusty
+    - env:
+        - SALT_NODE_ID=servo-master1
+        - SALT_FROM_SCRATCH=false
+      os: linux
+      sudo: required
+      dist: trusty
+      language: python
+      python: 3.5
+    # Not a Salt node, runs test suite instead
+    - env: SALT_NODE_ID=test
       os: linux
       sudo: required
       dist: trusty

.travis/dispatch.sh

@@ -3,24 +3,53 @@
 set -o errexit
 set -o nounset
 
-if [ "${SALT_NODE_ID}" = "test" ]; then
-    # Using .travis.yml to specify Python 3.5 to be preinstalled, just to check
-    printf "Using $(python3 --version) at $(which python3)\n"
+travis_fold_start () {
+    printf "travis_fold:start:$1\n"
+    printf "$2\n"
+}
 
-    # Run test suite separately for parallelism
-    ./test.py
-else
+travis_fold_end () {
+    printf "travis_fold:end:$1\n"
+}
+
+run_salt () {
+    travis_fold_start "salt.install.$1" 'Installing and configuring Salt'
     .travis/install_salt.sh -F -c .travis -- "${TRAVIS_OS_NAME}"
     .travis/setup_salt_roots.sh
+    travis_fold_end "salt.install.$1"
 
-    # For debugging, check the grains reported by the Travis builder
+    travis_fold_start "grains.items.$1" 'Printing Salt grains for debugging'
     sudo salt-call --id="${SALT_NODE_ID}" grains.items
+    travis_fold_end "grains.items.$1"
 
-    # Minimally validate YAML and Jinja at a basic level
+    travis_fold_start "state.show_highstate.$1" 'Performing basic YAML and Jinja validation'
     sudo salt-call --id="${SALT_NODE_ID}" --retcode-passthrough state.show_highstate
-    # Full on installation test
+    travis_fold_end "state.show_highstate.$1"
+
+    printf 'Running the full Salt highstate\n'
     sudo salt-call --id="${SALT_NODE_ID}" --retcode-passthrough --log-level=warning state.highstate
+}
+
+
+if [ "${SALT_NODE_ID}" = "test" ]; then
+    # Using .travis.yml to specify Python 3.5 to be preinstalled, just to check
+    printf "Using $(python3 --version) at $(which python3)\n"
+
+    # Run test suite separately for parallelism
+    ./test.py
+else
+    if [ "${SALT_FROM_SCRATCH}" = "true" ]; then
+        run_salt 'scratch'
+    else
+        git fetch origin master:master
+        git checkout master
+        run_salt 'old'
+
+        git checkout "${TRAVIS_COMMIT}"
+        run_salt 'upgrade'
+    fi
 
+    # Only run tests against the new configuration
     # TODO: don't hard-code this
     if [ "${SALT_NODE_ID}" = "servo-master1" ]; then
         ./test.py sls.buildbot.master sls.nginx

.travis/install_salt.sh

@@ -26,6 +26,7 @@ install_salt () {
 
 configure_salt () {
     printf "$0: copying Salt minion configuration from ${TEMPORARY_CONFIG_DIR}\n"
+    sudo rm -rf /etc/salt
     sudo mkdir -p /etc/salt
     sudo cp "${FORCE_FLAG}" -- "${TEMPORARY_CONFIG_DIR}/minion" /etc/salt/minion
 }

.travis/setup_salt_roots.sh

@@ -4,6 +4,7 @@ set -o errexit
 set -o nounset
 
 setup_salt_roots () {
+    sudo rm -rf /srv/salt
     sudo mkdir -p /srv/salt
     sudo cp -r . /srv/salt/states
     sudo cp -r .travis/test_pillars /srv/salt/pillars