ensure correct model for gubernator tests (#157)

* ensure correct model for gubernator tests

The gubernator tests rely on "juju switch" being in the right
model before processing other juju commands. This is fragile
because an admin or other job might switch the controller during
a test run.

Ensure all juju commands for gubernator tests specify the
'controller:model' option.

Note, this also involves a change in the e2e-gubernator-* jobs
in jenkins to export a new CONTROLLER envar before the test begins.

* remove cmd quotes for run_and_wait (they arent needed and cause issues when escaped by the shell)

* doh, destroy-model doesnt take a -m

tests/deploy-test-bundle.sh

@@ -7,27 +7,32 @@ set -o xtrace  # Print the commands that are executed.
 
 echo "${0} started at `date`."
 
-# First argument is the model namme for this build.
-MODEL=${1:-"model-is-undefined"}
-# The second argument is the bundle name.
-BUNDLE=${2:-"canonical-kubernetes"}
+# First argument is the controller name for this build.
+CONTROLLER=${1:-"controller-is-undefined"}
+# Second argument is the model name for this build.
+MODEL=${2:-"model-is-undefined"}
+# Third argument is the bundle name.
+BUNDLE=${3:-"canonical-kubernetes"}
+
 # Create a model just for this run of the tests.
-juju add-model ${MODEL}
+juju add-model -c ${CONTROLLER} ${MODEL}
 # Set test mode on the deployment so we dont bloat charm-store deployment count
-juju model-config -m ${MODEL} test-mode=true
+juju model-config -m ${CONTROLLER}:${MODEL} test-mode=true
 
 # Deploy the kubernetes bundle.
-juju deploy ${BUNDLE}
+juju deploy -m ${CONTROLLER}:${MODEL} ${BUNDLE}
 # TODO Check for a second worker, the bundle could already define one.
 # TODO Check for the e2e charm, the bundle could already define one.
 # Deploy the e2e charm and make the relations.
-juju deploy cs:~containers/kubernetes-e2e
-juju relate kubernetes-e2e easyrsa
-juju add-relation kubernetes-e2e:kube-control kubernetes-master:kube-control
-juju add-relation kubernetes-e2e:kubernetes-master kubernetes-master:kube-api-endpoint
+juju deploy -m ${CONTROLLER}:${MODEL} cs:~containers/kubernetes-e2e
+juju relate -m ${CONTROLLER}:${MODEL} kubernetes-e2e easyrsa
+juju relate -m ${CONTROLLER}:${MODEL} \
+  kubernetes-e2e:kube-control kubernetes-master:kube-control
+juju relate -m ${CONTROLLER}:${MODEL} \
+  kubernetes-e2e:kubernetes-master kubernetes-master:kube-api-endpoint
 
-juju config kubernetes-worker allow-privileged=true
-juju config kubernetes-master allow-privileged=true
+juju config -m ${CONTROLLER}:${MODEL} kubernetes-worker allow-privileged=true
+juju config -m ${CONTROLLER}:${MODEL} kubernetes-master allow-privileged=true
 
 # NOTE This script only deploys the bundle and charms, no waiting is done here!
 

tests/e2e-gubernator.sh

@@ -10,6 +10,7 @@ source "utils/retry.sh"
 
 # The cloud is an option for this script, default to gce.
 CLOUD=${CLOUD:-"gce"}
+CONTROLLER=${CONTROLLER:-"jenkins-ci-google"}
 # The directory to use for this script, should be WORKSPACE, but can be PWD.
 SCRIPT_DIRECTORY=${WORKSPACE:-${PWD}}
 
@@ -25,18 +26,18 @@ export JUJU_REPOSITORY=${SCRIPT_DIRECTORY}/build/charms
 
 mkdir -p ${JUJU_REPOSITORY}
 # Catch all EXITs from this script and make sure to destroy the model.
-trap "sleep 10 && juju destroy-model -y ${MODEL}" EXIT
+trap "sleep 10 && juju destroy-model -y ${CONTROLLER}:${MODEL}" EXIT
 
 # Deploy the bundle and add the kubernetes-e2e charm.
-${SCRIPT_DIRECTORY}/tests/deploy-test-bundle.sh ${MODEL} ${BUNDLE}
+${SCRIPT_DIRECTORY}/tests/deploy-test-bundle.sh ${CONTROLLER} ${MODEL} ${BUNDLE}
 
 # Let the deployment complete.
-${SCRIPT_DIRECTORY}/tests/wait-cluster-ready.sh
+${SCRIPT_DIRECTORY}/tests/wait-cluster-ready.sh ${CONTROLLER} ${MODEL}
 
 # Run the end to end tests and copy results to the output directory.
 # Retry 3 times. The second and third retries should be quick since
 # we have any images cached
-retry ${SCRIPT_DIRECTORY}/tests/run-e2e-tests.sh ${OUTPUT_DIRECTORY}
+retry ${SCRIPT_DIRECTORY}/tests/run-e2e-tests.sh ${CONTROLLER} ${MODEL} ${OUTPUT_DIRECTORY}
 
 # Formats the output data and upload to GCE.
 ${SCRIPT_DIRECTORY}/tests/upload-e2e-results-to-gubernator.sh ${OUTPUT_DIRECTORY}

tests/run-e2e-tests.sh

@@ -7,23 +7,27 @@ set -o xtrace  # Print the commands that are executed.
 
 echo "${0} started at `date`."
 
-# The first argument is the output directory.
-OUTPUT_DIRECTORY=${1:-"artifacts"}
+# First argument is the controller name for this build.
+CONTROLLER=${1:-"controller-is-undefined"}
+# Second argument is the model name for this build.
+MODEL=${2:-"model-is-undefined"}
+# Third argument is the output directory.
+OUTPUT_DIRECTORY=${3:-"artifacts"}
 
 # Create the output directory.
 mkdir -p ${OUTPUT_DIRECTORY}
 
 # Run the e2e test action.
-ACTION_ID=$(juju run-action kubernetes-e2e/0 test | cut -d " " -f 5)
-# Wait 2 hour for the action to complete.
-juju show-action-output --wait=2h ${ACTION_ID}
-# Print out the action result.
-outcome=`juju show-action-output ${ACTION_ID}`
-echo $outcome
+ACTION_ID=$(juju run-action -m ${CONTROLLER}:${MODEL} kubernetes-e2e/0 test | cut -d " " -f 5)
+# Show the action results (wait up to 2 hours for the action to finish)
+outcome=$(juju show-action-output -m ${CONTROLLER}:${MODEL} --wait=2h ${ACTION_ID})
+echo ${outcome}
 
 # Download results from the charm and move them to the the volume directory.
-juju scp kubernetes-e2e/0:${ACTION_ID}.log.tar.gz e2e.log.tar.gz
-juju scp kubernetes-e2e/0:${ACTION_ID}-junit.tar.gz e2e-junit.tar.gz
+juju scp -m ${CONTROLLER}:${MODEL} kubernetes-e2e/0:${ACTION_ID}.log.tar.gz \
+  e2e.log.tar.gz
+juju scp -m ${CONTROLLER}:${MODEL} kubernetes-e2e/0:${ACTION_ID}-junit.tar.gz \
+  e2e-junit.tar.gz
 
 if [[ "$outcome" == *"failed"* ]]
 then

tests/wait-cluster-ready.sh

@@ -10,15 +10,33 @@ echo "${0} started at `date`."
 # Define the utility functions such as run_and_wait.
 source ./utilities.sh
 
+# First argument is the controller name for this build.
+CONTROLLER=${1:-"controller-is-undefined"}
+# Second argument is the model name for this build.
+MODEL=${2:-"model-is-undefined"}
+
 # Wait in 10 second increments for the master charm to print running in status.
-run_and_wait "juju status kubernetes-master" "Kubernetes master running." 10
-wait_for_ready
+run_and_wait \
+  "juju status -m ${CONTROLLER}:${MODEL} kubernetes-master" \
+  "Kubernetes master running." \
+  10
+
+# Master is running; ensure the rest of the deployment is ready (nothing in
+# blocked, maintenance, error, etc).
+wait_for_ready \
+  "juju status -m ${CONTROLLER}:${MODEL}"
+
 # Print out a full juju status.
-juju status
+juju status -m ${CONTROLLER}:${MODEL}
 
 # Wait in 10 second increments for "KubeDNS" to show up in cluster-info output.
-run_and_wait 'juju run --application kubernetes-master /snap/bin/kubectl cluster-info' "KubeDNS" 10
+run_and_wait \
+  "juju run -m ${CONTROLLER}:${MODEL} --application kubernetes-master /snap/bin/kubectl cluster-info" \
+  "KubeDNS" \
+  10
+
 # Print out the cluster-info
-juju run --application kubernetes-master /snap/bin/kubectl cluster-info
+juju run -m ${CONTROLLER}:${MODEL} --application kubernetes-master \
+  "/snap/bin/kubectl cluster-info"
 
 echo "${0} completed successfully at `date`."

utilities.sh

@@ -40,7 +40,7 @@ function run_and_wait() {
   local sleep_seconds=${3:-5}
   local start_time=`date +"%s"`
   # Run the command in a loop looking for output.
-  until $(${cmd} | grep -q "${match}"); do 
+  until $(${cmd} | grep -q "${match}"); do
     # Check the time so this does not loop forever.
     check_time ${start_time} ${MAXIMUM_WAIT_SECONDS}
     sleep ${sleep_seconds}
@@ -49,7 +49,7 @@ function run_and_wait() {
 
 # Wait for deployment to get ready.
 function wait_for_ready() {
-  local cmd='juju status'
+  local cmd=${1:-"juju status"}
   local sleep_seconds=30
   local start_time=`date +"%s"`
   # NB: If lines == 0, grep $? > 0 (wc -l got no lines). This will fail scripts