Run kubernetes-e2e-g[ck]e-flaky inside Docker

hack/jenkins/dockerized-e2e-runner.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Copyright 2016 The Kubernetes Authors All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Save environment variables in $WORKSPACE/env.list and then run the Jenkins e2e
+# test runner inside the kubekins-test Docker image.
+
+env -u HOME -u PATH -u PWD -u WORKSPACE >${WORKSPACE}/env.list
+docker run --rm=true -i \
+  -v "${WORKSPACE}/_artifacts":/workspace/_artifacts \
+  -v /etc/localtime:/etc/localtime:ro \
+  -v /var/lib/jenkins/gce_keys:/workspace/.ssh:ro \
+  --env-file "${WORKSPACE}/env.list" \
+  -e "HOME=/workspace" \
+  -e "WORKSPACE=/workspace" \
+  gcr.io/google_containers/kubekins-test:0.9 \
+  bash -c "bash <(curl -fsS --retry 3 'https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh')"

hack/jenkins/e2e-runner.sh

@@ -21,6 +21,10 @@ set -o nounset
 set -o pipefail
 set -o xtrace
 
+function running_in_docker() {
+    grep -q docker /proc/self/cgroup
+}
+
 function check_dirty_workspace() {
     if [[ "${JENKINS_TOLERATE_DIRTY_WORKSPACE:-}" =~ ^[yY]$ ]]; then
         echo "Tolerating dirty workspace (because JENKINS_TOLERATE_DIRTY_WORKSPACE)."
@@ -143,6 +147,43 @@ function get_latest_trusty_image() {
     rm -rf .gsutil &> /dev/null
 }
 
+function install_google_cloud_sdk_tarball() {
+    local -r tarball=$1
+    local -r install_dir=$2
+    mkdir -p "${install_dir}"
+    tar xzf "${tarball}" -C "${install_dir}"
+
+    export CLOUDSDK_CORE_DISABLE_PROMPTS=1
+    "${install_dir}/google-cloud-sdk/install.sh" --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false
+    export PATH=${install_dir}/google-cloud-sdk/bin:${PATH}
+}
+
+### Pre Set Up ###
+if running_in_docker; then
+    curl -fsSL --retry 3 -o "${WORKSPACE}/google-cloud-sdk.tar.gz" 'https://dl.google.com/dl/cloudsdk/channels/rapid/google-cloud-sdk.tar.gz'
+    install_google_cloud_sdk_tarball "${WORKSPACE}/google-cloud-sdk.tar.gz" /
+fi
+
+# Install gcloud from a custom path if provided. Used to test GKE with gcloud
+# at HEAD, release candidate.
+# TODO: figure out how to avoid installing the cloud sdk twice if run inside Docker.
+if [[ -n "${CLOUDSDK_BUCKET:-}" ]]; then
+    # Retry the download a few times to mitigate transient server errors and
+    # race conditions where the bucket contents change under us as we download.
+    for n in $(seq 3); do
+        gsutil -mq cp -r "${CLOUDSDK_BUCKET}" ~ && break || sleep 1
+        # Delete any temporary files from the download so that we start from
+        # scratch when we retry.
+        rm -rf ~/.gsutil
+    done
+    rm -rf ~/repo ~/cloudsdk
+    mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo
+    export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json
+    install_google_cloud_sdk_tarball ~/repo/google-cloud-sdk.tar.gz ~/cloudsdk
+    # TODO: is this necessary? this won't work inside Docker currently.
+    export CLOUDSDK_CONFIG=/var/lib/jenkins/.config/gcloud
+fi
+
 # We get the image project and name for Trusty dynamically.
 if [[ "${JENKINS_USE_TRUSTY_IMAGES:-}" =~ ^[yY]$ ]]; then
   trusty_image_project="$(get_trusty_image_project)"
@@ -212,13 +253,23 @@ fi
 # # Move the permissions for the keys to Jenkins.
 # $ sudo chown -R jenkins /var/lib/jenkins/gce_keys/
 # $ sudo chgrp -R jenkins /var/lib/jenkins/gce_keys/
-if [[ "${KUBERNETES_PROVIDER}" == "aws" ]]; then
-    echo "Skipping SSH key copying for AWS"
-else
-    mkdir -p ${WORKSPACE}/.ssh/
-    cp /var/lib/jenkins/gce_keys/google_compute_engine ${WORKSPACE}/.ssh/
-    cp /var/lib/jenkins/gce_keys/google_compute_engine.pub ${WORKSPACE}/.ssh/
-fi
+case "${KUBERNETES_PROVIDER}" in
+    gce|gke|kubemark)
+        if ! running_in_docker; then
+            mkdir -p ${WORKSPACE}/.ssh/
+            cp /var/lib/jenkins/gce_keys/google_compute_engine ${WORKSPACE}/.ssh/
+            cp /var/lib/jenkins/gce_keys/google_compute_engine.pub ${WORKSPACE}/.ssh/
+        fi
+        if [[ ! -f ${WORKSPACE}/.ssh/google_compute_engine ]]; then
+            echo "google_compute_engine ssh key missing!"
+            exit 1
+        fi
+        ;;
+
+    default)
+        echo "Not copying ssh keys for ${KUBERNETES_PROVIDER}"
+        ;;
+esac
 
 cd kubernetes
 
@@ -231,6 +282,9 @@ fi
 # Have cmd/e2e run by goe2e.sh generate JUnit report in ${WORKSPACE}/junit*.xml
 ARTIFACTS=${WORKSPACE}/_artifacts
 mkdir -p ${ARTIFACTS}
+# When run inside Docker, we need to make sure all files are world-readable
+# (since they will be owned by root on the host).
+trap "chmod -R o+r '${ARTIFACTS}'" EXIT SIGINT SIGTERM
 export E2E_REPORT_DIR=${ARTIFACTS}
 declare -r gcp_list_resources_script="./cluster/gce/list-resources.sh"
 declare -r gcp_resources_before="${ARTIFACTS}/gcp-resources-before.txt"
@@ -244,29 +298,6 @@ else
   gcp_list_resources="false"
 fi
 
-### Pre Set Up ###
-# Install gcloud from a custom path if provided. Used to test GKE with gcloud
-# at HEAD, release candidate.
-if [[ -n "${CLOUDSDK_BUCKET:-}" ]]; then
-    # Retry the download a few times to mitigate transient server errors and
-    # race conditions where the bucket contents change under us as we download.
-    for n in $(seq 3); do
-        gsutil -mq cp -r "${CLOUDSDK_BUCKET}" ~ && break || sleep 1
-        # Delete any temporary files from the download so that we start from
-        # scratch when we retry.
-        rm -rf ~/.gsutil
-    done
-    rm -rf ~/repo ~/cloudsdk
-    mv ~/$(basename "${CLOUDSDK_BUCKET}") ~/repo
-    mkdir ~/cloudsdk
-    tar zxf ~/repo/google-cloud-sdk.tar.gz -C ~/cloudsdk
-    export CLOUDSDK_CORE_DISABLE_PROMPTS=1
-    export CLOUDSDK_COMPONENT_MANAGER_SNAPSHOT_URL=file://${HOME}/repo/components-2.json
-    ~/cloudsdk/google-cloud-sdk/install.sh --disable-installation-options --bash-completion=false --path-update=false --usage-reporting=false
-    export PATH=${HOME}/cloudsdk/google-cloud-sdk/bin:${PATH}
-    export CLOUDSDK_CONFIG=/var/lib/jenkins/.config/gcloud
-fi
-
 ### Set up ###
 if [[ "${E2E_UP,,}" == "true" ]]; then
     go run ./hack/e2e.go ${E2E_OPT:-} -v --down

hack/jenkins/job-configs/global.yaml

@@ -138,6 +138,7 @@
     branch: 'master'
     job-env: ''
     runner: bash <(curl -fsS --retry 3  "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/e2e-runner.sh")
+    dockerized-runner: bash <(curl -fsS --retry 3  "https://raw.githubusercontent.com/kubernetes/kubernetes/master/hack/jenkins/dockerized-e2e-runner.sh")
     old-runner-1-1: bash <(curl -fsS --retry 3  "https://raw.githubusercontent.com/kubernetes/kubernetes/release-1.1/hack/jenkins/e2e.sh")
     old-runner-1-0: bash <(curl -fsS --retry 3  "https://raw.githubusercontent.com/kubernetes/kubernetes/release-1.0/hack/jenkins/e2e.sh")
     provider-env: ''

hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml

@@ -15,6 +15,8 @@
     description: '{description} Test owner: {test-owner}.'
     logrotate:
         daysToKeep: 7
+    node: '{jenkins_node}'
+    jenkins_node: 'master'
     disabled: '{obj:disable_job}'
     builders:
         - shell: |
@@ -105,6 +107,8 @@
                 export KUBE_ADMISSION_CONTROL="NamespaceLifecycle,InitialResources,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota"
         - 'gce-flaky':
             description: 'Run the flaky tests on GCE, sequentially.'
+            jenkins_node: 'e2e'
+            runner: '{dockerized-runner}'
             timeout: 180
             job-env: |
                 export GINKGO_TEST_ARGS="--ginkgo.focus=\[Flaky\] \
@@ -213,6 +217,8 @@
                 - client (kubectl): ci/latest.txt<br>
                 - cluster (k8s): ci/latest.txt<br>
                 - tests: ci/latest.txt
+            jenkins_node: 'e2e'
+            runner: '{dockerized-runner}'
             timeout: 300
             job-env: |
                 export PROJECT="k8s-jkns-e2e-gke-ci-flaky"