Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add CI jobs to track the master and 1.2 branch with Trusty images #23086

Merged
merged 1 commit into from
Mar 18, 2016
Merged

Add CI jobs to track the master and 1.2 branch with Trusty images #23086

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
58 changes: 58 additions & 0 deletions hack/jenkins/e2e-runner.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -94,11 +94,69 @@ function unpack_binaries() {
tar -xzf kubernetes-test.tar.gz
}

# GCP Project to fetch Trusty images.
function get_trusty_image_project() {
local project=""
# Retry the gsutil command a couple times to mitigate the effect of
# transient server errors.
for n in $(seq 3); do
project="$(gsutil cat "gs://trusty-images/image-project.txt")" && break || sleep 1
done
if [[ -z "${project}" ]]; then
echo "Failed to find the image project for Trusty images."
exit 1
fi
echo "${project}"
# Clean up gsutil artifacts otherwise the later test stage will complain.
rm -rf .config &> /dev/null
rm -rf .gsutil &> /dev/null
}

# Get the latest Trusty image for a Jenkins job.
function get_latest_trusty_image() {
local image_project="$1"
local image_type="$2"
local image_index=""
if [[ "${image_type}" == head ]]; then
image_index="trusty-head"
elif [[ "${image_type}" == dev ]]; then
image_index="trusty-dev"
elif [[ "${image_type}" == beta ]]; then
image_index="trusty-beta"
elif [[ "${image_type}" == stable ]]; then
image_index="trusty-stable"
fi

local image=""
# Retry the gsutil command a couple times to mitigate the effect of
# transient server errors.
for n in $(seq 3); do
image="$(gsutil cat "gs://${image_project}/image-indices/latest-test-image-${image_index}")" && break || sleep 1
done
if [[ -z "${image}" ]]; then
echo "Failed to find Trusty image for ${image_type}"
exit 1
fi
echo "${image}"
# Clean up gsutil artifacts otherwise the later test stage will complain.
rm -rf .config &> /dev/null
rm -rf .gsutil &> /dev/null
}

echo "--------------------------------------------------------------------------------"
echo "Test Environment:"
printenv | sort
echo "--------------------------------------------------------------------------------"

# We get the image project and name for Trusty dynamically.
if [[ "${JENKINS_USE_TRUSTY_IMAGES:-}" =~ ^[yY]$ ]]; then
trusty_image_project="$(get_trusty_image_project)"
trusty_image="$(get_latest_trusty_image "${trusty_image_project}" "head")"
export KUBE_MASTER_IMAGE_PROJECT="${trusty_image_project}"
export KUBE_MASTER_IMAGE="${trusty_image}"
export KUBE_OS_DISTRIBUTION="trusty"
fi

# We get the Kubernetes tarballs unless we are going to use old ones
if [[ "${JENKINS_USE_EXISTING_BINARIES:-}" =~ ^[yY]$ ]]; then
echo "Using existing binaries; not cleaning, fetching, or unpacking new ones."
Expand Down
288 changes: 191 additions & 97 deletions hack/jenkins/job-configs/kubernetes-jenkins/kubernetes-e2e.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,25 +40,7 @@
- timestamps
- workspace-cleanup

# Template for e2e test jobs that run on GCE with Trusty images.
- job-template:
name: 'kubernetes-e2e-gce-trusty-{suffix}'
<<: *e2e_job_defaults
triggers:
- timed: 'H H/8 * * *'
publishers:
- e2e-publishers:
recipients: '{emails}'
- description-setter:
regexp: KUBE_GCE_MINION_IMAGE=(.*)
- groovy-postbuild:
script: |
def trustyImageMatcher = manager.getLogMatcher("KUBE_GCE_MINION_IMAGE=(.*)")
if(trustyImageMatcher?.matches()) manager.addShortText("<b>Trusty Image: " + trustyImageMatcher.group(1) + "</b>", "grey", "white", "0px", "white")
def k8sVersionMatcher = manager.getLogMatcher("Using\\spublished\\sversion\\s(.*)\\s\\(from.*")
if(k8sVersionMatcher?.matches()) manager.addShortText("<br><b>Kubernetes version: " + k8sVersionMatcher.group(1) + "</b>", "grey", "white", "0px", "white")

# Template for the rest of e2e test jobs.
# Template for most e2e test jobs.
- job-template:
name: 'kubernetes-e2e-{suffix}'
<<: *e2e_job_defaults
Expand Down Expand Up @@ -285,7 +267,7 @@
export TEST_CLUSTER_DELETE_COLLECTION_WORKERS="--delete-collection-workers=16"
jobs:
- 'kubernetes-e2e-{suffix}'

- project:
name: kubernetes-e2e-gke-1-2
trigger-job: 'kubernetes-build-1.2'
Expand Down Expand Up @@ -390,47 +372,6 @@
jobs:
- 'kubernetes-e2e-{suffix}'

- project:
name: kubernetes-e2e-gke-trusty
trigger-job: 'kubernetes-build-1.1'
test-owner: 'wonderfly@google.com'
branch: 'release-1.1'
runner: '{old-runner-1-1}'
post-env: ''
emails: 'wonderfly@google.com,qzheng@google.com'
suffix:
- 'gke-trusty-prod':
timeout: 180
description: |
Run e2e tests with Trusty as node image using the following config:<br>
- provider: GKE<br>
- api proxy: prod<br>
- borg job: prod<br>
- client (kubectl): release/stable.txt<br>
- cluster (k8s): release/stable.txt<br>
- tests: release/stable.txt
- 'gke-trusty-staging':
timeout: 300
description: |
Run e2e tests with Trusty as node image using the following config:<br>
- provider: GKE<br>
- api proxy: staging<br>
- borg job: staging<br>
- client (kubectl): release/stable.txt<br>
- cluster (k8s): release/stable.txt<br>
- tests: release/stable.txt
- 'gke-trusty-test':
timeout: 300
description: |
Run e2e tests with Trusty as node image using the following config:<br>
- provider: GKE<br>
- api proxy: staging<br>
- borg job: test<br>
- client (kubectl): release/stable.txt<br>
- cluster (k8s): release/stable.txt<br>
- tests: release/stable.txt
jobs:
- 'kubernetes-e2e-{suffix}'

- project:
name: kubernetes-e2e-gce-1.1
Expand Down Expand Up @@ -543,6 +484,154 @@
jobs:
- 'kubernetes-e2e-{suffix}'

- project:
name: kubernetes-e2e-gce-enormous-cluster
test-owner: 'gmarek'
emails: 'gmarek@google.com'
cron-string: 'H 8 * * *'
trigger-job: ''
description: 'Starts and deletes empty 1000 node cluster and runs Density 30 test on it. Does allow few Nodes to fail during startup.'
timeout: 480
branch: 'master'
suffix: 'gce-enormous-cluster'
provider-env: '{gce-provider-env}'
job-env: |
# XXX Not a unique project
export E2E_NAME="e2e-enormous-cluster"
export GINKGO_TEST_ARGS="--ginkgo.focus=\[Feature:Performance\]"
export PROJECT="kubernetes-scale"
export FAIL_ON_GCP_RESOURCE_LEAK="false"
# Override GCE defaults.
export NETWORK_PROVIDER="flannel"
# Temporarily switch of Heapster, as this will not schedule anywhere.
# TODO: Think of a solution to enable it.
export KUBE_ENABLE_CLUSTER_MONITORING="none"
export KUBE_GCE_ZONE="us-central1-c"
export MASTER_SIZE="n1-standard-32"
export NODE_SIZE="n1-standard-1"
export NODE_DISK_SIZE="50GB"
export NUM_NODES="1000"
export ALLOWED_NOTREADY_NODES="2"
export EXIT_ON_WEAK_ERROR="false"
# Reduce logs verbosity
export TEST_CLUSTER_LOG_LEVEL="--v=1"
export MAX_INSTANCES_PER_MIG="1000"
# Increase resync period to simulate production
export TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h"
# Increase delete collection parallelism
export TEST_CLUSTER_DELETE_COLLECTION_WORKERS="--delete-collection-workers=16"
jobs:
- 'kubernetes-e2e-{suffix}'

# ==============================================================================
# NOTE: From here on all jobs use Trusty as the image for master and/or nodes.
# Please add templates/groups/projects/jobs that use ContainerVm above/below
# this section (search "End of Trusty jobs" for the ending separator).

# Template for e2e test jobs that run on GCE with Trusty images against
# kubernetes HEAD.
- job-template:
name: 'kubernetes-e2e-gce-trusty-ci-{suffix}'
<<: *e2e_job_defaults
triggers:
- reverse:
jobs: '{trigger-job}'
result: success
- timed: '{cron-string}'
publishers:
- e2e-publishers:
recipients: '{emails}'
- description-setter:
regexp: KUBE_GCE_MASTER_IMAGE=(.*)
- groovy-postbuild:
script: |
def trustyImageMatcher = manager.getLogMatcher("KUBE_GCE_MASTER_IMAGE=(.*)")
if(trustyImageMatcher?.matches()) manager.addShortText("<b>Trusty Image: " + trustyImageMatcher.group(1) + "</b>", "grey", "white", "0px", "white")
def k8sVersionMatcher = manager.getLogMatcher("Using\\spublished\\sversion\\s(.*)\\s\\(from.*")
if(k8sVersionMatcher?.matches()) manager.addShortText("<br><b>Kubernetes version: " + k8sVersionMatcher.group(1) + "</b>", "grey", "white", "0px", "white")

# CI jobs that run on GCE with both master and nodes running Trusty images. We
# use these to guard k8s+Trusty compatibility.
- project:
name: kubernetes-e2e-gce-trusty-ci-master
trigger-job: 'kubernetes-build'
test-owner: 'wonderfly@google.com'
emails: 'wonderfly@google.com,qzheng@google.com'
provider-env: |
{gce-provider-env}
export JENKINS_USE_TRUSTY_IMAGES="y"
suffix:
# TODO(wonderfly): For Trusty, we currently only run CI and slow tests.
# More test coverage under way.
- 'master':
description: 'Runs all non-slow, non-serial, non-flaky, tests on GCE with Trusty images in parallel on the master branch.'
timeout: 30
job-env: |
export GINKGO_TEST_ARGS="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
export GINKGO_PARALLEL="y"
export PROJECT="e2e-gce-trusty-ci-master"
- 'slow-master':
description: 'Runs slow tests on GCE with Trusty images, sequentially on the master branch.'
timeout: 60
job-env: |
export GINKGO_TEST_ARGS="--ginkgo.focus=\[Slow\] \
--ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
export GINKGO_PARALLEL="y"
export PROJECT="e2e-gce-trusty-ci-master-slow"
jobs:
- 'kubernetes-e2e-gce-trusty-ci-{suffix}'

- project:
name: kubernetes-e2e-gce-trusty-ci-1-2
trigger-job: 'kubernetes-build-1.2'
test-owner: 'wonderfly@google.com'
emails: 'wonderfly@google.com,qzheng@google.com'
provider-env: |
{gce-provider-env}
export JENKINS_PUBLISHED_VERSION="ci/latest-1.2"
export JENKINS_USE_TRUSTY_IMAGES="y"
suffix:
# TODO(wonderfly): For Trusty, we currently only run CI and slow tests.
# More test coverage under way.
- 'release-1.2':
description: 'Runs all non-slow, non-serial, non-flaky, tests on GCE with Trusty images in parallel on the release-1.2 branch.'
timeout: 30
job-env: |
export GINKGO_TEST_ARGS="--ginkgo.skip=\[Slow\]|\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
export GINKGO_PARALLEL="y"
export PROJECT="e2e-gce-trusty-ci-1-2"
- 'slow-release-1.2':
description: 'Runs slow tests on GCE with Trusty images, sequentially on the release-1.2 branch.'
timeout: 60
job-env: |
export GINKGO_TEST_ARGS="--ginkgo.focus=\[Slow\] \
--ginkgo.skip=\[Serial\]|\[Disruptive\]|\[Flaky\]|\[Feature:.+\]"
export GINKGO_PARALLEL="y"
export PROJECT="e2e-gce-trusty-ci-slow-1-2"
jobs:
- 'kubernetes-e2e-gce-trusty-ci-{suffix}'

# Template for e2e test jobs that run on GCE with Trusty images against last
# released k8s version. We use these jobs to test Trusty's continuous build.
- job-template:
name: 'kubernetes-e2e-gce-trusty-{suffix}'
<<: *e2e_job_defaults
triggers:
- timed: 'H H/8 * * *'
publishers:
- e2e-publishers:
recipients: '{emails}'
- description-setter:
regexp: KUBE_GCE_MINION_IMAGE=(.*)
- groovy-postbuild:
script: |
def trustyImageMatcher = manager.getLogMatcher("KUBE_GCE_MINION_IMAGE=(.*)")
if(trustyImageMatcher?.matches()) manager.addShortText("<b>Trusty Image: " + trustyImageMatcher.group(1) + "</b>", "grey", "white", "0px", "white")
def k8sVersionMatcher = manager.getLogMatcher("Using\\spublished\\sversion\\s(.*)\\s\\(from.*")
if(k8sVersionMatcher?.matches()) manager.addShortText("<br><b>Kubernetes version: " + k8sVersionMatcher.group(1) + "</b>", "grey", "white", "0px", "white")

# Jobs that run e2e tests on GCE with Trusty images against the latest k8s
# release.
- project:
name: kubernetes-e2e-gce-trusty
test-owner: 'wonderfly@google.com'
Expand All @@ -569,6 +658,7 @@
- 'dev-slow':
# Constantly failing due to a known issue of the image. Disabled
# until the issue is resolved.
# TODO(wonderfly): Re-enable it once the internal issue is fixed.
disable_job: true
description: 'Run slow E2E tests on latest Trusty dev build.'
timeout: 270
Expand All @@ -581,42 +671,46 @@
jobs:
- 'kubernetes-e2e-gce-trusty-{suffix}'

# Jobs that run e2e tests on GKE with Trusty as node image on the release-1.1 branch.
- project:
name: kubernetes-e2e-gce-enormous-cluster
test-owner: 'gmarek'
emails: 'gmarek@google.com'
cron-string: 'H 8 * * *'
trigger-job: ''
description: 'Starts and deletes empty 1000 node cluster and runs Density 30 test on it. Does allow few Nodes to fail during startup.'
timeout: 480
branch: 'master'
suffix: 'gce-enormous-cluster'
provider-env: '{gce-provider-env}'
job-env: |
# XXX Not a unique project
export E2E_NAME="e2e-enormous-cluster"
export GINKGO_TEST_ARGS="--ginkgo.focus=\[Feature:Performance\]"
export PROJECT="kubernetes-scale"
export FAIL_ON_GCP_RESOURCE_LEAK="false"
# Override GCE defaults.
export NETWORK_PROVIDER="flannel"
# Temporarily switch of Heapster, as this will not schedule anywhere.
# TODO: Think of a solution to enable it.
export KUBE_ENABLE_CLUSTER_MONITORING="none"
export KUBE_GCE_ZONE="us-central1-c"
export MASTER_SIZE="n1-standard-32"
export NODE_SIZE="n1-standard-1"
export NODE_DISK_SIZE="50GB"
export NUM_NODES="1000"
export ALLOWED_NOTREADY_NODES="2"
export EXIT_ON_WEAK_ERROR="false"
# Reduce logs verbosity
export TEST_CLUSTER_LOG_LEVEL="--v=1"
export MAX_INSTANCES_PER_MIG="1000"
# Increase resync period to simulate production
export TEST_CLUSTER_RESYNC_PERIOD="--min-resync-period=12h"
# Increase delete collection parallelism
export TEST_CLUSTER_DELETE_COLLECTION_WORKERS="--delete-collection-workers=16"
name: kubernetes-e2e-gke-trusty
trigger-job: 'kubernetes-build-1.1'
test-owner: 'wonderfly@google.com'
branch: 'release-1.1'
runner: '{old-runner-1-1}'
post-env: ''
emails: 'wonderfly@google.com,qzheng@google.com'
suffix:
- 'gke-trusty-prod':
timeout: 180
description: |
Run e2e tests with Trusty as node image using the following config:<br>
- provider: GKE<br>
- api proxy: prod<br>
- borg job: prod<br>
- client (kubectl): release/stable.txt<br>
- cluster (k8s): release/stable.txt<br>
- tests: release/stable.txt
- 'gke-trusty-staging':
timeout: 300
description: |
Run e2e tests with Trusty as node image using the following config:<br>
- provider: GKE<br>
- api proxy: staging<br>
- borg job: staging<br>
- client (kubectl): release/stable.txt<br>
- cluster (k8s): release/stable.txt<br>
- tests: release/stable.txt
- 'gke-trusty-test':
timeout: 300
description: |
Run e2e tests with Trusty as node image using the following config:<br>
- provider: GKE<br>
- api proxy: staging<br>
- borg job: test<br>
- client (kubectl): release/stable.txt<br>
- cluster (k8s): release/stable.txt<br>
- tests: release/stable.txt
jobs:
- 'kubernetes-e2e-{suffix}'

#============================== End of Trusty jobs =============================