Skip to content

Commit

Permalink
Re-introduce Jenkins timeout for SI tests (#6714)
Browse files Browse the repository at this point in the history 
Summary:
We already use `timeout` for the SI test execution, however, it only covers the tests themselves and not cluster creation/removal. This leads to some "stuck jobs" in cases where e.g. dcos-launch is stuck removing the cluster. This is hardly a proper solution but at least we don't overload Jenkins with never-ending jobs.
  • Loading branch information
Aleksey Dukhovniy committed Dec 7, 2018
1 parent a17310a commit 93cc18d
Show file tree
Hide file tree
Showing 4 changed files with 161 additions and 153 deletions.
78 changes: 40 additions & 38 deletions Jenkinsfile.open.si
View file
Original file line number Diff line number Diff line change
@@ -1,45 +1,47 @@
#!/usr/bin/env groovy

node('py36') {
wrap([$class: 'MesosSingleUseSlave']) {
wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {
timeout(time: 3, unit: 'HOURS') {
node('py36') {
wrap([$class: 'MesosSingleUseSlave']) {
wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {

// Hardcoded default values are:
// DC/OS OS: channel="testing/pull/1739", for open cluster
// DC/OS EE: channel="testing/pull/1296", for strict or permissive clusters
// The channel can be passed to the job in Jenkins (Build with Parameters) so that this job can be executed against any PR/cluster.
properties([
// Run pipeline every 2h.
pipelineTriggers([cron('H H(0-23)/2 * * *')]),
parameters([
string(name: 'channel',
defaultValue: 'testing/pull/1739',
description: 'PR to start the cluster from e.g. testing/pull/1739'
)]
)
])
// Hardcoded default values are:
// DC/OS OS: channel="testing/pull/1739", for open cluster
// DC/OS EE: channel="testing/pull/1296", for strict or permissive clusters
// The channel can be passed to the job in Jenkins (Build with Parameters) so that this job can be executed against any PR/cluster.
properties([
// Run pipeline every 2h.
pipelineTriggers([cron('H H(0-23)/2 * * *')]),
parameters([
string(name: 'channel',
defaultValue: 'testing/pull/1739',
description: 'PR to start the cluster from e.g. testing/pull/1739'
)]
)
])

stage("Run System Integration Test Pipeline") {
try {
checkout scm
withCredentials(
[ [$class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'mesosphere-ci-marathon', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'],
file(credentialsId: '11fcc957-5156-4470-ae34-d433da88248a', variable: 'DOT_SHAKEDOWN'),
string(credentialsId: '7bdd2775-2911-41ba-918f-59c8ae52326d', variable: 'DOCKER_HUB_USERNAME'),
string(credentialsId: '42f2e3fb-3f4f-47b2-a128-10ac6d0f6825', variable: 'DOCKER_HUB_PASSWORD'),
string(credentialsId: '66c40969-a46d-470e-b8a2-6f04f2b3f2d5', variable: 'DATADOG_API_KEY')
]) {
sh """./ci/si_pipeline.sh $params.channel open"""
}
} finally {
junit allowEmptyResults: true, testResults: "**/shakedown.xml"
archive includes: "**/diagnostics.zip"
archive includes: "**/sandbox_*.tar.gz"
archive includes: "**/preflight*.log"
archive includes: "**/postflight*.log"
archive includes: "**/deploy*.log"
stage("Run System Integration Test Pipeline") {
try {
checkout scm
withCredentials(
[[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'mesosphere-ci-marathon', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'],
file(credentialsId: '11fcc957-5156-4470-ae34-d433da88248a', variable: 'DOT_SHAKEDOWN'),
string(credentialsId: '7bdd2775-2911-41ba-918f-59c8ae52326d', variable: 'DOCKER_HUB_USERNAME'),
string(credentialsId: '42f2e3fb-3f4f-47b2-a128-10ac6d0f6825', variable: 'DOCKER_HUB_PASSWORD'),
string(credentialsId: '66c40969-a46d-470e-b8a2-6f04f2b3f2d5', variable: 'DATADOG_API_KEY')
]) {
sh """./ci/si_pipeline.sh $params.channel open"""
}
} finally {
junit allowEmptyResults: true, testResults: "**/shakedown.xml"
archive includes: "**/diagnostics.zip"
archive includes: "**/sandbox_*.tar.gz"
archive includes: "**/preflight*.log"
archive includes: "**/postflight*.log"
archive includes: "**/deploy*.log"
}
}
}
}
}
}
}
}
84 changes: 43 additions & 41 deletions Jenkinsfile.permissive.si
View file
Original file line number Diff line number Diff line change
@@ -1,47 +1,49 @@
#!/usr/bin/env groovy

node('py36') {
wrap([$class: 'MesosSingleUseSlave']) {
wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {
timeout(time: 3, unit: 'HOURS') {
node('py36') {
wrap([$class: 'MesosSingleUseSlave']) {
wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {

// Hardcoded default values are:
// DC/OS OS: channel="testing/pull/1739", for open cluster
// DC/OS EE: channel="testing/pull/1296", for strict or permissive clusters
// The channel can be passed to the job in Jenkins (Build with Parameters) so that this job can be executed against any PR/cluster.
properties([
// Run pipeline every 2h.
pipelineTriggers([cron('H H(0-23)/2 * * *')]),
parameters([
string(name: 'channel',
defaultValue: 'testing/pull/1296',
description: 'PR to start the cluster from e.g. testing/pull/1739'
)]
)
])
// Hardcoded default values are:
// DC/OS OS: channel="testing/pull/1739", for open cluster
// DC/OS EE: channel="testing/pull/1296", for strict or permissive clusters
// The channel can be passed to the job in Jenkins (Build with Parameters) so that this job can be executed against any PR/cluster.
properties([
// Run pipeline every 2h.
pipelineTriggers([cron('H H(0-23)/2 * * *')]),
parameters([
string(name: 'channel',
defaultValue: 'testing/pull/1296',
description: 'PR to start the cluster from e.g. testing/pull/1739'
)]
)
])

stage("Run System Integration Test Pipeline") {
try {
checkout scm
withCredentials(
[ [$class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'mesosphere-ci-marathon', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'],
file(credentialsId: '11fcc957-5156-4470-ae34-d433da88248a', variable: 'DOT_SHAKEDOWN'),
usernamePassword(credentialsId: '323df884-742b-4099-b8b7-d764e5eb9674', passwordVariable: 'DCOS_PASSWORD', usernameVariable: 'DCOS_USERNAME'),
string(credentialsId: 'ca159ad3-7323-4564-818c-46a8f03e1389', variable: 'DCOS_LICENSE'),
string(credentialsId: '7bdd2775-2911-41ba-918f-59c8ae52326d', variable: 'DOCKER_HUB_USERNAME'),
string(credentialsId: '42f2e3fb-3f4f-47b2-a128-10ac6d0f6825', variable: 'DOCKER_HUB_PASSWORD'),
string(credentialsId: '66c40969-a46d-470e-b8a2-6f04f2b3f2d5', variable: 'DATADOG_API_KEY')
]) {
sh """./ci/si_pipeline.sh $params.channel permissive"""
}
} finally {
junit allowEmptyResults: true, testResults: "**/shakedown.xml"
archive includes: "**/diagnostics.zip"
archive includes: "**/sandbox_*.tar.gz"
archive includes: "**/preflight*.log"
archive includes: "**/postflight*.log"
archive includes: "**/deploy*.log"
stage("Run System Integration Test Pipeline") {
try {
checkout scm
withCredentials(
[[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'mesosphere-ci-marathon', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'],
file(credentialsId: '11fcc957-5156-4470-ae34-d433da88248a', variable: 'DOT_SHAKEDOWN'),
usernamePassword(credentialsId: '323df884-742b-4099-b8b7-d764e5eb9674', passwordVariable: 'DCOS_PASSWORD', usernameVariable: 'DCOS_USERNAME'),
string(credentialsId: 'ca159ad3-7323-4564-818c-46a8f03e1389', variable: 'DCOS_LICENSE'),
string(credentialsId: '7bdd2775-2911-41ba-918f-59c8ae52326d', variable: 'DOCKER_HUB_USERNAME'),
string(credentialsId: '42f2e3fb-3f4f-47b2-a128-10ac6d0f6825', variable: 'DOCKER_HUB_PASSWORD'),
string(credentialsId: '66c40969-a46d-470e-b8a2-6f04f2b3f2d5', variable: 'DATADOG_API_KEY')
]) {
sh """./ci/si_pipeline.sh $params.channel permissive"""
}
} finally {
junit allowEmptyResults: true, testResults: "**/shakedown.xml"
archive includes: "**/diagnostics.zip"
archive includes: "**/sandbox_*.tar.gz"
archive includes: "**/preflight*.log"
archive includes: "**/postflight*.log"
archive includes: "**/deploy*.log"
}
}
}
}
}
}
}
}
}
68 changes: 35 additions & 33 deletions Jenkinsfile.pr.si
View file
Original file line number Diff line number Diff line change
@@ -1,42 +1,44 @@
#!/usr/bin/env groovy

node('py36') {
wrap([$class: 'MesosSingleUseSlave']) {
wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {
timeout(time: 3, unit: 'HOURS') {
node('py36') {
wrap([$class: 'MesosSingleUseSlave']) {
wrap([$class: 'AnsiColorBuildWrapper', 'colorMapName': 'XTerm']) {

stage("Run System Integration Test Pipeline") {
try {
checkout scm
withCredentials([
[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'mesosphere-ci-marathon', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'],
usernamePassword(credentialsId: 'a7ac7f84-64ea-4483-8e66-bb204484e58f', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USER'),
file(credentialsId: '11fcc957-5156-4470-ae34-d433da88248a', variable: 'DOT_SHAKEDOWN'),
usernamePassword(credentialsId: '323df884-742b-4099-b8b7-d764e5eb9674', passwordVariable: 'DCOS_PASSWORD', usernameVariable: 'DCOS_USERNAME'),
string(credentialsId: 'ca159ad3-7323-4564-818c-46a8f03e1389', variable: 'DCOS_LICENSE'),
string(credentialsId: '7bdd2775-2911-41ba-918f-59c8ae52326d', variable: 'DOCKER_HUB_USERNAME'),
string(credentialsId: '42f2e3fb-3f4f-47b2-a128-10ac6d0f6825', variable: 'DOCKER_HUB_PASSWORD'),
string(credentialsId: '66c40969-a46d-470e-b8a2-6f04f2b3f2d5', variable: 'DATADOG_API_KEY')
]) {
// Checkout commit of pull request.
sshagent(credentials: ['4ff09dce-407b-41d3-847a-9e6609dd91b8']) {
sh """git fetch origin pull/$params.Pull_Request_Id/head:$params.Pull_Request_Id"""
sh """git checkout $params.Pull_Request_Id"""
}
sh """pip3 install requests"""
sh """python3 ./ci/github_status.py "$JOB_NAME/${params.Variant}" "$BUILD_URL" \$(git rev-parse HEAD) PENDING"""
stage("Run System Integration Test Pipeline") {
try {
checkout scm
withCredentials([
[$class: 'AmazonWebServicesCredentialsBinding', credentialsId: 'mesosphere-ci-marathon', accessKeyVariable: 'AWS_ACCESS_KEY_ID', secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'],
usernamePassword(credentialsId: 'a7ac7f84-64ea-4483-8e66-bb204484e58f', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USER'),
file(credentialsId: '11fcc957-5156-4470-ae34-d433da88248a', variable: 'DOT_SHAKEDOWN'),
usernamePassword(credentialsId: '323df884-742b-4099-b8b7-d764e5eb9674', passwordVariable: 'DCOS_PASSWORD', usernameVariable: 'DCOS_USERNAME'),
string(credentialsId: 'ca159ad3-7323-4564-818c-46a8f03e1389', variable: 'DCOS_LICENSE'),
string(credentialsId: '7bdd2775-2911-41ba-918f-59c8ae52326d', variable: 'DOCKER_HUB_USERNAME'),
string(credentialsId: '42f2e3fb-3f4f-47b2-a128-10ac6d0f6825', variable: 'DOCKER_HUB_PASSWORD'),
string(credentialsId: '66c40969-a46d-470e-b8a2-6f04f2b3f2d5', variable: 'DATADOG_API_KEY')
]) {
// Checkout commit of pull request.
sshagent(credentials: ['4ff09dce-407b-41d3-847a-9e6609dd91b8']) {
sh """git fetch origin pull/$params.Pull_Request_Id/head:$params.Pull_Request_Id"""
sh """git checkout $params.Pull_Request_Id"""
}
sh """pip3 install requests"""
sh """python3 ./ci/github_status.py "$JOB_NAME/${params.Variant}" "$BUILD_URL" \$(git rev-parse HEAD) PENDING"""

sh """./ci/si_pipeline.sh $params.Channel $params.Variant"""
}
} finally {
junit allowEmptyResults: true, testResults: "**/shakedown.xml"
archive includes: "**/diagnostics.zip"
archive includes: "**/sandbox_*.tar.gz"
sh """./ci/si_pipeline.sh $params.Channel $params.Variant"""
}
} finally {
junit allowEmptyResults: true, testResults: "**/shakedown.xml"
archive includes: "**/diagnostics.zip"
archive includes: "**/sandbox_*.tar.gz"

withCredentials([usernamePassword(credentialsId:'a7ac7f84-64ea-4483-8e66-bb204484e58f',passwordVariable:'GIT_PASSWORD', usernameVariable: 'GIT_USER')]) {
sh """python3 ./ci/github_status.py "$JOB_NAME/${params.Variant}" "$BUILD_URL" \$(git rev-parse HEAD) $currentBuild.result"""
withCredentials([usernamePassword(credentialsId: 'a7ac7f84-64ea-4483-8e66-bb204484e58f', passwordVariable: 'GIT_PASSWORD', usernameVariable: 'GIT_USER')]) {
sh """python3 ./ci/github_status.py "$JOB_NAME/${params.Variant}" "$BUILD_URL" \$(git rev-parse HEAD) $currentBuild.result"""
}
}
}
}
}
}
}
}
}
Loading

0 comments on commit 93cc18d

Please sign in to comment.