forked from NVIDIA/spark-rapids
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Jenkinsfile.integration
124 lines (111 loc) · 4.63 KB
/
Jenkinsfile.integration
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#!/usr/local/env groovy
/*
* Copyright (c) 2019-2020, NVIDIA CORPORATION.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
*
* Jenkins file for running spark3.0 integration tests
*
*/
@Library(['shared-libs', 'spark-jenkins-shared-lib']) _
def urmUrl="https://${ArtifactoryConstants.ARTIFACTORY_NAME}/artifactory/sw-spark-maven"
pipeline {
agent none
options {
ansiColor('xterm')
timestamps()
timeout(time: 240, unit: 'MINUTES')
buildDiscarder(logRotator(numToKeepStr: '10'))
}
parameters {
string(name: 'OVERWRITE_PARAMS', defaultValue: '',
description: 'parameters format XXX_VER=xxx;YYY_VER=yyy;')
string(name: 'CUDA_CLASSIFIER', defaultValue: 'cuda10-1',
description: '-Dclassifier=\n\n cuda10-1, cuda10-2, EMPTY as cuda10-1')
string(name: 'REF', defaultValue: 'branch-0.3', description: 'Commit to build')
}
environment {
JENKINS_ROOT = 'jenkins'
TEST_SCRIPT = '$JENKINS_ROOT/spark-tests.sh'
LIBCUDF_KERNEL_CACHE_PATH='/tmp/.cudf'
URM_CREDS = credentials("svcngcc_artifactory")
ARTIFACTORY_NAME = "${ArtifactoryConstants.ARTIFACTORY_NAME}"
URM_URL = "${urmUrl}"
MVN_URM_MIRROR='-s jenkins/settings.xml -P mirror-apache-to-urm'
}
stages {
stage('centos7 CUDA10.1') {
agent { label 'docker-gpu' }
steps {
script {
def CUDF_VER=sh(returnStdout: true,
script: '. jenkins/version-def.sh>&2 && echo -n $CUDF_VER') - "-SNAPSHOT"
def CUDA_NAME=sh(returnStdout: true,
script: '. jenkins/version-def.sh>&2 && echo -n $CUDA_CLASSIFIER | sed "s/-/./g"')
def IMAGE_NAME="$ARTIFACTORY_NAME/sw-spark-docker/plugin:it-centos7-$CUDA_NAME"
def CUDA_VER="$CUDA_NAME" - "cuda"
sh "docker pull $IMAGE_NAME"
def urmImageID=sh(returnStdout: true, script: "docker inspect -f {{'.Id'}} $IMAGE_NAME")
// Speed up Docker building via '--cache-from $IMAGE_NAME'
def buildImage=docker.build(IMAGE_NAME,
"-f jenkins/Dockerfile.integration.centos7 --build-arg CUDA_VER=$CUDA_VER \
--build-arg URM_URL=$URM_URL --build-arg CUDF_VER=$CUDF_VER --cache-from $IMAGE_NAME -t $IMAGE_NAME .")
def buildImageID=sh(returnStdout: true, script: "docker inspect -f {{'.Id'}} $IMAGE_NAME")
if (! buildImageID.equals(urmImageID)) {
echo "Dockerfile updated, upload docker image to URM"
uploadDocker(IMAGE_NAME)
}
buildImage.inside("--runtime=nvidia -v ${HOME}/.zinc:${HOME}/.zinc:rw") {
echo "Running integration tests on centos7 $CUDA_NAME"
sh "bash $TEST_SCRIPT"
}
}
}
}
} // end of stages
post {
always {
script {
def status = "failed"
if (currentBuild.currentResult == "SUCCESS") {
status = "success"
slack("#swrapids-spark-cicd", "Success", color: "#33CC33")
}
else {
slack("#swrapids-spark-cicd", "Failed", color: "#FF0000")
}
}
echo 'Pipeline finished!'
}
}
} // end of pipeline
void uploadDocker(String IMAGE_NAME) {
def DOCKER_CMD="docker --config $WORKSPACE/.docker"
sh """
echo $URM_CREDS_PSW | $DOCKER_CMD login $ARTIFACTORY_NAME -u $URM_CREDS_USR --password-stdin
$DOCKER_CMD push $IMAGE_NAME
$DOCKER_CMD logout $ARTIFACTORY_NAME
"""
}
void slack(Map params = [:], String channel, String message) {
Map defaultParams = [
color: "#000000",
baseUrl: "${SparkConstants.SLACK_API_ENDPOINT}",
tokenCredentialId: "slack_token"
]
params["channel"] = channel
params["message"] = "${BUILD_URL}\n" + message
slackSend(defaultParams << params)
}