Skip to content

Commit

Permalink
Have a way to compute wheels (#579)
Browse files Browse the repository at this point in the history 
Signed-off-by: Clement Fuji Tsang <cfujitsang@nvidia.com>
  • Loading branch information
@Caenorst
Caenorst committed Jun 5, 2022
1 parent 127bf32 commit 2e40204
Show file tree
Hide file tree
Showing 8 changed files with 384 additions and 108 deletions.
214 changes: 163 additions & 51 deletions ci/gitlab_jenkins_templates/core_ci.jenkins
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,94 @@ def ubuntu_cpuonly_configs = [
]
]

// Configs for building python wheels
def ubuntu_for_wheels_configs = [
[
'cudaVer': '10.1', 'cudnnVer': '7',
'torchVer': '1.8.0', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.8.0', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.8.0', 'archsToTest': 'TITAN_RTX'
],
[
'cudaVer': '10.1', 'cudnnVer': '7',
'torchVer': '1.8.1', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.8.1', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.8.1', 'archsToTest': 'TITAN_RTX'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.9.0', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.9.0', 'archsToTest': 'TITAN_RTX'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.9.1', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.9.1', 'archsToTest': 'TITAN_RTX'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.10.0', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.10.0', 'archsToTest': 'Tesla_T4'
],
[
'cudaVer': '11.3.1', 'cudnnVer': '8',
'torchVer': '1.10.0', 'archsToTest': 'A100_PCIE_40GB'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.10.1', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.10.1', 'archsToTest': 'Tesla_T4'
],
[
'cudaVer': '11.3.1', 'cudnnVer': '8',
'torchVer': '1.10.1', 'archsToTest': 'A100_PCIE_40GB'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.10.2', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.1.1', 'cudnnVer': '8',
'torchVer': '1.10.2', 'archsToTest': 'Tesla_T4'
],
[
'cudaVer': '11.3.1', 'cudnnVer': '8',
'torchVer': '1.10.2', 'archsToTest': 'A100_PCIE_40GB'
],
[
'cudaVer': '10.2', 'cudnnVer': '7',
'torchVer': '1.11.0', 'archsToTest': 'TITAN_V'
],
[
'cudaVer': '11.3.1', 'cudnnVer': '8',
'torchVer': '1.11.0', 'archsToTest': 'A100_PCIE_40GB'
]
]


dockerRegistryServer = 'gitlab-master.nvidia.com:5005'
dockerRegistryName = 'toronto_dl_lab/kaolin'
Expand Down Expand Up @@ -94,62 +182,84 @@ node {
commitHash = sh(script: "git log -1 --pretty=format:%h",
returnStdout: true).trim()
sh "echo ${commitHash}"
// Check if the last commit message has a [with custom] tag
def hasNoCustom = sh(script: "git log -1 | grep '.*\\[with custom\\].*'",
returnStatus: true)
// We try to build from cuda docker image if the commit has such tag
// or CI is applied on master
if (hasNoCustom == 0 || gitlabSourceBranch == "master") {
for (config in ubuntu_from_cuda_configs) {
def configName = "custom-torch${config['torchVer']}-" + \
"cuda${config['cudaVer']}-" +
"cudnn${config['cudnnVer']}-" +
"py${config['pythonVer']}"
jobMap["${configName}"] = prepareUbuntuFromCUDAJob(
if (gitlabMergeRequestTitle.contains("[for wheels]")) {
for (config in ubuntu_for_wheels_configs) {
for (pythonVer in ['3.7', '3.8', '3.9']) {
def configName = "custom-wheels-torch${config['torchVer']}-" + \
"cuda${config['cudaVer']}-" +
"cudnn${config['cudnnVer']}-" +
"py${pythonVer}"
jobMap["${configName}"] = prepareUbuntuFromCUDAJob(
configName,
config['cudaVer'],
config['cudnnVer'],
pythonVer,
config['torchVer'],
config['archsToTest'],
true
)
}
}
} else {
// Check if the last commit message has a [with custom] tag
def hasNoCustomInMess = sh(script: "git log -1 | grep '.*\\[with custom\\].*'",
returnStatus: true)
sh "echo ${gitlabMergeRequestTitle}"
// We try to build from cuda docker image if the commit has such tag
// or CI is applied on master
if (hasNoCustomInMess == 0 || gitlabSourceBranch == "master" ||
gitlabMergeRequestTitle.contains("[with custom]")) {
for (config in ubuntu_from_cuda_configs) {
def configName = "custom-torch${config['torchVer']}-" + \
"cuda${config['cudaVer']}-" +
"cudnn${config['cudnnVer']}-" +
"py${config['pythonVer']}"
jobMap["${configName}"] = prepareUbuntuFromCUDAJob(
configName,
config['cudaVer'],
config['cudnnVer'],
config['pythonVer'],
config['torchVer'],
config['archsToTest'],
false
)
}
}

for (config in ubuntu_from_pytorch_configs) {
def configName = "pytorch-torch${config['torchVer']}-" + \
"cuda${config['cudaVer']}-cudnn${config['cudnnVer']}"
def baseImageTag = "pytorch/pytorch:${config['torchVer']}-" + \
"cuda${config['cudaVer']}-" + \
"cudnn${config['cudnnVer']}-devel"
jobMap["${configName}"] = prepareUbuntuFromBaseImageJob(
configName,
config['cudaVer'],
config['cudnnVer'],
config['pythonVer'],
config['torchVer'],
baseImageTag,
config['archsToTest']
)
}
}

for (config in ubuntu_from_pytorch_configs) {
def configName = "pytorch-torch${config['torchVer']}-" + \
"cuda${config['cudaVer']}-cudnn${config['cudnnVer']}"
def baseImageTag = "pytorch/pytorch:${config['torchVer']}-" + \
"cuda${config['cudaVer']}-" + \
"cudnn${config['cudnnVer']}-devel"
jobMap["${configName}"] = prepareUbuntuFromBaseImageJob(
configName,
baseImageTag,
config['archsToTest']
)
}

for (config in ubuntu_from_nvcr_configs) {
def configName = "nvcr-${config['baseImageTag']}"
def baseImageTag = "nvcr.io/nvidia/pytorch:${config['baseImageTag']}"
jobMap["${configName}"] = prepareUbuntuFromBaseImageJob(
configName,
baseImageTag,
config['archsToTest']
)
}
for (config in ubuntu_from_nvcr_configs) {
def configName = "nvcr-${config['baseImageTag']}"
def baseImageTag = "nvcr.io/nvidia/pytorch:${config['baseImageTag']}"
jobMap["${configName}"] = prepareUbuntuFromBaseImageJob(
configName,
baseImageTag,
config['archsToTest']
)
}

for (config in ubuntu_cpuonly_configs) {
def torchVerLabel = config['torchVer'].split('\\.')[0..1].join('')
def pythonVerLabel = config['pythonVer'].split('\\.').join('')
def configName = "cpuonly-py${config['pythonVer']}-torch${config['torchVer']}"
jobMap["${configName}"] = prepareUbuntuCPUOnlyJob(
configName,
config['pythonVer'],
config['torchVer']
)
for (config in ubuntu_cpuonly_configs) {
def torchVerLabel = config['torchVer'].split('\\.')[0..1].join('')
def pythonVerLabel = config['pythonVer'].split('\\.').join('')
def configName = "cpuonly-py${config['pythonVer']}-torch${config['torchVer']}"
jobMap["${configName}"] = prepareUbuntuCPUOnlyJob(
configName,
config['pythonVer'],
config['torchVer']
)
}
}

stage('Launch builds') {
parallel jobMap
}
Expand Down Expand Up @@ -187,7 +297,8 @@ def prepareUbuntuFromBaseImageJob(configName, baseImageTag, archsToTest) {
}
}

def prepareUbuntuFromCUDAJob(configName, cudaVer, cudnnVer, pythonVer, torchVer, archsToTest) {
def prepareUbuntuFromCUDAJob(configName, cudaVer, cudnnVer, pythonVer, torchVer, archsToTest,
buildWheel) {
return {
stage("${configName}") {
// Notify Gitlab about the build and tests it will be running
Expand All @@ -209,7 +320,8 @@ def prepareUbuntuFromCUDAJob(configName, cudaVer, cudnnVer, pythonVer, torchVer,
string(name: 'sourceBranch', value: "${env.gitlabSourceBranch}"),
string(name: 'repoUrl', value: "${scm.userRemoteConfigs[0].url}" ),
string(name: 'archsToTest', value: "${archsToTest}"),
string(name: 'commitHash', value: "${commitHash}")
string(name: 'commitHash', value: "${commitHash}"),
booleanParam(name: 'buildWheel', value: "${buildWheel}")
],
// This node doesn't need to be held while builds and tests run.
wait: false,
Expand Down
46 changes: 46 additions & 0 deletions ci/gitlab_jenkins_templates/ubuntu_custom_build_CI.jenkins
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,15 @@ currentBuild.description = sourceBranch + ": " + commitHash

podTemplate(
cloud:'sc-ipp-blossom-prod',
//volumes: [persistentVolumeClaim(mountPath: '/mount_binaries', claimName: 'kaolin-pvc', readOnly: false)],
yaml:'''
apiVersion: v1
kind: Pod
spec:
volumes:
- name: pvc-mount
persistentVolumeClaim:
claimName: 'kaolin-pvc'
containers:
- name: docker
image: docker:19.03.1
Expand All @@ -22,6 +27,9 @@ spec:
env:
- name: DOCKER_HOST
value: tcp://localhost:2375
volumeMounts:
- mountPath: /mount_binaries
name: pvc-mount
- name: docker-daemon
image: docker:19.03.1-dind
securityContext:
Expand All @@ -36,6 +44,9 @@ spec:
limits:
memory: 32Gi
cpu: 12
volumeMounts:
- mountPath: /mount_binaries
name: pvc-mount
''') {
node(POD_LABEL) {
container("docker") {
Expand All @@ -60,6 +71,7 @@ spec:
])
}
docker.withRegistry("https://${docker_registry_server}", 'gitlab-credentials') {

stage("Build") {
baseImage = docker.build(
"${targetImageTag}-base",
Expand All @@ -77,7 +89,41 @@ spec:
.
""")
}
if (buildWheel) {
stage("Build wheel") {
cudaTag = cudaVer.split('\\.')[0..<2].join('')
targetImage.inside("-v /mount_binaries:/mount_binaries") {
sh """
mkdir -p /mount_binaries/torch-${torchVer}+cu${cudaTag}
python setup.py bdist_wheel --dist-dir /mount_binaries/torch-${torchVer}+cu${cudaTag}
"""
}
pythonVerTag = pythonVer.split('\\.').join('')
Integer pythonVerTagVal = pythonVerTag
sh """echo ${pythonVerTagVal}"""
pythonVerAbiTag = (pythonVerTagVal < 38) ? pythonVerTag + 'm' : pythonVerTag
kaolinVer = sh(script: "cat ./version.txt", returnStdout: true).trim()
baseWheelName = "kaolin-${kaolinVer}-cp${pythonVerTag}-cp${pythonVerAbiTag}"
srcWheelName = "${baseWheelName}-linux_x86_64.whl"
dstWheelName = "${baseWheelName}-manylinux1_x86_64.whl"
sh """
mv /mount_binaries/torch-${torchVer}+cu${cudaTag}/${srcWheelName} ./${dstWheelName}
"""
}
stage("Reinstall from wheel") {
targetImage = docker.build(
"${targetImageTag}",
"""--no-cache --network host -f ./tools/linux/Dockerfile.install_wheel \
--build-arg BASE_IMAGE=${targetImageTag}-base \
--build-arg WHEEL_NAME=${dstWheelName} \
.
""")
}
}
stage("Push") {
sh """
mv ./${dstWheelName} /mount_binaries/torch-${torchVer}+cu${cudaTag}/${dstWheelName}
"""
targetImage.push()
}
}
Expand Down

0 comments on commit 2e40204

Please sign in to comment.