Skip to content
This repository has been archived by the owner on Nov 17, 2023. It is now read-only.

Commit

Permalink
from master PR
Browse files Browse the repository at this point in the history
  • Loading branch information
mseth10 committed Feb 26, 2021
1 parent 089c53b commit 3249e55
Show file tree
Hide file tree
Showing 9 changed files with 34 additions and 111 deletions.
40 changes: 14 additions & 26 deletions cd/Jenkinsfile_cd_pipeline
Original file line number Diff line number Diff line change
Expand Up @@ -55,33 +55,21 @@ pipeline {
stage("MXNet Release") {
steps {
script {
cd_utils.error_checked_parallel([

"Static libmxnet based release": {
stage("Build") {
cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, "Build static libmxnet", "mxnet_lib/static", params.MXNET_VARIANTS)
}
stage("Releases") {
cd_utils.error_checked_parallel([
"PyPI Release": {
echo "Building PyPI Release"
cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, "Release PyPI Packages", "python/pypi", params.MXNET_VARIANTS)
},
"Python Docker Release": {
echo "Building Python Docker Release"
cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, "Release Python Docker Images", "python/docker", params.MXNET_VARIANTS)
}
])
stage("Build libmxnet") {
cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, "Build libmxnet", "mxnet_lib", params.MXNET_VARIANTS)
}
stage("Releases") {
cd_utils.error_checked_parallel([
"PyPI Release": {
echo "Building PyPI Release"
cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, "Release PyPI Packages", "python/pypi", params.MXNET_VARIANTS)
},
"Python Docker Release": {
echo "Building Python Docker Release"
cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, "Release Python Docker Images", "python/docker", params.MXNET_VARIANTS)
}
},

"Dynamic libmxnet based release": {
stage("Build") {
cd_utils.trigger_release_job(params.CD_RELEASE_JOB_NAME, "Build dynamic libmxnet", "mxnet_lib/dynamic", params.MXNET_VARIANTS)
}
}

])
])
}
}
}
}
Expand Down
5 changes: 2 additions & 3 deletions cd/Jenkinsfile_release_job
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ pipeline {
// Using string instead of choice parameter to keep the changes to the parameters minimal to avoid
// any disruption caused by different COMMIT_ID values chaning the job parameter configuration on
// Jenkins.
string(defaultValue: "mxnet_lib/static", description: "Pipeline to build", name: "RELEASE_JOB_TYPE")
string(defaultValue: "mxnet_lib", description: "Pipeline to build", name: "RELEASE_JOB_TYPE")
string(defaultValue: "cpu,native,cu100,cu101,cu102,cu110,cu112", description: "Comma separated list of variants", name: "MXNET_VARIANTS")
booleanParam(defaultValue: false, description: 'Whether this is a release build or not', name: "RELEASE_BUILD")
}
Expand Down Expand Up @@ -90,8 +90,7 @@ pipeline {

// Add new job types here
def valid_job_types = [
"mxnet_lib/static",
"mxnet_lib/dynamic",
"mxnet_lib",
"python/pypi",
"python/docker"
]
Expand Down
58 changes: 0 additions & 58 deletions cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy

This file was deleted.

17 changes: 8 additions & 9 deletions cd/python/docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,15 +23,14 @@
ARG BASE_IMAGE
FROM ${BASE_IMAGE}

RUN apt-get update || true
RUN apt-get install -y software-properties-common
RUN add-apt-repository -y ppa:deadsnakes/ppa
RUN apt-get update || true
RUN apt-get install -y python3.7-dev python3.7-distutils virtualenv wget
RUN ln -sf /usr/bin/python3.7 /usr/local/bin/python3

RUN wget -nv https://bootstrap.pypa.io/get-pip.py
RUN python3 get-pip.py
RUN apt-get update && \
apt-get install -y software-properties-common && \
add-apt-repository -y ppa:deadsnakes/ppa && \
apt-get update && \
apt-get install -y python3.7-dev python3.7-distutils virtualenv wget && \
ln -sf /usr/bin/python3.7 /usr/local/bin/python3 && \
wget -nv https://bootstrap.pypa.io/get-pip.py && \
python3 get-pip.py

ARG MXNET_COMMIT_ID
ENV MXNET_COMMIT_ID=${MXNET_COMMIT_ID}
Expand Down
3 changes: 0 additions & 3 deletions cd/python/docker/Dockerfile.test
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,6 @@
ARG BASE_IMAGE
FROM ${BASE_IMAGE}

# Install test dependencies
RUN pip install nose

ARG USER_ID=1001
ARG GROUP_ID=1001

Expand Down
18 changes: 8 additions & 10 deletions cd/python/docker/python_images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

set -xe

usage="Usage: python_images.sh <build|test|publish> MXNET-VARIANT"
usage="Usage: python_images.sh <build|test|push> MXNET-VARIANT"

command=${1:?$usage}
mxnet_variant=${2:?$usage}
Expand All @@ -39,8 +39,8 @@ image_name="${repository}:${main_tag}"

resources_path='cd/python/docker'

if [ ! -z "${RELEASE_DOCKERHUB_REPOSITORY}" ]; then
image_name="${RELEASE_DOCKERHUB_REPOSITORY}/${image_name}"
if [ ! -z "${RELEASE_PUBLIC_ECR_REPOSITORY}" ]; then
image_name="${RELEASE_PUBLIC_ECR_REPOSITORY}/${image_name}"
fi

build() {
Expand All @@ -57,26 +57,24 @@ test() {

# Ensure the correct context root is passed in when building - Dockerfile.test expects ci directory
docker build -t "${test_image_name}" --build-arg USER_ID=`id -u` --build-arg GROUP_ID=`id -g` --build-arg BASE_IMAGE="${image_name}" -f ${resources_path}/Dockerfile.test ./ci
./ci/safe_docker_run.py ${runtime_param} --cap-add "SYS_PTRACE" -u `id -u`:`id -g` -v `pwd`:/work/mxnet "${test_image_name}" ${resources_path}/test_python_image.sh "${mxnet_variant}"
}

push() {
if [ -z "${RELEASE_DOCKERHUB_REPOSITORY}" ]; then
echo "Cannot publish image without RELEASE_DOCKERHUB_REPOSITORY environment variable being set."
if [ -z "${RELEASE_PUBLIC_ECR_REPOSITORY}" ]; then
echo "Cannot publish image without RELEASE_PUBLIC_ECR_REPOSITORY environment variable being set."
exit 1
fi

# The secret name env var is set in the Jenkins configuration
# Manage Jenkins -> Configure System
./${ci_utils}/docker_login.py --secret-name "${RELEASE_DOCKERHUB_SECRET_NAME}"
# Retrieve an authentication token and authenticate Docker client to registry
aws ecr-public get-login-password --region us-east-1 | docker login --username AWS --password-stdin public.ecr.aws/w6z5f7h2

# Push image
docker push "${image_name}"

# Iterate over remaining tags, if any
for ((i=1;i<${#docker_tags[@]};i++)); do
local docker_tag="${docker_tags[${i}]}"
local latest_image_name="${RELEASE_DOCKERHUB_REPOSITORY}/${repository}:${docker_tag}_py3"
local latest_image_name="${RELEASE_PUBLIC_ECR_REPOSITORY}/${repository}:${docker_tag}_py3"

docker tag "${image_name}" "${latest_image_name}"
docker push "${latest_image_name}"
Expand Down
2 changes: 1 addition & 1 deletion cd/utils/docker_tag.sh
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@

mxnet_variant=${1:?"Please specify the mxnet variant as the first parameter"}
is_release=${RELEASE_BUILD:-false}
version=${VERSION:-nightly}
version=${VERSION:-nightly_v1.x}

# The docker tags will be in the form <version>_<hardware>(_mkl)
# Eg. nightly_cpu, 1.4.0_cpu_mkl, nightly_gpu_cu80_mkl, etc.
Expand Down
2 changes: 1 addition & 1 deletion cd/utils/mxnet_base_image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ case ${mxnet_variant} in
echo "nvidia/cuda:11.0-cudnn8-runtime-ubuntu16.04"
;;
cu112*)
echo "nvidia/cuda:11.2-cudnn8-runtime-ubuntu16.04"
echo "nvidia/cuda:11.2.1-cudnn8-runtime-ubuntu16.04"
;;
cpu)
echo "ubuntu:16.04"
Expand Down

0 comments on commit 3249e55

Please sign in to comment.