Skip to content

Commit

Permalink
[v1.x] Cherry-pick apache#17776 apache#17681 (apache#18465)
Browse files Browse the repository at this point in the history
* Fix CD (apache#17776)

* Fix cd/mxnet_lib/dynamic/Jenkins_pipeline.groovy

Fixes a regression in CD introduced by
apache#17645

* Fix whitespace

* Add NATIVE_ADDITIONAL.md

Fixes a regression in CD introduced by
apache#16899

* Update other $TYPE_ADDITIONAL.MD

* Fix cd/python/docker

Fixes regression introduced by
apache#15990

* [CD] update pypi description, setup.py (apache#17681)

* update pypi description, setup.py, use manylinux2010, use unified dist link for nightly

* Use manylinux2014

Co-authored-by: Leonard Lausen <leonard@lausen.nl>

* reverting .so path as per MAKE flow

Co-authored-by: Leonard Lausen <lausen@amazon.com>
Co-authored-by: Sheng Zha <szha@users.noreply.github.com>
Co-authored-by: Leonard Lausen <leonard@lausen.nl>
  • Loading branch information
4 people authored and ChaiBapchya committed Aug 15, 2020
1 parent 2472d4a commit afa1398
Show file tree
Hide file tree
Showing 23 changed files with 92 additions and 457 deletions.
2 changes: 1 addition & 1 deletion cd/mxnet_lib/mxnet_lib_pipeline.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ def get_pipeline(mxnet_variant, build_fn) {
}
}
}

parallel tests
}

Expand Down
2 changes: 1 addition & 1 deletion cd/python/docker/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
ARG BASE_IMAGE
FROM ${BASE_IMAGE}

ARG PYTHON_CMD=python
ARG PYTHON_CMD=python3
RUN apt-get update && \
apt-get install -y wget ${PYTHON_CMD}-dev gcc && \
wget https://bootstrap.pypa.io/get-pip.py && \
Expand Down
9 changes: 3 additions & 6 deletions cd/python/docker/Jenkins_pipeline.groovy
Original file line number Diff line number Diff line change
Expand Up @@ -50,24 +50,21 @@ def build(mxnet_variant) {
ci_utils.docker_run(environment, "cd_package_pypi ${mxnet_variant}", nvidia_docker)

// build python docker images
sh "./cd/python/docker/python_images.sh build ${mxnet_variant} py3"
sh "./cd/python/docker/python_images.sh build ${mxnet_variant} py2"
sh "./cd/python/docker/python_images.sh build ${mxnet_variant}"
}
}

def test(mxnet_variant) {
ws("workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}") {
// test python docker images
sh "./cd/python/docker/python_images.sh test ${mxnet_variant} py3"
sh "./cd/python/docker/python_images.sh test ${mxnet_variant} py2"
sh "./cd/python/docker/python_images.sh test ${mxnet_variant}"
}
}

def push(mxnet_variant) {
ws("workspace/python_docker/${mxnet_variant}/${env.BUILD_NUMBER}") {
// push python docker images
sh "./cd/python/docker/python_images.sh push ${mxnet_variant} py3"
sh "./cd/python/docker/python_images.sh push ${mxnet_variant} py2"
sh "./cd/python/docker/python_images.sh push ${mxnet_variant}"
}
}

Expand Down
37 changes: 7 additions & 30 deletions cd/python/docker/python_images.sh
Original file line number Diff line number Diff line change
Expand Up @@ -23,30 +23,16 @@

set -xe

usage="Usage: python_images.sh <build|test|publish> MXNET-VARIANT <py2|py3>"
usage="Usage: python_images.sh <build|test|publish> MXNET-VARIANT"

command=${1:?$usage}
mxnet_variant=${2:?$usage}
python_version=${3:?usage}

cd_utils='cd/utils'
ci_utils='ci/'

case ${python_version} in
py3)
python_cmd="python3"
;;
py2)
python_cmd="python"
;;
*)
echo "Error: specify python version with either 'py2' or 'py3'"
exit 1
;;
esac

docker_tags=($(./${cd_utils}/docker_tag.sh ${mxnet_variant}))
main_tag="${docker_tags[0]}_${python_version}"
main_tag="${docker_tags[0]}_py3"
base_image=$(./${cd_utils}/mxnet_base_image.sh ${mxnet_variant})
repository="python"
image_name="${repository}:${main_tag}"
Expand All @@ -59,7 +45,7 @@ fi

build() {
# NOTE: Ensure the correct context root is passed in when building - Dockerfile expects ./wheel_build
docker build -t "${image_name}" --build-arg PYTHON_CMD=${python_cmd} --build-arg BASE_IMAGE="${base_image}" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build
docker build -t "${image_name}" --build-arg --build-arg BASE_IMAGE="${base_image}" --build-arg MXNET_COMMIT_ID=${GIT_COMMIT} -f ${resources_path}/Dockerfile ./wheel_build
}

test() {
Expand All @@ -68,10 +54,10 @@ test() {
runtime_param="--runtime=nvidia"
fi
local test_image_name="${image_name}_test"

# Ensure the correct context root is passed in when building - Dockerfile.test expects ci directory
docker build -t "${test_image_name}" --build-arg USER_ID=`id -u` --build-arg GROUP_ID=`id -g` --build-arg BASE_IMAGE="${image_name}" -f ${resources_path}/Dockerfile.test ./ci
./ci/safe_docker_run.py ${runtime_param} --cap-add "SYS_PTRACE" -u `id -u`:`id -g` -v `pwd`:/work/mxnet "${test_image_name}" ${resources_path}/test_python_image.sh "${mxnet_variant}" "${python_cmd}"
./ci/safe_docker_run.py ${runtime_param} --cap-add "SYS_PTRACE" -u `id -u`:`id -g` -v `pwd`:/work/mxnet "${test_image_name}" ${resources_path}/test_python_image.sh "${mxnet_variant}"
}

push() {
Expand All @@ -90,23 +76,14 @@ push() {
# Iterate over remaining tags, if any
for ((i=1;i<${#docker_tags[@]};i++)); do
local docker_tag="${docker_tags[${i}]}"
local latest_image_name="${RELEASE_DOCKERHUB_REPOSITORY}/${repository}:${docker_tag}"

# latest and latest gpu should only be pushed for py3
if [[ ${docker_tag} == "latest" || ${docker_tag} == "latest_gpu" ]]; then
if [[ ${python_version} == "py2" ]]; then
continue
fi
else
latest_image_name="${latest_image_name}_${python_version}"
fi
local latest_image_name="${RELEASE_DOCKERHUB_REPOSITORY}/${repository}:${docker_tag}_py3"

docker tag "${image_name}" "${latest_image_name}"
docker push "${latest_image_name}"
echo "Successfully pushed ${latest_image_name}. Pull it with:"
echo "docker pull ${latest_image_name}"
echo "For a complete list of tags see https://hub.docker.com/u/${RELEASE_DOCKERHUB_REPOSITORY}/${repository}"
done
done
}

case ${command} in
Expand Down
7 changes: 3 additions & 4 deletions cd/python/docker/test_python_image.sh
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@ set -ex

# Variant parameter should be passed in
mxnet_variant=${1:?"Missing mxnet variant"}
python_cmd=${2:?"Missing python version (python or python3)"}

if [ -z "${MXNET_COMMIT_ID}" ]; then
echo "MXNET_COMMIT_ID environment variable is empty. Please rebuild the image with MXNET_COMMIT_ID build-arg specified."
Expand All @@ -39,9 +38,9 @@ if [[ $mxnet_variant == cu* ]]; then
fi

if [[ $mxnet_variant == cpu ]]; then
${python_cmd} tests/python/mkl/test_mkldnn.py
python3 tests/python/mkl/test_mkldnn.py
fi

${python_cmd} tests/python/train/test_conv.py ${test_conv_params}
${python_cmd} example/image-classification/train_mnist.py ${mnist_params}
python3 tests/python/train/test_conv.py ${test_conv_params}
python3 example/image-classification/train_mnist.py ${mnist_params}

20 changes: 11 additions & 9 deletions tools/pip/doc/CPU_ADDITIONAL.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,14 @@
Prerequisites
-------------
This package supports Linux, Mac OSX, and Windows platforms. You may also want to check:
- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
- [mxnet-cu102](https://pypi.python.org/pypi/mxnet-cu101/) with CUDA-10.2 support.
- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101/) with CUDA-10.1 support.
- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100/) with CUDA-10.0 support.
- [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
- [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
- [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
- [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
- [mxnet-native](https://pypi.python.org/pypi/mxnet-native/) CPU variant without MKLDNN.

To install for other platforms (e.g. Windows, Raspberry Pi/ARM) or other versions, check [Installing MXNet](https://mxnet.apache.org/versions/master/install/index.html) for instructions on building from source.

Expand All @@ -40,3 +35,10 @@ To install, use:
```bash
pip install mxnet
```

Nightly Builds
--------------
To install the latest nightly build, use:
```bash
pip install --pre mxnet -f https://dist.mxnet.io/python
```
44 changes: 0 additions & 44 deletions tools/pip/doc/CU100MKL_ADDITIONAL.md

This file was deleted.

16 changes: 8 additions & 8 deletions tools/pip/doc/CU100_ADDITIONAL.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,11 @@
Prerequisites
-------------
This package supports Linux and Windows platforms. You may also want to check:
- [mxnet-cu101](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support.
- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
- [mxnet-cu102](https://pypi.python.org/pypi/mxnet-cu101/) with CUDA-10.2 support.
- [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
- [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
- [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
- [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
- [mxnet](https://pypi.python.org/pypi/mxnet/).

To download CUDA, check [CUDA download](https://developer.nvidia.com/cuda-downloads). For more instructions, check [CUDA Toolkit online documentation](http://docs.nvidia.com/cuda/index.html).
Expand All @@ -42,3 +35,10 @@ To install:
```bash
pip install mxnet-cu100
```

Nightly Builds
--------------
To install the latest nightly build, use:
```bash
pip install --pre mxnet-cu100 -f https://dist.mxnet.io/python
```
44 changes: 0 additions & 44 deletions tools/pip/doc/CU101MKL_ADDITIONAL.md

This file was deleted.

16 changes: 8 additions & 8 deletions tools/pip/doc/CU101_ADDITIONAL.md
Original file line number Diff line number Diff line change
Expand Up @@ -18,18 +18,11 @@
Prerequisites
-------------
This package supports Linux and Windows platforms. You may also want to check:
- [mxnet-cu101mkl](https://pypi.python.org/pypi/mxnet-cu101mkl/) with CUDA-10.1 support and MKLDNN support.
- [mxnet-cu100](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support.
- [mxnet-cu100mkl](https://pypi.python.org/pypi/mxnet-cu100mkl/) with CUDA-10.0 support and MKLDNN support.
- [mxnet-cu102](https://pypi.python.org/pypi/mxnet-cu101/) with CUDA-10.2 support.
- [mxnet-cu92](https://pypi.python.org/pypi/mxnet-cu92/) with CUDA-9.2 support.
- [mxnet-cu92mkl](https://pypi.python.org/pypi/mxnet-cu92mkl/) with CUDA-9.2 support and MKLDNN support.
- [mxnet-cu90](https://pypi.python.org/pypi/mxnet-cu90/) with CUDA-9.0 support.
- [mxnet-cu90mkl](https://pypi.python.org/pypi/mxnet-cu90mkl/) with CUDA-9.0 support and MKLDNN support.
- [mxnet-cu80](https://pypi.python.org/pypi/mxnet-cu80/) with CUDA-8.0 support.
- [mxnet-cu80mkl](https://pypi.python.org/pypi/mxnet-cu80mkl/) with CUDA-8.0 support and MKLDNN support.
- [mxnet-cu75](https://pypi.python.org/pypi/mxnet-cu75/) with CUDA-7.5 support.
- [mxnet-cu75mkl](https://pypi.python.org/pypi/mxnet-cu75mkl/) with CUDA-7.5 support and MKLDNN support.
- [mxnet-mkl](https://pypi.python.org/pypi/mxnet-mkl/) with MKLDNN support.
- [mxnet](https://pypi.python.org/pypi/mxnet/).

To download CUDA, check [CUDA download](https://developer.nvidia.com/cuda-downloads). For more instructions, check [CUDA Toolkit online documentation](http://docs.nvidia.com/cuda/index.html).
Expand All @@ -42,3 +35,10 @@ To install:
```bash
pip install mxnet-cu101
```

Nightly Builds
--------------
To install the latest nightly build, use:
```bash
pip install --pre mxnet-cu101 -f https://dist.mxnet.io/python
```
46 changes: 0 additions & 46 deletions tools/pip/doc/CU102MKL_ADDITIONAL.md

This file was deleted.

Loading

0 comments on commit afa1398

Please sign in to comment.