[ML] Upgrade to PyTorch 1.11 on Linux

Together with elastic#2233 and elastic#2235 this completes the upgrade to PyTorch 1.11 for all platforms. A small change to the way things are done is that we now copy all MKL libraries into the Docker images, then copy the required subset into our release distribution. This means we can change the subset of MKL libraries that we redistribute without having to rebuild our Docker images. Fixes elastic#2126
droberts195 · Mar 22, 2022 · a8bf300 · a8bf300
1 parent f73a706
commit a8bf300
Show file tree

Hide file tree

Showing 12 changed files with 22 additions and 32 deletions.
diff --git a/3rd_party/3rd_party.sh b/3rd_party/3rd_party.sh
@@ -70,7 +70,7 @@ case `uname` in
                 MKL_LOCATION=/usr/local/gcc103/lib
                 MKL_EXTENSION=.so
                 MKL_PREFIX=libmkl_
-                MKL_LIBRARIES=`cd "$MKL_LOCATION" && ls $MKL_PREFIX*$MKL_EXTENSION`
+                MKL_LIBRARIES='avx avx2 avx512 avx512_mic core def gnu_thread intel_lp64 mc3 vml_avx vml_avx2 vml_avx512 vml_avx512_mic vml_cmpt vml_def vml_mc3'
             fi
             BOOST_EXTENSION=mt-${BOOST_ARCH}-1_77.so.1.77.0
             BOOST_LIBRARIES='atomic chrono date_time filesystem iostreams log log_setup program_options regex system thread unit_test_framework'
@@ -265,11 +265,11 @@ fi
 if [ ! -z "$MKL_LOCATION" ] ; then
     if [ ! -z "$MKL_LIBRARIES" ] ; then
         if [ -n "$INSTALL_DIR" ] ; then
+            rm -f $INSTALL_DIR/$MKL_PREFIX*$MKL_EXTENSION
             for LIBRARY in $MKL_LIBRARIES
             do
-                rm -f $INSTALL_DIR/$LIBRARY
-                cp $MKL_LOCATION/$LIBRARY $INSTALL_DIR
-                chmod u+wx $INSTALL_DIR/$LIBRARY
+                cp $MKL_LOCATION/$MKL_PREFIX$LIBRARY$MKL_EXTENSION $INSTALL_DIR
+                chmod u+wx $INSTALL_DIR/$MKL_PREFIX$LIBRARY$MKL_EXTENSION
             done
         fi
     else

diff --git a/3rd_party/licenses/pytorch-INFO.csv b/3rd_party/licenses/pytorch-INFO.csv
@@ -1,2 +1,2 @@
 name,version,revision,url,license,copyright,sourceURL
-PyTorch,1.8.0,37c1f4a7fef115d719104e871d0cf39434aa9d56,https://pytorch.org,BSD-3-Clause,,
+PyTorch,1.11.0,bc2c6edaf163b1a1330e37a6e34caf8c553e4755,https://pytorch.org,BSD-3-Clause,,
diff --git a/build-setup/linux.md b/build-setup/linux.md
@@ -274,18 +274,13 @@ sudo yum-config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-m
 sudo yum -y install intel-mkl-2020.4-912
 ```
 
-Then copy the required libraries to the system directory:
+Then copy the shared libraries to the system directory:
 
 ```
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_intel_lp64.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_core.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_def.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_avx*.so /usr/local/gcc103/lib
-sudo cp /opt/intel/mkl/lib/intel64/libmkl_vml*.so /usr/local/gcc103/lib
+sudo cp /opt/intel/mkl/lib/intel64/libmkl*.so /usr/local/gcc103/lib
 ```
 
-### PyTorch 1.9.0
+### PyTorch 1.11.0
 
 PyTorch requires that certain Python modules are installed. Install these modules with `pip` using the same Python version you will build PyTorch with. If you followed the instructions above and built Python from source use `python3.7`:
 
@@ -302,7 +297,7 @@ sudo /usr/local/gcc103/bin/python3.7 -m pip install install numpy pyyaml setupto
 Then obtain the PyTorch code:
 
 ```
-git clone --depth=1 --branch=v1.9.0 git@github.com:pytorch/pytorch.git
+git clone --depth=1 --branch=v1.11.0 git@github.com:pytorch/pytorch.git
 cd pytorch
 git submodule sync
 git submodule update --init --recursive
@@ -330,7 +325,7 @@ export USE_MKLDNN=ON
 export USE_QNNPACK=OFF
 export USE_PYTORCH_QNNPACK=OFF
 [ $(uname -m) = x86_64 ] && export USE_XNNPACK=OFF
-export PYTORCH_BUILD_VERSION=1.9.0
+export PYTORCH_BUILD_VERSION=1.11.0
 export PYTORCH_BUILD_NUMBER=1
 /usr/local/gcc103/bin/python3.7 setup.py install
 ```

diff --git a/dev-tools/docker/build_linux_aarch64_cross_build_image.sh b/dev-tools/docker/build_linux_aarch64_cross_build_image.sh
@@ -22,7 +22,7 @@
 HOST=docker.elastic.co
 ACCOUNT=ml-dev
 REPOSITORY=ml-linux-aarch64-cross-build
-VERSION=6
+VERSION=7
 
 set -e
 

diff --git a/dev-tools/docker/build_linux_aarch64_native_build_image.sh b/dev-tools/docker/build_linux_aarch64_native_build_image.sh
@@ -34,7 +34,7 @@ sleep 5
 HOST=docker.elastic.co
 ACCOUNT=ml-dev
 REPOSITORY=ml-linux-aarch64-native-build
-VERSION=6
+VERSION=7
 
 set -e
 

diff --git a/dev-tools/docker/build_linux_build_image.sh b/dev-tools/docker/build_linux_build_image.sh
@@ -34,7 +34,7 @@ sleep 5
 HOST=docker.elastic.co
 ACCOUNT=ml-dev
 REPOSITORY=ml-linux-build
-VERSION=20
+VERSION=21
 
 set -e
 

diff --git a/dev-tools/docker/linux_aarch64_cross_builder/Dockerfile b/dev-tools/docker/linux_aarch64_cross_builder/Dockerfile
@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:6
+FROM docker.elastic.co/ml-dev/ml-linux-aarch64-cross-build:7
 
 MAINTAINER David Roberts <dave.roberts@elastic.co>
 

diff --git a/dev-tools/docker/linux_aarch64_native_builder/Dockerfile b/dev-tools/docker/linux_aarch64_native_builder/Dockerfile
@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:6
+FROM docker.elastic.co/ml-dev/ml-linux-aarch64-native-build:7
 
 MAINTAINER David Roberts <dave.roberts@elastic.co>
 

diff --git a/dev-tools/docker/linux_aarch64_native_image/Dockerfile b/dev-tools/docker/linux_aarch64_native_image/Dockerfile
@@ -127,7 +127,7 @@ ENV PATH=${PATH}:/usr/local/cmake/bin
 # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION
 RUN \
   cd ${build_dir} && \
-  git -c advice.detachedHead=false clone --depth=1 --branch=v1.9.0 https://github.com/pytorch/pytorch.git && \
+  git -c advice.detachedHead=false clone --depth=1 --branch=v1.11.0 https://github.com/pytorch/pytorch.git && \
   cd pytorch && \
   git submodule sync && \
   git submodule update --init --recursive && \
@@ -140,7 +140,7 @@ RUN \
   export USE_MKLDNN=ON && \
   export USE_QNNPACK=OFF && \
   export USE_PYTORCH_QNNPACK=OFF && \
-  export PYTORCH_BUILD_VERSION=1.9.0 && \
+  export PYTORCH_BUILD_VERSION=1.11.0 && \
   export PYTORCH_BUILD_NUMBER=1 && \
   /usr/local/bin/python3.7 setup.py install && \
   mkdir /usr/local/gcc103/include/pytorch && \

diff --git a/dev-tools/docker/linux_builder/Dockerfile b/dev-tools/docker/linux_builder/Dockerfile
@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-build:20
+FROM docker.elastic.co/ml-dev/ml-linux-build:21
 
 MAINTAINER David Roberts <dave.roberts@elastic.co>
 

diff --git a/dev-tools/docker/linux_image/Dockerfile b/dev-tools/docker/linux_image/Dockerfile
@@ -128,18 +128,13 @@ ENV PATH=${PATH}:/usr/local/cmake/bin
 RUN \
   yum-config-manager --add-repo https://yum.repos.intel.com/mkl/setup/intel-mkl.repo && \
   yum -y install intel-mkl-2020.4-912 && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_intel_lp64.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_core.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_def.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_gnu_thread.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_avx*.so /usr/local/gcc103/lib && \
-  cp /opt/intel/mkl/lib/intel64/libmkl_vml*.so /usr/local/gcc103/lib
+  cp /opt/intel/mkl/lib/intel64/libmkl*.so /usr/local/gcc103/lib
 
 # Clone PyTorch and build LibTorch
 # If the PyTorch branch is changed also update PYTORCH_BUILD_VERSION
 RUN \
   cd ${build_dir} && \
-  git -c advice.detachedHead=false clone --depth=1 --branch=v1.9.0 https://github.com/pytorch/pytorch.git && \
+  git -c advice.detachedHead=false clone --depth=1 --branch=v1.11.0 https://github.com/pytorch/pytorch.git && \
   cd pytorch && \
   git submodule sync && \
   git submodule update --init --recursive && \
@@ -153,7 +148,7 @@ RUN \
   export USE_QNNPACK=OFF && \
   export USE_PYTORCH_QNNPACK=OFF && \
   export USE_XNNPACK=OFF && \
-  export PYTORCH_BUILD_VERSION=1.9.0 && \
+  export PYTORCH_BUILD_VERSION=1.11.0 && \
   export PYTORCH_BUILD_NUMBER=1 && \
   /usr/local/bin/python3.7 setup.py install && \
   mkdir /usr/local/gcc103/include/pytorch && \

diff --git a/dev-tools/docker/linux_tester/Dockerfile b/dev-tools/docker/linux_tester/Dockerfile
@@ -10,7 +10,7 @@
 #
 
 # Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-linux-build:20
+FROM docker.elastic.co/ml-dev/ml-linux-build:21
 
 MAINTAINER David Roberts <dave.roberts@elastic.co>