From 71d200b00f6b1c6256f4202a6a9bf4e8b01f9c2d Mon Sep 17 00:00:00 2001 From: Jacob Kahn Date: Fri, 11 Dec 2020 18:22:16 -0800 Subject: [PATCH] Update CPU and CUDA Dockerfiles Summary: Fix the Docker image build - Use multiple `RUN` commands with `docker prune` to allow checkingpointing the base image builds to make it easier to debug stuff in the future - Make `MKLROOT` a global env var - Remove the MKL BLAS vendor for oneDNN (apparently faster without MKL) - see https://github.com/microsoft/vcpkg/pull/15068#issuecomment-743478633 - Enable building tests in the CPU CI build since it works now and `gtest_discover_tests` is removed so we don't have runtime `libmlk_rt` issues. We also shouldn't have those issues anymore in general because we are no longer linking MKL to the MKL-DNN/oneDNN build Differential Revision: D25514612 fbshipit-source-id: d0f52bced60c5265f322f80363bd888e3d50ab5a --- .circleci/config.yml | 2 +- .docker/Dockerfile-CPU | 5 ++-- .docker/Dockerfile-CPU-Base | 47 ++++++++++++++++++------------------ .docker/Dockerfile-CUDA | 5 ++-- .docker/Dockerfile-CUDA-Base | 35 ++++++++++++++------------- 5 files changed, 49 insertions(+), 45 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 83d4a0943..ec8e4482e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -318,7 +318,7 @@ jobs: export CC=/usr/bin/gcc-5 && export CXX=/usr/bin/g++-5 export MKLROOT=/opt/intel/mkl && export KENLM_ROOT=/root/kenlm mkdir -p build && cd build - cmake .. -DFL_BACKEND=CPU -DFL_LIBRARIES_USE_CUDA=OFF -DFL_BUILD_TESTS=OFF + cmake .. -DFL_BACKEND=CPU -DFL_LIBRARIES_USE_CUDA=OFF make -j16 && make install workflows: diff --git a/.docker/Dockerfile-CPU b/.docker/Dockerfile-CPU index 58ebfa60a..f0a0bcd56 100644 --- a/.docker/Dockerfile-CPU +++ b/.docker/Dockerfile-CPU @@ -6,6 +6,8 @@ FROM flml/flashlight:cpu-base-consolidation-latest +ENV MKLROOT="/opt/intel/mkl" + # ================================================================== # flashlight with CPU backend # ------------------------------------------------------------------ @@ -14,7 +16,6 @@ RUN mkdir /root/flashlight COPY . /root/flashlight -RUN export MKLROOT=/opt/intel/mkl && \ - cd /root/flashlight && mkdir -p build && \ +RUN cd /root/flashlight && mkdir -p build && \ cd build && cmake .. -DFL_BACKEND=CPU -DFL_LIBRARIES_USE_CUDA=OFF && \ make -j$(nproc) && make install diff --git a/.docker/Dockerfile-CPU-Base b/.docker/Dockerfile-CPU-Base index 3eb7d571e..278e0ba97 100644 --- a/.docker/Dockerfile-CPU-Base +++ b/.docker/Dockerfile-CPU-Base @@ -18,8 +18,10 @@ FROM ubuntu:18.04 -RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ - apt-get update && \ +ENV APT_INSTALL="apt-get install -y --no-install-recommends" +ENV MKLROOT="/opt/intel/mkl" + +RUN apt-get update && \ DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ build-essential \ ca-certificates \ @@ -56,55 +58,54 @@ RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ # for glog libgoogle-glog-dev libgoogle-glog0v5 \ # for python sox - sox && \ + sox # ================================================================== -# cmake 3.10 (for MKLDNN) +# cmake 3.10 # ------------------------------------------------------------------ - apt-get purge -y cmake && \ +RUN apt-get purge -y cmake && \ # for cmake DEBIAN_FRONTEND=noninteractive $APT_INSTALL zlib1g-dev libcurl4-openssl-dev && \ cd /tmp && wget https://cmake.org/files/v3.10/cmake-3.10.3.tar.gz && \ tar -xzvf cmake-3.10.3.tar.gz && cd cmake-3.10.3 && \ ./bootstrap --system-curl && \ - make -j$(nproc) && make install && cmake --version && \ + make -j$(nproc) && make install && cmake --version # ================================================================== # MKL https://software.intel.com/en-us/mkl # ------------------------------------------------------------------ - cd /tmp && wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ +RUN cd /tmp && wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ # for build in March 2020 this row should be removed to prevent from the error # E: Failed to fetch https://apt.repos.intel.com/intelpython/binary/Packages Writing more data than expected (15520 > 15023) E: Some index files failed to download. They have been ignored, or old ones used instead. # wget https://apt.repos.intel.com/setup/intelproducts.list -O /etc/apt/sources.list.d/intelproducts.list && \ sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ - apt-get update && DEBIAN_FRONTEND=noninteractive $APT_INSTALL intel-mkl-64bit-2018.4-057 && \ - export MKLROOT=/opt/intel/mkl && \ + apt-get update && DEBIAN_FRONTEND=noninteractive $APT_INSTALL intel-mkl-64bit-2018.4-057 # ================================================================== # arrayfire with CPU backend https://github.com/arrayfire/arrayfire/wiki/ # ------------------------------------------------------------------ - cd /tmp && git clone --recursive https://github.com/arrayfire/arrayfire.git && \ +RUN cd /tmp && git clone --recursive https://github.com/arrayfire/arrayfire.git && \ wget https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.gz && tar xf boost_1_70_0.tar.gz && \ cd arrayfire && git checkout v3.7.1 && git submodule update --init --recursive && \ mkdir build && cd build && \ CXXFLAGS=-DOS_LNX cmake .. -DCMAKE_BUILD_TYPE=Release -DAF_BUILD_CUDA=OFF -DAF_BUILD_OPENCL=OFF -DAF_BUILD_EXAMPLES=OFF -DBOOST_INCLUDEDIR=/tmp/boost_1_70_0 && \ - make -j$(nproc) && make install && \ + make -j$(nproc) && make install # ================================================================== # oneDNN https://github.com/oneapi-src/oneDNN # ------------------------------------------------------------------ - cd /tmp && git clone https://github.com/oneapi-src/oneDNN && \ +RUN cd /tmp && git clone https://github.com/oneapi-src/oneDNN && \ cd oneDNN && git checkout v2.0 && mkdir -p build && cd build && \ - cmake .. -DDNNL_BLAS_VENDOR=MKL && \ - make -j$(nproc) && make install && \ + cmake .. && \ + make -j$(nproc) && make install # ================================================================== # Gloo https://github.com/facebookincubator/gloo.git # ------------------------------------------------------------------ - cd /tmp && git clone https://github.com/facebookincubator/gloo.git && \ +RUN cd /tmp && git clone https://github.com/facebookincubator/gloo.git && \ cd gloo && git checkout b7e0906 && mkdir build && cd build && \ cmake .. -DUSE_MPI=ON && \ - make -j$(nproc) && make install && \ + make -j$(nproc) && make install # ================================================================== # python (for bindings) # ------------------------------------------------------------------ - PIP_INSTALL="python3 -m pip --no-cache-dir install --upgrade" && \ +RUN PIP_INSTALL="python3 -m pip --no-cache-dir install --upgrade" && \ DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ software-properties-common \ && \ @@ -125,26 +126,26 @@ RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ $PIP_INSTALL \ numpy \ sox \ - tqdm && \ + tqdm # ================================================================== # libsndfile https://github.com/erikd/libsndfile.git # ------------------------------------------------------------------ - cd /tmp && git clone https://github.com/erikd/libsndfile.git && \ +RUN cd /tmp && git clone https://github.com/erikd/libsndfile.git && \ cd libsndfile && git checkout 4bdd7414602946a18799b514001b0570e8693a47 && \ ./autogen.sh && ./configure --enable-werror && \ - make && make check && make install && \ + make && make check && make install # ================================================================== # KenLM https://github.com/kpu/kenlm # ------------------------------------------------------------------ - cd /tmp && git clone https://github.com/kpu/kenlm.git && \ +RUN cd /tmp && git clone https://github.com/kpu/kenlm.git && \ cd kenlm && git checkout 4a277534fd33da323205e6ec256e8fd0ff6ee6fa && \ mkdir build && cd build && \ cmake .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON && \ - make -j$(nproc) && make install && \ + make -j$(nproc) && make install # ================================================================== # config & cleanup # ------------------------------------------------------------------ - ldconfig && \ +RUN ldconfig && \ apt-get clean && \ apt-get -y autoremove && \ rm -rf /var/lib/apt/lists/* /tmp/* diff --git a/.docker/Dockerfile-CUDA b/.docker/Dockerfile-CUDA index a24858723..6dee400ac 100644 --- a/.docker/Dockerfile-CUDA +++ b/.docker/Dockerfile-CUDA @@ -6,6 +6,8 @@ FROM flml/flashlight:cuda-base-consolidation-latest +ENV MKLROOT="/opt/intel/mkl" + # ================================================================== # flashlight with GPU backend # ------------------------------------------------------------------ @@ -14,7 +16,6 @@ RUN mkdir /root/flashlight COPY . /root/flashlight -RUN export MKLROOT=/opt/intel/mkl && \ - cd /root/flashlight && mkdir -p build && \ +RUN cd /root/flashlight && mkdir -p build && \ cd build && cmake .. -DFL_BACKEND=CUDA && \ make -j$(nproc) && make install diff --git a/.docker/Dockerfile-CUDA-Base b/.docker/Dockerfile-CUDA-Base index 2a835c2d0..bcc0857f8 100644 --- a/.docker/Dockerfile-CUDA-Base +++ b/.docker/Dockerfile-CUDA-Base @@ -18,8 +18,9 @@ FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu18.04 -RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ - rm -rf /var/lib/apt/lists/* \ +ENV APT_INSTALL="apt-get install -y --no-install-recommends" + +RUN rm -rf /var/lib/apt/lists/* \ /etc/apt/sources.list.d/cuda.list \ /etc/apt/sources.list.d/nvidia-ml.list && \ apt-get update && \ @@ -61,30 +62,30 @@ RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ # for receipts data processing sox \ # for python - python3-distutils && \ + python3-distutils # ================================================================== -# cmake 3.10 (for MKLDNN) +# cmake 3.10 # ------------------------------------------------------------------ - apt-get purge -y cmake && \ +RUN apt-get purge -y cmake && \ # for cmake DEBIAN_FRONTEND=noninteractive $APT_INSTALL zlib1g-dev libcurl4-openssl-dev && \ cd /tmp && wget https://cmake.org/files/v3.10/cmake-3.10.3.tar.gz && \ tar -xzvf cmake-3.10.3.tar.gz && cd cmake-3.10.3 && \ ./bootstrap --system-curl && \ - make -j$(nproc) && make install && cmake --version && \ + make -j$(nproc) && make install && cmake --version # ================================================================== # arrayfire https://github.com/arrayfire/arrayfire/wiki/ # ------------------------------------------------------------------ - cd /tmp && git clone --recursive https://github.com/arrayfire/arrayfire.git && \ +RUN cd /tmp && git clone --recursive https://github.com/arrayfire/arrayfire.git && \ wget https://dl.bintray.com/boostorg/release/1.70.0/source/boost_1_70_0.tar.gz && tar xf boost_1_70_0.tar.gz && \ cd arrayfire && git checkout v3.7.1 && git submodule update --init --recursive && \ mkdir build && cd build && \ CXXFLAGS=-DOS_LNX cmake .. -DCMAKE_BUILD_TYPE=Release -DAF_BUILD_CPU=OFF -DAF_BUILD_OPENCL=OFF -DAF_BUILD_EXAMPLES=OFF -DBOOST_INCLUDEDIR=/tmp/boost_1_70_0 && \ - make -j$(nproc) && make install && \ + make -j$(nproc) && make install # ================================================================== # python (for bindings) # ------------------------------------------------------------------ - PIP_INSTALL="python3 -m pip --no-cache-dir install --upgrade" && \ +RUN PIP_INSTALL="python3 -m pip --no-cache-dir install --upgrade" && \ DEBIAN_FRONTEND=noninteractive $APT_INSTALL \ software-properties-common \ && \ @@ -105,34 +106,34 @@ RUN APT_INSTALL="apt-get install -y --no-install-recommends" && \ $PIP_INSTALL \ sox \ tqdm \ - numpy && \ + numpy # ================================================================== # libsndfile https://github.com/erikd/libsndfile.git # ------------------------------------------------------------------ - cd /tmp && git clone https://github.com/erikd/libsndfile.git && \ +RUN cd /tmp && git clone https://github.com/erikd/libsndfile.git && \ cd libsndfile && git checkout 4bdd7414602946a18799b514001b0570e8693a47 && \ ./autogen.sh && ./configure --enable-werror && \ - make -j$(nproc) && make check && make install && \ + make -j$(nproc) && make check && make install # ================================================================== # MKL https://software.intel.com/en-us/mkl # ------------------------------------------------------------------ - cd /tmp && wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ +RUN cd /tmp && wget https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ apt-key add GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ # wget https://apt.repos.intel.com/setup/intelproducts.list -O /etc/apt/sources.list.d/intelproducts.list && \ sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ - apt-get update && DEBIAN_FRONTEND=noninteractive $APT_INSTALL intel-mkl-64bit-2018.4-057 && \ + apt-get update && DEBIAN_FRONTEND=noninteractive $APT_INSTALL intel-mkl-64bit-2018.4-057 # ================================================================== # KenLM https://github.com/kpu/kenlm # ------------------------------------------------------------------ - cd /tmp && git clone https://github.com/kpu/kenlm.git && \ +RUN cd /tmp && git clone https://github.com/kpu/kenlm.git && \ cd kenlm && git checkout 4a277534fd33da323205e6ec256e8fd0ff6ee6fa && \ mkdir build && cd build && \ cmake .. -DCMAKE_POSITION_INDEPENDENT_CODE=ON && \ - make -j$(nproc) && make install && \ + make -j$(nproc) && make install # ================================================================== # config & cleanup # ------------------------------------------------------------------ - ldconfig && \ +RUN ldconfig && \ apt-get clean && \ apt-get -y autoremove && \ rm -rf /var/lib/apt/lists/* /tmp/* && \