diff --git a/Jenkinsfile b/Jenkinsfile index f9137471..e1295305 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -227,7 +227,7 @@ try { stage('Build images') { docker.withRegistry('', dockerBaseImagePullCredId) { // Make sure we are using most recent version of external images - for (externalImage in ['docker/dockerfile:experimental', 'postgres:alpine', + for (externalImage in ['docker/dockerfile:1.2', 'postgres:alpine', 'redis:alpine', 'centos:7']) { try { sh "docker pull '$externalImage'" diff --git a/activemq/Dockerfile b/activemq/Dockerfile index d4537ce0..0abf7739 100644 --- a/activemq/Dockerfile +++ b/activemq/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.2 + ############################################################################# # NOTICE # # # @@ -32,6 +34,7 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] ENV LANG en_US.UTF-8 RUN yum update --assumeyes; \ + yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ yum install --assumeyes java-11-openjdk-headless; \ yum clean all; \ rm -rf /var/cache/yum/*; diff --git a/components/cpp_component_build/Dockerfile b/components/cpp_component_build/Dockerfile index 2e5da429..027a303e 100644 --- a/components/cpp_component_build/Dockerfile +++ b/components/cpp_component_build/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.2 + ############################################################################# # NOTICE # # # @@ -35,20 +37,32 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] ENV LANG en_US.UTF-8 -# Change config because the centos:7 image sets this yum property to en_US.utf8 which prevents other locales -# (including other English locales) from being installed with GCC. -RUN yum-config-manager --setopt=override_install_langs='' --save; \ +# Change override_install_langs because the centos:7 image sets this yum property to en_US.utf8 +# which prevents other locales (including other English locales) from being installed with GCC. +RUN yum-config-manager --setopt=override_install_langs='' \ + --setopt=skip_missing_names_on_install=False --save; \ yum update --assumeyes; \ + yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64; \ yum install --assumeyes epel-release \ - https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \ - yum install --assumeyes cmake3 make gcc gcc-c++ qt-devel log4cxx-devel boost-devel ffmpeg gtest-devel; \ + https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm \ + http://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; \ + yum install --assumeyes --nogpgcheck cmake3 make gcc gcc-c++ qt-devel log4cxx-devel \ + boost-devel ffmpeg gtest-devel openblas-devel cuda-minimal-build-10-2 \ + libcublas-devel-10.2.2.89-1 libcudnn7-devel-7.6.5.33-1.cuda10.2; \ yum clean all; \ - rm --recursive /var/cache/yum/*; - -COPY --from=openmpf_build /opt/opencv-3.4.7 /opt/opencv-3.4.7 - -# Normally components' CMakeList.txt file includes include(../ComponentSetup.cmake) but since that is outside of the -# components' build context we create it here. + rm --recursive /var/cache/yum/*; \ + ln --symbolic cuda-10.2 /usr/local/cuda; \ + # OpenCV doesn't use the statically compiled CUDA libraries except for libcudart and they are relatively large. + find /usr/local/cuda/lib64/ -name '*.a' -not -name 'libcudart_static.a' \ + -not -name 'libcudadevrt.a' -delete; \ + rm /usr/lib64/libcudnn*.a; \ + echo '/usr/local/cuda/lib64' >> /etc/ld.so.conf.d/cuda.conf; \ + ldconfig; + +COPY --from=openmpf_build /opt/opencv-4.5.0 /opt/opencv-4.5.0 + +# Normally components' CMakeList.txt file includes include(../ComponentSetup.cmake) but since that +# is outside of the components' build context we create it here. RUN mkdir /home/mpf; echo 'find_package(mpfCMakeHelpers REQUIRED)' > /home/mpf/ComponentSetup.cmake COPY --from=openmpf_build /build-artifacts/mpf-sdk-install/include /home/mpf/mpf-sdk-install/include diff --git a/components/cpp_executor/Dockerfile b/components/cpp_executor/Dockerfile index 7954bb92..e0021620 100644 --- a/components/cpp_executor/Dockerfile +++ b/components/cpp_executor/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:experimental +# syntax=docker/dockerfile:1.2 ############################################################################# # NOTICE # @@ -60,12 +60,21 @@ RUN cd /usr/local/bin; \ ln --symbolic /usr/local/lib/libpython3.8.so.1.0 /usr/lib64/libpython3.8.so.1.0; - RUN yum update --assumeyes; \ + yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ + yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64; \ yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \ - yum install --assumeyes ffmpeg; \ + yum install --assumeyes --nogpgcheck ffmpeg cuda-cudart-10-2; \ yum clean all; \ - rm --recursive /var/cache/yum/*; + rm --recursive /var/cache/yum/*; \ + ln --symbolic cuda-10.2 /usr/local/cuda; \ + echo '/usr/local/cuda/lib64' >> /etc/ld.so.conf.d/cuda.conf; \ + ldconfig; + + +# Environment variables required by nvidia runtime. +ENV NVIDIA_VISIBLE_DEVICES all +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility ENV MPF_HOME /opt/mpf diff --git a/components/java_component_build/Dockerfile b/components/java_component_build/Dockerfile index b787f903..dc69190d 100644 --- a/components/java_component_build/Dockerfile +++ b/components/java_component_build/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.2 + ############################################################################# # NOTICE # # # @@ -37,6 +39,7 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] ENV LANG en_US.UTF-8 RUN yum update --assumeyes; \ + yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \ yum install --assumeyes java-11-openjdk-devel ffmpeg; \ yum clean all; \ diff --git a/components/java_executor/Dockerfile b/components/java_executor/Dockerfile index ae56cf3e..a4eb5498 100644 --- a/components/java_executor/Dockerfile +++ b/components/java_executor/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.2 + ############################################################################# # NOTICE # # # @@ -58,6 +60,7 @@ RUN cd /usr/local/bin; \ RUN yum update --assumeyes; \ + yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \ yum install --assumeyes java-11-openjdk ffmpeg; \ yum clean all; \ diff --git a/components/python_component_build/Dockerfile b/components/python_component_build/Dockerfile index c6ab92f6..8bfeb480 100644 --- a/components/python_component_build/Dockerfile +++ b/components/python_component_build/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:experimental +# syntax=docker/dockerfile:1.2 ############################################################################# # NOTICE # @@ -66,6 +66,7 @@ ENV COMPONENT_VIRTUALENV $MPF_HOME/plugin-venv ENV PATH $COMPONENT_VIRTUALENV/bin:/scripts:$PATH RUN yum update --assumeyes; \ + yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \ yum install --assumeyes ffmpeg; \ yum clean all; \ diff --git a/components/python_executor/Dockerfile b/components/python_executor/Dockerfile index e2e7317f..d6adbd51 100644 --- a/components/python_executor/Dockerfile +++ b/components/python_executor/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:experimental +# syntax=docker/dockerfile:1.2 ############################################################################# # NOTICE # @@ -61,6 +61,7 @@ RUN cd /usr/local/bin; \ RUN yum update --assumeyes; \ + yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \ yum install --assumeyes ffmpeg; \ yum clean all; \ diff --git a/components/python_executor/README.md b/components/python_executor/README.md index b8fdb039..3306c6f2 100644 --- a/components/python_executor/README.md +++ b/components/python_executor/README.md @@ -53,7 +53,7 @@ FROM openmpf_python_component_build:latest as build_component # or build the dependencies here. Adding the dependencies prior the copying in your source code # allows you to take advantage of the Docker build cache to avoid re-installing the dependencies # every time your source code changes. -# e.g. RUN pip3 install --no-cache-dir 'opencv-python>=3.4.7' 'tensorflow>=2.1.0' +# e.g. RUN pip3 install --no-cache-dir 'opencv-python>=4.4.0' 'tensorflow>=2.1.0' # Copy in your source code COPY . . diff --git a/integration_tests/Dockerfile b/integration_tests/Dockerfile index 023ffd7c..c5daa233 100644 --- a/integration_tests/Dockerfile +++ b/integration_tests/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:experimental +# syntax=docker/dockerfile:1.2 ############################################################################# # NOTICE # @@ -37,7 +37,8 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] ENV LANG en_US.UTF-8 -RUN yum install -y java-11-openjdk; \ +RUN yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ + yum install -y java-11-openjdk; \ yum clean all; \ rm -rf /var/cache/yum/*; diff --git a/kibana/Dockerfile b/kibana/Dockerfile index 9545f0e6..9c59116d 100644 --- a/kibana/Dockerfile +++ b/kibana/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.2 + ############################################################################# # NOTICE # # # diff --git a/markup/Dockerfile b/markup/Dockerfile index 000bdf92..2509de49 100644 --- a/markup/Dockerfile +++ b/markup/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.2 + ############################################################################# # NOTICE # # # @@ -28,12 +30,6 @@ ARG BUILD_REGISTRY ARG BUILD_TAG=latest FROM ${BUILD_REGISTRY}openmpf_build:${BUILD_TAG} as openmpf_build -# Docker COPY only preserves symlinks when copying an entire directory so we copy the files and their symlinks in to -# a temporary directory -RUN mkdir /root/copy_libs64 \ - && cp --preserve=links --no-dereference /build-artifacts/install/lib/libopencv* /root/copy_libs64/ - - FROM centos:7 SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] @@ -41,12 +37,18 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] ENV LANG en_US.UTF-8 RUN yum update --assumeyes; \ + yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \ yum install --assumeyes java-11-openjdk ffmpeg libheif; \ yum clean all; \ rm -rf /var/cache/yum/*; -COPY --from=openmpf_build /root/copy_libs64 /usr/lib64/ +# Use mount and cp shell command because the docker COPY command resolves symlinks which then +# causes duplicate copies of the libraries to be added to the image. +RUN --mount=type=bind,from=openmpf_build,source=/build-artifacts/install/lib,target=/tmp/mpf-libs \ + cd /tmp/mpf-libs; \ + cp --preserve=links --no-dereference libopencv* libopenblas* libgfortran* libquadmath* \ + /usr/lib64; COPY --from=openmpf_build /build-artifacts/install/lib/libmpfopencvjni.so /usr/lib64/ diff --git a/openmpf_build/Dockerfile b/openmpf_build/Dockerfile index 7b2b857e..bd0165bb 100644 --- a/openmpf_build/Dockerfile +++ b/openmpf_build/Dockerfile @@ -1,4 +1,4 @@ -# syntax=docker/dockerfile:experimental +# syntax=docker/dockerfile:1.2 ############################################################################# # NOTICE # @@ -26,7 +26,6 @@ # limitations under the License. # ############################################################################# - FROM centos:7 as openmpf_build SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"] @@ -39,19 +38,38 @@ ENV LANG en_US.UTF-8 ENV PYTHONDONTWRITEBYTECODE=1 ENV PIP_COMPILE=0 + + RUN yum update --assumeyes; \ - yum install --assumeyes epel-release https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \ - yum install --assumeyes cmake3 make gcc gcc-c++ qt-devel log4cxx-devel boost-devel ffmpeg ffmpeg-devel gtest-devel \ - java-11-openjdk-devel activemq-cpp-devel protobuf-static protobuf-devel libheif-devel file \ + yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ + yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64; \ + yum install --assumeyes \ + epel-release \ + https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm \ + http://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; \ + CU_VERSION=10-2; \ + yum install --assumeyes --nogpgcheck \ + cmake3 make gcc gcc-c++ qt-devel log4cxx-devel boost-devel ffmpeg ffmpeg-devel \ + gtest-devel java-11-openjdk-devel activemq-cpp-devel protobuf-static protobuf-devel \ + libheif-devel file \ # Python build deps - openssl-devel libffi-devel; \ + openssl-devel libffi-devel \ + # OpenCV build deps + openblas-devel cuda-minimal-build-$CU_VERSION cuda-cufft-dev-$CU_VERSION cuda-npp-dev-$CU_VERSION \ + libcudnn7-devel-7.6.5.33-1.cuda10.2 libcublas-devel-10.2.2.89; \ yum clean all; \ rm --recursive /var/cache/yum/*; \ # For protobuf maven plugin - ln --symbolic /usr/bin/protoc /usr/local/bin/protoc; + ln --symbolic /usr/bin/protoc /usr/local/bin/protoc; \ + ln --symbolic cuda-10.2 /usr/local/cuda; \ + # OpenCV doesn't use the statically compiled CUDA libraries except for libcudart and they are relatively large. + find /usr/local/cuda/lib64/ -name '*.a' -not -name 'libcudart_static.a' -delete; \ + echo '/usr/local/cuda/lib64' >> /etc/ld.so.conf.d/cuda.conf; \ + ldconfig ENV JAVA_HOME=/etc/alternatives/java_sdk + RUN mkdir /tmp/python; \ cd /tmp/python; \ curl 'https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tar.xz' \ @@ -71,21 +89,31 @@ RUN mkdir /tmp/python; \ RUN mkdir /tmp/opencv-contrib; \ cd /tmp/opencv-contrib; \ - curl --location 'https://github.com/opencv/opencv_contrib/archive/3.4.7.tar.gz' \ + curl --location 'https://github.com/opencv/opencv_contrib/archive/4.5.0.tar.gz' \ | tar --extract --gzip; \ mkdir /tmp/opencv; \ cd /tmp/opencv; \ - curl --location 'https://github.com/opencv/opencv/archive/3.4.7.tar.gz' \ + curl --location 'https://github.com/opencv/opencv/archive/4.5.0.tar.gz' \ | tar --extract --gzip; \ - cd opencv-3.4.7; \ + cd opencv-4.5.0; \ mkdir build; \ cd build; \ - cmake3 -DCMAKE_INSTALL_PREFIX:PATH="/opt/opencv-3.4.7" -DWITH_IPP=false \ - -DBUILD_EXAMPLES=false -DBUILD_TESTS=false -DBUILD_PERF_TESTS=false \ - -DOPENCV_EXTRA_MODULES_PATH=/tmp/opencv-contrib/opencv_contrib-3.4.7/modules ..; \ + cmake3 -DCMAKE_INSTALL_PREFIX:PATH='/opt/opencv-4.5.0' \ + -DWITH_IPP=false \ + -DBUILD_EXAMPLES=false \ + -DBUILD_TESTS=false \ + -DBUILD_PERF_TESTS=false \ + -DWITH_CUDA=true \ + -DCUDA_ARCH_BIN=3.0 \ + -DCUDA_ARCH_PTX=3.0 \ + -DWITH_CUBLAS=true \ + -DWITH_CUDNN=true \ + -DOPENCV_DNN_CUDA=true \ + -DBUILD_CUDA_STUBS=true \ + -DOPENCV_EXTRA_MODULES_PATH=/tmp/opencv-contrib/opencv_contrib-4.5.0/modules \ + ..; \ make --jobs "$(nproc)" install; \ - ln --symbolic /opt/opencv-3.4.7/include/opencv2 /usr/local/include/opencv2; \ - ln --symbolic /opt/opencv-3.4.7/include/opencv /usr/local/include/opencv; \ + ln --symbolic '/opt/opencv-4.5.0/include/opencv4/opencv2' /usr/local/include/opencv2; \ rm --recursive /tmp/opencv /tmp/opencv-contrib; diff --git a/workflow_manager/Dockerfile b/workflow_manager/Dockerfile index 3e28222c..16526d8e 100644 --- a/workflow_manager/Dockerfile +++ b/workflow_manager/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1.2 + ############################################################################# # NOTICE # # # @@ -57,6 +59,7 @@ RUN cd /usr/local/bin; \ RUN yum update --assumeyes; \ + yum-config-manager --setopt=skip_missing_names_on_install=False --save; \ yum install --assumeyes --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \ yum install --assumeyes java-11-openjdk ffmpeg tomcat-native file; \ yum clean all; \