OpenCV4 upgrade with CUDA

Jenkinsfile

@@ -227,7 +227,7 @@ try {
     stage('Build images') {
         docker.withRegistry('', dockerBaseImagePullCredId) {
             // Make sure we are using most recent version of external images
-            for (externalImage in ['docker/dockerfile:experimental', 'postgres:alpine',
+            for (externalImage in ['docker/dockerfile:1.2', 'postgres:alpine',
                                    'redis:alpine', 'centos:7']) {
                 try {
                     sh "docker pull '$externalImage'"

activemq/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.2
+
 #############################################################################
 # NOTICE                                                                    #
 #                                                                           #
@@ -32,6 +34,7 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"]
 ENV LANG en_US.UTF-8
 
 RUN yum update --assumeyes; \
+    yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
     yum install --assumeyes java-11-openjdk-headless; \
     yum clean all; \
     rm -rf /var/cache/yum/*;

components/cpp_component_build/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.2
+
 #############################################################################
 # NOTICE                                                                    #
 #                                                                           #
@@ -35,20 +37,32 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"]
 
 ENV LANG en_US.UTF-8
 
-# Change config because the centos:7 image sets this yum property to en_US.utf8 which prevents other locales
-# (including other English locales) from being installed with GCC.
-RUN yum-config-manager --setopt=override_install_langs='' --save; \
+# Change override_install_langs because the centos:7 image sets this yum property to en_US.utf8
+# which prevents other locales (including other English locales) from being installed with GCC.
+RUN yum-config-manager --setopt=override_install_langs='' \
+                       --setopt=skip_missing_names_on_install=False --save; \
     yum update --assumeyes; \
+    yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64; \
     yum install --assumeyes epel-release \
-            https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \
-    yum install --assumeyes cmake3 make gcc gcc-c++ qt-devel log4cxx-devel boost-devel ffmpeg gtest-devel; \
+            https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm \
+            http://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; \
+    yum install --assumeyes  --nogpgcheck cmake3 make gcc gcc-c++ qt-devel log4cxx-devel \
+            boost-devel ffmpeg gtest-devel openblas-devel cuda-minimal-build-10-2 \
+            libcublas-devel-10.2.2.89-1 libcudnn7-devel-7.6.5.33-1.cuda10.2; \
     yum clean all; \
-    rm --recursive /var/cache/yum/*;
-
-COPY --from=openmpf_build /opt/opencv-3.4.7 /opt/opencv-3.4.7
-
-# Normally components' CMakeList.txt file includes include(../ComponentSetup.cmake) but since that is outside of the
-# components' build context we create it here.
+    rm --recursive /var/cache/yum/*; \
+    ln --symbolic cuda-10.2 /usr/local/cuda; \
+    # OpenCV doesn't use the statically compiled CUDA libraries except for libcudart and they are relatively large.
+    find /usr/local/cuda/lib64/ -name '*.a' -not -name 'libcudart_static.a' \
+                                            -not -name 'libcudadevrt.a' -delete; \
+    rm /usr/lib64/libcudnn*.a; \
+    echo '/usr/local/cuda/lib64' >> /etc/ld.so.conf.d/cuda.conf; \
+    ldconfig;
+
+COPY --from=openmpf_build /opt/opencv-4.5.0 /opt/opencv-4.5.0
+
+# Normally components' CMakeList.txt file includes include(../ComponentSetup.cmake) but since that
+# is outside of the components' build context we create it here.
 RUN mkdir /home/mpf; echo 'find_package(mpfCMakeHelpers REQUIRED)' > /home/mpf/ComponentSetup.cmake
 
 COPY --from=openmpf_build /build-artifacts/mpf-sdk-install/include /home/mpf/mpf-sdk-install/include

components/cpp_executor/Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:experimental
+# syntax=docker/dockerfile:1.2
 
 #############################################################################
 # NOTICE                                                                    #
@@ -60,12 +60,21 @@ RUN cd /usr/local/bin; \
     ln --symbolic /usr/local/lib/libpython3.8.so.1.0 /usr/lib64/libpython3.8.so.1.0;
 
 
-
 RUN yum update --assumeyes; \
+    yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
+    yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64; \
     yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \
-    yum install --assumeyes ffmpeg; \
+    yum install --assumeyes --nogpgcheck ffmpeg cuda-cudart-10-2; \
     yum clean all; \
-    rm --recursive /var/cache/yum/*;
+    rm --recursive /var/cache/yum/*; \
+    ln --symbolic cuda-10.2 /usr/local/cuda; \
+    echo '/usr/local/cuda/lib64' >> /etc/ld.so.conf.d/cuda.conf; \
+    ldconfig;
+
+
+# Environment variables required by nvidia runtime.
+ENV NVIDIA_VISIBLE_DEVICES all
+ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
 
 ENV MPF_HOME /opt/mpf
 

components/java_component_build/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.2
+
 #############################################################################
 # NOTICE                                                                    #
 #                                                                           #
@@ -37,6 +39,7 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"]
 ENV LANG en_US.UTF-8
 
 RUN yum update --assumeyes; \
+    yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
     yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \
     yum install --assumeyes java-11-openjdk-devel ffmpeg; \
     yum clean all; \

components/java_executor/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.2
+
 #############################################################################
 # NOTICE                                                                    #
 #                                                                           #
@@ -58,6 +60,7 @@ RUN cd /usr/local/bin; \
 
 
 RUN yum update --assumeyes; \
+    yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
     yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \
     yum install --assumeyes java-11-openjdk ffmpeg; \
     yum clean all; \

components/python_component_build/Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:experimental
+# syntax=docker/dockerfile:1.2
 
 #############################################################################
 # NOTICE                                                                    #
@@ -66,6 +66,7 @@ ENV COMPONENT_VIRTUALENV $MPF_HOME/plugin-venv
 ENV PATH $COMPONENT_VIRTUALENV/bin:/scripts:$PATH
 
 RUN yum update --assumeyes; \
+    yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
     yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \
     yum install --assumeyes ffmpeg; \
     yum clean all; \

components/python_executor/Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:experimental
+# syntax=docker/dockerfile:1.2
 
 #############################################################################
 # NOTICE                                                                    #
@@ -61,6 +61,7 @@ RUN cd /usr/local/bin; \
 
 
 RUN yum update --assumeyes; \
+    yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
     yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \
     yum install --assumeyes ffmpeg; \
     yum clean all; \

components/python_executor/README.md

@@ -53,7 +53,7 @@ FROM openmpf_python_component_build:latest as build_component
 # or build the dependencies here. Adding the dependencies prior the copying in your source code 
 # allows you to take advantage of the Docker build cache to avoid re-installing the dependencies 
 # every time your source code changes.
-# e.g. RUN pip3 install --no-cache-dir 'opencv-python>=3.4.7' 'tensorflow>=2.1.0'
+# e.g. RUN pip3 install --no-cache-dir 'opencv-python>=4.4.0' 'tensorflow>=2.1.0'
 
 # Copy in your source code
 COPY . .

integration_tests/Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:experimental
+# syntax=docker/dockerfile:1.2
 
 #############################################################################
 # NOTICE                                                                    #
@@ -37,7 +37,8 @@ SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"]
 
 ENV LANG en_US.UTF-8
 
-RUN yum install -y java-11-openjdk; \
+RUN yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
+    yum install -y java-11-openjdk; \
     yum clean all; \
     rm -rf /var/cache/yum/*;
 

kibana/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.2
+
 #############################################################################
 # NOTICE                                                                    #
 #                                                                           #

markup/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.2
+
 #############################################################################
 # NOTICE                                                                    #
 #                                                                           #
@@ -28,25 +30,25 @@ ARG BUILD_REGISTRY
 ARG BUILD_TAG=latest
 FROM ${BUILD_REGISTRY}openmpf_build:${BUILD_TAG} as openmpf_build
 
-# Docker COPY only preserves symlinks when copying an entire directory so we copy the files and their symlinks in to
-# a temporary directory
-RUN mkdir /root/copy_libs64 \
-    && cp --preserve=links --no-dereference /build-artifacts/install/lib/libopencv* /root/copy_libs64/
-
-
 FROM centos:7
 
 SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"]
 
 ENV LANG en_US.UTF-8
 
 RUN yum update --assumeyes; \
+    yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
     yum install --assumeyes https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \
     yum install --assumeyes java-11-openjdk ffmpeg libheif; \
     yum clean all; \
     rm -rf /var/cache/yum/*;
 
-COPY --from=openmpf_build /root/copy_libs64 /usr/lib64/
+# Use mount and cp shell command because the docker COPY command resolves symlinks which then
+# causes duplicate copies of the libraries to be added to the image.
+RUN --mount=type=bind,from=openmpf_build,source=/build-artifacts/install/lib,target=/tmp/mpf-libs \
+    cd /tmp/mpf-libs; \
+    cp --preserve=links --no-dereference libopencv* libopenblas* libgfortran* libquadmath* \
+        /usr/lib64;
 
 COPY --from=openmpf_build /build-artifacts/install/lib/libmpfopencvjni.so /usr/lib64/
 

openmpf_build/Dockerfile

@@ -1,4 +1,4 @@
-# syntax=docker/dockerfile:experimental
+# syntax=docker/dockerfile:1.2
 
 #############################################################################
 # NOTICE                                                                    #
@@ -26,7 +26,6 @@
 # limitations under the License.                                            #
 #############################################################################
 
-
 FROM centos:7 as openmpf_build
 
 SHELL ["/bin/bash", "-o", "errexit", "-o", "pipefail", "-c"]
@@ -39,19 +38,38 @@ ENV LANG en_US.UTF-8
 ENV PYTHONDONTWRITEBYTECODE=1
 ENV PIP_COMPILE=0
 
+
+
 RUN yum update --assumeyes; \
-    yum install --assumeyes epel-release https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \
-    yum install --assumeyes cmake3 make gcc gcc-c++ qt-devel log4cxx-devel boost-devel ffmpeg ffmpeg-devel gtest-devel \
-            java-11-openjdk-devel activemq-cpp-devel protobuf-static protobuf-devel libheif-devel file \
+    yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
+    yum-config-manager --add-repo http://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64; \
+    yum install --assumeyes \
+        epel-release \
+        https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm \
+        http://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/nvidia-machine-learning-repo-rhel7-1.0.0-1.x86_64.rpm; \
+    CU_VERSION=10-2; \
+    yum install --assumeyes --nogpgcheck \
+            cmake3 make gcc gcc-c++ qt-devel log4cxx-devel boost-devel ffmpeg ffmpeg-devel \
+            gtest-devel java-11-openjdk-devel activemq-cpp-devel protobuf-static protobuf-devel \
+            libheif-devel file \
             # Python build deps
-            openssl-devel libffi-devel; \
+            openssl-devel libffi-devel \
+            # OpenCV build deps
+            openblas-devel cuda-minimal-build-$CU_VERSION cuda-cufft-dev-$CU_VERSION cuda-npp-dev-$CU_VERSION \
+            libcudnn7-devel-7.6.5.33-1.cuda10.2 libcublas-devel-10.2.2.89; \
     yum clean all; \
     rm --recursive /var/cache/yum/*; \
     # For protobuf maven plugin
-    ln --symbolic /usr/bin/protoc /usr/local/bin/protoc;
+    ln --symbolic /usr/bin/protoc /usr/local/bin/protoc; \
+    ln --symbolic cuda-10.2 /usr/local/cuda; \
+    # OpenCV doesn't use the statically compiled CUDA libraries except for libcudart and they are relatively large.
+    find /usr/local/cuda/lib64/ -name '*.a' -not -name 'libcudart_static.a' -delete; \
+    echo '/usr/local/cuda/lib64' >> /etc/ld.so.conf.d/cuda.conf; \
+    ldconfig
 
 ENV JAVA_HOME=/etc/alternatives/java_sdk
 
+
 RUN mkdir /tmp/python; \
     cd /tmp/python; \
     curl 'https://www.python.org/ftp/python/3.8.2/Python-3.8.2.tar.xz' \
@@ -71,21 +89,31 @@ RUN mkdir /tmp/python; \
 
 RUN mkdir /tmp/opencv-contrib; \
     cd /tmp/opencv-contrib; \
-    curl --location 'https://github.com/opencv/opencv_contrib/archive/3.4.7.tar.gz' \
+    curl --location 'https://github.com/opencv/opencv_contrib/archive/4.5.0.tar.gz' \
         | tar --extract --gzip; \
     mkdir /tmp/opencv; \
     cd  /tmp/opencv; \
-    curl --location 'https://github.com/opencv/opencv/archive/3.4.7.tar.gz' \
+    curl --location 'https://github.com/opencv/opencv/archive/4.5.0.tar.gz' \
         | tar --extract --gzip; \
-    cd opencv-3.4.7; \
+    cd opencv-4.5.0; \
     mkdir build; \
     cd build; \
-    cmake3 -DCMAKE_INSTALL_PREFIX:PATH="/opt/opencv-3.4.7" -DWITH_IPP=false \
-        -DBUILD_EXAMPLES=false -DBUILD_TESTS=false -DBUILD_PERF_TESTS=false \
-        -DOPENCV_EXTRA_MODULES_PATH=/tmp/opencv-contrib/opencv_contrib-3.4.7/modules ..; \
+    cmake3 -DCMAKE_INSTALL_PREFIX:PATH='/opt/opencv-4.5.0' \
+        -DWITH_IPP=false \
+        -DBUILD_EXAMPLES=false \
+        -DBUILD_TESTS=false \
+        -DBUILD_PERF_TESTS=false \
+        -DWITH_CUDA=true \
+        -DCUDA_ARCH_BIN=3.0 \
+        -DCUDA_ARCH_PTX=3.0 \
+        -DWITH_CUBLAS=true \
+        -DWITH_CUDNN=true \
+        -DOPENCV_DNN_CUDA=true \
+        -DBUILD_CUDA_STUBS=true \
+        -DOPENCV_EXTRA_MODULES_PATH=/tmp/opencv-contrib/opencv_contrib-4.5.0/modules \
+        ..; \
     make --jobs "$(nproc)" install; \
-    ln --symbolic /opt/opencv-3.4.7/include/opencv2 /usr/local/include/opencv2; \
-    ln --symbolic /opt/opencv-3.4.7/include/opencv /usr/local/include/opencv; \
+    ln --symbolic '/opt/opencv-4.5.0/include/opencv4/opencv2' /usr/local/include/opencv2; \
     rm --recursive /tmp/opencv /tmp/opencv-contrib;
 
 

workflow_manager/Dockerfile

@@ -1,3 +1,5 @@
+# syntax=docker/dockerfile:1.2
+
 #############################################################################
 # NOTICE                                                                    #
 #                                                                           #
@@ -57,6 +59,7 @@ RUN cd /usr/local/bin; \
 
 
 RUN yum update --assumeyes; \
+    yum-config-manager --setopt=skip_missing_names_on_install=False --save; \
     yum install --assumeyes --nogpgcheck https://download1.rpmfusion.org/free/el/rpmfusion-free-release-7.noarch.rpm; \
     yum install --assumeyes java-11-openjdk ffmpeg tomcat-native file; \
     yum clean all; \