Merge pull request #550 from roflcoopter/dev

v2.2.0

.coveragerc

@@ -0,0 +1,7 @@
+[report]
+exclude_lines =
+    pragma: no cover
+    raise AssertionError
+    raise NotImplementedError
+    if TYPE_CHECKING:
+    @overload

.dockerignore

@@ -1,4 +1,5 @@
 *
+!/.coveragerc
 !tests/
 !tests/*
 !viseron/

.github/workflows/ci.yaml

@@ -338,7 +338,7 @@ jobs:
       - name: Check out code from GitHub
         uses: actions/checkout@v3.1.0
         with:
-          fetch-depth: 0
+          fetch-depth: 2
       - name: Set up Python ${{ env.DEFAULT_PYTHON }}
         uses: actions/setup-python@v4.3.0
         id: python
@@ -365,6 +365,7 @@ jobs:
           files: |
             requirements.txt
             docker/Dockerfile.wheels
+            azure-pipelines/.env
       - name: List all modified files
         run: |
           for file in "${{ steps.changed-files.outputs.all_modified_files }}"; do

.gitignore

@@ -136,6 +136,7 @@
 !/scripts/gen_docs/*
 
 # Repo
+!/.coveragerc
 !/.flake8
 !/.gitignore
 !/.isort.cfg

.pylintrc

@@ -302,6 +302,7 @@ function-naming-style=snake_case
 good-names=i,
            j,
            k,
+           id,
            ex,
            x1,
            x2,
@@ -406,7 +407,8 @@ redefining-builtins-modules=six.moves,past.builtins,future.builtins,io,builtins
 defining-attr-methods=__init__,
                       __new__,
                       setUp,
-                      initialize
+                      initialize, # for tornado
+                      setup_method # for pytest
 
 # List of member names, which should be excluded from the protected access
 # warning.

azure-pipelines/.env

@@ -3,9 +3,10 @@ UBUNTU_VERSION_NUMBER="20.04"
 BASE_VERSION="1.4.1"
 OPENCV_VERSION="4.6.0"
 OPENVINO_VERSION="2022.2.0"
-FFMPEG_VERSION="4.4"
-DLIB_VERSION="19.24"
-WHEELS_VERSION="1.2"
+FFMPEG_VERSION="5.1.2"
+DLIB_VERSION="19.22"
+SKLEARN_VERSION="1.2.2"
+WHEELS_VERSION="1.4"
 S6_OVERLAY_VERSION="2.1.0.2"
 CMAKE_VERSION=3.20.0
 MAKEFLAGS="-j 5"

azure-pipelines/docker-compose-build.yaml

@@ -138,6 +138,10 @@ services:
       bash -c "pytest --cov=viseron/ --cov-report term-missing -s tests
       && coverage xml"
 
+  amd64-viseron-tests-tox:
+    <<: *amd64-viseron-tests
+    entrypoint: pytest
+
   amd64-viseron-vulture:
     <<: *amd64-viseron-tests
     entrypoint: ""
@@ -286,14 +290,32 @@ services:
         - roflcoopter/rpi3-dlib:$DLIB_VERSION
     image: roflcoopter/rpi3-dlib:$DLIB_VERSION
 
+  # Sklearn does not have a prebuilt wheel for armv7l, and it takes forever to
+  # build from source. So we build it once and use it as a base for the other images
+  # No CI for this image, built locally and pushed to docker hub
+  rpi3-sklearn:
+    build:
+      context: ..
+      dockerfile: ./docker/rpi3/Dockerfile.sklearn
+      args:
+        UBUNTU_VERSION: "$UBUNTU_VERSION"
+        CMAKE_VERSION: "$CMAKE_VERSION"
+        SKLEARN_VERSION: "$SKLEARN_VERSION"
+        MAKEFLAGS: "$MAKEFLAGS"
+      cache_from:
+        - roflcoopter/rpi3-sklearn:$SKLEARN_VERSION
+    image: roflcoopter/rpi3-sklearn:$SKLEARN_VERSION
+
   rpi3-wheels:
     build:
       context: ..
       dockerfile: ./docker/Dockerfile.wheels
       args:
         ARCH: rpi3
-        BUILD_FROM: balenalib/raspberrypi3-ubuntu:$UBUNTU_VERSION-build
+        BUILD_FROM: roflcoopter/rpi3-sklearn:$SKLEARN_VERSION
         DLIB_VERSION: "$DLIB_VERSION"
+        EXTRA_PIP_ARGS: --extra-index-url https://www.piwheels.org/simple
+        CMAKE_VERSION: "$CMAKE_VERSION"
       cache_from:
         - roflcoopter/rpi3-wheels:$WHEELS_VERSION
     image: roflcoopter/rpi3-wheels:$WHEELS_VERSION

docker/Dockerfile

@@ -48,7 +48,6 @@ RUN \
   tzdata \
   python3 \
   python3-pip \
-  python3-sklearn \
   usbutils \
   # OpenCV and FFmpeg Dependencies
   libgomp1 \
@@ -72,6 +71,7 @@ RUN \
   gstreamer1.0-pulseaudio \
   && rm -rf /var/lib/apt/lists/* \
   \
+  && python3 -m pip install --upgrade pip \
   && pip3 install /wheels/*.whl \
   && rm -r /wheels \
   \

docker/Dockerfile.tests

@@ -10,5 +10,6 @@ ADD requirements_test.txt requirements_test.txt
 RUN \
   pip3 install -r requirements_test.txt
 
+COPY .coveragerc /src/
 COPY viseron /src/viseron/
 COPY tests /src/tests/

docker/Dockerfile.wheels

@@ -8,6 +8,7 @@ FROM ${BUILD_FROM} as build
 COPY --from=dlib /wheels /wheels
 
 ARG ARCH
+ARG EXTRA_PIP_ARGS=""
 
 ENV \
   DEBIAN_FRONTEND=noninteractive \
@@ -25,16 +26,20 @@ RUN apt-get -yqq update && apt-get install -yq --no-install-recommends \
   python3-pip \
   python3-setuptools \
   python3-wheel \
+  && python3 -m pip install --upgrade pip \
   && apt-get autoremove -y && apt-get clean -y
 
 ADD requirements.txt requirements.txt
-RUN \
-  mkdir -p /wheels \
+# We need to setup Rust compiler for bcrypt
+# A tmpfs is needed: https://github.com/rust-lang/cargo/issues/8719
+RUN --mount=type=tmpfs,target=/root/.cargo curl https://sh.rustup.rs -sSf | bash -s -- -y \
+  && . $HOME/.cargo/env \
+  && mkdir -p /wheels \
   && if [ "$ARCH" = "amd64" ] || [ "$ARCH" = "amd64-cuda" ]; then export PLATFORM="x86_64"; \
   elif [ "$ARCH" = "armhf" ] || [ "$ARCH" = "rpi3" ]; then export PLATFORM="armv7l"; \
   else export PLATFORM="$ARCH"; fi \
   \
-  && pip3 wheel --wheel-dir=/wheels --find-links=/wheels -r /requirements.txt \
+  && pip3 wheel --wheel-dir=/wheels --find-links=/wheels ${EXTRA_PIP_ARGS} -r /requirements.txt \
   \
   && ls -al /wheels/
 

docker/aarch64/Dockerfile.dlib

@@ -30,10 +30,9 @@ RUN buildDeps="autoconf \
 RUN \
   DIR=/tmp && mkdir -p ${DIR} && cd ${DIR} && \
   git clone --branch v${DLIB_VERSION} --depth 1 https://github.com/davisking/dlib.git && \
-  cd dlib; python3 setup.py bdist_wheel --dist-dir=/wheels
-
-RUN \
-  pip3 wheel face_recognition dlib==${DLIB_VERSION} --wheel-dir=/wheels/ --find-links=/wheels/
+  cd dlib; python3 setup.py bdist_wheel --dist-dir=/wheels \
+  \
+  && ls -al /wheels/
 
 RUN [ "cross-build-end" ]
 

docker/aarch64/Dockerfile.ffmpeg

@@ -512,7 +512,6 @@ RUN  \
   --disable-doc \
   --disable-ffplay \
   --enable-shared \
-  --enable-avresample \
   --enable-libopencore-amrnb \
   --enable-libopencore-amrwb \
   --enable-gpl \

docker/amd64-cuda/Dockerfile.dlib

@@ -30,10 +30,9 @@ RUN buildDeps="autoconf \
 RUN \
   DIR=/tmp && mkdir -p ${DIR} && cd ${DIR} && \
   git clone --branch v${DLIB_VERSION} --depth 1 https://github.com/davisking/dlib.git && \
-  cd dlib; python3 setup.py --set DLIB_USE_CUDA_COMPUTE_CAPABILITIES:STRING=35,50,52,53,60,61,70,72,75,80,86 bdist_wheel --dist-dir=/wheels
-
-RUN \
-  pip3 wheel face_recognition dlib==${DLIB_VERSION} --wheel-dir=/wheels/ --find-links=/wheels/
+  cd dlib; python3 setup.py --set DLIB_USE_CUDA_COMPUTE_CAPABILITIES:STRING=35,50,52,53,60,61,70,72,75,80,86 bdist_wheel --dist-dir=/wheels \
+  \
+  && ls -al /wheels/
 
 FROM scratch as scratch
 COPY --from=build /wheels /wheels/

docker/amd64-cuda/Dockerfile.ffmpeg

@@ -457,7 +457,6 @@ RUN  \
   --disable-doc \
   --disable-ffplay \
   --enable-shared \
-  --enable-avresample \
   --enable-libopencore-amrnb \
   --enable-libopencore-amrwb \
   --enable-gpl \

docker/amd64/Dockerfile.dlib

@@ -29,10 +29,9 @@ RUN buildDeps="autoconf \
 RUN \
   DIR=/tmp && mkdir -p ${DIR} && cd ${DIR} && \
   git clone --branch v${DLIB_VERSION} --depth 1 https://github.com/davisking/dlib.git && \
-  cd dlib; python3 setup.py bdist_wheel --dist-dir=/wheels
-
-RUN \
-  pip3 wheel face_recognition dlib==${DLIB_VERSION} --wheel-dir=/wheels/ --find-links=/wheels/
+  cd dlib; python3 setup.py bdist_wheel --dist-dir=/wheels \
+  \
+  && ls -al /wheels/
 
 FROM scratch as scratch
 COPY --from=build /wheels /wheels/

docker/amd64/Dockerfile.ffmpeg

@@ -439,7 +439,6 @@ RUN  \
   --disable-doc \
   --disable-ffplay \
   --enable-shared \
-  --enable-avresample \
   --enable-libopencore-amrnb \
   --enable-libopencore-amrwb \
   --enable-gpl \

docker/jetson-nano/Dockerfile.base

@@ -16,7 +16,8 @@ ARG JETPACK_VERSION_MAJOR
 ARG JETPACK_VERSION_MINOR
 ARG JETSON_NANO_FFMPEG_APT_VERSION
 ENV \
-  LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/tegra:/usr/lib/aarch64-linux-gnu/tegra-egl"
+  LD_LIBRARY_PATH="/usr/lib/aarch64-linux-gnu/tegra:/usr/lib/aarch64-linux-gnu/tegra-egl" \
+  LD_PRELOAD="/usr/lib/aarch64-linux-gnu/libgomp.so.1"
 
 RUN \
   cd /usr/local/bin && \

docker/jetson-nano/Dockerfile.dlib

@@ -69,10 +69,9 @@ RUN \
   git clone --branch v${DLIB_VERSION} --depth 1 https://github.com/davisking/dlib.git && \
   # Fix bug, explained here https://medium.com/@ageitgey/build-a-hardware-based-face-recognition-system-for-150-with-the-nvidia-jetson-nano-and-python-a25cb8c891fd
   sed -i -e 's/forward_algo = forward_best_algo;/\/\/forward_algo = forward_best_algo;/' dlib/dlib/cuda/cudnn_dlibapi.cpp && \
-  cd dlib; python3 setup.py bdist_wheel --dist-dir=/wheels
-
-RUN \
-  pip3 wheel face_recognition dlib==${DLIB_VERSION} --wheel-dir=/wheels/ --find-links=/wheels/
+  cd dlib; python3 setup.py bdist_wheel --dist-dir=/wheels \
+  \
+  && ls -al /wheels/
 
 RUN [ "cross-build-end" ]
 

docker/jetson-nano/Dockerfile.ffmpeg

@@ -574,7 +574,6 @@ RUN  \
   --disable-doc \
   --disable-ffplay \
   --enable-shared \
-  --enable-avresample \
   --enable-libopencore-amrnb \
   --enable-libopencore-amrwb \
   --enable-gpl \

docker/jetson-nano/Dockerfile.wheels

@@ -35,6 +35,7 @@ RUN apt-get -yqq update && apt-get install -yq --no-install-recommends \
   python3-pip \
   python3-setuptools \
   python3-wheel \
+  && python3 -m pip install --upgrade pip \
   && apt-get autoremove -y && apt-get clean -y \
   && pip install numpy pybind11
 
@@ -43,8 +44,6 @@ RUN \
   mkdir -p /wheels \
   \
   && pip3 wheel --wheel-dir=/wheels --find-links=/wheels -r /requirements.txt \
-  # We need to install sklearn from pip since we are using self supplied Python3 and not the one from apt
-  && pip3 wheel --wheel-dir=/wheels --find-links=/wheels sklearn \
   \
   && ls -al /wheels/
 

docker/rpi3/Dockerfile.dlib

@@ -34,10 +34,9 @@ RUN buildDeps="autoconf \
 RUN \
   DIR=/tmp && mkdir -p ${DIR} && cd ${DIR} && \
   git clone --branch v${DLIB_VERSION} --depth 1 https://github.com/davisking/dlib.git && \
-  cd dlib; python3 setup.py bdist_wheel --dist-dir=/wheels --compiler-flags "-mfpu=neon"
-
-RUN \
-  pip3 wheel face_recognition dlib==${DLIB_VERSION} --wheel-dir=/wheels/ --find-links=/wheels/
+  cd dlib; python3 setup.py bdist_wheel --dist-dir=/wheels --compiler-flags "-mfpu=neon" \
+  \
+  && ls -al /wheels/
 
 RUN [ "cross-build-end" ]
 

docker/rpi3/Dockerfile.ffmpeg

@@ -545,7 +545,6 @@ RUN  \
   --disable-doc \
   --disable-ffplay \
   --enable-shared \
-  --enable-avresample \
   --enable-libopencore-amrnb \
   --enable-libopencore-amrwb \
   --enable-gpl \

docker/rpi3/Dockerfile.sklearn

@@ -0,0 +1,52 @@
+
+# Sklearn does not have a prebuilt wheel for armv7l, and it takes forever to
+# build from source. So we build it once and use it as a base for the other images
+# No CI for this image, built locally and pushed to docker hub
+ARG CMAKE_VERSION
+ARG UBUNTU_VERSION
+FROM roflcoopter/rpi3-cmake:${CMAKE_VERSION} as cmake
+FROM balenalib/raspberrypi3-ubuntu:$UBUNTU_VERSION-build as build
+RUN [ "cross-build-start" ]
+
+COPY --from=cmake /usr/custom_cmake/bin /usr/custom_cmake/bin/
+COPY --from=cmake /usr/custom_cmake/share /usr/custom_cmake/share/
+
+ARG MAKEFLAGS="-j2"
+ARG SKLEARN_VERSION
+
+ENV \
+  DEBIAN_FRONTEND=noninteractive \
+  PIP_IGNORE_INSTALLED=0 \
+  PATH=/usr/custom_cmake/bin:$PATH
+
+RUN apt-get -yqq update && apt-get install -yq --no-install-recommends \
+  build-essential \
+  curl \
+  cython3 \
+  gcc \
+  gfortran \
+  g++ \
+  make \
+  ninja-build \
+  python3 \
+  python3-dev \
+  python3-pip \
+  python3-setuptools \
+  python3-wheel \
+  libopenblas-dev \
+  && python3 -m pip install --upgrade pip \
+  && apt-get autoremove -y && apt-get clean -y
+
+RUN mkdir -p /wheels \
+  \
+  && pip3 wheel --wheel-dir=/wheels --find-links=/wheels scikit-learn==${SKLEARN_VERSION} \
+  \
+  && ls -al /wheels/
+
+RUN [ "cross-build-end" ]
+
+FROM balenalib/raspberrypi3-ubuntu:$UBUNTU_VERSION-build
+
+# Copy scikit-learn + scipy wheels
+COPY --from=build /wheels/scikit_learn*.whl /wheels/
+COPY --from=build /wheels/scipy*.whl /wheels/