Remove model generation images

Latest rebase on dev brought in blakeblackshear#7006, which replaced the model
generation image with a trt-model-prepare service in the regular
frigate-tensorrt image. Follow the same paradigm for Jetsons.

.github/workflows/ci.yml

@@ -75,6 +75,7 @@ jobs:
           build-args:
             - BASE_IMAGE=timongentzsch/l4t-ubuntu20-opencv:latest
             - SLIM_BASE=timongentzsch/l4t-ubuntu20-opencv:latest
+            - TRT_BASE=timongentzsch/l4t-ubuntu20-opencv:latest
           tags: |
             ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ github.ref_name }}-${{ env.SHORT_SHA }}-jetson-jetpack4
           cache-from: type=gha
@@ -88,32 +89,7 @@ jobs:
           build-args:
             - BASE_IMAGE=nvcr.io/nvidia/l4t-tensorrt:r8.4.1-runtime
             - SLIM_BASE=nvcr.io/nvidia/l4t-tensorrt:r8.4.1-runtime
+            - TRT_BASE=nvcr.io/nvidia/l4t-tensorrt:r8.4.1-runtime
           tags: |
             ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ github.ref_name }}-${{ env.SHORT_SHA }}-jetson-jetpack5
           cache-from: type=gha
-      - name: Build and push TensorRT model conversion (Jetson Nano, TX1 & TX2)
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          push: true
-          platforms: linux/arm64
-          target: jetson-trt-models
-          build-args:
-            - BASE_IMAGE=timongentzsch/l4t-ubuntu20-opencv:latest
-            - SLIM_BASE=timongentzsch/l4t-ubuntu20-opencv:latest
-          tags: |
-            ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ github.ref_name }}-${{ env.SHORT_SHA }}-jetson-jetpack4-models
-          cache-from: type=gha
-      - name: Build and push TensorRT model conversion (Jetson Xavier & Orin)
-        uses: docker/build-push-action@v4
-        with:
-          context: .
-          push: true
-          platforms: linux/arm64
-          target: jetson-trt-models
-          build-args:
-            - BASE_IMAGE=nvcr.io/nvidia/l4t-tensorrt:r8.4.1-runtime
-            - SLIM_BASE=nvcr.io/nvidia/l4t-tensorrt:r8.4.1-runtime
-          tags: |
-            ghcr.io/${{ steps.lowercaseRepo.outputs.lowercase }}:${{ github.ref_name }}-${{ env.SHORT_SHA }}-jetson-jetpack5-models
-          cache-from: type=gha

Dockerfile

@@ -5,6 +5,7 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 ARG BASE_IMAGE=debian:11
 ARG SLIM_BASE=debian:11-slim
+ARG TRT_BASE=nvcr.io/nvidia/tensorrt:23.03-py3
 
 FROM ${BASE_IMAGE} AS base
 
@@ -196,10 +197,8 @@ ARG DEBIAN_FRONTEND
 RUN apt-get update \
     && apt-get install -y protobuf-compiler libprotobuf-dev \
     && rm -rf /var/lib/apt/lists/*
-
-# Match versions in tensorrt_models.sh
-RUN pip3 wheel --wheel-dir=/trt-model-wheels onnx==1.9.0 protobuf==3.20.3 numpy==1.23.*
-
+RUN --mount=type=bind,source=requirements-tensorrt-jetson-models.txt,target=/requirements-tensorrt-jetson-models.txt \
+    pip3 wheel --wheel-dir=/trt-model-wheels -r /requirements-tensorrt-jetson-models.txt
 
 FROM wget AS jetson-ffmpeg
 ARG DEBIAN_FRONTEND
@@ -316,68 +315,48 @@ WORKDIR /opt/frigate/
 COPY --from=rootfs / /
 
 # Build TensorRT-specific library
-FROM nvcr.io/nvidia/tensorrt:23.03-py3 AS trt-deps
-
+FROM ${TRT_BASE} AS trt-deps
+RUN apt-get update \
+    && apt-get install -y git build-essential cuda-nvcc-* cuda-nvtx-* libnvinfer-dev libnvinfer-plugin-dev libnvparsers-dev libnvonnxparsers-dev \
+    && rm -rf /var/lib/apt/lists/*
 RUN --mount=type=bind,source=docker/support/tensorrt_detector/tensorrt_libyolo.sh,target=/tensorrt_libyolo.sh \
     /tensorrt_libyolo.sh
 
 # Frigate w/ TensorRT Support as separate image
-FROM deps AS frigate-tensorrt
+FROM deps AS frigate-tensorrt-common
 
 #Disable S6 Global timeout
 ENV S6_CMD_WAIT_FOR_SERVICES_MAXTIME=0
 
-ENV TRT_VER=8.5.3
-ENV YOLO_MODELS="yolov7-tiny-416"
-
 COPY --from=trt-deps /usr/local/lib/libyolo_layer.so /usr/local/lib/libyolo_layer.so
 COPY --from=trt-deps /usr/local/src/tensorrt_demos /usr/local/src/tensorrt_demos
 COPY docker/support/tensorrt_detector/rootfs/ /
+ENV YOLO_MODELS="yolov7-tiny-416"
 
+FROM frigate-tensorrt-common AS frigate-tensorrt
+ENV TRT_VER=8.5.3
 RUN --mount=type=bind,from=trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \
     pip3 install -U /deps/trt-wheels/*.whl && \
     ldconfig
+
 WORKDIR /opt/frigate/
 COPY --from=rootfs / /
 
 # Frigate w/ TensorRT for NVIDIA Jetson platforms
-FROM deps AS frigate-jetson
+FROM frigate-tensorrt-common AS frigate-jetson
+RUN apt-get update \
+    && apt-get install -y python-is-python3 libprotobuf17  \
+    && rm -rf /var/lib/apt/lists/*
 COPY --from=jetson-ffmpeg /rootfs /
+COPY --from=jetson-trt-wheels /etc/TENSORRT_VER /etc/TENSORRT_VER
 RUN --mount=type=bind,from=jetson-trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \
-    pip3 install -U /deps/trt-wheels/*.whl \
+    --mount=type=bind,from=jetson-trt-model-wheels,source=/trt-model-wheels,target=/deps/trt-model-wheels \
+    pip3 install -U /deps/trt-wheels/*.whl /deps/trt-model-wheels/*.whl \
     && ldconfig
+
 WORKDIR /opt/frigate/
 COPY --from=rootfs / /
 
-# Image to generate TRT models (must run on target HW and with exact same TRT version)
-FROM wget AS tensorrt_demos
-RUN wget -q https://github.com/yeahme49/tensorrt_demos/archive/refs/heads/master.zip \
-    && unzip master.zip && mv tensorrt_demos-master /tensorrt_demos && rm master.zip \
-    && cd /tensorrt_demos/yolo && ./download_yolo.sh 2> /dev/null
-
-FROM deps AS jetson-trt-models
-RUN --mount=type=bind,from=jetson-trt-wheels,source=/trt-wheels,target=/deps/trt-wheels \
-    --mount=type=bind,from=jetson-trt-model-wheels,source=/trt-model-wheels,target=/deps/trt-model-wheels \
-    pip3 install -U /deps/trt-wheels/*.whl /deps/trt-model-wheels/*.whl
-
-# On Jetpack 4.6, libnvinfer is mounted from the host via the nvidia container runtime, and thus must
-# not be in the image, so don't add: {libnvinfer, libnvinfer-plugin, libnvparsers, libnvonnxparsers}
-# Otherwise model generation fails or produces models that don't load.
-# For Jetpack 5.0, these libraries must be installed in the image.
-RUN apt-get update \
-    && apt-get install -y python-is-python3 libprotobuf17 build-essential cuda-nvcc-* \
-    && if [ ${CUDA_VERSION%%.*} -ge 11 ]; then \
-         apt-get install -y libnvinfer-dev libnvinfer-plugin-dev libnvparsers-dev libnvonnxparsers-dev; \
-       else \
-         [ ! -e /usr/lib/aarch64-linux-gnu/libnvinfer.so ]; \
-       fi \
-    && rm -rf /var/lib/apt/lists/*
-
-COPY --from=tensorrt_demos /tensorrt_demos /tensorrt_demos
-WORKDIR /tensorrt_demos/yolo
-COPY docker/tensorrt_models.sh /tensorrt_models.sh
-ENTRYPOINT ["/tensorrt_models.sh"]
-
 # Dev Container w/ TRT
 FROM devcontainer AS devcontainer-trt
 

Makefile

@@ -8,8 +8,8 @@ CURRENT_GID := $(shell id -g)
 
 JETPACK4_BASE ?= timongentzsch/l4t-ubuntu20-opencv:latest
 JETPACK5_BASE ?= nvcr.io/nvidia/l4t-tensorrt:r8.4.1-runtime
-JETPACK4_ARGS := --build-arg BASE_IMAGE=$(JETPACK4_BASE) --build-arg SLIM_BASE=$(JETPACK4_BASE)
-JETPACK5_ARGS := --build-arg BASE_IMAGE=$(JETPACK5_BASE) --build-arg SLIM_BASE=$(JETPACK5_BASE)
+JETPACK4_ARGS := --build-arg BASE_IMAGE=$(JETPACK4_BASE) --build-arg SLIM_BASE=$(JETPACK4_BASE) --build-arg TRT_BASE=$(JETPACK4_BASE)
+JETPACK5_ARGS := --build-arg BASE_IMAGE=$(JETPACK5_BASE) --build-arg SLIM_BASE=$(JETPACK5_BASE) --build-arg TRT_BASE=$(JETPACK5_BASE)
 
 version:
 	echo 'VERSION = "$(VERSION)-$(COMMIT_HASH)"' > frigate/version.py
@@ -30,15 +30,9 @@ arm64:
 jetson-jetpack4: version
 	docker buildx build --platform linux/arm64 --target=frigate-jetson --tag $(IMAGE_REPO):$(VERSION)-$(COMMIT_HASH)-jetson-jetpack4 $(JETPACK4_ARGS) .
 
-jetson-jetpack4-models:
-	docker buildx build --platform linux/arm64 --target=jetson-trt-models --tag $(IMAGE_REPO):$(VERSION)-$(COMMIT_HASH)-jetson-jetpack4-models $(JETPACK4_ARGS) .
-
 jetson-jetpack5: version
 	docker buildx build --platform linux/arm64 --target=frigate-jetson --tag $(IMAGE_REPO):$(VERSION)-$(COMMIT_HASH)-jetson-jetpack5 $(JETPACK5_ARGS) .
 
-jetson-jetpack5-models:
-	docker buildx build --platform linux/arm64 --target=jetson-trt-models --tag $(IMAGE_REPO):$(VERSION)-$(COMMIT_HASH)-jetson-jetpack5-models $(JETPACK5_ARGS) .
-
 build: version amd64 arm64 jetson-jetpack4 jetson-jetpack5
 	docker buildx build --platform linux/arm64/v8,linux/amd64 --target=frigate --tag $(IMAGE_REPO):$(VERSION)-$(COMMIT_HASH) .
 

docker-compose.yml

@@ -29,8 +29,6 @@ services:
       - /etc/localtime:/etc/localtime:ro
       - ./config:/config
       - ./debug:/media/frigate
-      # Create the trt-models folder using the documented method of generating TRT models
-      # - ./debug/trt-models:/trt-models
       - /dev/bus/usb:/dev/bus/usb
   mqtt:
     container_name: mqtt

docker/support/tensorrt_detector/rootfs/etc/s6-overlay/s6-rc.d/trt-model-prepare/run

@@ -4,7 +4,23 @@
 
 set -o errexit -o nounset -o pipefail
 
+# On Jetpack 4.6, the nvidia container runtime will mount several host nvidia libraries into the
+# container which should not be present in the image - if they are, TRT model generation will
+# fail or produce invalid models. Thus we must request the user to install them on the host in
+# order to build libyolo here.
+# On Jetpack 5.0, these libraries are not mounted by the runtime and are supplied by the image.
+if [[ "$(arch)" == "aarch64" &&
+      ( ! -e /usr/lib/aarch64-linux-gnu/libnvinfer.so.8 ||
+        ! -e /usr/lib/aarch64-linux-gnu/libnvinfer_plugin.so.8 ||
+        ! -e /usr/lib/aarch64-linux-gnu/libnvparsers.so.8 ||
+        ! -e /usr/lib/aarch64-linux-gnu/libnvonnxparser.so.8 ) ]]; then
+    echo "Please run the following on the HOST:"
+    echo "  sudo apt install libnvinfer8 libnvinfer-plugin8 libnvparsers8 libnvonnxparsers8 nvidia-container"
+    exit 1
+fi
+
 MODEL_CACHE_DIR=${MODEL_CACHE_DIR:-"/config/model_cache/tensorrt"}
+TRT_VER=${TRT_VER:-$(cat /etc/TENSORRT_VER)}
 OUTPUT_FOLDER="${MODEL_CACHE_DIR}/${TRT_VER}"
 
 # Create output folder
@@ -17,7 +33,8 @@ for model in ${YOLO_MODELS//,/ }
 do
     # Remove old link in case path/version changed
     rm -f ${MODEL_CACHE_DIR}/${model}.trt
-
+    rm -f ${MODEL_CACHE_DIR}/${model}_dla.trt
+
     if [[ ! -f ${OUTPUT_FOLDER}/${model}.trt ]]; then
         if [[ ${FIRST_MODEL} = true ]]; then
             MODEL_CONVERT="${model}"
@@ -27,6 +44,9 @@ do
         fi
     else
         ln -s ${OUTPUT_FOLDER}/${model}.trt ${MODEL_CACHE_DIR}/${model}.trt
+        if [[ -f ${OUTPUT_FOLDER}/${model}_dla.trt ]]; then
+            ln -s ${OUTPUT_FOLDER}/${model}_dla.trt ${MODEL_CACHE_DIR}/${model}_dla.trt
+        fi
     fi
 done
 
@@ -45,9 +65,26 @@ cd /usr/local/src/tensorrt_demos/yolo
 
 for model in ${MODEL_CONVERT//,/ }
 do
-    echo "Converting ${model} model"
     python3 yolo_to_onnx.py -m ${model} > /dev/null
+
+    echo -e "\nGenerating ${model}.trt. This may take a few minutes.\n"; start=$(date +%s)
     python3 onnx_to_tensorrt.py -m ${model} > /dev/null
-    cp ${model}.trt ${OUTPUT_FOLDER}/${model}.trt
+    mv ${model}.trt ${OUTPUT_FOLDER}/${model}.trt;
     ln -s ${OUTPUT_FOLDER}/${model}.trt ${MODEL_CACHE_DIR}/${model}.trt
+    echo "Generated ${model}.trt in $(($(date +%s)-start)) seconds"
+
+    if [ -e /dev/nvhost-nvdla0 ]; then
+        # enable DLA0
+        echo -e "\nGenerating ${model}_dla.trt. This may take a few minutes.\n"; start=$(date +%s)
+        python3 onnx_to_tensorrt.py -m ${model} --dla_core 0 > /dev/null
+        mv ${model}.trt ${OUTPUT_FOLDER}/${model}_dla.trt;
+        ln -s ${OUTPUT_FOLDER}/${model}_dla.trt ${MODEL_CACHE_DIR}/${model}_dla.trt
+        echo "Generated ${model}_dla.trt in $(($(date +%s)-start)) seconds"
+
+        # no use in making a dla core 1 model because frigate doesn't have a mechanism
+        # to specify the dla core id, so a core 1 model will still run on core 0
+    fi
 done
+
+echo "Available tensorrt models:"
+cd ${OUTPUT_FOLDER} && ls *.trt;

docker/support/tensorrt_detector/tensorrt_libyolo.sh

@@ -8,7 +8,10 @@ SCRIPT_DIR="/usr/local/src/tensorrt_demos"
 git clone --depth 1 https://github.com/NateMeyer/tensorrt_demos.git -b conditional_download
 
 # Build libyolo
-cd ./tensorrt_demos/plugins && make all
+if [ ! -e /usr/local/cuda ]; then
+    ln -s /usr/local/cuda-* /usr/local/cuda
+fi
+cd ./tensorrt_demos/plugins && make all -j$(nproc)
 cp libyolo_layer.so /usr/local/lib/libyolo_layer.so
 
 # Store yolo scripts for later conversion

docs/docs/configuration/object_detectors.md

@@ -237,12 +237,6 @@ frigate:
     - USE_FP16=false
 ```
 
-**For Jetson platforms**, models must be generated beforehand. It is recommended to use the following command instead of the above:
-
-```
-docker run --runtime=nvidia --rm -v `pwd`/trt-models:/tensorrt_models madsciencetist/jetson-trt-models:jetpack5.0.2 <model>
-```
-
 ### Configuration Parameters
 
 The TensorRT detector can be selected by specifying `tensorrt` as the model type. The GPU will need to be passed through to the docker container using the same methods described in the [Hardware Acceleration](hardware_acceleration.md#nvidia-gpu) section. If you pass through multiple GPUs, you can select which GPU is used for a detector with the `device` configuration parameter. The `device` parameter is an integer value of the GPU index, as shown by `nvidia-smi` within the container.

requirements-tensorrt-jetson-models.txt

@@ -0,0 +1,3 @@
+onnx == 1.9.0; platform_machine == 'aarch64'
+protobuf == 3.20.3; platform_machine == 'aarch64'
+numpy == 1.23.*; platform_machine == 'aarch64'