feat(ml)!: cuda and openvino acceleration (#5619)

* cuda and openvino ep, refactor, update dockerfile * updated workflow * typing fixes * added tests * updated ml test gh action * updated README * updated docker-compose * added compute to hwaccel.yml * updated gh matrix updated gh matrix updated gh matrix updated gh matrix updated gh matrix give up * remove cuda/arm64 build * add hwaccel image tags to docker-compose * remove unnecessary quotes * add suffix to git tag * fixed kwargs in base model * armnn ld_library_path * update pyproject.toml * add armnn workflow * formatting * consolidate hwaccel files, update docker compose * update hw transcoding docs * add ml hwaccel docs * update dev and prod docker-compose * added armnn prerequisite docs * support 3.10 * updated docker-compose comments * formatting * test coverage * don't set arena extend strategy for openvino * working openvino * formatting * fix dockerfile * added type annotation * add wsl configuration for openvino * updated lock file * copy python3 * comment out extends section * fix platforms * simplify workflow suffix tagging * simplify aio transcoding doc * update docs and workflow for `hwaccel.yml` change * revert docs
immich-app · Jan 21, 2024 · 95cfe22 · 95cfe22
1 parent 6b419a9
commit 95cfe22
Show file tree

Hide file tree

Showing 23 changed files with 964 additions and 462 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -25,15 +25,38 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - context: "machine-learning"
-            file: "machine-learning/Dockerfile"
-            image: "immich-machine-learning"
-            platforms: "linux/amd64,linux/arm64"
-          - context: "."
-            file: "server/Dockerfile"
-            image: "immich-server"
-            platforms: "linux/arm64,linux/amd64"
+          - image: immich-machine-learning
+            context: machine-learning
+            file: machine-learning/Dockerfile
+            platforms: linux/amd64,linux/arm64
+            device: cpu
 
+          - image: immich-machine-learning
+            context: machine-learning
+            file: machine-learning/Dockerfile
+            platforms: linux/amd64
+            device: cuda
+            suffix: -cuda
+
+          - image: immich-machine-learning
+            context: machine-learning
+            file: machine-learning/Dockerfile
+            platforms: linux/amd64
+            device: openvino
+            suffix: -openvino
+
+          - image: immich-machine-learning
+            context: machine-learning
+            file: machine-learning/Dockerfile
+            platforms: linux/arm64
+            device: armnn
+            suffix: -armnn
+
+          - image: immich-server
+            context: .
+            file: server/Dockerfile
+            platforms: linux/amd64,linux/arm64
+            device: cpu
     steps:
       - name: Checkout
         uses: actions/checkout@v4
@@ -79,12 +102,12 @@ jobs:
             name=altran1502/${{matrix.image}},enable=${{ github.event_name == 'release' }}
           tags: |
             # Tag with branch name
-            type=ref,event=branch
+            type=ref,event=branch,suffix=${{ matrix.suffix }}
             # Tag with pr-number
-            type=ref,event=pr
+            type=ref,event=pr,suffix=${{ matrix.suffix }}
             # Tag with git tag on release
-            type=ref,event=tag
-            type=raw,value=release,enable=${{ github.event_name == 'release' }}
+            type=ref,event=tag,suffix=${{ matrix.suffix }}
+            type=raw,value=release,enable=${{ github.event_name == 'release' }},suffix=${{ matrix.suffix }}
 
       - name: Determine build cache output
         id: cache-target
@@ -106,5 +129,7 @@ jobs:
           push: ${{ !github.event.pull_request.head.repo.fork }}
           cache-from: type=registry,ref=ghcr.io/${{ github.repository_owner }}/immich-build-cache:${{matrix.image}}
           cache-to: ${{ steps.cache-target.outputs.cache-to }}
+          build-args: |
+            DEVICE=${{ matrix.device }}
           tags: ${{ steps.metadata.outputs.tags }}
           labels: ${{ steps.metadata.outputs.labels }}
diff --git a/.github/workflows/prepare-release.yml b/.github/workflows/prepare-release.yml
@@ -83,5 +83,6 @@ jobs:
           files: |
             docker/docker-compose.yml
             docker/example.env
-            docker/hwaccel.yml
+            docker/hwaccel.ml.yml
+            docker/hwaccel.transcoding.yml
             *.apk
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -225,7 +225,7 @@ jobs:
           cache: "poetry"
       - name: Install dependencies
         run: |
-          poetry install --with dev
+          poetry install --with dev --with cpu
       - name: Lint with ruff
         run: |
           poetry run ruff check --output-format=github app export

diff --git a/docker/docker-compose.dev.yml b/docker/docker-compose.dev.yml
@@ -44,8 +44,8 @@ services:
     command: [ "/usr/src/app/bin/immich-dev", "microservices" ]
     <<: *server-common
     # extends:
-    #   file: hwaccel.yml
-    #   service: hwaccel
+    #   file: hwaccel.transcoding.yml
+    #   service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
     ports:
       - 9231:9230
     depends_on:
@@ -79,9 +79,14 @@ services:
   immich-machine-learning:
     container_name: immich_machine_learning
     image: immich-machine-learning-dev:latest
+    # extends:
+    #   file: hwaccel.ml.yml
+    #   service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
     build:
       context: ../machine-learning
       dockerfile: Dockerfile
+      args:
+        - DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
     ports:
       - 3003:3003
     volumes:

diff --git a/docker/docker-compose.prod.yml b/docker/docker-compose.prod.yml
@@ -30,8 +30,8 @@ services:
     command: [ "./start-microservices.sh" ]
     <<: *server-common
     # extends:
-    #   file: hwaccel.yml
-    #   service: hwaccel
+    #   file: hwaccel.transcoding.yml
+    #   service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
     depends_on:
       - redis
       - database
@@ -40,9 +40,14 @@ services:
   immich-machine-learning:
     container_name: immich_machine_learning
     image: immich-machine-learning:latest
+    # extends:
+    #   file: hwaccel.ml.yml
+    #   service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
     build:
       context: ../machine-learning
       dockerfile: Dockerfile
+      args:
+        - DEVICE=cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference
     volumes:
       - model-cache:/cache
     env_file:

diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -30,9 +30,9 @@ services:
   immich-microservices:
     container_name: immich_microservices
     image: ghcr.io/immich-app/immich-server:${IMMICH_VERSION:-release}
-    # extends:
-    #   file: hwaccel.yml
-    #   service: hwaccel
+    # extends: # uncomment this section for hardware acceleration - see https://immich.app/docs/features/hardware-transcoding
+    #   file: hwaccel.transcoding.yml 
+    #   service: cpu # set to one of [nvenc, quicksync, rkmpp, vaapi, vaapi-wsl] for accelerated transcoding
     command: [ "start.sh", "microservices" ]
     volumes:
       - ${UPLOAD_LOCATION}:/usr/src/app/upload
@@ -46,7 +46,12 @@ services:
 
   immich-machine-learning:
     container_name: immich_machine_learning
+    # For hardware acceleration, add one of -[armnn, cuda, openvino] to the image tag.
+    # Example tag: ${IMMICH_VERSION:-release}-cuda
     image: ghcr.io/immich-app/immich-machine-learning:${IMMICH_VERSION:-release}
+    # extends: # uncomment this section for hardware acceleration - see https://immich.app/docs/features/ml-hardware-acceleration
+    #   file: hwaccel.ml.yml
+    #   service: cpu # set to one of [armnn, cuda, openvino, openvino-wsl] for accelerated inference - use the `-wsl` version for WSL2 where applicable
     volumes:
       - model-cache:/cache
     env_file:

diff --git a/docker/hwaccel-rkmpp.yml b/docker/hwaccel-rkmpp.yml
diff --git a/docker/hwaccel.ml.yml b/docker/hwaccel.ml.yml
@@ -0,0 +1,47 @@
+version: "3.8"
+
+# Configurations for hardware-accelerated machine learning
+
+# If using Unraid or another platform that doesn't allow multiple Compose files,
+# you can inline the config for a backend by copying its contents 
+# into the immich-machine-learning service in the docker-compose.yml file.
+
+# See https://immich.app/docs/features/ml-hardware-acceleration for info on usage.
+
+services:
+  armnn:
+    devices:
+      - /dev/mali0:/dev/mali0
+    volumes:
+      - /lib/firmware/mali_csffw.bin:/lib/firmware/mali_csffw.bin:ro # Mali firmware for your chipset (not always required depending on the driver)
+      - /usr/lib/libmali.so:/usr/lib/libmali.so:ro # Mali driver for your chipset (always required)
+
+  cpu:
+
+  cuda:
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities:
+                - gpu
+                - compute
+                - video
+
+  openvino:
+    device_cgroup_rules:
+      - "c 189:* rmw"
+    devices:
+      - /dev/dri:/dev/dri
+    volumes:
+      - /dev/bus/usb:/dev/bus/usb
+
+  openvino-wsl:
+    devices:
+      - /dev/dri:/dev/dri
+      - /dev/dxg:/dev/dxg
+    volumes:
+      - /dev/bus/usb:/dev/bus/usb
+      - /usr/lib/wsl:/usr/lib/wsl
diff --git a/docker/hwaccel.transcoding.yml b/docker/hwaccel.transcoding.yml
@@ -0,0 +1,59 @@
+version: "3.8"
+
+# Configurations for hardware-accelerated transcoding 
+
+# If using Unraid or another platform that doesn't allow multiple Compose files,
+# you can inline the config for a backend by copying its contents 
+# into the immich-microservices service in the docker-compose.yml file.
+
+# See https://immich.app/docs/features/hardware-transcoding for more info on using hardware transcoding.
+
+services:
+  cpu:
+
+  nvenc:
+    deploy:
+      resources:
+        reservations:
+          devices:
+            - driver: nvidia
+              count: 1
+              capabilities:
+                - gpu
+                - compute
+                - video
+
+  quicksync:
+    devices:
+      - /dev/dri:/dev/dri
+
+  rkmpp:
+    security_opt: # enables full access to /sys and /proc, still far better than privileged: true
+      - systempaths=unconfined
+      - apparmor=unconfined
+    group_add:
+      - video
+    devices:
+      - /dev/rga:/dev/rga
+      - /dev/dri:/dev/dri
+      - /dev/dma_heap:/dev/dma_heap
+      - /dev/mpp_service:/dev/mpp_service
+    volumes:
+      - /usr/bin/ffmpeg:/usr/bin/ffmpeg_mpp:ro
+      - /lib/aarch64-linux-gnu:/lib/ffmpeg-mpp:ro
+      - /lib/aarch64-linux-gnu/libblas.so.3:/lib/ffmpeg-mpp/libblas.so.3:ro # symlink is resolved by mounting
+      - /lib/aarch64-linux-gnu/liblapack.so.3:/lib/ffmpeg-mpp/liblapack.so.3:ro # symlink is resolved by mounting
+      - /lib/aarch64-linux-gnu/pulseaudio/libpulsecommon-15.99.so:/lib/ffmpeg-mpp/libpulsecommon-15.99.so:ro
+
+  vaapi:
+    devices:
+      - /dev/dri:/dev/dri
+
+  vaapi-wsl: # use this for VAAPI if you're running Immich in WSL2
+    devices:
+      - /dev/dri:/dev/dri
+    volumes:
+      - /usr/lib/wsl:/usr/lib/wsl
+    environment:
+      - LD_LIBRARY_PATH=/usr/lib/wsl/lib
+      - LIBVA_DRIVER_NAME=d3d12
diff --git a/docker/hwaccel.yml b/docker/hwaccel.yml
diff --git a/docker/mlaccel-armnn.yml b/docker/mlaccel-armnn.yml