chore(deps): update onnxruntime-openvino

machine-learning/Dockerfile

@@ -2,7 +2,7 @@ ARG DEVICE=cpu
 
 FROM python:3.11-bookworm@sha256:8e697181d24bd77cc4251fdd37e4cdd6d725c5de2ed63b9bc8db77357400c5e2 as builder-cpu
 
-FROM openvino/ubuntu22_runtime:2023.1.0@sha256:002842a9005ba01543b7169ff6f14ecbec82287f09c4d1dd37717f0a8e8754a7 as builder-openvino
+FROM openvino/ubuntu22_runtime:2023.3.0@sha256:176646df619032ea6c10faf842867119c393e7497b7f88b5e307e932a0fd5aa8 as builder-openvino
 USER root
 RUN apt-get update && apt-get install -y --no-install-recommends python3-dev
 
@@ -21,10 +21,12 @@ FROM builder-${DEVICE} as builder
 
 ARG DEVICE
 ENV PYTHONDONTWRITEBYTECODE=1 \
-  PYTHONUNBUFFERED=1 \
-  PIP_NO_CACHE_DIR=true \
-  VIRTUAL_ENV="/opt/venv" \
-  PATH="/opt/venv/bin:${PATH}"
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=true \
+    VIRTUAL_ENV="/opt/venv" \
+    PATH="/opt/venv/bin:${PATH}"
+
+RUN apt-get update && apt-get install -y --no-install-recommends g++
 
 RUN pip install --upgrade pip && pip install poetry
 RUN poetry config installer.max-workers 10 && \
@@ -36,7 +38,7 @@ RUN poetry install --sync --no-interaction --no-ansi --no-root --with ${DEVICE}
 
 FROM python:3.11-slim-bookworm@sha256:ce81dc539f0aedc9114cae640f8352fad83d37461c24a3615b01f081d0c0583a as prod-cpu
 
-FROM openvino/ubuntu22_runtime:2023.1.0@sha256:002842a9005ba01543b7169ff6f14ecbec82287f09c4d1dd37717f0a8e8754a7 as prod-openvino
+FROM openvino/ubuntu22_runtime:2023.3.0@sha256:176646df619032ea6c10faf842867119c393e7497b7f88b5e307e932a0fd5aa8 as prod-openvino
 USER root
 
 FROM nvidia/cuda:12.2.2-cudnn8-runtime-ubuntu22.04@sha256:2d913b09e6be8387e1a10976933642c73c840c0b735f0bf3c28d97fc9bc422e0 as prod-cuda
@@ -56,14 +58,14 @@ RUN apt-get update && apt-get install -y --no-install-recommends ocl-icd-libopen
     mkdir /opt/armnn
 
 COPY --from=builder-armnn \
-      /opt/armnn/libarmnn.so.?? \
-      /opt/armnn/libarmnnOnnxParser.so.?? \
-      /opt/armnn/libarmnnDeserializer.so.?? \
-      /opt/armnn/libarmnnTfLiteParser.so.?? \
-      /opt/armnn/libprotobuf.so.?.??.?.? \
-      /opt/ann/libann.s[o] \
-      /opt/ann/build.sh \
-      /opt/armnn/
+    /opt/armnn/libarmnn.so.?? \
+    /opt/armnn/libarmnnOnnxParser.so.?? \
+    /opt/armnn/libarmnnDeserializer.so.?? \
+    /opt/armnn/libarmnnTfLiteParser.so.?? \
+    /opt/armnn/libprotobuf.so.?.??.?.? \
+    /opt/ann/libann.s[o] \
+    /opt/ann/build.sh \
+    /opt/armnn/
 
 FROM prod-${DEVICE} as prod
 
@@ -73,11 +75,12 @@ RUN apt-get update && \
 
 WORKDIR /usr/src/app
 ENV NODE_ENV=production \
-  TRANSFORMERS_CACHE=/cache \
-  PYTHONDONTWRITEBYTECODE=1 \
-  PYTHONUNBUFFERED=1 \
-  PATH="/opt/venv/bin:$PATH" \
-  PYTHONPATH=/usr/src
+    TRANSFORMERS_CACHE=/cache \
+    PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PATH="/opt/venv/bin:$PATH" \
+    PYTHONPATH=/usr/src \
+    DEVICE=${DEVICE}
 
 # prevent core dumps
 RUN echo "hard core 0" >> /etc/security/limits.conf && \

machine-learning/app/models/base.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import os
 from abc import ABC, abstractmethod
 from pathlib import Path
 from shutil import rmtree
@@ -115,17 +114,12 @@ def _make_session(self, model_path: Path) -> AnnSession | ort.InferenceSession:
             case ".armnn":
                 session = AnnSession(model_path)
             case ".onnx":
-                cwd = os.getcwd()
-                try:
-                    os.chdir(model_path.parent)
-                    session = ort.InferenceSession(
-                        model_path.as_posix(),
-                        sess_options=self.sess_options,
-                        providers=self.providers,
-                        provider_options=self.provider_options,
-                    )
-                finally:
-                    os.chdir(cwd)
+                session = ort.InferenceSession(
+                    model_path.as_posix(),
+                    sess_options=self.sess_options,
+                    providers=self.providers,
+                    provider_options=self.provider_options,
+                )
             case _:
                 raise ValueError(f"Unsupported model file type: {model_path.suffix}")
         return session

machine-learning/app/test_main.py

@@ -262,7 +262,6 @@ def test_make_session_return_ort_if_available_and_ann_is_not(self, mocker: Mocke
 
         mock_ann = mocker.patch("app.models.base.AnnSession")
         mock_ort = mocker.patch("app.models.base.ort.InferenceSession")
-        mocker.patch("app.models.base.os.chdir")
 
         encoder = OpenCLIPEncoder("ViT-B-32__openai")
         encoder._make_session(mock_armnn_path)
@@ -285,26 +284,6 @@ def test_make_session_raises_exception_if_path_does_not_exist(self, mocker: Mock
         mock_ann.assert_not_called()
         mock_ort.assert_not_called()
 
-    def test_make_session_changes_cwd(self, mocker: MockerFixture) -> None:
-        mock_model_path = mocker.Mock()
-        mock_model_path.is_file.return_value = True
-        mock_model_path.suffix = ".onnx"
-        mock_model_path.parent = "model_parent"
-        mock_model_path.with_suffix.return_value = mock_model_path
-        mock_ort = mocker.patch("app.models.base.ort.InferenceSession")
-        mock_chdir = mocker.patch("app.models.base.os.chdir")
-
-        encoder = OpenCLIPEncoder("ViT-B-32__openai")
-        encoder._make_session(mock_model_path)
-
-        mock_chdir.assert_has_calls(
-            [
-                mock.call(mock_model_path.parent),
-                mock.call(os.getcwd()),
-            ]
-        )
-        mock_ort.assert_called_once()
-
     def test_download(self, mocker: MockerFixture) -> None:
         mock_snapshot_download = mocker.patch("app.models.base.snapshot_download")
 

machine-learning/pyproject.toml

@@ -51,7 +51,7 @@ onnxruntime-gpu = {version = "^1.17.0", source = "cuda12"}
 optional = true
 
 [tool.poetry.group.openvino.dependencies]
-onnxruntime-openvino = ">=1.15.0,<1.16.0"
+onnxruntime-openvino = "^1.17.1"
 
 [tool.poetry.group.armnn]
 optional = true

machine-learning/start.sh

@@ -1,8 +1,11 @@
 #!/usr/bin/env sh
 
 lib_path="/usr/lib/$(arch)-linux-gnu/libmimalloc.so.2"
-export LD_PRELOAD="$lib_path"
-export LD_BIND_NOW=1
+# mimalloc seems to increase memory usage dramatically with openvino, need to investigate
+if ! [ "$DEVICE" = "openvino" ]; then 
+	export LD_PRELOAD="$lib_path"
+	export LD_BIND_NOW=1
+fi
 
 : "${MACHINE_LEARNING_HOST:=0.0.0.0}"
 : "${MACHINE_LEARNING_PORT:=3003}"