build: Lock GitHub runners' OS

This was motivated by our macOS jobs failing [2] because colima is missing. It looks like this is because the latest versions of the macOS runner no longer have colima installed by default [1]. colima is now explicitly installed. [1] actions/runner-images#6216 [2] `/Users/runner/work/_temp/f19ffbff-27a9-4fc7-80b6-97791d2de141.sh: line 9: colima: command not found`
SeldonIO · May 29, 2024 · 08d9e7d · 08d9e7d
1 parent fcce136
commit 08d9e7d
Show file tree

Hide file tree

Showing 2 changed files with 83 additions and 38 deletions.
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -15,7 +15,7 @@ jobs:
       matrix:
         python-version: ["3.9", "3.10"]
         target: ["dataplane", "model-repository"]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -36,8 +36,10 @@ jobs:
   lint:
     strategy:
       matrix:
-        python-version: ["3.9", "3.10"]
-    runs-on: ubuntu-latest
+        python-version:
+          - "3.9"
+          - "3.10"
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
@@ -57,32 +59,38 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
+        os:
+          - ubuntu-22.04
+          - macos-13
         # NOTE: There's no pre-built `grpcio` wheel for Python 3.11 yet
         # https://github.com/grpc/grpc/issues/32454
-        python-version: ["3.9", "3.10"]
+        python-version:
+          - "3.9"
+          - "3.10"
         is-pr:
           - ${{ github.event_name == 'pull_request' }}
         exclude:
           # MacOS tests take a lot of time, so we will run them only on merge
           # From https://github.com/orgs/community/discussions/26253
           - is-pr: true
-            os: macos-latest
+            os: macos-13
     runs-on: ${{ matrix.os }}
     steps:
-      - name: Setup docker (missing on MacOS)
+      - name: Setup Docker (missing on MacOS)
         if: runner.os == 'macos'
         run: |
           # From https://github.com/actions/runner-images/issues/17#issuecomment-1537238473
           # From https://github.com/abiosoft/colima/discussions/273#discussioncomment-4959736
-          # NOTE: Use the `slirp` driver rather than the default to avoid
-          # random connection errors.
-          # From https://github.com/abiosoft/colima/issues/577
           brew install docker docker-buildx
           mkdir -p $HOME/.docker/cli-plugins
           ln -sfn $(which docker-buildx) $HOME/.docker/cli-plugins/docker-buildx
-          colima start --memory 5 --network-driver slirp
+          # colima version: 0.6.8
+          brew install colima
+          colima start --memory 5
           sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
+      - name: Install OpenMP (in MacOS)
+        if: runner.os == 'macOS'
+        run: brew install libomp
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
@@ -97,16 +105,30 @@ jobs:
         with:
           virtualenvs-create: false
       - name: Install Dependencies
-        run: poetry install --sync --only dev
+        run: |
+          if [[ ${{ runner.os == 'macOS' }} ]]; then
+            export LDFLAGS="-L/usr/local/opt/libomp/lib"
+            export CPPFLAGS="-I/usr/local/opt/libomp/include"
+          fi
+          poetry install --sync --only dev
       - name: Test
-        run: tox -e mlserver
+        run: |
+          if [[ ${{ runner.os == 'macOS' }} ]]; then
+            export LDFLAGS="-L/usr/local/opt/libomp/lib"
+            export CPPFLAGS="-I/usr/local/opt/libomp/include"
+          fi
+          tox -e mlserver
 
   runtimes:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: ["3.9", "3.10"]
+        os:
+          - ubuntu-22.04
+          - macos-13
+        python-version:
+          - "3.9"
+          - "3.10"
         tox-environment:
           - sklearn
           - xgboost
@@ -122,9 +144,12 @@ jobs:
           # MacOS tests take a lot of time, so we will run them only on merge
           # From https://github.com/orgs/community/discussions/26253
           - is-pr: true
-            os: macos-latest
+            os: macos-13
     runs-on: ${{ matrix.os }}
     steps:
+      - name: Install OpenMP (in MacOS)
+        if: runner.os == 'macOS'
+        run: brew install libomp
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
@@ -139,18 +164,32 @@ jobs:
         with:
           virtualenvs-create: false
       - name: Install Dependencies
-        run: poetry install --sync --only dev
+        run: |
+          if [[ ${{ runner.os == 'macOS' }} ]]; then
+            export LDFLAGS="-L/usr/local/opt/libomp/lib"
+            export CPPFLAGS="-I/usr/local/opt/libomp/include"
+          fi
+          poetry install --sync --only dev
       - name: Test
-        run: tox -c ./runtimes/${{ matrix.tox-environment }}
+        run: |
+          if [[ ${{ runner.os == 'macOS' }} ]]; then
+            export LDFLAGS="-L/usr/local/opt/libomp/lib"
+            export CPPFLAGS="-I/usr/local/opt/libomp/include"
+          fi
+          tox -c ./runtimes/${{ matrix.tox-environment }}
 
   # Ensure that having all the runtimes installed together works
   all-runtimes:
     if: github.event_name == 'push'
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-latest]
-        python-version: ["3.9", "3.10"]
+        os:
+          - ubuntu-22.04
+          - macos-13
+        python-version:
+          - "3.9"
+          - "3.10"
     runs-on: ${{ matrix.os }}
     steps:
       - name: Maximize build space
@@ -164,29 +203,21 @@ jobs:
           remove-docker-images: 'true'
           overprovision-lvm: 'true'
           swap-size-mb: 1024
-      - name: Setup docker (missing on MacOS)
+      - name: Setup Docker (missing on MacOS)
         if: runner.os == 'macOS'
         run: |
           # From https://github.com/actions/runner-images/issues/17#issuecomment-1537238473
           # From https://github.com/abiosoft/colima/discussions/273#discussioncomment-4959736
-          # NOTE: Use the `slirp` driver rather than the default to avoid
-          # random connection errors.
-          # From https://github.com/abiosoft/colima/issues/577
           brew install docker docker-buildx
           mkdir -p $HOME/.docker/cli-plugins
           ln -sfn $(which docker-buildx) $HOME/.docker/cli-plugins/docker-buildx
-          colima start --memory 5 --network-driver slirp
+          # 0.6.8
+          brew install colima
+          colima start --memory 5
           sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock
-      - name: Downgrade OpenMP (in MacOS)
+      - name: Install OpenMP (in MacOS)
         if: runner.os == 'macOS'
-        run: |
-          # Recent versions of OpenMP cause segfaults in MacOS when training
-          # LightGBM / XGBoost models (but only when Torch is present)
-          # https://github.com/microsoft/LightGBM/issues/4229
-          # https://github.com/autogluon/autogluon/issues/1442
-          wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
-          brew unlink libomp
-          brew install libomp.rb
+        run: brew install libomp
       - uses: actions/checkout@v4
       - name: Set up Python ${{ matrix.python-version }}
         uses: actions/setup-python@v5
@@ -201,6 +232,16 @@ jobs:
         with:
           virtualenvs-create: false
       - name: Install Dependencies
-        run: poetry install --sync --only dev
+        run: |
+          if [[ ${{ runner.os == 'macOS' }} ]]; then
+            export LDFLAGS="-L/usr/local/opt/libomp/lib"
+            export CPPFLAGS="-I/usr/local/opt/libomp/include"
+          fi
+          poetry install --sync --only dev
       - name: Test
-        run: tox -e all-runtimes
+        run: |
+          if [[ ${{ runner.os == 'macOS' }} ]]; then
+            export LDFLAGS="-L/usr/local/opt/libomp/lib"
+            export CPPFLAGS="-I/usr/local/opt/libomp/include"
+          fi
+          tox -e all-runtimes
diff --git a/tox.ini b/tox.ini
@@ -11,13 +11,15 @@ allowlist_externals = poetry
 
 [testenv:py3]
 commands_pre =
-    poetry install --sync --no-root
+    poetry install --sync --no-root \
+        --with dev
 commands =
     python -m pytest {posargs} -n auto
 
 [testenv:mlserver]
 commands_pre =
-    poetry install --sync --no-root
+    poetry install --sync --no-root \
+        --with dev
 commands =
     python -m pytest {posargs} -n auto \
         {toxinidir}/tests
@@ -29,6 +31,7 @@ set_env =
 [testenv:all-runtimes]
 commands_pre =
     poetry install --sync --no-root \
+        --with dev \
         --with all-runtimes \
         --with all-runtimes-dev
 commands =
@@ -46,6 +49,7 @@ commands =
 [testenv:licenses]
 commands_pre =
     poetry install --sync --no-root \
+        --with dev
         --with all-runtimes \
         --with all-runtimes-dev
 commands =