From 4bc9dfbbc20b369c24916d84f770cd3b3251706d Mon Sep 17 00:00:00 2001 From: Jesse Claven Date: Thu, 16 May 2024 17:01:27 +0100 Subject: [PATCH 1/5] build: Lock GitHub runners' OS This was motivated by our macOS jobs failing [2] because colima is missing. It looks like this is because the latest versions of the macOS runner no longer have colima installed by default [1]. colima is now explicitly installed. [1] https://github.com/actions/runner-images/issues/6216 [2] `/Users/runner/work/_temp/f19ffbff-27a9-4fc7-80b6-97791d2de141.sh: line 9: colima: command not found` --- .github/workflows/tests.yml | 113 ++++++++++++++++++++++++------------ 1 file changed, 77 insertions(+), 36 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 55bf2ebf5..c7bc0f6d4 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -15,7 +15,7 @@ jobs: matrix: python-version: ["3.9", "3.10"] target: ["dataplane", "model-repository"] - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -36,8 +36,10 @@ jobs: lint: strategy: matrix: - python-version: ["3.9", "3.10"] - runs-on: ubuntu-latest + python-version: + - "3.9" + - "3.10" + runs-on: ubuntu-22.04 steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} @@ -57,32 +59,38 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] + os: + - ubuntu-22.04 + - macos-13 # NOTE: There's no pre-built `grpcio` wheel for Python 3.11 yet # https://github.com/grpc/grpc/issues/32454 - python-version: ["3.9", "3.10"] + python-version: + - "3.9" + - "3.10" is-pr: - ${{ github.event_name == 'pull_request' }} exclude: # MacOS tests take a lot of time, so we will run them only on merge # From https://github.com/orgs/community/discussions/26253 - is-pr: true - os: macos-latest + os: macos-13 runs-on: ${{ matrix.os }} steps: - - name: Setup docker (missing on MacOS) + - name: Setup Docker (missing on MacOS) if: runner.os == 'macos' run: | # From https://github.com/actions/runner-images/issues/17#issuecomment-1537238473 # From https://github.com/abiosoft/colima/discussions/273#discussioncomment-4959736 - # NOTE: Use the `slirp` driver rather than the default to avoid - # random connection errors. - # From https://github.com/abiosoft/colima/issues/577 brew install docker docker-buildx mkdir -p $HOME/.docker/cli-plugins ln -sfn $(which docker-buildx) $HOME/.docker/cli-plugins/docker-buildx - colima start --memory 5 --network-driver slirp + # colima version: 0.6.8 + brew install colima + colima start --memory 5 sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + - name: Install OpenMP (in MacOS) + if: runner.os == 'macOS' + run: brew install libomp - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -97,16 +105,30 @@ jobs: with: virtualenvs-create: false - name: Install Dependencies - run: poetry install --sync --only dev + run: | + if [[ ${{ runner.os == 'macOS' }} ]]; then + export LDFLAGS="-L/usr/local/opt/libomp/lib" + export CPPFLAGS="-I/usr/local/opt/libomp/include" + fi + poetry install --sync --only dev - name: Test - run: tox -e mlserver + run: | + if [[ ${{ runner.os == 'macOS' }} ]]; then + export LDFLAGS="-L/usr/local/opt/libomp/lib" + export CPPFLAGS="-I/usr/local/opt/libomp/include" + fi + tox -e mlserver runtimes: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - python-version: ["3.9", "3.10"] + os: + - ubuntu-22.04 + - macos-13 + python-version: + - "3.9" + - "3.10" tox-environment: - sklearn - xgboost @@ -122,9 +144,12 @@ jobs: # MacOS tests take a lot of time, so we will run them only on merge # From https://github.com/orgs/community/discussions/26253 - is-pr: true - os: macos-latest + os: macos-13 runs-on: ${{ matrix.os }} steps: + - name: Install OpenMP (in MacOS) + if: runner.os == 'macOS' + run: brew install libomp - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -139,9 +164,19 @@ jobs: with: virtualenvs-create: false - name: Install Dependencies - run: poetry install --sync --only dev + run: | + if [[ ${{ runner.os == 'macOS' }} ]]; then + export LDFLAGS="-L/usr/local/opt/libomp/lib" + export CPPFLAGS="-I/usr/local/opt/libomp/include" + fi + poetry install --sync --only dev - name: Test - run: tox -c ./runtimes/${{ matrix.tox-environment }} + run: | + if [[ ${{ runner.os == 'macOS' }} ]]; then + export LDFLAGS="-L/usr/local/opt/libomp/lib" + export CPPFLAGS="-I/usr/local/opt/libomp/include" + fi + tox -c ./runtimes/${{ matrix.tox-environment }} # Ensure that having all the runtimes installed together works all-runtimes: @@ -149,8 +184,12 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-latest] - python-version: ["3.9", "3.10"] + os: + - ubuntu-22.04 + - macos-13 + python-version: + - "3.9" + - "3.10" runs-on: ${{ matrix.os }} steps: - name: Maximize build space @@ -164,29 +203,21 @@ jobs: remove-docker-images: 'true' overprovision-lvm: 'true' swap-size-mb: 1024 - - name: Setup docker (missing on MacOS) + - name: Setup Docker (missing on MacOS) if: runner.os == 'macOS' run: | # From https://github.com/actions/runner-images/issues/17#issuecomment-1537238473 # From https://github.com/abiosoft/colima/discussions/273#discussioncomment-4959736 - # NOTE: Use the `slirp` driver rather than the default to avoid - # random connection errors. - # From https://github.com/abiosoft/colima/issues/577 brew install docker docker-buildx mkdir -p $HOME/.docker/cli-plugins ln -sfn $(which docker-buildx) $HOME/.docker/cli-plugins/docker-buildx - colima start --memory 5 --network-driver slirp + # 0.6.8 + brew install colima + colima start --memory 5 sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - - name: Downgrade OpenMP (in MacOS) + - name: Install OpenMP (in MacOS) if: runner.os == 'macOS' - run: | - # Recent versions of OpenMP cause segfaults in MacOS when training - # LightGBM / XGBoost models (but only when Torch is present) - # https://github.com/microsoft/LightGBM/issues/4229 - # https://github.com/autogluon/autogluon/issues/1442 - wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb - brew unlink libomp - brew install libomp.rb + run: brew install libomp - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 @@ -201,6 +232,16 @@ jobs: with: virtualenvs-create: false - name: Install Dependencies - run: poetry install --sync --only dev + run: | + if [[ ${{ runner.os == 'macOS' }} ]]; then + export LDFLAGS="-L/usr/local/opt/libomp/lib" + export CPPFLAGS="-I/usr/local/opt/libomp/include" + fi + poetry install --sync --only dev - name: Test - run: tox -e all-runtimes + run: | + if [[ ${{ runner.os == 'macOS' }} ]]; then + export LDFLAGS="-L/usr/local/opt/libomp/lib" + export CPPFLAGS="-I/usr/local/opt/libomp/include" + fi + tox -e all-runtimes From af3c1d673fd9aee3c5255bf49a9dc604fdafd9cc Mon Sep 17 00:00:00 2001 From: Jesse Claven Date: Wed, 29 May 2024 15:09:05 +0100 Subject: [PATCH 2/5] build: Lock Colima --- .github/workflows/tests.yml | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c7bc0f6d4..65660935b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -84,9 +84,16 @@ jobs: brew install docker docker-buildx mkdir -p $HOME/.docker/cli-plugins ln -sfn $(which docker-buildx) $HOME/.docker/cli-plugins/docker-buildx - # colima version: 0.6.8 - brew install colima - colima start --memory 5 + # Install Lima + brew install lima + limactl start + # Install Colima + sudo mkdir -p /usr/local/bin + sudo curl -L -o /usr/local/bin/colima https://github.com/abiosoft/colima/releases/download/v0.5.6/colima-Darwin-x86_64 && sudo chmod +x /usr/local/bin/colima + # NOTE: Use the `slirp` driver rather than the default to avoid + # random connection errors. + # From https://github.com/abiosoft/colima/issues/577 + colima start --memory 5 --network-driver slirp sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - name: Install OpenMP (in MacOS) if: runner.os == 'macOS' @@ -147,10 +154,10 @@ jobs: os: macos-13 runs-on: ${{ matrix.os }} steps: + - uses: actions/checkout@v4 - name: Install OpenMP (in MacOS) if: runner.os == 'macOS' run: brew install libomp - - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -211,9 +218,16 @@ jobs: brew install docker docker-buildx mkdir -p $HOME/.docker/cli-plugins ln -sfn $(which docker-buildx) $HOME/.docker/cli-plugins/docker-buildx - # 0.6.8 - brew install colima - colima start --memory 5 + # Install Lima + brew install lima + limactl start + # Install Colima + sudo mkdir -p /usr/local/bin + sudo curl -L -o /usr/local/bin/colima https://github.com/abiosoft/colima/releases/download/v0.5.6/colima-Darwin-x86_64 && sudo chmod +x /usr/local/bin/colima + # NOTE: Use the `slirp` driver rather than the default to avoid + # random connection errors. + # From https://github.com/abiosoft/colima/issues/577 + colima start --memory 5 --network-driver slirp sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock - name: Install OpenMP (in MacOS) if: runner.os == 'macOS' From 60b452ac6c399d44567e3d821c40a4392eb9286f Mon Sep 17 00:00:00 2001 From: Jesse Claven Date: Wed, 29 May 2024 15:20:01 +0100 Subject: [PATCH 3/5] build: Move macOS Docker installation to script --- .github/workflows/tests.yml | 40 ++++-------------------------------- hack/install-docker-macos.sh | 27 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 36 deletions(-) create mode 100755 hack/install-docker-macos.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 65660935b..a84132403 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -76,29 +76,13 @@ jobs: os: macos-13 runs-on: ${{ matrix.os }} steps: + - uses: actions/checkout@v4 - name: Setup Docker (missing on MacOS) if: runner.os == 'macos' - run: | - # From https://github.com/actions/runner-images/issues/17#issuecomment-1537238473 - # From https://github.com/abiosoft/colima/discussions/273#discussioncomment-4959736 - brew install docker docker-buildx - mkdir -p $HOME/.docker/cli-plugins - ln -sfn $(which docker-buildx) $HOME/.docker/cli-plugins/docker-buildx - # Install Lima - brew install lima - limactl start - # Install Colima - sudo mkdir -p /usr/local/bin - sudo curl -L -o /usr/local/bin/colima https://github.com/abiosoft/colima/releases/download/v0.5.6/colima-Darwin-x86_64 && sudo chmod +x /usr/local/bin/colima - # NOTE: Use the `slirp` driver rather than the default to avoid - # random connection errors. - # From https://github.com/abiosoft/colima/issues/577 - colima start --memory 5 --network-driver slirp - sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + run: ./hack/install-docker-macos.sh - name: Install OpenMP (in MacOS) if: runner.os == 'macOS' run: brew install libomp - - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: @@ -210,29 +194,13 @@ jobs: remove-docker-images: 'true' overprovision-lvm: 'true' swap-size-mb: 1024 + - uses: actions/checkout@v4 - name: Setup Docker (missing on MacOS) if: runner.os == 'macOS' - run: | - # From https://github.com/actions/runner-images/issues/17#issuecomment-1537238473 - # From https://github.com/abiosoft/colima/discussions/273#discussioncomment-4959736 - brew install docker docker-buildx - mkdir -p $HOME/.docker/cli-plugins - ln -sfn $(which docker-buildx) $HOME/.docker/cli-plugins/docker-buildx - # Install Lima - brew install lima - limactl start - # Install Colima - sudo mkdir -p /usr/local/bin - sudo curl -L -o /usr/local/bin/colima https://github.com/abiosoft/colima/releases/download/v0.5.6/colima-Darwin-x86_64 && sudo chmod +x /usr/local/bin/colima - # NOTE: Use the `slirp` driver rather than the default to avoid - # random connection errors. - # From https://github.com/abiosoft/colima/issues/577 - colima start --memory 5 --network-driver slirp - sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock + run: ./hack/install-docker-macos.sh - name: Install OpenMP (in MacOS) if: runner.os == 'macOS' run: brew install libomp - - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: diff --git a/hack/install-docker-macos.sh b/hack/install-docker-macos.sh new file mode 100755 index 000000000..5a54185b2 --- /dev/null +++ b/hack/install-docker-macos.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +# Install Docker +# From https://github.com/actions/runner-images/issues/17#issuecomment-1537238473 +# From https://github.com/abiosoft/colima/discussions/273#discussioncomment-4959736 +brew install docker docker-buildx +mkdir -p $HOME/.docker/cli-plugins +ln -sfn $(which docker-buildx) $HOME/.docker/cli-plugins/docker-buildx + +# Install Lima +brew install lima +# Don't explicitly start Lima here (e.g. with `limactl start`). +# Let Colina do that, otherwise, # Colima seems to hang +# waiting for the SSH requirement to be met. + +# Install Colima +sudo mkdir -p /usr/local/bin +sudo curl -L -o /usr/local/bin/colima https://github.com/abiosoft/colima/releases/download/v0.5.6/colima-Darwin-x86_64 && sudo chmod +x /usr/local/bin/colima +# NOTE: Use the `slirp` driver rather than the default to avoid +# random connection errors. +# From https://github.com/abiosoft/colima/issues/577 +colima start --memory 5 --network-driver slirp --runtime docker + +# Link Colima and Docker +sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock From d372323228d73731f365652d7ea6e3604902bcb7 Mon Sep 17 00:00:00 2001 From: Jesse Claven Date: Wed, 29 May 2024 15:24:50 +0100 Subject: [PATCH 4/5] build: Move macOS libomp activation to script --- .github/workflows/tests.yml | 18 ++++++------------ hack/activate-libomp-macos.sh | 6 ++++++ 2 files changed, 12 insertions(+), 12 deletions(-) create mode 100755 hack/activate-libomp-macos.sh diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a84132403..aec13497e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -98,15 +98,13 @@ jobs: - name: Install Dependencies run: | if [[ ${{ runner.os == 'macOS' }} ]]; then - export LDFLAGS="-L/usr/local/opt/libomp/lib" - export CPPFLAGS="-I/usr/local/opt/libomp/include" + source ./hack/activate-libomp-macos.sh fi poetry install --sync --only dev - name: Test run: | if [[ ${{ runner.os == 'macOS' }} ]]; then - export LDFLAGS="-L/usr/local/opt/libomp/lib" - export CPPFLAGS="-I/usr/local/opt/libomp/include" + source ./hack/activate-libomp-macos.sh fi tox -e mlserver @@ -157,15 +155,13 @@ jobs: - name: Install Dependencies run: | if [[ ${{ runner.os == 'macOS' }} ]]; then - export LDFLAGS="-L/usr/local/opt/libomp/lib" - export CPPFLAGS="-I/usr/local/opt/libomp/include" + source ./hack/activate-libomp-macos.sh fi poetry install --sync --only dev - name: Test run: | if [[ ${{ runner.os == 'macOS' }} ]]; then - export LDFLAGS="-L/usr/local/opt/libomp/lib" - export CPPFLAGS="-I/usr/local/opt/libomp/include" + source ./hack/activate-libomp-macos.sh fi tox -c ./runtimes/${{ matrix.tox-environment }} @@ -216,14 +212,12 @@ jobs: - name: Install Dependencies run: | if [[ ${{ runner.os == 'macOS' }} ]]; then - export LDFLAGS="-L/usr/local/opt/libomp/lib" - export CPPFLAGS="-I/usr/local/opt/libomp/include" + source ./hack/activate-libomp-macos.sh fi poetry install --sync --only dev - name: Test run: | if [[ ${{ runner.os == 'macOS' }} ]]; then - export LDFLAGS="-L/usr/local/opt/libomp/lib" - export CPPFLAGS="-I/usr/local/opt/libomp/include" + source ./hack/activate-libomp-macos.sh fi tox -e all-runtimes diff --git a/hack/activate-libomp-macos.sh b/hack/activate-libomp-macos.sh new file mode 100755 index 000000000..6b3813c19 --- /dev/null +++ b/hack/activate-libomp-macos.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -Eeuo pipefail + +export LDFLAGS="-L/usr/local/opt/libomp/lib" +export CPPFLAGS="-I/usr/local/opt/libomp/include" From b827da6767056a900de58e8a5ecb4ab21af793c1 Mon Sep 17 00:00:00 2001 From: Jesse Claven Date: Thu, 30 May 2024 11:49:35 +0100 Subject: [PATCH 5/5] build: Use latest Colima The > 0.6.0 releases actually fix the issue we have linked [1][2][3]. [1] abiosoft/colima#577 [2] https://github.com/jesse-c/MLServer/blob/c3acd60995a72141027eff506e4fd330fe824179/hack/install-docker-macos.sh#L18-L20 [3] > Switch to new user-v2 network. Fixes abiosoft/colima#648, abiosoft/colima#603, abiosoft/colima#577, abiosoft/colima#779, abiosoft/colima#137, abiosoft/colima#740. --- hack/install-docker-macos.sh | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/hack/install-docker-macos.sh b/hack/install-docker-macos.sh index 5a54185b2..87e5b6fbe 100755 --- a/hack/install-docker-macos.sh +++ b/hack/install-docker-macos.sh @@ -17,11 +17,8 @@ brew install lima # Install Colima sudo mkdir -p /usr/local/bin -sudo curl -L -o /usr/local/bin/colima https://github.com/abiosoft/colima/releases/download/v0.5.6/colima-Darwin-x86_64 && sudo chmod +x /usr/local/bin/colima -# NOTE: Use the `slirp` driver rather than the default to avoid -# random connection errors. -# From https://github.com/abiosoft/colima/issues/577 -colima start --memory 5 --network-driver slirp --runtime docker +sudo curl -L -o /usr/local/bin/colima https://github.com/abiosoft/colima/releases/download/v0.6.9/colima-Darwin-x86_64 && sudo chmod +x /usr/local/bin/colima +colima start --memory 5 --runtime docker # Link Colima and Docker sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock