From 8e2892deb15263e668c42853515f9df2176f29de Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Fri, 8 Nov 2024 15:02:39 +1300
Subject: [PATCH 01/16] [ML] Build solely on macos Ventura aarch64

From version 9.0.0 we drop support for macos x86_64 builds.
Therefore all support for cross compiling for macos x86_64 has also been
dropped.
Builds have been migrated to be executed solely on macos 13 (Ventura)
aarch64. In CI, these builds are performed on updated Orka VMs.

In addition, 3rd part dependencies (boost, pytorch) are now baked into
the Orka VM, so downloading these dependencies at run time is no longer
required,
---
 .buildkite/pipelines/build_macos.json.py     | 24 +----
 .buildkite/scripts/steps/build_and_test.sh   | 23 +----
 .ci/orka/README.md                           | 29 +++---
 .ci/orka/install.sh                          | 18 ++--
 .ci/orka/orka-macos-13-arm.pkr.hcl           | 63 +++++++++++++
 .ci/orka/third_party_deps.sh                 | 76 ++++++++++++++++
 build-setup/macos.md                         | 75 +++++-----------
 build-setup/macos_cross_compiled.md          | 93 --------------------
 cmake/compiler/clang.cmake                   | 35 ++------
 dev-tools/build_macos_third_party_deps.sh    |  1 +
 dev-tools/docker/README.md                   | 21 -----
 dev-tools/docker/build_macosx_build_image.sh | 40 ---------
 dev-tools/docker/macosx_builder/Dockerfile   | 37 --------
 dev-tools/docker/macosx_image/Dockerfile     | 59 -------------
 dev-tools/docker_build.sh                    |  7 +-
 dev-tools/download_macos_deps.sh             | 49 -----------
 dev-tools/strip_binaries.sh                  | 26 +-----
 17 files changed, 195 insertions(+), 481 deletions(-)
 create mode 100644 .ci/orka/orka-macos-13-arm.pkr.hcl
 create mode 100644 .ci/orka/third_party_deps.sh
 delete mode 100644 build-setup/macos_cross_compiled.md
 create mode 120000 dev-tools/build_macos_third_party_deps.sh
 delete mode 100755 dev-tools/docker/build_macosx_build_image.sh
 delete mode 100644 dev-tools/docker/macosx_builder/Dockerfile
 delete mode 100644 dev-tools/docker/macosx_image/Dockerfile
 delete mode 100755 dev-tools/download_macos_deps.sh

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 5342007d2b..0fb25c31d2 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -21,8 +21,7 @@
 from itertools import product
 
 archs = [
-  "aarch64",
-  "x86_64",
+  "aarch64"
 ]
 build_types = [
   "RelWithDebInfo",
@@ -32,13 +31,9 @@
   "debug"
 ]
 agents = {
-   "x86_64": {
-      "provider": "orka",
-      "image": "ml-macos-12-x86_64-001.img"
-   },
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-12-arm-001.orkasi"
+      "image": "ml-macos-13-arm-001.orkasi"
    }
 }
 envs = {
@@ -51,16 +46,6 @@
       "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-aarch64.cmake",
       "RUN_TESTS": "true",
       "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
-    },
-    "x86_64": {
-      "TMPDIR": "/tmp",
-      "HOMEBREW_PREFIX": "/opt/homebrew",
-      "PATH": "/opt/homebrew/bin:$PATH",
-      "ML_DEBUG": "0",
-      "CPP_CROSS_COMPILE": "",
-      "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-x86_64.cmake",
-      "RUN_TESTS": "true",
-      "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
     }
 }
 
@@ -122,11 +107,6 @@ def main(args):
                         action='store_true',
                         default=False,
                         help="Build for aarch64?.")
-    parser.add_argument("--build-x86_64",
-                        required=False,
-                        action='store_true',
-                        default=False,
-                        help="Build for x86_64?")
 
     args = parser.parse_args()
 
diff --git a/.buildkite/scripts/steps/build_and_test.sh b/.buildkite/scripts/steps/build_and_test.sh
index af1c912052..5fc133874c 100755
--- a/.buildkite/scripts/steps/build_and_test.sh
+++ b/.buildkite/scripts/steps/build_and_test.sh
@@ -82,7 +82,6 @@ if [[ `uname` = "Linux" ]]; then # Linux x86_64
     fi
   fi
 else # Darwin (macOS)
-  sudo -E ${REPO_ROOT}/dev-tools/download_macos_deps.sh
   if [[ "$HARDWARE_ARCH" = aarch64 ]] ; then # Darwin aarch64
     # For macOS aarch64, build directly on the machine using gradle
     if [ "$RUN_TESTS" = false ] ; then
@@ -90,29 +89,11 @@ else # Darwin (macOS)
     else
        TASKS="clean buildZip buildZipSymbols check"
     fi
-    # For macOS we usually only use a particular version as our build platform
-    # once Xcode has stopped receiving updates for it. However, with Big Sur
-    # on ARM we couldn't do this, as Big Sur was the first macOS version for
-    # ARM. Therefore, the compiler may get upgraded on a CI server, and we
-    # need to hardcode the version that was used to build Boost for that
-    # version of Elasticsearch.
-    if [ "$HARDWARE_ARCH" = aarch64 ] ; then
-       export BOOSTCLANGVER=13
-    fi
 
     (cd ${REPO_ROOT} && ./gradlew --info -Dbuild.version_qualifier=${VERSION_QUALIFIER:-} -Dbuild.snapshot=$BUILD_SNAPSHOT -Dbuild.ml_debug=$ML_DEBUG $TASKS) || TEST_OUTCOME=$?
   else # Darwin x86_64
-    # For macOS x86_64 we re-use existing Docker scripts and build directly on the machine
-    function nproc() {
-        sysctl -n hw.logicalcpu
-    }
-    export -f nproc
-    if [ "$RUN_TESTS" = "true" ]; then
-      ${REPO_ROOT}/dev-tools/docker/docker_entrypoint.sh --test
-      grep passed build/test_status.txt || TEST_OUTCOME=$?
-    else
-      ${REPO_ROOT}/dev-tools/docker/docker_entrypoint.sh
-    fi
+    echo "Unsupported architecture - macos x86_64"
+    exit 1
   fi
 fi
 
diff --git a/.ci/orka/README.md b/.ci/orka/README.md
index 91978c43c5..7d76cf379a 100644
--- a/.ci/orka/README.md
+++ b/.ci/orka/README.md
@@ -17,11 +17,7 @@
 If you haven't run these before, run the following once so packer downloads the `vault` integration:
 
 ```
-packer init orka-macos-12-arm.pkr.hcl
-```
-or
-```
-packer init orka-macos-12-x86_64.pkr.hcl
+packer init orka-macos-13-arm.pkr.hcl
 ```
 
 ## Build
@@ -33,7 +29,7 @@ Packer requires access to secrets in vault, where VAULT_ADDR=https://vault-ci-pr
 Run the following to create the image (MacOS 12 ARM in this example):
 
 ```
-packer build orka-macos-12-arm.pkr.hcl
+packer build orka-macos-13-arm.pkr.hcl
 ```
 
 ## Versioning
@@ -42,35 +38,32 @@ The name of the resulting images are hard-coded (currently), and end in a sequen
 
 ## Source Images
 
-The source images used for the MacOS builds are slightly modified copies of the standard Orka images, e.g. 90GBMontereySSH.orkasi:
+The source images used for the MacOS builds are slightly modified copies of the standard Orka images, e.g. 90GBVenturaSSH.orkasi:
 
 The source images are named:
- * `ml-macos-12-base-arm-fundamental.orkasi`
- * `ml-macos-12-base-x86_64-fundamental.img`
+ * `ml-macos-13-base-arm-fundamental.orkasi`
 
 The source image only has the following changes on it:
     * Adding passwordless `sudo` for the default `admin` user
     * Configured `admin` user to be automatically logged in
-    * Installed Xcode command line tools version 13 (by running `clang++ --version` and clicking through the dialogues)
+    * Installed Xcode command line tools version 14 (by running `clang++ --version` and clicking through the dialogues)
 
 ## Packer Install Steps
 
-The packer script does the following:
+The packer scripts do the following:
     * Install Brew `4.2.21`
-    * Install JDK `11.0.23`
+    * Install JDK `11.0.25`
     * Install python3
-    * Install vault
     * Install jq
     * Install Google Cloud SDK into `~admin/google-cloud-sdk/`
-    * Install CMake 3.23.3
+    * Install CMake 3.30.5
     * Install `gobld-bootstrap.sh` script to run at system startup
         * This script pulls down and runs another script from a static location to do the following:
             * Unseal one-time vault token from gobld
             * Install and run the latest `buildkite-agent`
+    * Install Boost 1.83.0 from source
+    * Install PyTorch 2.3.1 from source
 
 ## Caveats
 
-* Prior to the dependency on PyTorch 2.3.1 we only needed Orka for ARM builds (CI and dependencies), x86_64 builds were
-  performed via cross-compilation. However, PyTorch 2.3.1 now requires a more modern version of `clang` that our cross
-  compilation framework provided. As a suitable Orka base image is available for x86_64, it is now simpler to compile
-  natively for that architecture.
+* As of version 9.0.0 support for macos x86_64 builds has been dropped. It is necessary to checkout and work on previous branches in order to maintain x86_64 Orka VMs.
diff --git a/.ci/orka/install.sh b/.ci/orka/install.sh
index 22cac895e9..abccab3710 100644
--- a/.ci/orka/install.sh
+++ b/.ci/orka/install.sh
@@ -4,8 +4,8 @@ export PATH=/opt/homebrew/bin:/usr/local/bin:$PATH
 
 MACHINE_TYPE=$(uname -m)
 
-PYTHON3_URL=https://www.python.org/ftp/python/3.10.8/python-3.10.8-macos11.pkg
-PYTHON3_FILE=python-3.10.8-macos11.pkg
+PYTHON3_URL=https://www.python.org/ftp/python/3.10.10/python-3.10.10-macos11.pkg
+PYTHON3_FILE=python-3.10.10-macos11.pkg
 
 GCLOUD_SDK_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-408.0.1-darwin-x86_64.tar.gz
 GCLOUD_SDK_FILE=google-cloud-cli.tar.gz
@@ -29,13 +29,11 @@ eval "$(brew shellenv)"
 
 if ! java --version 2> /dev/null ; then
   echo 'install jdk 11'
-  brew install openjdk@11
-  sudo ln -sfn /opt/homebrew/opt/openjdk@11/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-11.jdk
-fi
-
-if ! command -v vault 2> /dev/null ; then
-    echo "Install vault"
-    brew install vault
+  # Don't use brew to install java, it brings in many unnecessary libraries that e.g. Boost will link against if
+  # present.
+  curl https://cdn.azul.com/zulu/bin/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64.tar.gz | sudo tar xvzf - -C /Library/Java/JavaVirtualMachines && \
+  sudo mv /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64/zulu-11.jdk /Library/Java/JavaVirtualMachines && \
+  sudo rm -rf /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64
 fi
 
 if ! command -v jq 2> /dev/null ; then
@@ -71,7 +69,7 @@ sudo cp /tmp/gobld-bootstrap.plist /Users/admin
 
 # Install CMake
 echo "Install CMake"
-curl -v -L https://github.com/Kitware/CMake/releases/download/v3.23.3/cmake-3.23.3-macos-universal.tar.gz | tar xvzf - --strip-components 1 -C /Applications
+curl -v -L https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-macos-universal.tar.gz | tar xvzf - --strip-components 1 -C /Applications
 sudo ln -sf /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake
 
 # Make sure all changes are written to disk
diff --git a/.ci/orka/orka-macos-13-arm.pkr.hcl b/.ci/orka/orka-macos-13-arm.pkr.hcl
new file mode 100644
index 0000000000..541648a1f0
--- /dev/null
+++ b/.ci/orka/orka-macos-13-arm.pkr.hcl
@@ -0,0 +1,63 @@
+packer {
+  required_plugins {
+    macstadium-orka = {
+      version = "= 2.3.0"
+      source  = "github.com/macstadium/macstadium-orka"
+    }
+  }
+}
+
+locals {
+  orka_endpoint = vault("secret/ci/elastic-ml-cpp/orka", "orka_endpoint")
+  orka_user     = vault("secret/ci/elastic-ml-cpp/orka", "orka_user")
+  orka_password = vault("secret/ci/elastic-ml-cpp/orka", "orka_password")
+  ssh_username  = vault("secret/ci/elastic-ml-cpp/orka", "ssh_username")
+  ssh_password  = vault("secret/ci/elastic-ml-cpp/orka", "ssh_password")
+  sensitive     = true
+}
+
+source "macstadium-orka" "image" {
+  source_image     = "ml-macos-13-base-arm-fundamental.orkasi"
+  image_name       = "ml-macos-13-arm-001.orkasi"
+  orka_endpoint    = local.orka_endpoint
+  orka_user        = local.orka_user
+  orka_password    = local.orka_password
+  ssh_username     = local.ssh_username
+  ssh_password     = local.ssh_password
+  orka_vm_cpu_core = 4
+  no_delete_vm     = false
+}
+
+build {
+  sources = [
+    "macstadium-orka.image"
+  ]
+  provisioner "file" {
+    source = "install.sh"
+    destination = "/tmp/install.sh"
+  }
+  provisioner "file" {
+    source = "third_party_deps.sh"
+    destination = "/tmp/third_party_deps.sh"
+  }
+  provisioner "file" {
+    source = "gobld-bootstrap.sh"
+    destination = "/tmp/gobld-bootstrap.sh"
+  }
+  provisioner "file" {
+    source = "gobld-bootstrap.plist"
+    destination = "/tmp/gobld-bootstrap.plist"
+  }
+  provisioner "shell" {
+    inline = [
+      "chmod u+x /tmp/install.sh",
+      "/tmp/install.sh",
+    ]
+  }
+  provisioner "shell" {
+    inline = [
+      "chmod u+x /tmp/third_party_deps.sh",
+      "/tmp/third_party_deps.sh",
+    ]
+  }
+}
diff --git a/.ci/orka/third_party_deps.sh b/.ci/orka/third_party_deps.sh
new file mode 100644
index 0000000000..dee54af13b
--- /dev/null
+++ b/.ci/orka/third_party_deps.sh
@@ -0,0 +1,76 @@
+#!/bin/bash
+set -xe
+export PATH=/opt/homebrew/bin:/usr/local/bin:$PATH
+
+MACHINE_TYPE=$(uname -m)
+if [ ${MACHINE_TYPE} != "arm64" ]; then
+    echo "Unsupported architecture: ${MACHINE_TYPE}"
+    echo "Expected arm64"
+    exit 1
+fi
+
+export CPP='clang -E'
+export CC=clang
+export CFLAGS="-O3"
+export CXX='clang++ -std=c++17 -stdlib=libc++'
+export CXXFLAGS="-O3"
+export CXXCPP='clang++ -std=c++17 -E'
+export LDFLAGS=-Wl,-headerpad_max_install_names
+unset CPATH
+unset C_INCLUDE_PATH
+unset CPLUS_INCLUDE_PATH
+unset LIBRARY_PATH
+
+# Build and install boost 1.83.0
+curl -L https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.tar.bz2 | tar xvjf - && \
+cd boost_1_83_0 && \
+./bootstrap.sh --with-toolset=clang --without-libraries=context --without-libraries=coroutine --without-libraries=graph_parallel --without-libraries=mpi --without-libraries=python --without-icu && \
+sed -i -e 's|(13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)( \\|(3ul)(13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)(       \\|' boost/unordered/detail/prime_fmod.hpp
+./b2 -j8 --layout=versioned --disable-icu cxxflags="-std=c++17 -stdlib=libc++" linkflags="-std=c++17 -stdlib=libc++ -Wl,-headerpad_max_install_names" optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC && \
+sudo ./b2 install --layout=versioned --disable-icu cxxflags="-std=c++17 -stdlib=libc++" linkflags="-std=c++17 -stdlib=libc++ -Wl,-headerpad_max_install_names" optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC && \
+cd .. && \
+sudo rm -rf boost_1_83_0
+
+# Install python modules required by PyTorch
+sudo pip3 install numpy ninja pyyaml setuptools cffi typing_extensions future six requests dataclasses
+
+# Build and install PyTorch
+git clone --depth=1 --branch=v2.3.1 https://github.com/pytorch/pytorch.git && \
+cd pytorch && \
+git submodule sync && \
+git submodule update --init --recursive && \
+sed -i -e 's/system(/strlen(/' torch/csrc/jit/codegen/fuser/cpu/fused_kernel.cpp && \
+sed -i -e '/CUDNN_ROOT/a\
+           "DNNL_TARGET_ARCH",
+' tools/setup_helpers/cmake.py && \
+sed -i -e 's/CMAKE_LINK_WHAT_YOU_USE TRUE/CMAKE_LINK_WHAT_YOU_USE FALSE/' CMakeLists.txt && \
+sed -i -e '189i\
+inline void to_bytes(bytestring& bytes, const unsigned long arg) {\
+  auto as_cstring = reinterpret_cast<const char*>(&arg);\
+  bytes.append(as_cstring, sizeof(unsigned long));\
+}\
+' third_party/ideep/include/ideep/utils.hpp
+
+export BLAS=vecLib
+export BUILD_TEST=OFF
+export BUILD_CAFFE2=OFF
+export USE_NUMPY=OFF
+export USE_DISTRIBUTED=OFF
+export DNNL_TARGET_ARCH=AARCH64
+export USE_MKLDNN=ON
+export USE_QNNPACK=OFF
+export USE_PYTORCH_QNNPACK=OFF
+export PYTORCH_BUILD_VERSION=2.3.1
+export PYTORCH_BUILD_NUMBER=1
+python3 setup.py install
+
+sudo mkdir -p /usr/local/lib && \
+sudo mkdir -p /usr/local/include/pytorch && \
+sudo cp -r torch/include/* /usr/local/include/pytorch/ && \
+sudo cp torch/lib/libtorch_cpu.dylib /usr/local/lib/ && \
+sudo cp torch/lib/libc10.dylib /usr/local/lib/ && \
+cd .. && \
+rm -rf pytorch
+
+# Make sure all changes are written to disk
+sync
diff --git a/build-setup/macos.md b/build-setup/macos.md
index 9d3719384f..f56e907fa5 100644
--- a/build-setup/macos.md
+++ b/build-setup/macos.md
@@ -16,7 +16,7 @@ For example, you might create a `.bashrc` file in your home directory containing
 
 ```
 umask 0002
-export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk-17.0.1.jdk/Contents/Home
+export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk-21.0.1.jdk/Contents/Home
 export PYTHONHOME=/Library/Frameworks/Python.framework/Versions/3.10
 export PATH=$JAVA_HOME/bin:$PYTHONHOME/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
 # Only required if building the C++ code directly using cmake - adjust depending on the location of your Git clone
@@ -66,6 +66,25 @@ xcode-select --install
 
 at the command prompt.
 
+### CMake
+
+CMake version 3.19.2 is the minimum required to build ml-cpp.  Download the graphical installer for version 3.30.5 from <https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-macos-universal.dmg> (or get a more recent version).
+
+Open the `.dmg` and install the application it by dragging it to the `Applications` folder.
+
+Then make the `cmake` program accessible to programs that look in `/usr/local/bin`:
+
+```
+sudo mkdir -p /usr/local/bin
+sudo ln -s /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake
+```
+
+## Third Party Dependencies
+
+The following instructions are for building and installing 3rd party dependencies required for the `ml-cpp` build. They
+can either be performed manually, or if preferred, running the `dev-tools/build_macos_third_party_deps.sh` automates the
+process.
+
 ### Boost 1.83.0
 
 Download version 1.83.0 of Boost from <https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.tar.bz2>. You must get this exact version, as the Machine Learning build system requires it.
@@ -106,24 +125,11 @@ sudo ./b2 install --layout=versioned --disable-icu cxxflags="-std=c++17 -stdlib=
 
 to install the Boost headers and libraries.
 
-### CMake
-
-CMake version 3.19.2 is the minimum required to build ml-cpp.  Download the graphical installer for version 3.23.2 from <https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-macos-universal.dmg> (or get a more recent version).
-
-Open the `.dmg` and install the application it by dragging it to the `Applications` folder.
-
-Then make the `cmake` program accessible to programs that look in `/usr/local/bin`:
-
-```
-sudo mkdir -p /usr/local/bin
-sudo ln -s /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake
-```
-
 ### Python 3.10
 
 PyTorch currently requires Python 3.7 or higher; we use version 3.10.
 
-Download the graphical installer for Python 3.10.9 from <https://www.python.org/ftp/python/3.10.9/python-3.10.9-macos11.pkg>.
+Download the graphical installer for Python 3.10.10 from <https://www.python.org/ftp/python/3.10.10/python-3.10.10-macos11.pkg>.
 
 Install using all the default options.  When the installer completes a Finder window pops up.  Double click the `Install Certificates.command` file in this folder to install the SSL certificates Python needs.
 
@@ -170,44 +176,6 @@ inline void to_bytes(bytestring& bytes, const unsigned long arg) {
 
 at around line 189. This is necessary to resolve a template specialization issue.
 
-For compilation on `macOS Mojave (10.14)`
-
-* Edit `torch/csrc/jit/api/module.h` and change
-  ```
-  Module(Module&&) noexcept = default;`
-  Module& operator=(Module&&) noexcept = default;
-  ``` 
-  to
-  ```
-  Module(Module&&) = default;
-  Module& operator=(Module&&) noexcept = default;
-  ```
-* Edit `third_party/onnx/CMakeLists.txt` and
-    * add `find_package(Boost 1.83.0 EXACT REQUIRED COMPONENTS filesystem)`
-      before `target_include_directories(onnx PUBLIC` (around line 531)
-    * add `$<BUILD_INTERFACE:${Boost_INCLUDE_DIRS}>` after `$<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>` (around line
-      536)
-* Edit `third_party/onnx/onnx/common/file_utils.h` and change the lines
-  ```
-  #include <filesystem>
-  #include <fstream>
-  ```
-  to
-  ```
-  #include <boost/filesystem/operations.hpp>
-  #include <boost/filesystem/fstream.hpp>
-  ```
-  and change
-  ```
-  std::filesystem::path proto_u8_path = std::filesystem::u8path(proto_path);
-  std::fstream proto_stream(proto_u8_path, std::ios::in | std::ios::binary);
-  ```
-  to
-  ```
-  boost::filesystem::path proto_u8_path = boost::filesystem::u8path(proto_path);
-  boost::filesystem::fstream proto_stream(proto_u8_path, std::ios::in | std::ios::binary);
-  ```
-
 Build as follows:
 
 ```
@@ -229,6 +197,7 @@ export PYTORCH_BUILD_NUMBER=1
 Once built copy headers and libraries to system directories:
 
 ```
+sudo mkdir -p /usr/local/lib
 sudo mkdir -p /usr/local/include/pytorch
 sudo cp -r torch/include/* /usr/local/include/pytorch/
 sudo cp torch/lib/libtorch_cpu.dylib /usr/local/lib/
diff --git a/build-setup/macos_cross_compiled.md b/build-setup/macos_cross_compiled.md
deleted file mode 100644
index b6305b93ba..0000000000
--- a/build-setup/macos_cross_compiled.md
+++ /dev/null
@@ -1,93 +0,0 @@
-# Machine Learning Build Machine Setup for macOS cross compiled on Linux
-
-You will need the following environment variables to be defined:
-
-- `JAVA_HOME` - Should point to the JDK you want to use to run Gradle.
-- `CPP_CROSS_COMPILE` - Should be set to "macosx".
-- `CPP_SRC_HOME` - Only required if building the C++ code directly using `cmake`, as Gradle sets it automatically.
-
-For example, you might create a .bashrc file in your home directory containing this:
-
-```
-umask 0002
-export JAVA_HOME=/usr/local/jdk1.8.0_121
-export PATH=$JAVA_HOME/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
-# Only required if building the C++ code directly using cmake - adjust depending on the location of your Git clone
-export CPP_SRC_HOME=$HOME/ml-cpp
-export CPP_CROSS_COMPILE=macosx
-```
-
-### Initial Preparation
-
-Start by configuring a native macOS build server as described in [macos.md](macos.md).
-
-The remainder of these instructions assume the macOS build server you have configured is for macOS 10.14 (Mojave).  This is what builds for distribution are currently built on.
-
-On the fully configured macOS build server, run the following commands:
-
-```
-cd /usr
-tar jcvf ~/usr-x86_64-apple-macosx10.14.tar.bz2 lib local
-cd /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr
-tar jcvf ~/xcode-x86_64-apple-macosx10.14.tar.bz2 include
-cd `xcrun --show-sdk-path`/usr
-tar jcvf ~/sdk-x86_64-apple-macosx10.14.tar.bz2 include lib
-```
-
-These instructions also assume the host platform is Ubuntu 18.04.  It makes life much easier if the host platform is a version of Ubuntu that's new enough to run the official binary distribution of clang/LLVM (otherwise it would be necessary to build clang/LLVM from source).
-
-Transfer the three archives created in your home directory on the macOS build server, `usr-x86_64-apple-macosx10.14.tar.bz2`, `xcode-x86_64-apple-macosx10.14.tar.bz2` and `sdk-x86_64-apple-macosx10.14.tar.bz2`, to your home directory on the cross compilation host build server.
-
-### OS Packages
-
-You need clang 8, plus a number of other build tools.  They can be installed on modern Ubuntu as follows:
-
-```
-sudo apt-get install automake autogen build-essential bzip2 git gobjc libtool software-properties-common unzip wget
-
-wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
-sudo apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal main"
-sudo apt-get install clang-8 clang-8-doc libclang1-8 libllvm8 lldb-8 llvm-8 llvm-8-doc llvm-8-runtime
-```
-
-(It is strongly recommended NOT to attempt to create a cross compile environment on an old version of Linux, because you will have to build clang from source and you need a modern C++ compiler to build clang.  So you would probably end up first building a modern version of gcc using the system default gcc, then building clang using the modern gcc.)
-
-### Transferred Build Dependencies
-
-Add the dependencies that you copied from the fully configured macOS build server in the "Initial Preparation" step.
-
-```
-sudo mkdir -p /usr/local/sysroot-x86_64-apple-macosx10.14/usr
-cd /usr/local/sysroot-x86_64-apple-macosx10.14/usr
-sudo tar jxvf ~/usr-x86_64-apple-macosx10.14.tar.bz2
-sudo tar jxvf ~/xcode-x86_64-apple-macosx10.14.tar.bz2
-sudo tar jxvf ~/sdk-x86_64-apple-macosx10.14.tar.bz2
-```
-
-### CMake
-
-CMake version 3.19.2 is the minimum required to build ml-cpp. Download version 3.23.2 from <https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-Linux-x86_64.sh> and install:
-
-```
-chmod +x cmake-3.23.2-Linux-x86_64.sh
-sudo ./cmake-3.23.2-Linux-x86_64.sh --skip-license --prefix=/usr/local
-```
-
-### cctools-port
-
-You need to obtain Linux ports of several Apple development tools.  The easiest way to get them is to use the [cctools-port project on GitHub](https://github.com/tpoechtrager/cctools-port):
-
-```
-git clone https://github.com/tpoechtrager/cctools-port.git
-cd cctools-port/cctools
-git checkout 949.0.1-ld64-530
-export CC=clang-8
-export CXX=clang++-8
-./autogen.sh
-./configure --target=x86_64-apple-macosx10.14 --with-llvm-config=/usr/bin/llvm-config-8
-make
-sudo make install
-```
-
-The "949.0.1-ld64-530" branch in the [cctools-port repository](https://github.com/tpoechtrager/cctools-port) corresponds to the tools for macOS 10.14 Mojave and clang 8.  (A different branch would be required for newer versions of the OS/compiler.)
-
diff --git a/cmake/compiler/clang.cmake b/cmake/compiler/clang.cmake
index bd9ff030db..1749ad0a89 100644
--- a/cmake/compiler/clang.cmake
+++ b/cmake/compiler/clang.cmake
@@ -9,36 +9,11 @@
 # limitation.
 #
 
-# which compilers to use for C and C++
-if(DEFINED ENV{CPP_CROSS_COMPILE} AND NOT "$ENV{CPP_CROSS_COMPILE}" STREQUAL "")
-  message(STATUS "Cross compiling: CPP_CROSS_COMPILE = $ENV{CPP_CROSS_COMPILE}")
-
-  set(CROSS_FLAGS --sysroot=${SYSROOT} -B /usr/local/bin -target ${CROSS_TARGET_PLATFORM} -stdlib=libc++)
-  set(ML_SHARED_LINKER_FLAGS ${CROSS_FLAGS})
-  set(ML_EXE_LINKER_FLAGS    ${CROSS_FLAGS})
-
-  # which compilers to use for C and C++
-  set(CMAKE_C_COMPILER   "clang-8")
-  set(CMAKE_CXX_COMPILER "clang++-8")
-
-  set(CMAKE_AR  "/usr/local/bin/${CROSS_TARGET_PLATFORM}-ar")
-  set(CMAKE_RANLIB  "/usr/local/bin/${CROSS_TARGET_PLATFORM}-ranlib")
-  set(CMAKE_STRIP  "/usr/local/bin/${CROSS_TARGET_PLATFORM}-strip")
-  set(CMAKE_LD  "/usr/local/bin/${CROSS_TARGET_PLATFORM}-ld")
-
-  set(CMAKE_CXX_ARCHIVE_CREATE "<CMAKE_AR> -ru <TARGET> <OBJECTS>")
-
-  # where is the target environment located
-  set(CMAKE_FIND_ROOT_PATH  /usr/local/sysroot-${CROSS_TARGET_PLATFORM})
-else()
-  set(CMAKE_C_COMPILER   "clang")
-  set(CMAKE_CXX_COMPILER "clang++")
-  set(CMAKE_AR  "ar")
-  set(CMAKE_RANLIB  "ranlib")
-  set(CMAKE_STRIP  "strip")
-
-  #set(Boost_COMPILER "-clang-darwin13")
-endif()
+set(CMAKE_C_COMPILER   "clang")
+set(CMAKE_CXX_COMPILER "clang++")
+set(CMAKE_AR  "ar")
+set(CMAKE_RANLIB  "ranlib")
+set(CMAKE_STRIP  "strip")
 
 
 list(APPEND ML_C_FLAGS 
diff --git a/dev-tools/build_macos_third_party_deps.sh b/dev-tools/build_macos_third_party_deps.sh
new file mode 120000
index 0000000000..4a9fada635
--- /dev/null
+++ b/dev-tools/build_macos_third_party_deps.sh
@@ -0,0 +1 @@
+../.ci/orka/third_party_deps.sh
\ No newline at end of file
diff --git a/dev-tools/docker/README.md b/dev-tools/docker/README.md
index 4d5716e083..4e4e9f26c4 100644
--- a/dev-tools/docker/README.md
+++ b/dev-tools/docker/README.md
@@ -129,24 +129,3 @@ This image is not intended to be built regularly.  When changing the
  
 ### Build script: dev-tools/docker/build_check_style_image.sh
 
-
-
-## REPOSITORY: ml-macosx-build
-
-### VERSION: 18
-
-### Comments
-A Docker image that can be used to **cross compile** the machine learning
-C++ code for Intel macOS
-
-This image is not intended to be built regularly.  When changing the tools
-or 3rd party components required to build the machine learning C++ code:
-
-
- 1. increment the version
- 2.  Change the Dockerfile and build a new image to be
-used for subsequent builds on this branch.
- 3. Update the version to be used for builds in *dev-tools/docker/macosx_builder/Dockerfile*.
-
-### Build script: dev-tools/docker/build_macosx_build_image.sh
-
diff --git a/dev-tools/docker/build_macosx_build_image.sh b/dev-tools/docker/build_macosx_build_image.sh
deleted file mode 100755
index 3c915f62da..0000000000
--- a/dev-tools/docker/build_macosx_build_image.sh
+++ /dev/null
@@ -1,40 +0,0 @@
-#!/bin/bash
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License
-# 2.0 and the following additional limitation. Functionality enabled by the
-# files subject to the Elastic License 2.0 may only be used in production when
-# invoked by an Elasticsearch process with a license key installed that permits
-# use of machine learning features. You may not use this file except in
-# compliance with the Elastic License 2.0 and the foregoing additional
-# limitation.
-#
-
-# Builds the Docker image that can be used to compile the machine learning
-# C++ code for Intel macOS
-#
-# This script is not intended to be run regularly.  When changing the tools
-# or 3rd party components required to build the machine learning C++ code
-# increment the version, change the Dockerfile and build a new image to be
-# used for subsequent builds on this branch.  Then update the version to be
-# used for builds in docker/macosx_builder/Dockerfile.
-
-HOST=docker.elastic.co
-ACCOUNT=ml-dev
-REPOSITORY=ml-macosx-build
-VERSION=19
-
-set -e
-
-cd `dirname $0`
-
-. ./prefetch_docker_image.sh
-CONTEXT=macosx_image
-prefetch_docker_base_image $CONTEXT/Dockerfile
-docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT
-# Get a username and password for this by visiting
-# https://docker-auth.elastic.co and allowing it to authenticate against your
-# GitHub account
-docker login $HOST
-docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION
-
diff --git a/dev-tools/docker/macosx_builder/Dockerfile b/dev-tools/docker/macosx_builder/Dockerfile
deleted file mode 100644
index 74f49698b4..0000000000
--- a/dev-tools/docker/macosx_builder/Dockerfile
+++ /dev/null
@@ -1,37 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License
-# 2.0 and the following additional limitation. Functionality enabled by the
-# files subject to the Elastic License 2.0 may only be used in production when
-# invoked by an Elasticsearch process with a license key installed that permits
-# use of machine learning features. You may not use this file except in
-# compliance with the Elastic License 2.0 and the foregoing additional
-# limitation.
-#
-
-# Increment the version here when a new tools/3rd party components image is built
-FROM docker.elastic.co/ml-dev/ml-macosx-build:19
-
-MAINTAINER David Roberts <dave.roberts@elastic.co>
-
-# Copy the current Git repository into the container
-COPY . /ml-cpp/
-
-# Tell the build we want to cross compile
-ENV CPP_CROSS_COMPILE macosx
-
-ENV CMAKE_FLAGS -DCMAKE_TOOLCHAIN_FILE=/ml-cpp/cmake/darwin-x86_64.cmake
-
-# Pass through any version qualifier (default none)
-ARG VERSION_QUALIFIER=
-
-# Pass through whether this is a snapshot build (default yes if not specified)
-ARG SNAPSHOT=yes
-
-# Pass through ML debug option (default blank)
-ARG ML_DEBUG=
-
-# Run the build
-RUN \
-  /ml-cpp/dev-tools/docker/docker_entrypoint.sh
-
diff --git a/dev-tools/docker/macosx_image/Dockerfile b/dev-tools/docker/macosx_image/Dockerfile
deleted file mode 100644
index 4f5db531a0..0000000000
--- a/dev-tools/docker/macosx_image/Dockerfile
+++ /dev/null
@@ -1,59 +0,0 @@
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License
-# 2.0 and the following additional limitation. Functionality enabled by the
-# files subject to the Elastic License 2.0 may only be used in production when
-# invoked by an Elasticsearch process with a license key installed that permits
-# use of machine learning features. You may not use this file except in
-# compliance with the Elastic License 2.0 and the foregoing additional
-# limitation.
-#
-
-FROM ubuntu:20.04
-
-# This is basically automating the setup instructions in build-setup/macos_cross_compiled.md
-
-MAINTAINER David Roberts <dave.roberts@elastic.co>
-
-# Make sure apt-get is up to date and required packages are installed
-RUN \
-  export DEBIAN_FRONTEND=noninteractive && \
-  apt-get update && \
-  apt-get install --no-install-recommends -y apt-utils automake autogen build-essential bzip2 git gobjc gpg-agent libtool software-properties-common unzip wget zip
-
-# Install clang
-RUN \
-  wget --quiet -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \
-  apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal main" && \
-  apt-get install --no-install-recommends -y clang-8 libclang1-8 libllvm8 llvm-8 llvm-8-runtime
-
-# Add build dependencies transferred from native Mac build server
-RUN \
-  mkdir -p /usr/local/sysroot-x86_64-apple-macosx10.14/usr && \
-  cd /usr/local/sysroot-x86_64-apple-macosx10.14/usr && \
-  wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-x86_64-apple-macosx10.14-10.tar.bz2 | tar jxf - && \
-  wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/xcode-x86_64-apple-macosx10.14-1.tar.bz2 | tar jxf - && \
-  wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/sdk-x86_64-apple-macosx10.14-1.tar.bz2 | tar jxf -
-
-# Build cctools-port
-RUN \
-  git clone https://github.com/tpoechtrager/cctools-port.git && \
-  cd cctools-port/cctools && \
-  git checkout 949.0.1-ld64-530 && \
-  export CC=clang-8 && \
-  export CXX=clang++-8 && \
-  ./autogen.sh && \
-  ./configure --target=x86_64-apple-macosx10.14 --with-llvm-config=/usr/bin/llvm-config-8 && \
-  make -j`nproc` && \
-  make install && \
-  cd ../.. && \
-  rm -rf cctools-port
-
-# Install CMake
-# v3.19.2 minimum is required
-RUN \
-  wget --quiet https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-Linux-x86_64.sh && \
-  chmod +x cmake-3.23.2-Linux-x86_64.sh && \
-  ./cmake-3.23.2-Linux-x86_64.sh --skip-license --prefix=/usr/local && \
-  rm -f cmake-3.23.2-Linux-x86_64.sh
-
diff --git a/dev-tools/docker_build.sh b/dev-tools/docker_build.sh
index 47f1064f91..fc1eac34aa 100755
--- a/dev-tools/docker_build.sh
+++ b/dev-tools/docker_build.sh
@@ -10,8 +10,7 @@
 # limitation.
 #
 
-# Builds the machine learning C++ code for Linux or macOS in a Docker
-# container.
+# Builds the machine learning C++ code for Linux in a Docker container.
 #
 # The output .zip files are then copied out of the container to the
 # location in the current repository that they'd be in had they been
@@ -20,7 +19,7 @@
 # Finally, the Docker container used for the build is deleted.
 
 usage() {
-    echo "Usage: $0 linux|linux_aarch64_cross|linux_aarch64_native|macosx ..."
+    echo "Usage: $0 linux|linux_aarch64_cross|linux_aarch64_native ..."
     exit 1
 }
 
@@ -30,7 +29,7 @@ while [ -n "$1" ]
 do
 
     case "$1" in
-        linux|linux_aarch64_cross|linux_aarch64_native|macosx)
+        linux|linux_aarch64_cross|linux_aarch64_native)
             PLATFORMS="$1 $PLATFORMS"
             ;;
         *)
diff --git a/dev-tools/download_macos_deps.sh b/dev-tools/download_macos_deps.sh
deleted file mode 100755
index ea2f7d29ce..0000000000
--- a/dev-tools/download_macos_deps.sh
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
-# or more contributor license agreements. Licensed under the Elastic License
-# 2.0 and the following additional limitation. Functionality enabled by the
-# files subject to the Elastic License 2.0 may only be used in production when
-# invoked by an Elasticsearch process with a license key installed that permits
-# use of machine learning features. You may not use this file except in
-# compliance with the Elastic License 2.0 and the foregoing additional
-# limitation.
-#
-
-# Script to download 3rd party dependencies built on a reference macOS
-# build server to a CI machine.
-
-set -e
-
-if [ `uname` != Darwin ] ; then
-    echo "This script is intended for use on macOS"
-    exit 1
-fi
-
-DEST=/usr
-
-case `uname -m` in
-
-    x86_64)
-        ARCHIVE=local-x86_64-apple-macosx12.0-1.tar.bz2
-        ;;
-    arm64)
-        ARCHIVE=local-arm64-apple-macosx11.1-10.tar.bz2
-        ;;
-
-    *)
-        echo "No archive is available for this architecture:" `uname -m 2>&1`
-        exit 2
-        ;;
-
-esac
-
-URL="https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/$ARCHIVE"
-
-echo "Downloading dependencies from $URL"
-cd "$TMPDIR" && curl -s -S --retry 5 -O "$URL"
-echo "Extracting dependencies from $ARCHIVE"
-cd "$DEST" && tar -jmxf "$TMPDIR/$ARCHIVE"
-echo "Cleaning up dependency archive"
-rm -f "$TMPDIR/$ARCHIVE"
-
diff --git a/dev-tools/strip_binaries.sh b/dev-tools/strip_binaries.sh
index eef5933474..ca4d7103fe 100755
--- a/dev-tools/strip_binaries.sh
+++ b/dev-tools/strip_binaries.sh
@@ -21,13 +21,8 @@ case `uname` in
         ;;
 
     Linux)
-        if [ "$CPP_CROSS_COMPILE" = macosx ] ; then
-            EXE_DIR="$ML_APP_NAME.app/Contents/MacOS"
-            DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib"
-        else
-            EXE_DIR=bin
-            DYNAMIC_LIB_DIR=lib
-        fi
+        EXE_DIR=bin
+        DYNAMIC_LIB_DIR=lib
         ;;
 
 esac
@@ -91,23 +86,6 @@ case `uname` in
                 strip --strip-unneeded $LIBRARY
                 objcopy --add-gnu-debuglink="$LIBRARY-debug" "$LIBRARY"
             done
-        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
-            CROSS_TARGET_PLATFORM=x86_64-apple-macosx10.14
-            for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'`
-            do
-                echo "Stripping $PROGRAM"
-                dsymutil-8 $PROGRAM
-                /usr/local/bin/$CROSS_TARGET_PLATFORM-strip -u -r $PROGRAM
-            done
-            for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'`
-            do
-                echo "Stripping $LIBRARY"
-                case $LIBRARY in
-                    *Ml*)
-                        dsymutil-8 $LIBRARY
-                esac
-                /usr/local/bin/$CROSS_TARGET_PLATFORM-strip -x $LIBRARY
-            done
         else
             CROSS_TARGET_PLATFORM=$CPP_CROSS_COMPILE-linux-gnu
             for PROGRAM in `ls -1 "$EXE_DIR"/* | egrep -v "$EXE_DIR"'/core|-debug$'`

From 472591733e8f4d3f3392df2c16fe9f327e8f5043 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Fri, 8 Nov 2024 15:18:10 +1300
Subject: [PATCH 02/16] Remove macos x86_64 from config generation

---
 .buildkite/pipeline.json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py
index 790add24cf..f410e31ab4 100755
--- a/.buildkite/pipeline.json.py
+++ b/.buildkite/pipeline.json.py
@@ -45,7 +45,7 @@ def main():
         build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64)
         pipeline_steps.append(build_windows)
     if config.build_macos:
-        build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, config.build_x86_64)
+        build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, "")
         pipeline_steps.append(build_macos)
     if config.build_linux:
         build_linux = pipeline_steps.generate_step_template("Linux", config.action, config.build_aarch64, config.build_x86_64)

From 51c76f69804e6cd2cd4a4f41c8b6f7f64a1206a4 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Fri, 8 Nov 2024 17:00:18 +1300
Subject: [PATCH 03/16] Install vault

---
 .ci/orka/install.sh | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.ci/orka/install.sh b/.ci/orka/install.sh
index abccab3710..8f239325c4 100644
--- a/.ci/orka/install.sh
+++ b/.ci/orka/install.sh
@@ -41,6 +41,12 @@ if ! command -v jq 2> /dev/null ; then
     brew install jq
 fi
 
+if ! command -v vault 2> /dev/null ; then
+    echo "install vault"
+    brew tap hashicorp/tap
+    brew install hashicorp/tap/vault
+fi
+
 if ! command -v orka-vm-tools 2> /dev/null ; then
     echo "Install orka-vm-tools"
     brew install orka-vm-tools

From d65ff6f52970dd1b09d9f592f43ef911b6c91b42 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Mon, 11 Nov 2024 12:01:11 +1300
Subject: [PATCH 04/16] Trial a new image

---
 .buildkite/pipelines/build_macos.json.py |  2 +-
 .ci/orka/install.sh                      | 18 +++++++++++++-----
 .ci/orka/orka-macos-13-arm.pkr.hcl       |  2 +-
 3 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 0fb25c31d2..b4c763d005 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-13-arm-001.orkasi"
+      "image": "ml-macos-13-arm-002.orkasi"
    }
 }
 envs = {
diff --git a/.ci/orka/install.sh b/.ci/orka/install.sh
index 8f239325c4..fe77803162 100644
--- a/.ci/orka/install.sh
+++ b/.ci/orka/install.sh
@@ -36,22 +36,30 @@ if ! java --version 2> /dev/null ; then
   sudo rm -rf /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64
 fi
 
-if ! command -v jq 2> /dev/null ; then
-    echo "Install jq"
-    brew install jq
-fi
-
 if ! command -v vault 2> /dev/null ; then
     echo "install vault"
     brew tap hashicorp/tap
     brew install hashicorp/tap/vault
 fi
 
+if ! command -v jq 2> /dev/null ; then
+    echo "Install jq"
+    brew install jq
+fi
+
 if ! command -v orka-vm-tools 2> /dev/null ; then
     echo "Install orka-vm-tools"
     brew install orka-vm-tools
 fi
 
+if ! command -v buildkite-agent 2> /dev/null ; then
+  echo "Downloading buildkite-agent..."
+  curl -sL https://raw.githubusercontent.com/elastic/buildkite-agent/main/install.sh -o /tmp/buildkite-install.sh
+  echo "Done downloading buildkite-agent..."
+
+  bash -c "$(cat /tmp/buildkite-install.sh)" > /tmp/insty 2>&1 || true
+fi
+
 echo "Install google cloud sdk in home dir"
 pushd ~
 /bin/bash -c "$(curl -s ${GCLOUD_SDK_URL} --output ${GCLOUD_SDK_FILE})"
diff --git a/.ci/orka/orka-macos-13-arm.pkr.hcl b/.ci/orka/orka-macos-13-arm.pkr.hcl
index 541648a1f0..47e444a149 100644
--- a/.ci/orka/orka-macos-13-arm.pkr.hcl
+++ b/.ci/orka/orka-macos-13-arm.pkr.hcl
@@ -18,7 +18,7 @@ locals {
 
 source "macstadium-orka" "image" {
   source_image     = "ml-macos-13-base-arm-fundamental.orkasi"
-  image_name       = "ml-macos-13-arm-001.orkasi"
+  image_name       = "ml-macos-13-arm-002.orkasi"
   orka_endpoint    = local.orka_endpoint
   orka_user        = local.orka_user
   orka_password    = local.orka_password

From fd38f69dc89b9eaafd6d11354d2adef4e14de92c Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Mon, 11 Nov 2024 12:10:18 +1300
Subject: [PATCH 05/16] Experiment with generic VM image

---
 .buildkite/pipelines/build_macos.json.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index b4c763d005..2c4b17c756 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,8 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-13-arm-002.orkasi"
+      "image": "generic-13-ventura-arm-002.orkasi"
+      #"image": "ml-macos-13-arm-002.orkasi"
    }
 }
 envs = {

From 57d6b1b6da99f7721bb9deb4418015bd3f9fe0b9 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Mon, 11 Nov 2024 13:48:09 +1300
Subject: [PATCH 06/16] Trial another image

---
 .buildkite/pipelines/build_macos.json.py |  2 +-
 .ci/orka/orka-macos-13-arm.pkr.hcl       | 14 +++++++-------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 2c4b17c756..a1abe7ac1b 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "generic-13-ventura-arm-002.orkasi"
+      "image": "ml-macos-13-arm-003.orkasi"
       #"image": "ml-macos-13-arm-002.orkasi"
    }
 }
diff --git a/.ci/orka/orka-macos-13-arm.pkr.hcl b/.ci/orka/orka-macos-13-arm.pkr.hcl
index 47e444a149..a3805944f9 100644
--- a/.ci/orka/orka-macos-13-arm.pkr.hcl
+++ b/.ci/orka/orka-macos-13-arm.pkr.hcl
@@ -18,7 +18,7 @@ locals {
 
 source "macstadium-orka" "image" {
   source_image     = "ml-macos-13-base-arm-fundamental.orkasi"
-  image_name       = "ml-macos-13-arm-002.orkasi"
+  image_name       = "ml-macos-13-arm-003.orkasi"
   orka_endpoint    = local.orka_endpoint
   orka_user        = local.orka_user
   orka_password    = local.orka_password
@@ -54,10 +54,10 @@ build {
       "/tmp/install.sh",
     ]
   }
-  provisioner "shell" {
-    inline = [
-      "chmod u+x /tmp/third_party_deps.sh",
-      "/tmp/third_party_deps.sh",
-    ]
-  }
+  #provisioner "shell" {
+  #  inline = [
+  #    "chmod u+x /tmp/third_party_deps.sh",
+  #    "/tmp/third_party_deps.sh",
+  #  ]
+  #}
 }

From 733fbfa53927bbf6a5a11d846d9a45b260dd68e7 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Mon, 11 Nov 2024 15:28:58 +1300
Subject: [PATCH 07/16] Yet another image to try

---
 .buildkite/pipelines/build_macos.json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index a1abe7ac1b..f3657d2564 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-13-arm-003.orkasi"
+      "image": "ml-macos-12-arm-002.orkasi"
       #"image": "ml-macos-13-arm-002.orkasi"
    }
 }

From aac97c71e8abd77efe9cd512e80ae8f64e607393 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Mon, 11 Nov 2024 17:05:17 +1300
Subject: [PATCH 08/16] Another attempt...

---
 .buildkite/pipelines/build_macos.json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index f3657d2564..0f1468b0ec 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-12-arm-002.orkasi"
+      "image": "ml-macos-12-arm-003.orkasi"
       #"image": "ml-macos-13-arm-002.orkasi"
    }
 }

From 4e138d3aa8dab26d98773f635e4e273b764bced0 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Tue, 12 Nov 2024 11:07:38 +1300
Subject: [PATCH 09/16] Try with generic ventura image

---
 .buildkite/pipelines/build_macos.json.py | 2 +-
 .ci/orka/install.sh                      | 3 +++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 0f1468b0ec..548ab83243 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-12-arm-003.orkasi"
+      "image": "generic-13-ventura-arm-001.orkasi"
       #"image": "ml-macos-13-arm-002.orkasi"
    }
 }
diff --git a/.ci/orka/install.sh b/.ci/orka/install.sh
index fe77803162..d6a7640388 100644
--- a/.ci/orka/install.sh
+++ b/.ci/orka/install.sh
@@ -26,6 +26,9 @@ if ! command -v brew 2> /dev/null ; then
 fi
 
 eval "$(brew shellenv)"
+echo 'export PATH="$HOMEBREW_PREFIX/bin:$PATH"' >> ~/.zshrc
+echo 'export PATH="$HOMEBREW_PREFIX/bin:$PATH"' >> ~/.bash_profile
+export PATH="$HOMEBREW_PREFIX/bin:$PATH"
 
 if ! java --version 2> /dev/null ; then
   echo 'install jdk 11'

From cd4378a5926295bf61a1b340c6d68489b13ff82a Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Tue, 12 Nov 2024 11:54:25 +1300
Subject: [PATCH 10/16] Trial updated generic image

---
 .buildkite/pipelines/build_macos.json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 548ab83243..45d27254a5 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "generic-13-ventura-arm-001.orkasi"
+      "image": "generic-13-ventura-update-arm-003.orkasi"
       #"image": "ml-macos-13-arm-002.orkasi"
    }
 }

From 0163c4bef359afdf0efaa2550ed981cba593d25b Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Tue, 12 Nov 2024 12:01:18 +1300
Subject: [PATCH 11/16] Revert to macos 12 image for sanity check

---
 .buildkite/pipelines/build_macos.json.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 45d27254a5..7748e4291f 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,8 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "generic-13-ventura-update-arm-003.orkasi"
-      #"image": "ml-macos-13-arm-002.orkasi"
+      "image": "ml-macos-12-arm-001.orkasi"
    }
 }
 envs = {

From 8b15dcec4a40221559c5f9630875003492d5c2dc Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Tue, 12 Nov 2024 14:04:04 +1300
Subject: [PATCH 12/16] Try with rebuilt image

---
 .buildkite/pipelines/build_macos.json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 7748e4291f..0fb25c31d2 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-12-arm-001.orkasi"
+      "image": "ml-macos-13-arm-001.orkasi"
    }
 }
 envs = {

From c22e7e30e18fb1deafd6f7233509f5d22e80f18d Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Wed, 13 Nov 2024 09:41:25 +1300
Subject: [PATCH 13/16] Try with existing macos 12 image

---
 .buildkite/pipelines/build_macos.json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 0fb25c31d2..7748e4291f 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-13-arm-001.orkasi"
+      "image": "ml-macos-12-arm-001.orkasi"
    }
 }
 envs = {

From ed43e1fa963e47cc7dc257ed5d5a3e4a63bf2ba6 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Wed, 13 Nov 2024 09:54:01 +1300
Subject: [PATCH 14/16] Try generic-13-ventura-arm-002.orkasi image (already
 deployed to node)

---
 .buildkite/pipelines/build_macos.json.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 7748e4291f..c9467ae360 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-12-arm-001.orkasi"
+      "image": "generic-13-ventura-arm-002.orkasi"
    }
 }
 envs = {

From d1272b50f01b8902a2611c43312cae391b2e7da5 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Wed, 13 Nov 2024 10:59:20 +1300
Subject: [PATCH 15/16] Newly created minimal image

---
 .buildkite/pipelines/build_macos.json.py |  2 +-
 .ci/orka/install.sh                      | 26 ++++++++----------------
 2 files changed, 9 insertions(+), 19 deletions(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index c9467ae360..29d19e0c5a 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "generic-13-ventura-arm-002.orkasi"
+      "image": "ml-macos-13-arm-003.orkasi"
    }
 }
 envs = {
diff --git a/.ci/orka/install.sh b/.ci/orka/install.sh
index d6a7640388..f83bed0cba 100644
--- a/.ci/orka/install.sh
+++ b/.ci/orka/install.sh
@@ -25,18 +25,16 @@ if ! command -v brew 2> /dev/null ; then
     /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
 fi
 
-eval "$(brew shellenv)"
-echo 'export PATH="$HOMEBREW_PREFIX/bin:$PATH"' >> ~/.zshrc
-echo 'export PATH="$HOMEBREW_PREFIX/bin:$PATH"' >> ~/.bash_profile
-export PATH="$HOMEBREW_PREFIX/bin:$PATH"
+eval "$(/opt/homebrew/bin/brew shellenv)"
+echo 'export PATH=$PATH' >> ~/.zprofile
 
 if ! java --version 2> /dev/null ; then
-  echo 'install jdk 11'
-  # Don't use brew to install java, it brings in many unnecessary libraries that e.g. Boost will link against if
-  # present.
-  curl https://cdn.azul.com/zulu/bin/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64.tar.gz | sudo tar xvzf - -C /Library/Java/JavaVirtualMachines && \
-  sudo mv /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64/zulu-11.jdk /Library/Java/JavaVirtualMachines && \
-  sudo rm -rf /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64
+    echo 'install jdk 11'
+    # Don't use brew to install java, it brings in many unnecessary libraries that e.g. Boost will link against if
+    # present.
+    curl https://cdn.azul.com/zulu/bin/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64.tar.gz | sudo tar xvzf - -C /Library/Java/JavaVirtualMachines && \
+    sudo mv /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64/zulu-11.jdk /Library/Java/JavaVirtualMachines && \
+    sudo rm -rf /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64
 fi
 
 if ! command -v vault 2> /dev/null ; then
@@ -55,14 +53,6 @@ if ! command -v orka-vm-tools 2> /dev/null ; then
     brew install orka-vm-tools
 fi
 
-if ! command -v buildkite-agent 2> /dev/null ; then
-  echo "Downloading buildkite-agent..."
-  curl -sL https://raw.githubusercontent.com/elastic/buildkite-agent/main/install.sh -o /tmp/buildkite-install.sh
-  echo "Done downloading buildkite-agent..."
-
-  bash -c "$(cat /tmp/buildkite-install.sh)" > /tmp/insty 2>&1 || true
-fi
-
 echo "Install google cloud sdk in home dir"
 pushd ~
 /bin/bash -c "$(curl -s ${GCLOUD_SDK_URL} --output ${GCLOUD_SDK_FILE})"

From 35f9edcfb07b118802779a05b8df5cba28b3c970 Mon Sep 17 00:00:00 2001
From: Ed Savage <ed.savage@elastic.co>
Date: Thu, 14 Nov 2024 11:42:23 +1300
Subject: [PATCH 16/16] Final tidy up

---
 .buildkite/pipelines/build_macos.json.py |  2 +-
 .ci/orka/README.md                       | 41 +++++++++----------
 .ci/orka/install.sh                      | 50 +-----------------------
 .ci/orka/orka-macos-13-arm.pkr.hcl       | 16 ++++----
 4 files changed, 31 insertions(+), 78 deletions(-)

diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py
index 29d19e0c5a..3a2c334a80 100755
--- a/.buildkite/pipelines/build_macos.json.py
+++ b/.buildkite/pipelines/build_macos.json.py
@@ -33,7 +33,7 @@
 agents = {
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-13-arm-003.orkasi"
+      "imagePrefix": "ml-macos-13-arm"
    }
 }
 envs = {
diff --git a/.ci/orka/README.md b/.ci/orka/README.md
index 7d76cf379a..733fbf7f71 100644
--- a/.ci/orka/README.md
+++ b/.ci/orka/README.md
@@ -38,31 +38,32 @@ The name of the resulting images are hard-coded (currently), and end in a sequen
 
 ## Source Images
 
-The source images used for the MacOS builds are slightly modified copies of the standard Orka images, e.g. 90GBVenturaSSH.orkasi:
+We make use of an image - `generic-13-ventura-arm-002.orkasi` - that is configured such that it:
 
-The source images are named:
- * `ml-macos-13-base-arm-fundamental.orkasi`
+    * Adds passwordless `sudo` for the default `admin` user
+    * Configures `the admin` user to be automatically logged in
+    * Installs Xcode command line tools version 14 (by running `clang++ --version` and clicking through the dialogues)
 
-The source image only has the following changes on it:
-    * Adding passwordless `sudo` for the default `admin` user
-    * Configured `admin` user to be automatically logged in
-    * Installed Xcode command line tools version 14 (by running `clang++ --version` and clicking through the dialogues)
+The generic image has the following packages installed:
+
+ * brew `4.0.28`
+ * vault `1.14.0`
+ * python3 `3.10.8`
+ * jq `1.6`
+ * orka-vm-tools
+ * Google Cloud SDK into `~admin/google-cloud-sdk/`
+ * `gobld-bootstrap.sh` script to run at system startup
+    * This script pulls down and runs another script from a static location to do the following:
+      * Unseal one-time vault token from gobld
+      * Install and run the latest `buildkite-agent`
 
 ## Packer Install Steps
 
-The packer scripts do the following:
-    * Install Brew `4.2.21`
-    * Install JDK `11.0.25`
-    * Install python3
-    * Install jq
-    * Install Google Cloud SDK into `~admin/google-cloud-sdk/`
-    * Install CMake 3.30.5
-    * Install `gobld-bootstrap.sh` script to run at system startup
-        * This script pulls down and runs another script from a static location to do the following:
-            * Unseal one-time vault token from gobld
-            * Install and run the latest `buildkite-agent`
-    * Install Boost 1.83.0 from source
-    * Install PyTorch 2.3.1 from source
+The ML packer scripts do the following:
+ * Install JDK `11.0.25`
+ * Install CMake `3.30.5`
+ * Install Boost `1.83.0` from source
+ * Install PyTorch `2.3.1` from source
 
 ## Caveats
 
diff --git a/.ci/orka/install.sh b/.ci/orka/install.sh
index f83bed0cba..40d17297ad 100644
--- a/.ci/orka/install.sh
+++ b/.ci/orka/install.sh
@@ -4,12 +4,6 @@ export PATH=/opt/homebrew/bin:/usr/local/bin:$PATH
 
 MACHINE_TYPE=$(uname -m)
 
-PYTHON3_URL=https://www.python.org/ftp/python/3.10.10/python-3.10.10-macos11.pkg
-PYTHON3_FILE=python-3.10.10-macos11.pkg
-
-GCLOUD_SDK_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-408.0.1-darwin-x86_64.tar.gz
-GCLOUD_SDK_FILE=google-cloud-cli.tar.gz
-
 if [ ${MACHINE_TYPE} == "x86_64" ]; then
     echo "Running on x86_64"
 
@@ -21,12 +15,7 @@ else
     exit 1
 fi
 
-if ! command -v brew 2> /dev/null ; then
-    /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-fi
-
-eval "$(/opt/homebrew/bin/brew shellenv)"
-echo 'export PATH=$PATH' >> ~/.zprofile
+echo "export PATH=$PATH" >> .zshrc
 
 if ! java --version 2> /dev/null ; then
     echo 'install jdk 11'
@@ -37,43 +26,6 @@ if ! java --version 2> /dev/null ; then
     sudo rm -rf /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64
 fi
 
-if ! command -v vault 2> /dev/null ; then
-    echo "install vault"
-    brew tap hashicorp/tap
-    brew install hashicorp/tap/vault
-fi
-
-if ! command -v jq 2> /dev/null ; then
-    echo "Install jq"
-    brew install jq
-fi
-
-if ! command -v orka-vm-tools 2> /dev/null ; then
-    echo "Install orka-vm-tools"
-    brew install orka-vm-tools
-fi
-
-echo "Install google cloud sdk in home dir"
-pushd ~
-/bin/bash -c "$(curl -s ${GCLOUD_SDK_URL} --output ${GCLOUD_SDK_FILE})"
-tar zxf ${GCLOUD_SDK_FILE}
-rm -f ${GCLOUD_SDK_FILE}
-popd
-
-echo "Install python 3"
-/bin/bash -c "$(curl -s ${PYTHON3_URL} --output ${PYTHON3_FILE})"
-sudo installer -pkg ${PYTHON3_FILE} -target /
-rm -f ${PYTHON3_FILE}
-
-# Install the gobld bootstrap script
-echo "Install gobld bootstrap callout script"
-sudo mkdir -p /usr/local/bin
-sudo cp /tmp/gobld-bootstrap.sh /usr/local/bin/gobld-bootstrap.sh
-sudo chmod +x /usr/local/bin/gobld-bootstrap.sh
-sudo cp /tmp/gobld-bootstrap.plist /Library/LaunchDaemons/gobld-bootstrap.plist
-sudo launchctl bootstrap system /Library/LaunchDaemons/gobld-bootstrap.plist
-sudo cp /tmp/gobld-bootstrap.plist /Users/admin
-
 # Install CMake
 echo "Install CMake"
 curl -v -L https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-macos-universal.tar.gz | tar xvzf - --strip-components 1 -C /Applications
diff --git a/.ci/orka/orka-macos-13-arm.pkr.hcl b/.ci/orka/orka-macos-13-arm.pkr.hcl
index a3805944f9..defdf81b6c 100644
--- a/.ci/orka/orka-macos-13-arm.pkr.hcl
+++ b/.ci/orka/orka-macos-13-arm.pkr.hcl
@@ -17,8 +17,8 @@ locals {
 }
 
 source "macstadium-orka" "image" {
-  source_image     = "ml-macos-13-base-arm-fundamental.orkasi"
-  image_name       = "ml-macos-13-arm-003.orkasi"
+  source_image     = "generic-13-ventura-arm-002.orkasi"
+  image_name       = "ml-macos-13-arm-001.orkasi"
   orka_endpoint    = local.orka_endpoint
   orka_user        = local.orka_user
   orka_password    = local.orka_password
@@ -54,10 +54,10 @@ build {
       "/tmp/install.sh",
     ]
   }
-  #provisioner "shell" {
-  #  inline = [
-  #    "chmod u+x /tmp/third_party_deps.sh",
-  #    "/tmp/third_party_deps.sh",
-  #  ]
-  #}
+  provisioner "shell" {
+    inline = [
+      "chmod u+x /tmp/third_party_deps.sh",
+      "/tmp/third_party_deps.sh",
+    ]
+  }
 }