[ML] Build solely on macos 13 (Ventura) aarch64

.buildkite/pipeline.json.py

@@ -45,7 +45,7 @@ def main():
         build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64)
         pipeline_steps.append(build_windows)
     if config.build_macos:
-        build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, config.build_x86_64)
+        build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, "")
         pipeline_steps.append(build_macos)
     if config.build_linux:
         build_linux = pipeline_steps.generate_step_template("Linux", config.action, config.build_aarch64, config.build_x86_64)

.buildkite/pipelines/build_macos.json.py

@@ -21,8 +21,7 @@
 from itertools import product
 
 archs = [
-  "aarch64",
-  "x86_64",
+  "aarch64"
 ]
 build_types = [
   "RelWithDebInfo",
@@ -32,13 +31,9 @@
   "debug"
 ]
 agents = {
-   "x86_64": {
-      "provider": "orka",
-      "image": "ml-macos-12-x86_64-001.img"
-   },
    "aarch64": {
       "provider": "orka",
-      "image": "ml-macos-12-arm-001.orkasi"
+      "imagePrefix": "ml-macos-13-arm"
    }
 }
 envs = {
@@ -51,16 +46,6 @@
       "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-aarch64.cmake",
       "RUN_TESTS": "true",
       "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
-    },
-    "x86_64": {
-      "TMPDIR": "/tmp",
-      "HOMEBREW_PREFIX": "/opt/homebrew",
-      "PATH": "/opt/homebrew/bin:$PATH",
-      "ML_DEBUG": "0",
-      "CPP_CROSS_COMPILE": "",
-      "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-x86_64.cmake",
-      "RUN_TESTS": "true",
-      "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit",
     }
 }
 
@@ -122,11 +107,6 @@ def main(args):
                         action='store_true',
                         default=False,
                         help="Build for aarch64?.")
-    parser.add_argument("--build-x86_64",
-                        required=False,
-                        action='store_true',
-                        default=False,
-                        help="Build for x86_64?")
 
     args = parser.parse_args()
 

.buildkite/scripts/steps/build_and_test.sh

@@ -82,37 +82,18 @@ if [[ `uname` = "Linux" ]]; then # Linux x86_64
     fi
   fi
 else # Darwin (macOS)
-  sudo -E ${REPO_ROOT}/dev-tools/download_macos_deps.sh
   if [[ "$HARDWARE_ARCH" = aarch64 ]] ; then # Darwin aarch64
     # For macOS aarch64, build directly on the machine using gradle
     if [ "$RUN_TESTS" = false ] ; then
        TASKS="clean buildZip buildZipSymbols"
     else
        TASKS="clean buildZip buildZipSymbols check"
     fi
-    # For macOS we usually only use a particular version as our build platform
-    # once Xcode has stopped receiving updates for it. However, with Big Sur
-    # on ARM we couldn't do this, as Big Sur was the first macOS version for
-    # ARM. Therefore, the compiler may get upgraded on a CI server, and we
-    # need to hardcode the version that was used to build Boost for that
-    # version of Elasticsearch.
-    if [ "$HARDWARE_ARCH" = aarch64 ] ; then
-       export BOOSTCLANGVER=13
-    fi
 
     (cd ${REPO_ROOT} && ./gradlew --info -Dbuild.version_qualifier=${VERSION_QUALIFIER:-} -Dbuild.snapshot=$BUILD_SNAPSHOT -Dbuild.ml_debug=$ML_DEBUG $TASKS) || TEST_OUTCOME=$?
   else # Darwin x86_64
-    # For macOS x86_64 we re-use existing Docker scripts and build directly on the machine
-    function nproc() {
-        sysctl -n hw.logicalcpu
-    }
-    export -f nproc
-    if [ "$RUN_TESTS" = "true" ]; then
-      ${REPO_ROOT}/dev-tools/docker/docker_entrypoint.sh --test
-      grep passed build/test_status.txt || TEST_OUTCOME=$?
-    else
-      ${REPO_ROOT}/dev-tools/docker/docker_entrypoint.sh
-    fi
+    echo "Unsupported architecture - macos x86_64"
+    exit 1
   fi
 fi
 

.ci/orka/README.md

@@ -17,11 +17,7 @@
 If you haven't run these before, run the following once so packer downloads the `vault` integration:
 
 ```
-packer init orka-macos-12-arm.pkr.hcl
-```
-or
-```
-packer init orka-macos-12-x86_64.pkr.hcl
+packer init orka-macos-13-arm.pkr.hcl
 ```
 
 ## Build
@@ -33,7 +29,7 @@ Packer requires access to secrets in vault, where VAULT_ADDR=https://vault-ci-pr
 Run the following to create the image (MacOS 12 ARM in this example):
 
 ```
-packer build orka-macos-12-arm.pkr.hcl
+packer build orka-macos-13-arm.pkr.hcl
 ```
 
 ## Versioning
@@ -42,35 +38,33 @@ The name of the resulting images are hard-coded (currently), and end in a sequen
 
 ## Source Images
 
-The source images used for the MacOS builds are slightly modified copies of the standard Orka images, e.g. 90GBMontereySSH.orkasi:
+We make use of an image - `generic-13-ventura-arm-002.orkasi` - that is configured such that it:
+
+    * Adds passwordless `sudo` for the default `admin` user
+    * Configures `the admin` user to be automatically logged in
+    * Installs Xcode command line tools version 14 (by running `clang++ --version` and clicking through the dialogues)
 
-The source images are named:
- * `ml-macos-12-base-arm-fundamental.orkasi`
- * `ml-macos-12-base-x86_64-fundamental.img`
+The generic image has the following packages installed:
 
-The source image only has the following changes on it:
-    * Adding passwordless `sudo` for the default `admin` user
-    * Configured `admin` user to be automatically logged in
-    * Installed Xcode command line tools version 13 (by running `clang++ --version` and clicking through the dialogues)
+ * brew `4.0.28`
+ * vault `1.14.0`
+ * python3 `3.10.8`
+ * jq `1.6`
+ * orka-vm-tools
+ * Google Cloud SDK into `~admin/google-cloud-sdk/`
+ * `gobld-bootstrap.sh` script to run at system startup
+    * This script pulls down and runs another script from a static location to do the following:
+      * Unseal one-time vault token from gobld
+      * Install and run the latest `buildkite-agent`
 
 ## Packer Install Steps
 
-The packer script does the following:
-    * Install Brew `4.2.21`
-    * Install JDK `11.0.23`
-    * Install python3
-    * Install vault
-    * Install jq
-    * Install Google Cloud SDK into `~admin/google-cloud-sdk/`
-    * Install CMake 3.23.3
-    * Install `gobld-bootstrap.sh` script to run at system startup
-        * This script pulls down and runs another script from a static location to do the following:
-            * Unseal one-time vault token from gobld
-            * Install and run the latest `buildkite-agent`
+The ML packer scripts do the following:
+ * Install JDK `11.0.25`
+ * Install CMake `3.30.5`
+ * Install Boost `1.83.0` from source
+ * Install PyTorch `2.3.1` from source
 
 ## Caveats
 
-* Prior to the dependency on PyTorch 2.3.1 we only needed Orka for ARM builds (CI and dependencies), x86_64 builds were
-  performed via cross-compilation. However, PyTorch 2.3.1 now requires a more modern version of `clang` that our cross
-  compilation framework provided. As a suitable Orka base image is available for x86_64, it is now simpler to compile
-  natively for that architecture.
+* As of version 9.0.0 support for macos x86_64 builds has been dropped. It is necessary to checkout and work on previous branches in order to maintain x86_64 Orka VMs.

.ci/orka/install.sh

@@ -4,12 +4,6 @@ export PATH=/opt/homebrew/bin:/usr/local/bin:$PATH
 
 MACHINE_TYPE=$(uname -m)
 
-PYTHON3_URL=https://www.python.org/ftp/python/3.10.8/python-3.10.8-macos11.pkg
-PYTHON3_FILE=python-3.10.8-macos11.pkg
-
-GCLOUD_SDK_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-408.0.1-darwin-x86_64.tar.gz
-GCLOUD_SDK_FILE=google-cloud-cli.tar.gz
-
 if [ ${MACHINE_TYPE} == "x86_64" ]; then
     echo "Running on x86_64"
 
@@ -21,57 +15,20 @@ else
     exit 1
 fi
 
-if ! command -v brew 2> /dev/null ; then
-    /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
-fi
-
-eval "$(brew shellenv)"
+echo "export PATH=$PATH" >> .zshrc
 
 if ! java --version 2> /dev/null ; then
-  echo 'install jdk 11'
-  brew install openjdk@11
-  sudo ln -sfn /opt/homebrew/opt/openjdk@11/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-11.jdk
+    echo 'install jdk 11'
+    # Don't use brew to install java, it brings in many unnecessary libraries that e.g. Boost will link against if
+    # present.
+    curl https://cdn.azul.com/zulu/bin/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64.tar.gz | sudo tar xvzf - -C /Library/Java/JavaVirtualMachines && \
+    sudo mv /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64/zulu-11.jdk /Library/Java/JavaVirtualMachines && \
+    sudo rm -rf /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64
 fi
 
-if ! command -v vault 2> /dev/null ; then
-    echo "Install vault"
-    brew install vault
-fi
-
-if ! command -v jq 2> /dev/null ; then
-    echo "Install jq"
-    brew install jq
-fi
-
-if ! command -v orka-vm-tools 2> /dev/null ; then
-    echo "Install orka-vm-tools"
-    brew install orka-vm-tools
-fi
-
-echo "Install google cloud sdk in home dir"
-pushd ~
-/bin/bash -c "$(curl -s ${GCLOUD_SDK_URL} --output ${GCLOUD_SDK_FILE})"
-tar zxf ${GCLOUD_SDK_FILE}
-rm -f ${GCLOUD_SDK_FILE}
-popd
-
-echo "Install python 3"
-/bin/bash -c "$(curl -s ${PYTHON3_URL} --output ${PYTHON3_FILE})"
-sudo installer -pkg ${PYTHON3_FILE} -target /
-rm -f ${PYTHON3_FILE}
-
-# Install the gobld bootstrap script
-echo "Install gobld bootstrap callout script"
-sudo mkdir -p /usr/local/bin
-sudo cp /tmp/gobld-bootstrap.sh /usr/local/bin/gobld-bootstrap.sh
-sudo chmod +x /usr/local/bin/gobld-bootstrap.sh
-sudo cp /tmp/gobld-bootstrap.plist /Library/LaunchDaemons/gobld-bootstrap.plist
-sudo launchctl bootstrap system /Library/LaunchDaemons/gobld-bootstrap.plist
-sudo cp /tmp/gobld-bootstrap.plist /Users/admin
-
 # Install CMake
 echo "Install CMake"
-curl -v -L https://github.com/Kitware/CMake/releases/download/v3.23.3/cmake-3.23.3-macos-universal.tar.gz | tar xvzf - --strip-components 1 -C /Applications
+curl -v -L https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-macos-universal.tar.gz | tar xvzf - --strip-components 1 -C /Applications
 sudo ln -sf /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake
 
 # Make sure all changes are written to disk

.ci/orka/orka-macos-13-arm.pkr.hcl

@@ -0,0 +1,63 @@
+packer {
+  required_plugins {
+    macstadium-orka = {
+      version = "= 2.3.0"
+      source  = "github.com/macstadium/macstadium-orka"
+    }
+  }
+}
+
+locals {
+  orka_endpoint = vault("secret/ci/elastic-ml-cpp/orka", "orka_endpoint")
+  orka_user     = vault("secret/ci/elastic-ml-cpp/orka", "orka_user")
+  orka_password = vault("secret/ci/elastic-ml-cpp/orka", "orka_password")
+  ssh_username  = vault("secret/ci/elastic-ml-cpp/orka", "ssh_username")
+  ssh_password  = vault("secret/ci/elastic-ml-cpp/orka", "ssh_password")
+  sensitive     = true
+}
+
+source "macstadium-orka" "image" {
+  source_image     = "generic-13-ventura-arm-002.orkasi"
+  image_name       = "ml-macos-13-arm-001.orkasi"
+  orka_endpoint    = local.orka_endpoint
+  orka_user        = local.orka_user
+  orka_password    = local.orka_password
+  ssh_username     = local.ssh_username
+  ssh_password     = local.ssh_password
+  orka_vm_cpu_core = 4
+  no_delete_vm     = false
+}
+
+build {
+  sources = [
+    "macstadium-orka.image"
+  ]
+  provisioner "file" {
+    source = "install.sh"
+    destination = "/tmp/install.sh"
+  }
+  provisioner "file" {
+    source = "third_party_deps.sh"
+    destination = "/tmp/third_party_deps.sh"
+  }
+  provisioner "file" {
+    source = "gobld-bootstrap.sh"
+    destination = "/tmp/gobld-bootstrap.sh"
+  }
+  provisioner "file" {
+    source = "gobld-bootstrap.plist"
+    destination = "/tmp/gobld-bootstrap.plist"
+  }
+  provisioner "shell" {
+    inline = [
+      "chmod u+x /tmp/install.sh",
+      "/tmp/install.sh",
+    ]
+  }
+  provisioner "shell" {
+    inline = [
+      "chmod u+x /tmp/third_party_deps.sh",
+      "/tmp/third_party_deps.sh",
+    ]
+  }
+}