diff --git a/.buildkite/pipeline.json.py b/.buildkite/pipeline.json.py index 790add24cf..f410e31ab4 100755 --- a/.buildkite/pipeline.json.py +++ b/.buildkite/pipeline.json.py @@ -45,7 +45,7 @@ def main(): build_windows = pipeline_steps.generate_step_template("Windows", config.action, "", config.build_x86_64) pipeline_steps.append(build_windows) if config.build_macos: - build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, config.build_x86_64) + build_macos = pipeline_steps.generate_step_template("MacOS", config.action, config.build_aarch64, "") pipeline_steps.append(build_macos) if config.build_linux: build_linux = pipeline_steps.generate_step_template("Linux", config.action, config.build_aarch64, config.build_x86_64) diff --git a/.buildkite/pipelines/build_macos.json.py b/.buildkite/pipelines/build_macos.json.py index 5342007d2b..3a2c334a80 100755 --- a/.buildkite/pipelines/build_macos.json.py +++ b/.buildkite/pipelines/build_macos.json.py @@ -21,8 +21,7 @@ from itertools import product archs = [ - "aarch64", - "x86_64", + "aarch64" ] build_types = [ "RelWithDebInfo", @@ -32,13 +31,9 @@ "debug" ] agents = { - "x86_64": { - "provider": "orka", - "image": "ml-macos-12-x86_64-001.img" - }, "aarch64": { "provider": "orka", - "image": "ml-macos-12-arm-001.orkasi" + "imagePrefix": "ml-macos-13-arm" } } envs = { @@ -51,16 +46,6 @@ "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-aarch64.cmake", "RUN_TESTS": "true", "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit", - }, - "x86_64": { - "TMPDIR": "/tmp", - "HOMEBREW_PREFIX": "/opt/homebrew", - "PATH": "/opt/homebrew/bin:$PATH", - "ML_DEBUG": "0", - "CPP_CROSS_COMPILE": "", - "CMAKE_FLAGS": "-DCMAKE_TOOLCHAIN_FILE=cmake/darwin-x86_64.cmake", - "RUN_TESTS": "true", - "BOOST_TEST_OUTPUT_FORMAT_FLAGS": "--logger=JUNIT,error,boost_test_results.junit", } } @@ -122,11 +107,6 @@ def main(args): action='store_true', default=False, help="Build for aarch64?.") - parser.add_argument("--build-x86_64", - required=False, - action='store_true', - default=False, - help="Build for x86_64?") args = parser.parse_args() diff --git a/.buildkite/scripts/steps/build_and_test.sh b/.buildkite/scripts/steps/build_and_test.sh index af1c912052..5fc133874c 100755 --- a/.buildkite/scripts/steps/build_and_test.sh +++ b/.buildkite/scripts/steps/build_and_test.sh @@ -82,7 +82,6 @@ if [[ `uname` = "Linux" ]]; then # Linux x86_64 fi fi else # Darwin (macOS) - sudo -E ${REPO_ROOT}/dev-tools/download_macos_deps.sh if [[ "$HARDWARE_ARCH" = aarch64 ]] ; then # Darwin aarch64 # For macOS aarch64, build directly on the machine using gradle if [ "$RUN_TESTS" = false ] ; then @@ -90,29 +89,11 @@ else # Darwin (macOS) else TASKS="clean buildZip buildZipSymbols check" fi - # For macOS we usually only use a particular version as our build platform - # once Xcode has stopped receiving updates for it. However, with Big Sur - # on ARM we couldn't do this, as Big Sur was the first macOS version for - # ARM. Therefore, the compiler may get upgraded on a CI server, and we - # need to hardcode the version that was used to build Boost for that - # version of Elasticsearch. - if [ "$HARDWARE_ARCH" = aarch64 ] ; then - export BOOSTCLANGVER=13 - fi (cd ${REPO_ROOT} && ./gradlew --info -Dbuild.version_qualifier=${VERSION_QUALIFIER:-} -Dbuild.snapshot=$BUILD_SNAPSHOT -Dbuild.ml_debug=$ML_DEBUG $TASKS) || TEST_OUTCOME=$? else # Darwin x86_64 - # For macOS x86_64 we re-use existing Docker scripts and build directly on the machine - function nproc() { - sysctl -n hw.logicalcpu - } - export -f nproc - if [ "$RUN_TESTS" = "true" ]; then - ${REPO_ROOT}/dev-tools/docker/docker_entrypoint.sh --test - grep passed build/test_status.txt || TEST_OUTCOME=$? - else - ${REPO_ROOT}/dev-tools/docker/docker_entrypoint.sh - fi + echo "Unsupported architecture - macos x86_64" + exit 1 fi fi diff --git a/.ci/orka/README.md b/.ci/orka/README.md index 91978c43c5..733fbf7f71 100644 --- a/.ci/orka/README.md +++ b/.ci/orka/README.md @@ -17,11 +17,7 @@ If you haven't run these before, run the following once so packer downloads the `vault` integration: ``` -packer init orka-macos-12-arm.pkr.hcl -``` -or -``` -packer init orka-macos-12-x86_64.pkr.hcl +packer init orka-macos-13-arm.pkr.hcl ``` ## Build @@ -33,7 +29,7 @@ Packer requires access to secrets in vault, where VAULT_ADDR=https://vault-ci-pr Run the following to create the image (MacOS 12 ARM in this example): ``` -packer build orka-macos-12-arm.pkr.hcl +packer build orka-macos-13-arm.pkr.hcl ``` ## Versioning @@ -42,35 +38,33 @@ The name of the resulting images are hard-coded (currently), and end in a sequen ## Source Images -The source images used for the MacOS builds are slightly modified copies of the standard Orka images, e.g. 90GBMontereySSH.orkasi: +We make use of an image - `generic-13-ventura-arm-002.orkasi` - that is configured such that it: + + * Adds passwordless `sudo` for the default `admin` user + * Configures `the admin` user to be automatically logged in + * Installs Xcode command line tools version 14 (by running `clang++ --version` and clicking through the dialogues) -The source images are named: - * `ml-macos-12-base-arm-fundamental.orkasi` - * `ml-macos-12-base-x86_64-fundamental.img` +The generic image has the following packages installed: -The source image only has the following changes on it: - * Adding passwordless `sudo` for the default `admin` user - * Configured `admin` user to be automatically logged in - * Installed Xcode command line tools version 13 (by running `clang++ --version` and clicking through the dialogues) + * brew `4.0.28` + * vault `1.14.0` + * python3 `3.10.8` + * jq `1.6` + * orka-vm-tools + * Google Cloud SDK into `~admin/google-cloud-sdk/` + * `gobld-bootstrap.sh` script to run at system startup + * This script pulls down and runs another script from a static location to do the following: + * Unseal one-time vault token from gobld + * Install and run the latest `buildkite-agent` ## Packer Install Steps -The packer script does the following: - * Install Brew `4.2.21` - * Install JDK `11.0.23` - * Install python3 - * Install vault - * Install jq - * Install Google Cloud SDK into `~admin/google-cloud-sdk/` - * Install CMake 3.23.3 - * Install `gobld-bootstrap.sh` script to run at system startup - * This script pulls down and runs another script from a static location to do the following: - * Unseal one-time vault token from gobld - * Install and run the latest `buildkite-agent` +The ML packer scripts do the following: + * Install JDK `11.0.25` + * Install CMake `3.30.5` + * Install Boost `1.83.0` from source + * Install PyTorch `2.3.1` from source ## Caveats -* Prior to the dependency on PyTorch 2.3.1 we only needed Orka for ARM builds (CI and dependencies), x86_64 builds were - performed via cross-compilation. However, PyTorch 2.3.1 now requires a more modern version of `clang` that our cross - compilation framework provided. As a suitable Orka base image is available for x86_64, it is now simpler to compile - natively for that architecture. +* As of version 9.0.0 support for macos x86_64 builds has been dropped. It is necessary to checkout and work on previous branches in order to maintain x86_64 Orka VMs. diff --git a/.ci/orka/install.sh b/.ci/orka/install.sh index 22cac895e9..40d17297ad 100644 --- a/.ci/orka/install.sh +++ b/.ci/orka/install.sh @@ -4,12 +4,6 @@ export PATH=/opt/homebrew/bin:/usr/local/bin:$PATH MACHINE_TYPE=$(uname -m) -PYTHON3_URL=https://www.python.org/ftp/python/3.10.8/python-3.10.8-macos11.pkg -PYTHON3_FILE=python-3.10.8-macos11.pkg - -GCLOUD_SDK_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-cli-408.0.1-darwin-x86_64.tar.gz -GCLOUD_SDK_FILE=google-cloud-cli.tar.gz - if [ ${MACHINE_TYPE} == "x86_64" ]; then echo "Running on x86_64" @@ -21,57 +15,20 @@ else exit 1 fi -if ! command -v brew 2> /dev/null ; then - /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" -fi - -eval "$(brew shellenv)" +echo "export PATH=$PATH" >> .zshrc if ! java --version 2> /dev/null ; then - echo 'install jdk 11' - brew install openjdk@11 - sudo ln -sfn /opt/homebrew/opt/openjdk@11/libexec/openjdk.jdk /Library/Java/JavaVirtualMachines/openjdk-11.jdk + echo 'install jdk 11' + # Don't use brew to install java, it brings in many unnecessary libraries that e.g. Boost will link against if + # present. + curl https://cdn.azul.com/zulu/bin/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64.tar.gz | sudo tar xvzf - -C /Library/Java/JavaVirtualMachines && \ + sudo mv /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64/zulu-11.jdk /Library/Java/JavaVirtualMachines && \ + sudo rm -rf /Library/Java/JavaVirtualMachines/zulu11.76.21-ca-jdk11.0.25-macosx_aarch64 fi -if ! command -v vault 2> /dev/null ; then - echo "Install vault" - brew install vault -fi - -if ! command -v jq 2> /dev/null ; then - echo "Install jq" - brew install jq -fi - -if ! command -v orka-vm-tools 2> /dev/null ; then - echo "Install orka-vm-tools" - brew install orka-vm-tools -fi - -echo "Install google cloud sdk in home dir" -pushd ~ -/bin/bash -c "$(curl -s ${GCLOUD_SDK_URL} --output ${GCLOUD_SDK_FILE})" -tar zxf ${GCLOUD_SDK_FILE} -rm -f ${GCLOUD_SDK_FILE} -popd - -echo "Install python 3" -/bin/bash -c "$(curl -s ${PYTHON3_URL} --output ${PYTHON3_FILE})" -sudo installer -pkg ${PYTHON3_FILE} -target / -rm -f ${PYTHON3_FILE} - -# Install the gobld bootstrap script -echo "Install gobld bootstrap callout script" -sudo mkdir -p /usr/local/bin -sudo cp /tmp/gobld-bootstrap.sh /usr/local/bin/gobld-bootstrap.sh -sudo chmod +x /usr/local/bin/gobld-bootstrap.sh -sudo cp /tmp/gobld-bootstrap.plist /Library/LaunchDaemons/gobld-bootstrap.plist -sudo launchctl bootstrap system /Library/LaunchDaemons/gobld-bootstrap.plist -sudo cp /tmp/gobld-bootstrap.plist /Users/admin - # Install CMake echo "Install CMake" -curl -v -L https://github.com/Kitware/CMake/releases/download/v3.23.3/cmake-3.23.3-macos-universal.tar.gz | tar xvzf - --strip-components 1 -C /Applications +curl -v -L https://github.com/Kitware/CMake/releases/download/v3.30.5/cmake-3.30.5-macos-universal.tar.gz | tar xvzf - --strip-components 1 -C /Applications sudo ln -sf /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake # Make sure all changes are written to disk diff --git a/.ci/orka/orka-macos-13-arm.pkr.hcl b/.ci/orka/orka-macos-13-arm.pkr.hcl new file mode 100644 index 0000000000..defdf81b6c --- /dev/null +++ b/.ci/orka/orka-macos-13-arm.pkr.hcl @@ -0,0 +1,63 @@ +packer { + required_plugins { + macstadium-orka = { + version = "= 2.3.0" + source = "github.com/macstadium/macstadium-orka" + } + } +} + +locals { + orka_endpoint = vault("secret/ci/elastic-ml-cpp/orka", "orka_endpoint") + orka_user = vault("secret/ci/elastic-ml-cpp/orka", "orka_user") + orka_password = vault("secret/ci/elastic-ml-cpp/orka", "orka_password") + ssh_username = vault("secret/ci/elastic-ml-cpp/orka", "ssh_username") + ssh_password = vault("secret/ci/elastic-ml-cpp/orka", "ssh_password") + sensitive = true +} + +source "macstadium-orka" "image" { + source_image = "generic-13-ventura-arm-002.orkasi" + image_name = "ml-macos-13-arm-001.orkasi" + orka_endpoint = local.orka_endpoint + orka_user = local.orka_user + orka_password = local.orka_password + ssh_username = local.ssh_username + ssh_password = local.ssh_password + orka_vm_cpu_core = 4 + no_delete_vm = false +} + +build { + sources = [ + "macstadium-orka.image" + ] + provisioner "file" { + source = "install.sh" + destination = "/tmp/install.sh" + } + provisioner "file" { + source = "third_party_deps.sh" + destination = "/tmp/third_party_deps.sh" + } + provisioner "file" { + source = "gobld-bootstrap.sh" + destination = "/tmp/gobld-bootstrap.sh" + } + provisioner "file" { + source = "gobld-bootstrap.plist" + destination = "/tmp/gobld-bootstrap.plist" + } + provisioner "shell" { + inline = [ + "chmod u+x /tmp/install.sh", + "/tmp/install.sh", + ] + } + provisioner "shell" { + inline = [ + "chmod u+x /tmp/third_party_deps.sh", + "/tmp/third_party_deps.sh", + ] + } +} diff --git a/.ci/orka/third_party_deps.sh b/.ci/orka/third_party_deps.sh new file mode 100644 index 0000000000..dee54af13b --- /dev/null +++ b/.ci/orka/third_party_deps.sh @@ -0,0 +1,76 @@ +#!/bin/bash +set -xe +export PATH=/opt/homebrew/bin:/usr/local/bin:$PATH + +MACHINE_TYPE=$(uname -m) +if [ ${MACHINE_TYPE} != "arm64" ]; then + echo "Unsupported architecture: ${MACHINE_TYPE}" + echo "Expected arm64" + exit 1 +fi + +export CPP='clang -E' +export CC=clang +export CFLAGS="-O3" +export CXX='clang++ -std=c++17 -stdlib=libc++' +export CXXFLAGS="-O3" +export CXXCPP='clang++ -std=c++17 -E' +export LDFLAGS=-Wl,-headerpad_max_install_names +unset CPATH +unset C_INCLUDE_PATH +unset CPLUS_INCLUDE_PATH +unset LIBRARY_PATH + +# Build and install boost 1.83.0 +curl -L https://boostorg.jfrog.io/artifactory/main/release/1.83.0/source/boost_1_83_0.tar.bz2 | tar xvjf - && \ +cd boost_1_83_0 && \ +./bootstrap.sh --with-toolset=clang --without-libraries=context --without-libraries=coroutine --without-libraries=graph_parallel --without-libraries=mpi --without-libraries=python --without-icu && \ +sed -i -e 's|(13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)( \\|(3ul)(13ul)(29ul)(53ul)(97ul)(193ul)(389ul)(769ul)(1543ul)(3079ul)(6151ul)( \\|' boost/unordered/detail/prime_fmod.hpp +./b2 -j8 --layout=versioned --disable-icu cxxflags="-std=c++17 -stdlib=libc++" linkflags="-std=c++17 -stdlib=libc++ -Wl,-headerpad_max_install_names" optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC && \ +sudo ./b2 install --layout=versioned --disable-icu cxxflags="-std=c++17 -stdlib=libc++" linkflags="-std=c++17 -stdlib=libc++ -Wl,-headerpad_max_install_names" optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC && \ +cd .. && \ +sudo rm -rf boost_1_83_0 + +# Install python modules required by PyTorch +sudo pip3 install numpy ninja pyyaml setuptools cffi typing_extensions future six requests dataclasses + +# Build and install PyTorch +git clone --depth=1 --branch=v2.3.1 https://github.com/pytorch/pytorch.git && \ +cd pytorch && \ +git submodule sync && \ +git submodule update --init --recursive && \ +sed -i -e 's/system(/strlen(/' torch/csrc/jit/codegen/fuser/cpu/fused_kernel.cpp && \ +sed -i -e '/CUDNN_ROOT/a\ + "DNNL_TARGET_ARCH", +' tools/setup_helpers/cmake.py && \ +sed -i -e 's/CMAKE_LINK_WHAT_YOU_USE TRUE/CMAKE_LINK_WHAT_YOU_USE FALSE/' CMakeLists.txt && \ +sed -i -e '189i\ +inline void to_bytes(bytestring& bytes, const unsigned long arg) {\ + auto as_cstring = reinterpret_cast(&arg);\ + bytes.append(as_cstring, sizeof(unsigned long));\ +}\ +' third_party/ideep/include/ideep/utils.hpp + +export BLAS=vecLib +export BUILD_TEST=OFF +export BUILD_CAFFE2=OFF +export USE_NUMPY=OFF +export USE_DISTRIBUTED=OFF +export DNNL_TARGET_ARCH=AARCH64 +export USE_MKLDNN=ON +export USE_QNNPACK=OFF +export USE_PYTORCH_QNNPACK=OFF +export PYTORCH_BUILD_VERSION=2.3.1 +export PYTORCH_BUILD_NUMBER=1 +python3 setup.py install + +sudo mkdir -p /usr/local/lib && \ +sudo mkdir -p /usr/local/include/pytorch && \ +sudo cp -r torch/include/* /usr/local/include/pytorch/ && \ +sudo cp torch/lib/libtorch_cpu.dylib /usr/local/lib/ && \ +sudo cp torch/lib/libc10.dylib /usr/local/lib/ && \ +cd .. && \ +rm -rf pytorch + +# Make sure all changes are written to disk +sync diff --git a/build-setup/macos.md b/build-setup/macos.md index 9d3719384f..f56e907fa5 100644 --- a/build-setup/macos.md +++ b/build-setup/macos.md @@ -16,7 +16,7 @@ For example, you might create a `.bashrc` file in your home directory containing ``` umask 0002 -export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk-17.0.1.jdk/Contents/Home +export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk-21.0.1.jdk/Contents/Home export PYTHONHOME=/Library/Frameworks/Python.framework/Versions/3.10 export PATH=$JAVA_HOME/bin:$PYTHONHOME/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin # Only required if building the C++ code directly using cmake - adjust depending on the location of your Git clone @@ -66,6 +66,25 @@ xcode-select --install at the command prompt. +### CMake + +CMake version 3.19.2 is the minimum required to build ml-cpp. Download the graphical installer for version 3.30.5 from (or get a more recent version). + +Open the `.dmg` and install the application it by dragging it to the `Applications` folder. + +Then make the `cmake` program accessible to programs that look in `/usr/local/bin`: + +``` +sudo mkdir -p /usr/local/bin +sudo ln -s /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake +``` + +## Third Party Dependencies + +The following instructions are for building and installing 3rd party dependencies required for the `ml-cpp` build. They +can either be performed manually, or if preferred, running the `dev-tools/build_macos_third_party_deps.sh` automates the +process. + ### Boost 1.83.0 Download version 1.83.0 of Boost from . You must get this exact version, as the Machine Learning build system requires it. @@ -106,24 +125,11 @@ sudo ./b2 install --layout=versioned --disable-icu cxxflags="-std=c++17 -stdlib= to install the Boost headers and libraries. -### CMake - -CMake version 3.19.2 is the minimum required to build ml-cpp. Download the graphical installer for version 3.23.2 from (or get a more recent version). - -Open the `.dmg` and install the application it by dragging it to the `Applications` folder. - -Then make the `cmake` program accessible to programs that look in `/usr/local/bin`: - -``` -sudo mkdir -p /usr/local/bin -sudo ln -s /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake -``` - ### Python 3.10 PyTorch currently requires Python 3.7 or higher; we use version 3.10. -Download the graphical installer for Python 3.10.9 from . +Download the graphical installer for Python 3.10.10 from . Install using all the default options. When the installer completes a Finder window pops up. Double click the `Install Certificates.command` file in this folder to install the SSL certificates Python needs. @@ -170,44 +176,6 @@ inline void to_bytes(bytestring& bytes, const unsigned long arg) { at around line 189. This is necessary to resolve a template specialization issue. -For compilation on `macOS Mojave (10.14)` - -* Edit `torch/csrc/jit/api/module.h` and change - ``` - Module(Module&&) noexcept = default;` - Module& operator=(Module&&) noexcept = default; - ``` - to - ``` - Module(Module&&) = default; - Module& operator=(Module&&) noexcept = default; - ``` -* Edit `third_party/onnx/CMakeLists.txt` and - * add `find_package(Boost 1.83.0 EXACT REQUIRED COMPONENTS filesystem)` - before `target_include_directories(onnx PUBLIC` (around line 531) - * add `$` after `$` (around line - 536) -* Edit `third_party/onnx/onnx/common/file_utils.h` and change the lines - ``` - #include - #include - ``` - to - ``` - #include - #include - ``` - and change - ``` - std::filesystem::path proto_u8_path = std::filesystem::u8path(proto_path); - std::fstream proto_stream(proto_u8_path, std::ios::in | std::ios::binary); - ``` - to - ``` - boost::filesystem::path proto_u8_path = boost::filesystem::u8path(proto_path); - boost::filesystem::fstream proto_stream(proto_u8_path, std::ios::in | std::ios::binary); - ``` - Build as follows: ``` @@ -229,6 +197,7 @@ export PYTORCH_BUILD_NUMBER=1 Once built copy headers and libraries to system directories: ``` +sudo mkdir -p /usr/local/lib sudo mkdir -p /usr/local/include/pytorch sudo cp -r torch/include/* /usr/local/include/pytorch/ sudo cp torch/lib/libtorch_cpu.dylib /usr/local/lib/ diff --git a/build-setup/macos_cross_compiled.md b/build-setup/macos_cross_compiled.md deleted file mode 100644 index b6305b93ba..0000000000 --- a/build-setup/macos_cross_compiled.md +++ /dev/null @@ -1,93 +0,0 @@ -# Machine Learning Build Machine Setup for macOS cross compiled on Linux - -You will need the following environment variables to be defined: - -- `JAVA_HOME` - Should point to the JDK you want to use to run Gradle. -- `CPP_CROSS_COMPILE` - Should be set to "macosx". -- `CPP_SRC_HOME` - Only required if building the C++ code directly using `cmake`, as Gradle sets it automatically. - -For example, you might create a .bashrc file in your home directory containing this: - -``` -umask 0002 -export JAVA_HOME=/usr/local/jdk1.8.0_121 -export PATH=$JAVA_HOME/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin -# Only required if building the C++ code directly using cmake - adjust depending on the location of your Git clone -export CPP_SRC_HOME=$HOME/ml-cpp -export CPP_CROSS_COMPILE=macosx -``` - -### Initial Preparation - -Start by configuring a native macOS build server as described in [macos.md](macos.md). - -The remainder of these instructions assume the macOS build server you have configured is for macOS 10.14 (Mojave). This is what builds for distribution are currently built on. - -On the fully configured macOS build server, run the following commands: - -``` -cd /usr -tar jcvf ~/usr-x86_64-apple-macosx10.14.tar.bz2 lib local -cd /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr -tar jcvf ~/xcode-x86_64-apple-macosx10.14.tar.bz2 include -cd `xcrun --show-sdk-path`/usr -tar jcvf ~/sdk-x86_64-apple-macosx10.14.tar.bz2 include lib -``` - -These instructions also assume the host platform is Ubuntu 18.04. It makes life much easier if the host platform is a version of Ubuntu that's new enough to run the official binary distribution of clang/LLVM (otherwise it would be necessary to build clang/LLVM from source). - -Transfer the three archives created in your home directory on the macOS build server, `usr-x86_64-apple-macosx10.14.tar.bz2`, `xcode-x86_64-apple-macosx10.14.tar.bz2` and `sdk-x86_64-apple-macosx10.14.tar.bz2`, to your home directory on the cross compilation host build server. - -### OS Packages - -You need clang 8, plus a number of other build tools. They can be installed on modern Ubuntu as follows: - -``` -sudo apt-get install automake autogen build-essential bzip2 git gobjc libtool software-properties-common unzip wget - -wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - -sudo apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal main" -sudo apt-get install clang-8 clang-8-doc libclang1-8 libllvm8 lldb-8 llvm-8 llvm-8-doc llvm-8-runtime -``` - -(It is strongly recommended NOT to attempt to create a cross compile environment on an old version of Linux, because you will have to build clang from source and you need a modern C++ compiler to build clang. So you would probably end up first building a modern version of gcc using the system default gcc, then building clang using the modern gcc.) - -### Transferred Build Dependencies - -Add the dependencies that you copied from the fully configured macOS build server in the "Initial Preparation" step. - -``` -sudo mkdir -p /usr/local/sysroot-x86_64-apple-macosx10.14/usr -cd /usr/local/sysroot-x86_64-apple-macosx10.14/usr -sudo tar jxvf ~/usr-x86_64-apple-macosx10.14.tar.bz2 -sudo tar jxvf ~/xcode-x86_64-apple-macosx10.14.tar.bz2 -sudo tar jxvf ~/sdk-x86_64-apple-macosx10.14.tar.bz2 -``` - -### CMake - -CMake version 3.19.2 is the minimum required to build ml-cpp. Download version 3.23.2 from and install: - -``` -chmod +x cmake-3.23.2-Linux-x86_64.sh -sudo ./cmake-3.23.2-Linux-x86_64.sh --skip-license --prefix=/usr/local -``` - -### cctools-port - -You need to obtain Linux ports of several Apple development tools. The easiest way to get them is to use the [cctools-port project on GitHub](https://github.com/tpoechtrager/cctools-port): - -``` -git clone https://github.com/tpoechtrager/cctools-port.git -cd cctools-port/cctools -git checkout 949.0.1-ld64-530 -export CC=clang-8 -export CXX=clang++-8 -./autogen.sh -./configure --target=x86_64-apple-macosx10.14 --with-llvm-config=/usr/bin/llvm-config-8 -make -sudo make install -``` - -The "949.0.1-ld64-530" branch in the [cctools-port repository](https://github.com/tpoechtrager/cctools-port) corresponds to the tools for macOS 10.14 Mojave and clang 8. (A different branch would be required for newer versions of the OS/compiler.) - diff --git a/cmake/compiler/clang.cmake b/cmake/compiler/clang.cmake index bd9ff030db..1749ad0a89 100644 --- a/cmake/compiler/clang.cmake +++ b/cmake/compiler/clang.cmake @@ -9,36 +9,11 @@ # limitation. # -# which compilers to use for C and C++ -if(DEFINED ENV{CPP_CROSS_COMPILE} AND NOT "$ENV{CPP_CROSS_COMPILE}" STREQUAL "") - message(STATUS "Cross compiling: CPP_CROSS_COMPILE = $ENV{CPP_CROSS_COMPILE}") - - set(CROSS_FLAGS --sysroot=${SYSROOT} -B /usr/local/bin -target ${CROSS_TARGET_PLATFORM} -stdlib=libc++) - set(ML_SHARED_LINKER_FLAGS ${CROSS_FLAGS}) - set(ML_EXE_LINKER_FLAGS ${CROSS_FLAGS}) - - # which compilers to use for C and C++ - set(CMAKE_C_COMPILER "clang-8") - set(CMAKE_CXX_COMPILER "clang++-8") - - set(CMAKE_AR "/usr/local/bin/${CROSS_TARGET_PLATFORM}-ar") - set(CMAKE_RANLIB "/usr/local/bin/${CROSS_TARGET_PLATFORM}-ranlib") - set(CMAKE_STRIP "/usr/local/bin/${CROSS_TARGET_PLATFORM}-strip") - set(CMAKE_LD "/usr/local/bin/${CROSS_TARGET_PLATFORM}-ld") - - set(CMAKE_CXX_ARCHIVE_CREATE " -ru ") - - # where is the target environment located - set(CMAKE_FIND_ROOT_PATH /usr/local/sysroot-${CROSS_TARGET_PLATFORM}) -else() - set(CMAKE_C_COMPILER "clang") - set(CMAKE_CXX_COMPILER "clang++") - set(CMAKE_AR "ar") - set(CMAKE_RANLIB "ranlib") - set(CMAKE_STRIP "strip") - - #set(Boost_COMPILER "-clang-darwin13") -endif() +set(CMAKE_C_COMPILER "clang") +set(CMAKE_CXX_COMPILER "clang++") +set(CMAKE_AR "ar") +set(CMAKE_RANLIB "ranlib") +set(CMAKE_STRIP "strip") list(APPEND ML_C_FLAGS diff --git a/dev-tools/build_macos_third_party_deps.sh b/dev-tools/build_macos_third_party_deps.sh new file mode 120000 index 0000000000..4a9fada635 --- /dev/null +++ b/dev-tools/build_macos_third_party_deps.sh @@ -0,0 +1 @@ +../.ci/orka/third_party_deps.sh \ No newline at end of file diff --git a/dev-tools/docker/README.md b/dev-tools/docker/README.md index 4d5716e083..4e4e9f26c4 100644 --- a/dev-tools/docker/README.md +++ b/dev-tools/docker/README.md @@ -129,24 +129,3 @@ This image is not intended to be built regularly. When changing the ### Build script: dev-tools/docker/build_check_style_image.sh - - -## REPOSITORY: ml-macosx-build - -### VERSION: 18 - -### Comments -A Docker image that can be used to **cross compile** the machine learning -C++ code for Intel macOS - -This image is not intended to be built regularly. When changing the tools -or 3rd party components required to build the machine learning C++ code: - - - 1. increment the version - 2. Change the Dockerfile and build a new image to be -used for subsequent builds on this branch. - 3. Update the version to be used for builds in *dev-tools/docker/macosx_builder/Dockerfile*. - -### Build script: dev-tools/docker/build_macosx_build_image.sh - diff --git a/dev-tools/docker/build_macosx_build_image.sh b/dev-tools/docker/build_macosx_build_image.sh deleted file mode 100755 index 3c915f62da..0000000000 --- a/dev-tools/docker/build_macosx_build_image.sh +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/bash -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the following additional limitation. Functionality enabled by the -# files subject to the Elastic License 2.0 may only be used in production when -# invoked by an Elasticsearch process with a license key installed that permits -# use of machine learning features. You may not use this file except in -# compliance with the Elastic License 2.0 and the foregoing additional -# limitation. -# - -# Builds the Docker image that can be used to compile the machine learning -# C++ code for Intel macOS -# -# This script is not intended to be run regularly. When changing the tools -# or 3rd party components required to build the machine learning C++ code -# increment the version, change the Dockerfile and build a new image to be -# used for subsequent builds on this branch. Then update the version to be -# used for builds in docker/macosx_builder/Dockerfile. - -HOST=docker.elastic.co -ACCOUNT=ml-dev -REPOSITORY=ml-macosx-build -VERSION=19 - -set -e - -cd `dirname $0` - -. ./prefetch_docker_image.sh -CONTEXT=macosx_image -prefetch_docker_base_image $CONTEXT/Dockerfile -docker build --no-cache -t $HOST/$ACCOUNT/$REPOSITORY:$VERSION $CONTEXT -# Get a username and password for this by visiting -# https://docker-auth.elastic.co and allowing it to authenticate against your -# GitHub account -docker login $HOST -docker push $HOST/$ACCOUNT/$REPOSITORY:$VERSION - diff --git a/dev-tools/docker/macosx_builder/Dockerfile b/dev-tools/docker/macosx_builder/Dockerfile deleted file mode 100644 index 74f49698b4..0000000000 --- a/dev-tools/docker/macosx_builder/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the following additional limitation. Functionality enabled by the -# files subject to the Elastic License 2.0 may only be used in production when -# invoked by an Elasticsearch process with a license key installed that permits -# use of machine learning features. You may not use this file except in -# compliance with the Elastic License 2.0 and the foregoing additional -# limitation. -# - -# Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-macosx-build:19 - -MAINTAINER David Roberts - -# Copy the current Git repository into the container -COPY . /ml-cpp/ - -# Tell the build we want to cross compile -ENV CPP_CROSS_COMPILE macosx - -ENV CMAKE_FLAGS -DCMAKE_TOOLCHAIN_FILE=/ml-cpp/cmake/darwin-x86_64.cmake - -# Pass through any version qualifier (default none) -ARG VERSION_QUALIFIER= - -# Pass through whether this is a snapshot build (default yes if not specified) -ARG SNAPSHOT=yes - -# Pass through ML debug option (default blank) -ARG ML_DEBUG= - -# Run the build -RUN \ - /ml-cpp/dev-tools/docker/docker_entrypoint.sh - diff --git a/dev-tools/docker/macosx_image/Dockerfile b/dev-tools/docker/macosx_image/Dockerfile deleted file mode 100644 index 4f5db531a0..0000000000 --- a/dev-tools/docker/macosx_image/Dockerfile +++ /dev/null @@ -1,59 +0,0 @@ -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the following additional limitation. Functionality enabled by the -# files subject to the Elastic License 2.0 may only be used in production when -# invoked by an Elasticsearch process with a license key installed that permits -# use of machine learning features. You may not use this file except in -# compliance with the Elastic License 2.0 and the foregoing additional -# limitation. -# - -FROM ubuntu:20.04 - -# This is basically automating the setup instructions in build-setup/macos_cross_compiled.md - -MAINTAINER David Roberts - -# Make sure apt-get is up to date and required packages are installed -RUN \ - export DEBIAN_FRONTEND=noninteractive && \ - apt-get update && \ - apt-get install --no-install-recommends -y apt-utils automake autogen build-essential bzip2 git gobjc gpg-agent libtool software-properties-common unzip wget zip - -# Install clang -RUN \ - wget --quiet -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - apt-add-repository "deb http://apt.llvm.org/focal/ llvm-toolchain-focal main" && \ - apt-get install --no-install-recommends -y clang-8 libclang1-8 libllvm8 llvm-8 llvm-8-runtime - -# Add build dependencies transferred from native Mac build server -RUN \ - mkdir -p /usr/local/sysroot-x86_64-apple-macosx10.14/usr && \ - cd /usr/local/sysroot-x86_64-apple-macosx10.14/usr && \ - wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/usr-x86_64-apple-macosx10.14-10.tar.bz2 | tar jxf - && \ - wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/xcode-x86_64-apple-macosx10.14-1.tar.bz2 | tar jxf - && \ - wget --quiet -O - https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/sdk-x86_64-apple-macosx10.14-1.tar.bz2 | tar jxf - - -# Build cctools-port -RUN \ - git clone https://github.com/tpoechtrager/cctools-port.git && \ - cd cctools-port/cctools && \ - git checkout 949.0.1-ld64-530 && \ - export CC=clang-8 && \ - export CXX=clang++-8 && \ - ./autogen.sh && \ - ./configure --target=x86_64-apple-macosx10.14 --with-llvm-config=/usr/bin/llvm-config-8 && \ - make -j`nproc` && \ - make install && \ - cd ../.. && \ - rm -rf cctools-port - -# Install CMake -# v3.19.2 minimum is required -RUN \ - wget --quiet https://github.com/Kitware/CMake/releases/download/v3.23.2/cmake-3.23.2-Linux-x86_64.sh && \ - chmod +x cmake-3.23.2-Linux-x86_64.sh && \ - ./cmake-3.23.2-Linux-x86_64.sh --skip-license --prefix=/usr/local && \ - rm -f cmake-3.23.2-Linux-x86_64.sh - diff --git a/dev-tools/docker_build.sh b/dev-tools/docker_build.sh index 47f1064f91..fc1eac34aa 100755 --- a/dev-tools/docker_build.sh +++ b/dev-tools/docker_build.sh @@ -10,8 +10,7 @@ # limitation. # -# Builds the machine learning C++ code for Linux or macOS in a Docker -# container. +# Builds the machine learning C++ code for Linux in a Docker container. # # The output .zip files are then copied out of the container to the # location in the current repository that they'd be in had they been @@ -20,7 +19,7 @@ # Finally, the Docker container used for the build is deleted. usage() { - echo "Usage: $0 linux|linux_aarch64_cross|linux_aarch64_native|macosx ..." + echo "Usage: $0 linux|linux_aarch64_cross|linux_aarch64_native ..." exit 1 } @@ -30,7 +29,7 @@ while [ -n "$1" ] do case "$1" in - linux|linux_aarch64_cross|linux_aarch64_native|macosx) + linux|linux_aarch64_cross|linux_aarch64_native) PLATFORMS="$1 $PLATFORMS" ;; *) diff --git a/dev-tools/download_macos_deps.sh b/dev-tools/download_macos_deps.sh deleted file mode 100755 index ea2f7d29ce..0000000000 --- a/dev-tools/download_macos_deps.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -# -# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -# or more contributor license agreements. Licensed under the Elastic License -# 2.0 and the following additional limitation. Functionality enabled by the -# files subject to the Elastic License 2.0 may only be used in production when -# invoked by an Elasticsearch process with a license key installed that permits -# use of machine learning features. You may not use this file except in -# compliance with the Elastic License 2.0 and the foregoing additional -# limitation. -# - -# Script to download 3rd party dependencies built on a reference macOS -# build server to a CI machine. - -set -e - -if [ `uname` != Darwin ] ; then - echo "This script is intended for use on macOS" - exit 1 -fi - -DEST=/usr - -case `uname -m` in - - x86_64) - ARCHIVE=local-x86_64-apple-macosx12.0-1.tar.bz2 - ;; - arm64) - ARCHIVE=local-arm64-apple-macosx11.1-10.tar.bz2 - ;; - - *) - echo "No archive is available for this architecture:" `uname -m 2>&1` - exit 2 - ;; - -esac - -URL="https://s3-eu-west-2.amazonaws.com/ml-cpp-artifacts/dependencies/$ARCHIVE" - -echo "Downloading dependencies from $URL" -cd "$TMPDIR" && curl -s -S --retry 5 -O "$URL" -echo "Extracting dependencies from $ARCHIVE" -cd "$DEST" && tar -jmxf "$TMPDIR/$ARCHIVE" -echo "Cleaning up dependency archive" -rm -f "$TMPDIR/$ARCHIVE" - diff --git a/dev-tools/strip_binaries.sh b/dev-tools/strip_binaries.sh index eef5933474..ca4d7103fe 100755 --- a/dev-tools/strip_binaries.sh +++ b/dev-tools/strip_binaries.sh @@ -21,13 +21,8 @@ case `uname` in ;; Linux) - if [ "$CPP_CROSS_COMPILE" = macosx ] ; then - EXE_DIR="$ML_APP_NAME.app/Contents/MacOS" - DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib" - else - EXE_DIR=bin - DYNAMIC_LIB_DIR=lib - fi + EXE_DIR=bin + DYNAMIC_LIB_DIR=lib ;; esac @@ -91,23 +86,6 @@ case `uname` in strip --strip-unneeded $LIBRARY objcopy --add-gnu-debuglink="$LIBRARY-debug" "$LIBRARY" done - elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then - CROSS_TARGET_PLATFORM=x86_64-apple-macosx10.14 - for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'` - do - echo "Stripping $PROGRAM" - dsymutil-8 $PROGRAM - /usr/local/bin/$CROSS_TARGET_PLATFORM-strip -u -r $PROGRAM - done - for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'` - do - echo "Stripping $LIBRARY" - case $LIBRARY in - *Ml*) - dsymutil-8 $LIBRARY - esac - /usr/local/bin/$CROSS_TARGET_PLATFORM-strip -x $LIBRARY - done else CROSS_TARGET_PLATFORM=$CPP_CROSS_COMPILE-linux-gnu for PROGRAM in `ls -1 "$EXE_DIR"/* | egrep -v "$EXE_DIR"'/core|-debug$'`