diff --git a/3rd_party/3rd_party.sh b/3rd_party/3rd_party.sh index e038f6f422..32b31ef8ba 100755 --- a/3rd_party/3rd_party.sh +++ b/3rd_party/3rd_party.sh @@ -60,7 +60,7 @@ case `uname` in STL_EXTENSION=.so.6 ZLIB_LOCATION= elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then - SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.11 + SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.13 BOOST_LOCATION=$SYSROOT/usr/local/lib BOOST_COMPILER=clang BOOST_EXTENSION=mt-x64-1_71.dylib diff --git a/build-setup/macos.md b/build-setup/macos.md index 7de3a01739..ca09791e1c 100644 --- a/build-setup/macos.md +++ b/build-setup/macos.md @@ -24,15 +24,15 @@ Note, that bash doesn't read `~/.bashrc` for login shells (which is what you get ### General settings for building the tools -Most of the tools are built via a GNU "configure" script. There are some environment variables that affect the behaviour of this. Therefore, when building ANY tool on macOS, set the following environment variables: +Some tools may be built via a GNU "configure" script. There are some environment variables that affect the behaviour of this. Therefore, when building ANY tool on macOS, set the following environment variables: ``` export CPP='clang -E' export CC=clang export CFLAGS='-O3 -msse4.2' -export CXX='clang++ -std=c++14 -stdlib=libc++' +export CXX='clang++ -std=c++17 -stdlib=libc++' export CXXFLAGS='-O3 -msse4.2' -export CXXCPP='clang++ -std=c++14 -E' +export CXXCPP='clang++ -std=c++17 -E' export LDFLAGS=-Wl,-headerpad_max_install_names unset CPATH unset C_INCLUDE_PATH @@ -46,13 +46,12 @@ The above environment variables only need to be set when building tools on macOS The first major piece of development software to install is Apple's development environment, Xcode, which can be downloaded from . You will need to register as a developer with Apple. Alternatively, you can get the latest version of Xcode from the App Store. -- If you are using Yosemite, you must install Xcode 7.2.x -- If you are using El Capitan, you must install Xcode 8.2.x -- If you are using Sierra, you must install Xcode 9.2.x +For C++17 Xcode 10 is required, and this requires macOS High Sierra or above. Therefore you must be running macOS High Sierra (10.13) or above to build the Machine Learning C++ code. + - If you are using High Sierra, you must install Xcode 10.1.x - If you are using Mojave or Catalina, you must install Xcode 11.2.x -Older versions of Xcode are installed by dragging the app from the `.dmg` file to the `/Applications` directory on your Mac (or if you got it from the App Store it will already be in the `/Applications` directory). More modern versions of Xcode are distributed as a `.xip` file; simply double click the `.xip` file to expand it, then drag `Xcode.app` to your `/Applications` directory. +Xcode is distributed as a `.xip` file; simply double click the `.xip` file to expand it, then drag `Xcode.app` to your `/Applications` directory. (Older versions of Xcode can be downloaded from [here](https://developer.apple.com/download/more/), provided you are signed in with your Apple ID.) There are no command line tools out-of-the-box, so you'll need to install them following installation of Xcode. You can do this by running: @@ -96,8 +95,8 @@ to: To complete the build, type: ``` -./b2 -j8 --layout=versioned --disable-icu cxxflags="-std=c++14 -stdlib=libc++" linkflags="-std=c++14 -stdlib=libc++ -Wl,-headerpad_max_install_names" optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC -sudo ./b2 install --layout=versioned --disable-icu cxxflags="-std=c++14 -stdlib=libc++" linkflags="-std=c++14 -stdlib=libc++ -Wl,-headerpad_max_install_names" optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC +./b2 -j8 --layout=versioned --disable-icu cxxflags="-std=c++17 -stdlib=libc++ -msse4.2" linkflags="-std=c++17 -stdlib=libc++ -Wl,-headerpad_max_install_names" optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC +sudo ./b2 install --layout=versioned --disable-icu cxxflags="-std=c++17 -stdlib=libc++ -msse4.2" linkflags="-std=c++17 -stdlib=libc++ -Wl,-headerpad_max_install_names" optimization=speed inlining=full define=BOOST_MATH_NO_LONG_DOUBLE_MATH_FUNCTIONS define=BOOST_LOG_WITHOUT_DEBUG_OUTPUT define=BOOST_LOG_WITHOUT_EVENT_LOG define=BOOST_LOG_WITHOUT_SYSLOG define=BOOST_LOG_WITHOUT_IPC ``` to install the Boost headers and libraries. diff --git a/build-setup/macos_cross_compiled.md b/build-setup/macos_cross_compiled.md index 00ee151148..d067bcc64d 100644 --- a/build-setup/macos_cross_compiled.md +++ b/build-setup/macos_cross_compiled.md @@ -21,31 +21,31 @@ export CPP_CROSS_COMPILE=macosx Start by configuring a native macOS build server as described in [macos.md](macos.md). -The remainder of these instructions assume the macOS build server you have configured is for macOS 10.11 (El Capitan). This is what builds for distribution are currently built on. +The remainder of these instructions assume the macOS build server you have configured is for macOS 10.13 (High Sierra). This is what builds for distribution are currently built on. On the fully configured macOS build server, run the following commands: ``` cd /usr -tar jcvf ~/usr-x86_64-apple-macosx10.11.tar.bz2 include lib local +tar jcvf ~/usr-x86_64-apple-macosx10.13.tar.bz2 include lib local cd /Applications/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr -tar jcvf ~/xcode-x86_64-apple-macosx10.11.tar.bz2 include +tar jcvf ~/xcode-x86_64-apple-macosx10.13.tar.bz2 include ``` -These instructions also assume the host platform is Ubuntu 16.04. It makes life much easier if the host platform is a version of Ubuntu that's new enough to run the official binary distribution of clang/LLVM (otherwise it would be necessary to build clang/LLVM from source). +These instructions also assume the host platform is Ubuntu 18.04. It makes life much easier if the host platform is a version of Ubuntu that's new enough to run the official binary distribution of clang/LLVM (otherwise it would be necessary to build clang/LLVM from source). -Transfer the two archives created in your home directory on the macOS build server, `usr-x86_64-apple-macosx10.11.tar.bz2` and `xcode-x86_64-apple-macosx10.11.tar.bz2`, to your home directory on the cross compilation host build server. +Transfer the two archives created in your home directory on the macOS build server, `usr-x86_64-apple-macosx10.13.tar.bz2` and `xcode-x86_64-apple-macosx10.13.tar.bz2`, to your home directory on the cross compilation host build server. ### OS Packages -You need clang 3.9, plus a number of other build tools. They can be installed on modern Ubuntu as follows: +You need clang 6.0, plus a number of other build tools. They can be installed on modern Ubuntu as follows: ``` sudo apt-get install automake autogen build-essential bzip2 git gobjc libtool software-properties-common unzip wget wget -O - http://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add - -sudo apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial main" -sudo apt-get install clang-3.9 clang-3.9-doc libclang1-3.9 libllvm3.9 lldb-3.9 llvm-3.9 llvm-3.9-doc llvm-3.9-runtime +sudo apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" +sudo apt-get install clang-6.0 clang-6.0-doc libclang1-6.0 libllvm6.0 lldb-6.0 llvm-6.0 llvm-6.0-doc llvm-6.0-runtime ``` (It is strongly recommended NOT to attempt to create a cross compile environment on an old version of Linux, because you will have to build clang from source and you need a modern C++ compiler to build clang. So you would probably end up first building a modern version of gcc using the system default gcc, then building clang using the modern gcc.) @@ -55,10 +55,10 @@ sudo apt-get install clang-3.9 clang-3.9-doc libclang1-3.9 libllvm3.9 lldb-3.9 l Add the dependencies that you copied from the fully configured macOS build server in the "Initial Preparation" step. ``` -sudo mkdir -p /usr/local/sysroot-x86_64-apple-macosx10.11/usr -cd /usr/local/sysroot-x86_64-apple-macosx10.11/usr -sudo tar jxvf ~/usr-x86_64-apple-macosx10.11.tar.bz2 -sudo tar jxvf ~/xcode-x86_64-apple-macosx10.11.tar.bz2 +sudo mkdir -p /usr/local/sysroot-x86_64-apple-macosx10.13/usr +cd /usr/local/sysroot-x86_64-apple-macosx10.13/usr +sudo tar jxvf ~/usr-x86_64-apple-macosx10.13.tar.bz2 +sudo tar jxvf ~/xcode-x86_64-apple-macosx10.13.tar.bz2 ``` ### cctools-port @@ -68,12 +68,15 @@ You need to obtain Linux ports of several Apple development tools. The easiest ``` git clone https://github.com/tpoechtrager/cctools-port.git cd cctools-port/cctools -git checkout 895-ld64-274.2 +git checkout 921-ld64-409.12 +sed -i -e 's/autoconf/autoreconf -fi/' autogen.sh +export CC=clang-6.0 +export CXX=clang++-6.0 ./autogen.sh -./configure --target=x86_64-apple-macosx10.11 --with-llvm-config=/usr/bin/llvm-config-3.9 +./configure --target=x86_64-apple-macosx10.13 --with-llvm-config=/usr/bin/llvm-config-6.0 make sudo make install ``` -The "895-ld64-274.2" branch in the [cctools-port repository](https://github.com/tpoechtrager/cctools-port) corresponds to the tools for macOS 10.11 El Capitan and clang 3.9. (A different branch would be required for newer versions of the OS/compiler.) +The "921-ld64-409.12" branch in the [cctools-port repository](https://github.com/tpoechtrager/cctools-port) corresponds to the tools for macOS 10.13 High Sierra and clang 6.0. (A different branch would be required for newer versions of the OS/compiler.) diff --git a/dev-tools/docker/build_macosx_build_image.sh b/dev-tools/docker/build_macosx_build_image.sh index 7c1866620d..89fddee89f 100755 --- a/dev-tools/docker/build_macosx_build_image.sh +++ b/dev-tools/docker/build_macosx_build_image.sh @@ -17,7 +17,7 @@ HOST=push.docker.elastic.co ACCOUNT=ml-dev REPOSITORY=ml-macosx-build -VERSION=7 +VERSION=8 set -e diff --git a/dev-tools/docker/macosx_builder/Dockerfile b/dev-tools/docker/macosx_builder/Dockerfile index 36ab8dcf92..b991e8bab1 100644 --- a/dev-tools/docker/macosx_builder/Dockerfile +++ b/dev-tools/docker/macosx_builder/Dockerfile @@ -5,7 +5,7 @@ # # Increment the version here when a new tools/3rd party components image is built -FROM docker.elastic.co/ml-dev/ml-macosx-build:7 +FROM docker.elastic.co/ml-dev/ml-macosx-build:8 MAINTAINER David Roberts diff --git a/dev-tools/docker/macosx_image/Dockerfile b/dev-tools/docker/macosx_image/Dockerfile index 3d04f01e1e..c57c45f388 100644 --- a/dev-tools/docker/macosx_image/Dockerfile +++ b/dev-tools/docker/macosx_image/Dockerfile @@ -4,7 +4,7 @@ # you may not use this file except in compliance with the Elastic License. # -FROM ubuntu:16.04 +FROM ubuntu:18.04 # This is basically automating the setup instructions in build-setup/macos_cross_compiled.md @@ -13,33 +13,31 @@ MAINTAINER David Roberts # Make sure apt-get is up to date and required packages are installed RUN \ apt-get update && \ - apt-get install --no-install-recommends -y apt-utils automake autogen build-essential bzip2 git gobjc libtool software-properties-common unzip wget zip + apt-get install --no-install-recommends -y apt-utils automake autogen build-essential bzip2 git gobjc gpg-agent libtool software-properties-common unzip wget zip # Install clang -# Note: apt-key cannot update /etc/apt/trusted.gpg on the Ubuntu 16.04 Docker image, so it's updated in /tmp RUN \ - cp /etc/apt/trusted.gpg /tmp && \ - wget --quiet -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key --keyring /tmp/trusted.gpg add - && \ - mv /tmp/trusted.gpg /etc/apt && \ - rm /tmp/trusted.gpg~ && \ - apt-add-repository "deb http://apt.llvm.org/xenial/ llvm-toolchain-xenial main" && \ - apt-get install --no-install-recommends -y clang-3.9 libclang1-3.9 libllvm3.9 llvm-3.9 llvm-3.9-runtime + wget --quiet -O - http://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ + apt-add-repository "deb http://apt.llvm.org/bionic/ llvm-toolchain-bionic main" && \ + apt-get install --no-install-recommends -y clang-6.0 libclang1-6.0 libllvm6.0 llvm-6.0 llvm-6.0-runtime # Add build dependencies transferred from native Mac build server RUN \ - mkdir -p /usr/local/sysroot-x86_64-apple-macosx10.11/usr && \ - cd /usr/local/sysroot-x86_64-apple-macosx10.11/usr && \ - wget --quiet -O - https://s3-eu-west-1.amazonaws.com/prelert-artifacts/dependencies/usr-x86_64-apple-macosx10.11-3.tar.bz2 | tar jxf - && \ - wget --quiet -O - https://s3-eu-west-1.amazonaws.com/prelert-artifacts/dependencies/xcode-x86_64-apple-macosx10.11-1.tar.bz2 | tar jxf - + mkdir -p /usr/local/sysroot-x86_64-apple-macosx10.13/usr && \ + cd /usr/local/sysroot-x86_64-apple-macosx10.13/usr && \ + wget --quiet -O - https://s3-eu-west-1.amazonaws.com/prelert-artifacts/dependencies/usr-x86_64-apple-macosx10.13-1.tar.bz2 | tar jxf - && \ + wget --quiet -O - https://s3-eu-west-1.amazonaws.com/prelert-artifacts/dependencies/xcode-x86_64-apple-macosx10.13-1.tar.bz2 | tar jxf - # Build cctools-port RUN \ git clone https://github.com/tpoechtrager/cctools-port.git && \ cd cctools-port/cctools && \ - git checkout 895-ld64-274.2 && \ + git checkout 921-ld64-409.12 && \ sed -i -e 's/autoconf/autoreconf -fi/' autogen.sh && \ + export CC=clang-6.0 && \ + export CXX=clang++-6.0 && \ ./autogen.sh && \ - ./configure --target=x86_64-apple-macosx10.11 --with-llvm-config=/usr/bin/llvm-config-3.9 && \ + ./configure --target=x86_64-apple-macosx10.13 --with-llvm-config=/usr/bin/llvm-config-6.0 && \ make -j`nproc` && \ make install && \ cd ../.. && \ diff --git a/dev-tools/strip_binaries.sh b/dev-tools/strip_binaries.sh index 0831dd9c8b..41ca3455b0 100755 --- a/dev-tools/strip_binaries.sh +++ b/dev-tools/strip_binaries.sh @@ -93,17 +93,17 @@ case `uname` in for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'` do echo "Stripping $PROGRAM" - llvm-dsymutil-3.9 $PROGRAM - /usr/local/bin/x86_64-apple-macosx10.11-strip -u -r $PROGRAM + llvm-dsymutil-6.0 $PROGRAM + /usr/local/bin/x86_64-apple-macosx10.13-strip -u -r $PROGRAM done for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'` do echo "Stripping $LIBRARY" case $LIBRARY in *Ml*) - llvm-dsymutil-3.9 $LIBRARY + llvm-dsymutil-6.0 $LIBRARY esac - /usr/local/bin/x86_64-apple-macosx10.11-strip -x $LIBRARY + /usr/local/bin/x86_64-apple-macosx10.13-strip -x $LIBRARY done else echo "Cannot cross compile to $CPP_CROSS_COMPILE" diff --git a/docs/CHANGELOG.asciidoc b/docs/CHANGELOG.asciidoc index ded182efd7..98864fd0ec 100644 --- a/docs/CHANGELOG.asciidoc +++ b/docs/CHANGELOG.asciidoc @@ -28,6 +28,13 @@ //=== Regressions +== {es} version 8.0.0 + +=== Enhancements + +* The macOS build platform for the {ml} C++ code is now High Sierra running Xcode 10.1, + or Ubuntu 18.04 running clang 6.0 for cross compilation. (See {ml-pull}867[#867].) + == {es} version 7.6.0 === Enhancements diff --git a/mk/linux_crosscompile_macosx.mk b/mk/linux_crosscompile_macosx.mk index 0233eac532..7e899f0924 100644 --- a/mk/linux_crosscompile_macosx.mk +++ b/mk/linux_crosscompile_macosx.mk @@ -10,9 +10,9 @@ CPP_PLATFORM_HOME=$(CPP_DISTRIBUTION_HOME)/platform/darwin-x86_64 ML_APP_NAME=controller APP_CONTENTS=$(ML_APP_NAME).app/Contents -CROSS_TARGET_PLATFORM=x86_64-apple-macosx10.11 +CROSS_TARGET_PLATFORM=x86_64-apple-macosx10.13 SYSROOT=/usr/local/sysroot-$(CROSS_TARGET_PLATFORM) -CLANGVER=3.9 +CLANGVER=6.0 # We use a natively compiled Boost even when cross compiling our own source # code, and Apple's clang versions are different to LLVM's clang versions, so # the natively built library file names will contain different versions. Then @@ -20,10 +20,10 @@ CLANGVER=3.9 # 3.8 -> 70 (Xcode 7.2) # 3.9 -> 80 (Xcode 8.2) # 6.0 -> 100 (Xcode 10.1) -BOOSTCLANGVER=80 +BOOSTCLANGVER=100 CROSS_FLAGS=--sysroot=$(SYSROOT) -B /usr/local/bin -target $(CROSS_TARGET_PLATFORM) CC=clang-$(CLANGVER) $(CROSS_FLAGS) -CXX=clang++-$(CLANGVER) $(CROSS_FLAGS) -std=c++14 -stdlib=libc++ +CXX=clang++-$(CLANGVER) $(CROSS_FLAGS) -std=c++17 -stdlib=libc++ ifndef ML_DEBUG OPTCFLAGS=-O3 @@ -37,8 +37,8 @@ endif endif # Start by enabling all warnings and then disable the really pointless/annoying ones -CFLAGS=-g $(OPTCFLAGS) -msse4.2 -fstack-protector -Weverything -Werror-switch -Wno-deprecated -Wno-disabled-macro-expansion -Wno-documentation-deprecated-sync -Wno-documentation-unknown-command -Wno-float-equal -Wno-gnu -Wno-missing-prototypes -Wno-padded -Wno-sign-conversion -Wno-unreachable-code -Wno-used-but-marked-unused $(COVERAGE) -CXXFLAGS=$(CFLAGS) -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-exit-time-destructors -Wno-global-constructors -Wno-undefined-reinterpret-cast -Wno-unused-member-function -Wno-weak-vtables +CFLAGS=-g $(OPTCFLAGS) -msse4.2 -fstack-protector -Weverything -Werror-switch -Wno-deprecated -Wno-disabled-macro-expansion -Wno-documentation-deprecated-sync -Wno-documentation-unknown-command -Wno-float-equal -Wno-missing-prototypes -Wno-padded -Wno-sign-conversion -Wno-unreachable-code -Wno-used-but-marked-unused $(COVERAGE) +CXXFLAGS=$(CFLAGS) -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-exit-time-destructors -Wno-global-constructors -Wno-unused-member-function -Wno-weak-vtables CPPFLAGS=-isystem $(SYSROOT)/usr/include/c++/v1 -isystem $(CPP_SRC_HOME)/3rd_party/include -isystem $(SYSROOT)/usr/local/include -D$(OS) $(OPTCPPFLAGS) ANALYZEFLAGS=--analyze CDEPFLAGS=-MM diff --git a/mk/macosx.mk b/mk/macosx.mk index 7f95deaed6..313b3924e1 100644 --- a/mk/macosx.mk +++ b/mk/macosx.mk @@ -11,7 +11,7 @@ ML_APP_NAME=controller APP_CONTENTS=$(ML_APP_NAME).app/Contents CC=clang -CXX=clang++ -std=c++14 -stdlib=libc++ +CXX=clang++ -std=c++17 -stdlib=libc++ ifndef ML_DEBUG OPTCFLAGS=-O3 @@ -26,8 +26,8 @@ endif SDK_PATH:=$(shell xcrun --show-sdk-path) # Start by enabling all warnings and then disable the really pointless/annoying ones -CFLAGS=-g $(OPTCFLAGS) -msse4.2 -fstack-protector -Weverything -Werror-switch -Wno-deprecated -Wno-disabled-macro-expansion -Wno-documentation-deprecated-sync -Wno-documentation-unknown-command -Wno-float-equal -Wno-gnu -Wno-missing-prototypes -Wno-padded -Wno-sign-conversion -Wno-unreachable-code -Wno-used-but-marked-unused $(COVERAGE) -CXXFLAGS=$(CFLAGS) -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-exit-time-destructors -Wno-global-constructors -Wno-undefined-reinterpret-cast -Wno-unused-member-function -Wno-weak-vtables +CFLAGS=-g $(OPTCFLAGS) -msse4.2 -fstack-protector -Weverything -Werror-switch -Wno-deprecated -Wno-disabled-macro-expansion -Wno-documentation-deprecated-sync -Wno-documentation-unknown-command -Wno-float-equal -Wno-missing-prototypes -Wno-padded -Wno-sign-conversion -Wno-unreachable-code -Wno-used-but-marked-unused $(COVERAGE) +CXXFLAGS=$(CFLAGS) -Wno-c++98-compat -Wno-c++98-compat-pedantic -Wno-exit-time-destructors -Wno-global-constructors -Wno-unused-member-function -Wno-weak-vtables CPPFLAGS=-isystem $(CPP_SRC_HOME)/3rd_party/include -isystem /usr/local/include -D$(OS) $(OPTCPPFLAGS) ANALYZEFLAGS=--analyze CDEPFLAGS=-MM