[6.8][ML] Make the ML native code more ".app"-like on macOS

3rd_party/3rd_party.sh

@@ -52,67 +52,44 @@ case `uname` in
 
     Linux)
         if [ -z "$CPP_CROSS_COMPILE" ] ; then
-            ldd --version 2>&1 | grep musl > /dev/null
-            if [ $? -ne 0 ] ; then
-                APR_LOCATION=/usr/local/gcc62/lib
-                APR_EXTENSION=1.so.0
-                BOOST_LOCATION=/usr/local/gcc62/lib
-                BOOST_COMPILER=gcc
-                BOOST_EXTENSION=mt-1_65_1.so.1.65.1
-                BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
-                LOG4CXX_LOCATION=/usr/local/gcc62/lib
-                LOG4CXX_EXTENSION=.so.10
-                XML_LOCATION=/usr/local/gcc62/lib
-                XML_EXTENSION=.so.2
-                # Ship the version of expat that came with the apr-util library
-                EXPAT_LOCATION=/usr/local/gcc62/lib
-                EXPAT_EXTENSION=.so.0
-                GCC_RT_LOCATION=/usr/local/gcc62/lib64
-                GCC_RT_EXTENSION=.so.1
-                STL_LOCATION=/usr/local/gcc62/lib64
-                STL_PREFIX=libstdc++
-                STL_EXTENSION=.so.6
-                ATOMIC_LOCATION=
-                ZLIB_LOCATION=
-            else
-                APR_LOCATION=/usr/local/lib
-                APR_EXTENSION=1.so.0
-                BOOST_LOCATION=/usr/local/lib
-                BOOST_COMPILER=gcc
-                BOOST_EXTENSION=mt-1_65_1.so.1.65.1
-                BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
-                LOG4CXX_LOCATION=/usr/local/lib
-                LOG4CXX_EXTENSION=.so.10
-                XML_LOCATION=/usr/local/lib
-                XML_EXTENSION=.so.2
-                # Ship the version of expat that came with the apr-util library
-                EXPAT_LOCATION=/usr/local/lib
-                EXPAT_EXTENSION=.so.0
-                GCC_RT_LOCATION=
-                STL_LOCATION=
-                ATOMIC_LOCATION=
-                ZLIB_LOCATION=
-            fi
+            APR_LOCATION=/usr/local/gcc62/lib
+            APR_EXTENSION=1.so.0
+            BOOST_LOCATION=/usr/local/gcc62/lib
+            BOOST_COMPILER=gcc
+            BOOST_EXTENSION=mt-1_65_1.so.1.65.1
+            BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
+            LOG4CXX_LOCATION=/usr/local/gcc62/lib
+            LOG4CXX_EXTENSION=.so.10
+            XML_LOCATION=/usr/local/gcc62/lib
+            XML_EXTENSION=.so.2
+            # Ship the version of expat that came with the apr-util library
+            EXPAT_LOCATION=/usr/local/gcc62/lib
+            EXPAT_EXTENSION=.so.0
+            GCC_RT_LOCATION=/usr/local/gcc62/lib64
+            GCC_RT_EXTENSION=.so.1
+            STL_LOCATION=/usr/local/gcc62/lib64
+            STL_PREFIX=libstdc++
+            STL_EXTENSION=.so.6
+            ATOMIC_LOCATION=
+            ZLIB_LOCATION=
+        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
+            SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.10
+            APR_LOCATION=
+            BOOST_LOCATION=$SYSROOT/usr/local/lib
+            BOOST_COMPILER=clang
+            BOOST_EXTENSION=mt-1_65_1.dylib
+            BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
+            LOG4CXX_LOCATION=$SYSROOT/usr/local/lib
+            LOG4CXX_EXTENSION=.10.dylib
+            XML_LOCATION=
+            EXPAT_LOCATION=
+            GCC_RT_LOCATION=
+            STL_LOCATION=
+            ATOMIC_LOCATION=
+            ZLIB_LOCATION=
         else
-            if [ "$CPP_CROSS_COMPILE" = macosx ] ; then
-                SYSROOT=/usr/local/sysroot-x86_64-apple-macosx10.10
-                APR_LOCATION=
-                BOOST_LOCATION=$SYSROOT/usr/local/lib
-                BOOST_COMPILER=clang
-                BOOST_EXTENSION=mt-1_65_1.dylib
-                BOOST_LIBRARIES='date_time filesystem iostreams program_options regex system thread'
-                LOG4CXX_LOCATION=$SYSROOT/usr/local/lib
-                LOG4CXX_EXTENSION=.10.dylib
-                XML_LOCATION=
-                EXPAT_LOCATION=
-                GCC_RT_LOCATION=
-                STL_LOCATION=
-                ATOMIC_LOCATION=
-                ZLIB_LOCATION=
-            else
-                echo "Cannot cross compile to $CPP_CROSS_COMPILE"
-                exit 3
-            fi
+            echo "Cannot cross compile to $CPP_CROSS_COMPILE"
+            exit 3
         fi
         ;;
 
@@ -293,27 +270,5 @@ case `uname` in
         fi
         ;;
 
-    SunOS)
-        if [ -n "$INSTALL_DIR" ] ; then
-            cd "$INSTALL_DIR"
-            for FILE in `find . -type f | egrep -v '^core|-debug$|libMl'`
-            do
-                # Replace RPATH for 3rd party libraries that already have one
-                elfedit -r -e 'dyn:runpath' $FILE >/dev/null 2>&1 && chmod u+wx $FILE && elfedit -e 'dyn:runpath $ORIGIN/.' $FILE
-                if [ $? -eq 0 ] ; then
-                    echo "Set RPATH in $FILE"
-                else
-                    # Set RPATH for 3rd party libraries that reference other libraries we ship
-                    ldd $FILE | grep /usr/local/lib >/dev/null 2>&1 && chmod u+wx $FILE && elfedit -e 'dyn:runpath $ORIGIN/.' $FILE
-                    if [ $? -eq 0 ] ; then
-                        echo "Set RPATH in $FILE"
-                    else
-                        echo "Did not set RPATH in $FILE"
-                    fi
-                fi
-            done
-        fi
-        ;;
-
 esac
 

bin/autoconfig/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/autoconfig$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API) $(LIB_ML_CONFIG)
 

bin/autodetect/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/autodetect$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

bin/categorize/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/categorize$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

bin/controller/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/controller$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE)
 
@@ -16,14 +16,19 @@ USE_BOOST_PROGRAMOPTIONS_LIBS=1
 
 LIBS=$(ML_LIBS)
 
-all: build
+all:: build
 
 SRCS= \
     Main.cc \
     CBlockingCallCancellerThread.cc \
     CCmdLineParser.cc \
     CCommandProcessor.cc \
 
+ifneq ($(PLIST_FILE),)
+all:: $(CPP_SRC_HOME)/gradle.properties $(CPP_SRC_HOME)/mk/make_info_plist.sh
+	$(MKDIR) $(INSTALL_DIR)
+	$(CPP_SRC_HOME)/mk/make_info_plist.sh "$(ML_APP_NAME)" true > $(INSTALL_DIR)/../$(notdir $(PLIST_FILE))
+endif
 
 include $(CPP_SRC_HOME)/mk/stdapp.mk
 

bin/normalize/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=$(OBJS_DIR)/normalize$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/bin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 ML_LIBS=$(LIB_ML_CORE) $(LIB_ML_MATHS) $(LIB_ML_MODEL) $(LIB_ML_API)
 

dev-tools/strip_binaries.sh

@@ -5,24 +5,47 @@
 # you may not use this file except in compliance with the Elastic License.
 #
 
-# Strips Ml native code binaries to make them smaller before distribution.
+# Strips ML native code binaries to make them smaller before distribution.
+
+ML_APP_NAME=controller
+case `uname` in
+
+    Darwin)
+        EXE_DIR="$ML_APP_NAME.app/Contents/MacOS"
+        DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib"
+        ;;
+
+    Linux)
+        if [ -z "$CPP_CROSS_COMPILE" ] ; then
+            EXE_DIR=bin
+            DYNAMIC_LIB_DIR=lib
+        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
+            EXE_DIR="$ML_APP_NAME.app/Contents/MacOS"
+            DYNAMIC_LIB_DIR="$ML_APP_NAME.app/Contents/lib"
+        else
+            echo "Cannot cross compile to $CPP_CROSS_COMPILE"
+            exit 1
+        fi
+        ;;
+
+esac
 
 # Ensure $CPP_PLATFORM_HOME is set
 if [ -z "$CPP_PLATFORM_HOME" ] ; then
     echo '$CPP_PLATFORM_HOME is not set'
-    exit 1
+    exit 2
 fi
 
-# Ensure the bin folder has been created.
-if [ ! -d "$CPP_PLATFORM_HOME/bin" ] ; then
-    echo '$CPP_PLATFORM_HOME/bin does not exist'
-    exit 2
+# Ensure the executable programs folder has been created.
+if [ ! -d "$CPP_PLATFORM_HOME/$EXE_DIR" ] ; then
+    echo "$CPP_PLATFORM_HOME/$EXE_DIR does not exist"
+    exit 3
 fi
 
 # Ensure the lib folder has been created.
-if [ ! -d "$CPP_PLATFORM_HOME/lib" ] ; then
-    echo '$CPP_PLATFORM_HOME/lib does not exist'
-    exit 3
+if [ ! -d "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR" ] ; then
+    echo "$CPP_PLATFORM_HOME/$DYNAMIC_LIB_DIR does not exist"
+    exit 4
 fi
 
 cd "$CPP_PLATFORM_HOME"
@@ -32,13 +55,13 @@ cd "$CPP_PLATFORM_HOME"
 case `uname` in
 
     Darwin)
-        for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'`
+        for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'`
         do
             echo "Stripping $PROGRAM"
             dsymutil $PROGRAM
             strip -u -r $PROGRAM
         done
-        for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'`
+        for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'`
         do
             echo "Stripping $LIBRARY"
             case $LIBRARY in
@@ -51,42 +74,40 @@ case `uname` in
 
     Linux)
         if [ -z "$CPP_CROSS_COMPILE" ] ; then
-            for PROGRAM in `ls -1 bin/* | egrep -v 'bin/core|-debug$'`
+            for PROGRAM in `ls -1 "$EXE_DIR"/* | egrep -v "$EXE_DIR"'/core|-debug$'`
             do
                 echo "Stripping $PROGRAM"
                 objcopy --only-keep-debug "$PROGRAM" "$PROGRAM-debug"
                 strip --strip-all $PROGRAM
                 objcopy --add-gnu-debuglink="$PROGRAM-debug" "$PROGRAM"
                 chmod -x "$PROGRAM-debug"
             done
-            for LIBRARY in `ls -1 lib/* | egrep -v 'lib/core|-debug$'`
+            for LIBRARY in `ls -1 "$DYNAMIC_LIB_DIR"/* | egrep -v 'lib/core|-debug$'`
             do
                 echo "Stripping $LIBRARY"
                 objcopy --only-keep-debug "$LIBRARY" "$LIBRARY-debug"
                 strip --strip-unneeded $LIBRARY
                 objcopy --add-gnu-debuglink="$LIBRARY-debug" "$LIBRARY"
             done
+        elif [ "$CPP_CROSS_COMPILE" = macosx ] ; then
+            for PROGRAM in `ls -1d "$EXE_DIR"/* | grep -v '\.dSYM$'`
+            do
+                echo "Stripping $PROGRAM"
+                llvm-dsymutil-3.8 $PROGRAM
+                /usr/local/bin/x86_64-apple-macosx10.10-strip -u -r $PROGRAM
+            done
+            for LIBRARY in `ls -1d "$DYNAMIC_LIB_DIR"/* | grep -v '\.dSYM$'`
+            do
+                echo "Stripping $LIBRARY"
+                case $LIBRARY in
+                    *Ml*)
+                        llvm-dsymutil-3.8 $LIBRARY
+                esac
+                /usr/local/bin/x86_64-apple-macosx10.10-strip -x $LIBRARY
+            done
         else
-            if [ "$CPP_CROSS_COMPILE" = macosx ] ; then
-                for PROGRAM in `ls -1d bin/* | grep -v '\.dSYM$'`
-                do
-                    echo "Stripping $PROGRAM"
-                    llvm-dsymutil-3.8 $PROGRAM
-                    /usr/local/bin/x86_64-apple-macosx10.10-strip -u -r $PROGRAM
-                done
-                for LIBRARY in `ls -1d lib/* | grep -v '\.dSYM$'`
-                do
-                    echo "Stripping $LIBRARY"
-                    case $LIBRARY in
-                        *Ml*)
-                            llvm-dsymutil-3.8 $LIBRARY
-                    esac
-                    /usr/local/bin/x86_64-apple-macosx10.10-strip -x $LIBRARY
-                done
-            else
-                echo "Cannot cross compile to $CPP_CROSS_COMPILE"
-                exit 4
-            fi
+            echo "Cannot cross compile to $CPP_CROSS_COMPILE"
+            exit 5
         fi
         ;;
 

devbin/analyze_test/Makefile

@@ -7,7 +7,7 @@ include $(CPP_SRC_HOME)/mk/defines.mk
 
 TARGET=analyze_test$(EXE_EXT)
 
-INSTALL_DIR=$(CPP_PLATFORM_HOME)/devbin
+INSTALL_DIR=$(CPP_PLATFORM_HOME)/$(EXE_DIR)
 
 all: build
 

docs/CHANGELOG.asciidoc

@@ -28,6 +28,13 @@
 
 //=== Regressions
 
+== {es} version 6.8.5
+
+=== Enhancements
+
+* The {ml} native processes are now arranged in a .app directory structure on
+  macOS, to allow for notarization on macOS Catalina. (See {ml-pull}593[#593].)
+
 == {es} version 6.8.4
 
 === Bug Fixes

include/core/CLogger.h

@@ -116,12 +116,11 @@ class CORE_EXPORT CLogger : private CNonCopyable {
     CLogger();
     ~CLogger();
 
-    //! Replace Ml specific patterns in log4cxx properties.  In
-    //! addition to the patterns usually supported by log4cxx, Ml will
+    //! Replace ML specific patterns in log4cxx properties.  In
+    //! addition to the patterns usually supported by log4cxx, ML will
     //! substitute:
-    //! 1) %D with the path to the Ml base log directory
-    //! 2) %N with the program's name
-    //! 3) %P with the program's process ID
+    //! 1) %N with the program's name
+    //! 2) %P with the program's process ID
     void massageProperties(log4cxx::helpers::Properties& props) const;
 
     using TLogCharLogStrMap = std::map<log4cxx::logchar, log4cxx::LogString>;

include/core/CResourceLocator.h

@@ -36,9 +36,6 @@ class CORE_EXPORT CResourceLocator : private CNonInstantiatable {
     //! details and dictionary words.
     static std::string resourceDir();
 
-    //! Get the directory that stores log files.
-    static std::string logDir();
-
     //! Get the root directory for the C++ section of the source tree.
     //! (Obviously this should only be used in test code!)
     static std::string cppRootDir();