From 3892e1b48cde7a078f5c7ea7d5dfd73c27439985 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 30 Oct 2024 12:38:26 -0600 Subject: [PATCH 01/12] enable m1 tests and fixing native lookup --- .../Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs | 2 +- test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs index 7cb1a73ef7..86ab438b0c 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs @@ -19,7 +19,7 @@ public static bool NativeLibraryExists(string name) string extension = default; string prefix = "lib"; - if (Environment.OSVersion.Platform == PlatformID.MacOSX) + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) extension = ".dylib"; else if (Environment.OSVersion.Platform == PlatformID.Unix) extension = ".so"; diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs index fb717075c8..d56ab680e8 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs @@ -69,6 +69,10 @@ private bool TryLocateNativeAssetFromDeps(string name, out string appLocalNative List allRIDs = new List(); allRIDs.Add(currentRID); + if (currentRID.Contains("arm64")) + { + allRIDs.Add("osx-arm64"); + } if (!AddFallbacks(allRIDs, currentRID, defaultContext.RuntimeGraph)) { #pragma warning disable MSML_ParameterLocalVarName // Parameter or local variable name not standard From 3b00a139224f3c95ef11a9a898c45f52d66bbd81 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 30 Oct 2024 14:46:57 -0600 Subject: [PATCH 02/12] Mac testing --- src/Native/SymSgdNative/CMakeLists.txt | 2 +- .../Utility/NativeLibrary.cs | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 01652d2aab..5507b5c17e 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -11,7 +11,7 @@ if(APPLE) # these commands are added to sepcifically handle the Apple Clang scenario # If the LLVM version of clang is used for Apple builds, this can be removed # and the else condition can be used instead. - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp -liomp5") SET(OPENMP_LIBRARY "omp") include_directories("/usr/local/opt/libomp/include") link_directories("/usr/local/opt/libomp/lib") diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs index 86ab438b0c..92eb50743f 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs @@ -43,7 +43,17 @@ public static bool NativeLibraryExists(string name) } catch { - return false; + // If that didn't load, dispose of the first attempt and try appending lib_ in front + try + { + nativeLibrary?.Dispose(); + nativeLibrary = new NativeLibrary(prefix + "_" + name + extension); + return true; + } + catch + { + return false; + } } } finally From 10aa46d1a16df7c28b95d15c60dc1a00042e4928 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 30 Oct 2024 15:06:04 -0600 Subject: [PATCH 03/12] mac test --- src/Native/SymSgdNative/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 5507b5c17e..4338afb69d 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -11,7 +11,7 @@ if(APPLE) # these commands are added to sepcifically handle the Apple Clang scenario # If the LLVM version of clang is used for Apple builds, this can be removed # and the else condition can be used instead. - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp -liomp5") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp -lomp") SET(OPENMP_LIBRARY "omp") include_directories("/usr/local/opt/libomp/include") link_directories("/usr/local/opt/libomp/lib") From 1452b062e2e31ffe1a0fa793bb43f377f463e7ed Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 30 Oct 2024 15:18:23 -0600 Subject: [PATCH 04/12] macos test --- src/Native/SymSgdNative/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 4338afb69d..004ba7ab68 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -5,19 +5,19 @@ set(SOURCES SymSgdNative.cpp ) -if(APPLE) +#if(APPLE) # CMake has support for OpenMP, however, Apple has a version of Clang # that does not support openMP out of the box. Therefore # these commands are added to sepcifically handle the Apple Clang scenario # If the LLVM version of clang is used for Apple builds, this can be removed # and the else condition can be used instead. - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp -lomp") - SET(OPENMP_LIBRARY "omp") - include_directories("/usr/local/opt/libomp/include") - link_directories("/usr/local/opt/libomp/lib") +# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") +# SET(OPENMP_LIBRARY "omp") +# include_directories("/usr/local/opt/libomp/include") +# link_directories("/usr/local/opt/libomp/lib") - list(APPEND SOURCES ${VERSION_FILE_PATH}) -else() +# list(APPEND SOURCES ${VERSION_FILE_PATH}) +#else() find_package(OpenMP) if (OPENMP_FOUND) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") @@ -31,7 +31,7 @@ else() SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) SET(CMAKE_INSTALL_RPATH "$ORIGIN/") endif() -endif() +#endif() if(NOT ${ARCHITECTURE} MATCHES "arm.*") find_library(MKL_LIBRARY MklImports HINTS ${MKL_LIB_PATH}) From b3f57f1c754a68e6b87351f986392668dc2f13bc Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Thu, 31 Oct 2024 12:46:44 -0600 Subject: [PATCH 05/12] mac testing --- src/Native/SymSgdNative/CMakeLists.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 004ba7ab68..1b8e38b9df 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -5,7 +5,7 @@ set(SOURCES SymSgdNative.cpp ) -#if(APPLE) +if(APPLE) # CMake has support for OpenMP, however, Apple has a version of Clang # that does not support openMP out of the box. Therefore # these commands are added to sepcifically handle the Apple Clang scenario @@ -13,10 +13,11 @@ set(SOURCES # and the else condition can be used instead. # SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") # SET(OPENMP_LIBRARY "omp") -# include_directories("/usr/local/opt/libomp/include") -# link_directories("/usr/local/opt/libomp/lib") + include_directories("/usr/local/opt/libomp/include") + link_directories("/usr/local/opt/libomp/lib") -# list(APPEND SOURCES ${VERSION_FILE_PATH}) + list(APPEND SOURCES ${VERSION_FILE_PATH}) +endif() #else() find_package(OpenMP) if (OPENMP_FOUND) From d0b9b0c09e68229a48f72d69a75c77a5604b5763 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 16:30:34 -0600 Subject: [PATCH 06/12] reverting mac changes --- src/Native/SymSgdNative/CMakeLists.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 1b8e38b9df..6049909efb 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -11,14 +11,13 @@ if(APPLE) # these commands are added to sepcifically handle the Apple Clang scenario # If the LLVM version of clang is used for Apple builds, this can be removed # and the else condition can be used instead. -# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") -# SET(OPENMP_LIBRARY "omp") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") + SET(OPENMP_LIBRARY "omp") include_directories("/usr/local/opt/libomp/include") link_directories("/usr/local/opt/libomp/lib") list(APPEND SOURCES ${VERSION_FILE_PATH}) -endif() -#else() +else() find_package(OpenMP) if (OPENMP_FOUND) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") From ac4e06605c6c49be7f0c255ea22c7c36b972d084 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 16:33:12 -0600 Subject: [PATCH 07/12] testing mac temp fix --- eng/helix.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix.proj b/eng/helix.proj index effc0c5e3d..9d4b758af6 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -104,7 +104,7 @@ $(HelixPreCommands);export PATH=$HELIX_CORRELATION_PAYLOAD/$(DotNetCliDestination):$PATH $(HelixPreCommands);set PATH=%HELIX_CORRELATION_PAYLOAD%\$(DotNetCliDestination)%3B%PATH% - $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT + $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE $(HelixPreCommands);sudo apt update;sudo apt-get install libomp-dev libomp5 -y From 791cad01ea257e008380a2783a393aba4e8e1c51 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 16:38:38 -0600 Subject: [PATCH 08/12] fixed cmake --- src/Native/SymSgdNative/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 6049909efb..01652d2aab 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -31,7 +31,7 @@ else() SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) SET(CMAKE_INSTALL_RPATH "$ORIGIN/") endif() -#endif() +endif() if(NOT ${ARCHITECTURE} MATCHES "arm.*") find_library(MKL_LIBRARY MklImports HINTS ${MKL_LIB_PATH}) From d9c112a691279e697a8f27a8d86c15f63523d460 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 17:08:52 -0600 Subject: [PATCH 09/12] print DYLD libraries --- eng/helix.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix.proj b/eng/helix.proj index 9d4b758af6..1fe97b6039 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -104,7 +104,7 @@ $(HelixPreCommands);export PATH=$HELIX_CORRELATION_PAYLOAD/$(DotNetCliDestination):$PATH $(HelixPreCommands);set PATH=%HELIX_CORRELATION_PAYLOAD%\$(DotNetCliDestination)%3B%PATH% - $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE + $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE;export DYLD_PRINT_LIBRARIES=1 $(HelixPreCommands);sudo apt update;sudo apt-get install libomp-dev libomp5 -y From 43a535daf75b56440f87b62f63d5ed4818c8fd7a Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 17:14:10 -0600 Subject: [PATCH 10/12] osx lightgbm test --- eng/helix.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix.proj b/eng/helix.proj index 1fe97b6039..a9f6af73cb 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -128,7 +128,7 @@ Command="install_name_tool -change "/usr/local/opt/libomp/lib/libomp.dylib" "@loader_path/libomp.dylib" $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/libSymSgdNative.dylib" /> + Command="install_name_tool -change "/usr/local/opt/libomp/lib/libomp.dylib" "@rpath/libomp.dylib" $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/runtimes/osx-x64/native/lib_lightgbm.dylib" /> From 0e12aef54fabb3b9947812257cbf359422649946 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 17:59:43 -0600 Subject: [PATCH 11/12] lightgbm dyld log --- eng/helix.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix.proj b/eng/helix.proj index a9f6af73cb..f7fb259148 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -104,7 +104,7 @@ $(HelixPreCommands);export PATH=$HELIX_CORRELATION_PAYLOAD/$(DotNetCliDestination):$PATH $(HelixPreCommands);set PATH=%HELIX_CORRELATION_PAYLOAD%\$(DotNetCliDestination)%3B%PATH% - $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE;export DYLD_PRINT_LIBRARIES=1 + $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE;export DYLD_PRINT_LIBRARIES=1;otool -L $HELIX_WORKITEM_ROOT/runtimes/osx-x64/native/lib_lightgbm.dylib $(HelixPreCommands);sudo apt update;sudo apt-get install libomp-dev libomp5 -y From 96d64565b90965cce21095750c793ac18eb33646 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 11 Nov 2024 12:40:16 -0700 Subject: [PATCH 12/12] excluding osx x64 from some tests --- .../Attributes/NativeDependencyFactAttribute.cs | 3 +++ .../Attributes/NativeDependencyTheoryAttribute.cs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs index 29bd0baa17..f40f8e1be2 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs @@ -19,6 +19,9 @@ public NativeDependencyFactAttribute(string library) : base($"This test requires /// protected override bool IsEnvironmentSupported() { + // Starting to drop native support for X64 OSX since intel no longer makes them. + if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX) && System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture == System.Runtime.InteropServices.Architecture.X64) + return false; return NativeLibrary.NativeLibraryExists(_library); } } diff --git a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs index 444e17b5e6..1f4e1e6b1a 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs @@ -19,6 +19,9 @@ public NativeDependencyTheoryAttribute(string library) : base($"This test requir /// protected override bool IsEnvironmentSupported() { + // Starting to drop native support for X64 OSX since intel no longer makes them. + if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX) && System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture == System.Runtime.InteropServices.Architecture.X64) + return false; return NativeLibrary.NativeLibraryExists(_library); } }