From 3b8f2c9ba3920bf005303a22a8b58317b4d40996 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 30 Oct 2024 12:38:26 -0600 Subject: [PATCH 01/12] enable m1 tests and fixing native lookup --- .../Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs | 2 +- test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs index 7cb1a73ef7..86ab438b0c 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs @@ -19,7 +19,7 @@ public static bool NativeLibraryExists(string name) string extension = default; string prefix = "lib"; - if (Environment.OSVersion.Platform == PlatformID.MacOSX) + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) extension = ".dylib"; else if (Environment.OSVersion.Platform == PlatformID.Unix) extension = ".so"; diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs index fb717075c8..d56ab680e8 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/PathResolver.cs @@ -69,6 +69,10 @@ private bool TryLocateNativeAssetFromDeps(string name, out string appLocalNative List allRIDs = new List(); allRIDs.Add(currentRID); + if (currentRID.Contains("arm64")) + { + allRIDs.Add("osx-arm64"); + } if (!AddFallbacks(allRIDs, currentRID, defaultContext.RuntimeGraph)) { #pragma warning disable MSML_ParameterLocalVarName // Parameter or local variable name not standard From 23b7c5ccf49423d1a3aa90af677d19dc0cd58b59 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 30 Oct 2024 14:46:57 -0600 Subject: [PATCH 02/12] Mac testing --- src/Native/SymSgdNative/CMakeLists.txt | 2 +- .../Utility/NativeLibrary.cs | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 01652d2aab..5507b5c17e 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -11,7 +11,7 @@ if(APPLE) # these commands are added to sepcifically handle the Apple Clang scenario # If the LLVM version of clang is used for Apple builds, this can be removed # and the else condition can be used instead. - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp -liomp5") SET(OPENMP_LIBRARY "omp") include_directories("/usr/local/opt/libomp/include") link_directories("/usr/local/opt/libomp/lib") diff --git a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs index 86ab438b0c..92eb50743f 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Utility/NativeLibrary.cs @@ -43,7 +43,17 @@ public static bool NativeLibraryExists(string name) } catch { - return false; + // If that didn't load, dispose of the first attempt and try appending lib_ in front + try + { + nativeLibrary?.Dispose(); + nativeLibrary = new NativeLibrary(prefix + "_" + name + extension); + return true; + } + catch + { + return false; + } } } finally From 4147307c6cfb8bac2a393347dd62b247f0d85d71 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 30 Oct 2024 15:06:04 -0600 Subject: [PATCH 03/12] mac test --- src/Native/SymSgdNative/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 5507b5c17e..4338afb69d 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -11,7 +11,7 @@ if(APPLE) # these commands are added to sepcifically handle the Apple Clang scenario # If the LLVM version of clang is used for Apple builds, this can be removed # and the else condition can be used instead. - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp -liomp5") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp -lomp") SET(OPENMP_LIBRARY "omp") include_directories("/usr/local/opt/libomp/include") link_directories("/usr/local/opt/libomp/lib") From bd269d260435f7b44e38d2ad4476749626754969 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Wed, 30 Oct 2024 15:18:23 -0600 Subject: [PATCH 04/12] macos test --- src/Native/SymSgdNative/CMakeLists.txt | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 4338afb69d..004ba7ab68 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -5,19 +5,19 @@ set(SOURCES SymSgdNative.cpp ) -if(APPLE) +#if(APPLE) # CMake has support for OpenMP, however, Apple has a version of Clang # that does not support openMP out of the box. Therefore # these commands are added to sepcifically handle the Apple Clang scenario # If the LLVM version of clang is used for Apple builds, this can be removed # and the else condition can be used instead. - SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp -lomp") - SET(OPENMP_LIBRARY "omp") - include_directories("/usr/local/opt/libomp/include") - link_directories("/usr/local/opt/libomp/lib") +# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") +# SET(OPENMP_LIBRARY "omp") +# include_directories("/usr/local/opt/libomp/include") +# link_directories("/usr/local/opt/libomp/lib") - list(APPEND SOURCES ${VERSION_FILE_PATH}) -else() +# list(APPEND SOURCES ${VERSION_FILE_PATH}) +#else() find_package(OpenMP) if (OPENMP_FOUND) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") @@ -31,7 +31,7 @@ else() SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) SET(CMAKE_INSTALL_RPATH "$ORIGIN/") endif() -endif() +#endif() if(NOT ${ARCHITECTURE} MATCHES "arm.*") find_library(MKL_LIBRARY MklImports HINTS ${MKL_LIB_PATH}) From b1c95cbfb4afd744a7b746cbbb60e80d7efc7ed8 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Thu, 31 Oct 2024 12:46:44 -0600 Subject: [PATCH 05/12] mac testing --- src/Native/SymSgdNative/CMakeLists.txt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 004ba7ab68..1b8e38b9df 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -5,7 +5,7 @@ set(SOURCES SymSgdNative.cpp ) -#if(APPLE) +if(APPLE) # CMake has support for OpenMP, however, Apple has a version of Clang # that does not support openMP out of the box. Therefore # these commands are added to sepcifically handle the Apple Clang scenario @@ -13,10 +13,11 @@ set(SOURCES # and the else condition can be used instead. # SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") # SET(OPENMP_LIBRARY "omp") -# include_directories("/usr/local/opt/libomp/include") -# link_directories("/usr/local/opt/libomp/lib") + include_directories("/usr/local/opt/libomp/include") + link_directories("/usr/local/opt/libomp/lib") -# list(APPEND SOURCES ${VERSION_FILE_PATH}) + list(APPEND SOURCES ${VERSION_FILE_PATH}) +endif() #else() find_package(OpenMP) if (OPENMP_FOUND) From 0d28b23d12e48f54e7ec4b861e205a2dfa7b4284 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 16:30:34 -0600 Subject: [PATCH 06/12] reverting mac changes --- src/Native/SymSgdNative/CMakeLists.txt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 1b8e38b9df..6049909efb 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -11,14 +11,13 @@ if(APPLE) # these commands are added to sepcifically handle the Apple Clang scenario # If the LLVM version of clang is used for Apple builds, this can be removed # and the else condition can be used instead. -# SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") -# SET(OPENMP_LIBRARY "omp") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xpreprocessor -fopenmp") + SET(OPENMP_LIBRARY "omp") include_directories("/usr/local/opt/libomp/include") link_directories("/usr/local/opt/libomp/lib") list(APPEND SOURCES ${VERSION_FILE_PATH}) -endif() -#else() +else() find_package(OpenMP) if (OPENMP_FOUND) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}") From 62ba4d9eb61bf206c3901351c3a3812a8bd1ee29 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 16:33:12 -0600 Subject: [PATCH 07/12] testing mac temp fix --- eng/helix.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix.proj b/eng/helix.proj index effc0c5e3d..9d4b758af6 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -104,7 +104,7 @@ $(HelixPreCommands);export PATH=$HELIX_CORRELATION_PAYLOAD/$(DotNetCliDestination):$PATH $(HelixPreCommands);set PATH=%HELIX_CORRELATION_PAYLOAD%\$(DotNetCliDestination)%3B%PATH% - $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT + $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE $(HelixPreCommands);sudo apt update;sudo apt-get install libomp-dev libomp5 -y From f61cc370ec91d3adffff73a1dd939a13afc5127a Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 16:38:38 -0600 Subject: [PATCH 08/12] fixed cmake --- src/Native/SymSgdNative/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Native/SymSgdNative/CMakeLists.txt b/src/Native/SymSgdNative/CMakeLists.txt index 6049909efb..01652d2aab 100644 --- a/src/Native/SymSgdNative/CMakeLists.txt +++ b/src/Native/SymSgdNative/CMakeLists.txt @@ -31,7 +31,7 @@ else() SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE) SET(CMAKE_INSTALL_RPATH "$ORIGIN/") endif() -#endif() +endif() if(NOT ${ARCHITECTURE} MATCHES "arm.*") find_library(MKL_LIBRARY MklImports HINTS ${MKL_LIB_PATH}) From 950555725ad02b2bdf145f0f5f39c0317024db20 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 17:08:52 -0600 Subject: [PATCH 09/12] print DYLD libraries --- eng/helix.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix.proj b/eng/helix.proj index 9d4b758af6..1fe97b6039 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -104,7 +104,7 @@ $(HelixPreCommands);export PATH=$HELIX_CORRELATION_PAYLOAD/$(DotNetCliDestination):$PATH $(HelixPreCommands);set PATH=%HELIX_CORRELATION_PAYLOAD%\$(DotNetCliDestination)%3B%PATH% - $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE + $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE;export DYLD_PRINT_LIBRARIES=1 $(HelixPreCommands);sudo apt update;sudo apt-get install libomp-dev libomp5 -y From 65d907c0f7be568e5693ac57bcabcc98df83a6bc Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 17:14:10 -0600 Subject: [PATCH 10/12] osx lightgbm test --- eng/helix.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix.proj b/eng/helix.proj index 1fe97b6039..a9f6af73cb 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -128,7 +128,7 @@ Command="install_name_tool -change "/usr/local/opt/libomp/lib/libomp.dylib" "@loader_path/libomp.dylib" $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/libSymSgdNative.dylib" /> + Command="install_name_tool -change "/usr/local/opt/libomp/lib/libomp.dylib" "@rpath/libomp.dylib" $(BUILD_SOURCESDIRECTORY)/artifacts/bin/%(ProjectsWithTargetFramework.Filename)/$(BuildConfig)/%(ProjectsWithTargetFramework.TargetFrameworks)$(PublishFolder)/runtimes/osx-x64/native/lib_lightgbm.dylib" /> From c4faf1fe531f44222a24c818cb66cc51dd62cc19 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Fri, 1 Nov 2024 17:59:43 -0600 Subject: [PATCH 11/12] lightgbm dyld log --- eng/helix.proj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/helix.proj b/eng/helix.proj index a9f6af73cb..f7fb259148 100644 --- a/eng/helix.proj +++ b/eng/helix.proj @@ -104,7 +104,7 @@ $(HelixPreCommands);export PATH=$HELIX_CORRELATION_PAYLOAD/$(DotNetCliDestination):$PATH $(HelixPreCommands);set PATH=%HELIX_CORRELATION_PAYLOAD%\$(DotNetCliDestination)%3B%PATH% - $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE;export DYLD_PRINT_LIBRARIES=1 + $(HelixPreCommands);export LD_LIBRARY_PATH=/opt/homebrew/opt/mono-libgdiplus/lib;ls /usr/lib;ls $HELIX_WORKITEM_ROOT;export KMP_DUPLICATE_LIB_OK=TRUE;export DYLD_PRINT_LIBRARIES=1;otool -L $HELIX_WORKITEM_ROOT/runtimes/osx-x64/native/lib_lightgbm.dylib $(HelixPreCommands);sudo apt update;sudo apt-get install libomp-dev libomp5 -y From e36310673c258365ea49c13e3d893c3d87dadbd4 Mon Sep 17 00:00:00 2001 From: Michael Sharp Date: Mon, 11 Nov 2024 12:40:16 -0700 Subject: [PATCH 12/12] excluding osx x64 from some tests --- .../Attributes/NativeDependencyFactAttribute.cs | 3 +++ .../Attributes/NativeDependencyTheoryAttribute.cs | 3 +++ 2 files changed, 6 insertions(+) diff --git a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs index 29bd0baa17..f40f8e1be2 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyFactAttribute.cs @@ -19,6 +19,9 @@ public NativeDependencyFactAttribute(string library) : base($"This test requires /// protected override bool IsEnvironmentSupported() { + // Starting to drop native support for X64 OSX since intel no longer makes them. + if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX) && System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture == System.Runtime.InteropServices.Architecture.X64) + return false; return NativeLibrary.NativeLibraryExists(_library); } } diff --git a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs index 444e17b5e6..1f4e1e6b1a 100644 --- a/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs +++ b/test/Microsoft.ML.TestFrameworkCommon/Attributes/NativeDependencyTheoryAttribute.cs @@ -19,6 +19,9 @@ public NativeDependencyTheoryAttribute(string library) : base($"This test requir /// protected override bool IsEnvironmentSupported() { + // Starting to drop native support for X64 OSX since intel no longer makes them. + if (System.Runtime.InteropServices.RuntimeInformation.IsOSPlatform(System.Runtime.InteropServices.OSPlatform.OSX) && System.Runtime.InteropServices.RuntimeInformation.ProcessArchitecture == System.Runtime.InteropServices.Architecture.X64) + return false; return NativeLibrary.NativeLibraryExists(_library); } }