Added Linux & Mac changes for Arcade (dotnet#5479)

* Initial Windows, Linux, Macos builds test * Add Linux/MacOS specific CI requirements * Run Arcade CI tests on MacOS/Linux * Fix final package building * Add benchmark download to benchmars .csporj file * Print detailed status of each unit test * Install CentOS & Ubuntu build dependencies * Use container names to differenciate between Ubuntu & CentOS * Remove sudo usage in CentOS * Fix Linux build dependencies * Add -y param to apt install * Remove installation of Linux dependencies * Minor additions * Rename Benchmarks to PerformanceTests for Arcade * Changes * Added benchmark doc changes * Pre-merge changes
michaelgsharp · Nov 9, 2020 · 7795cb9 · 7795cb9
1 parent 49f3ff3
commit 7795cb9
Show file tree

Hide file tree

Showing 39 changed files with 173 additions and 142 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,6 +6,7 @@
 # Tool Runtime Dir
 /[Tt]ools/
 /.dotnet/
+/.packages/
 
 # User-specific files
 *.suo

diff --git a/.vsts-dotnet-ci.yml b/.vsts-dotnet-ci.yml
@@ -11,42 +11,42 @@ resources:
     image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8
 
 jobs:
-# - template: /build/ci/job-template.yml
-#   parameters:
-#     name: Centos_x64_NetCoreApp31
-#     buildScript: ./build.sh
-#     container: CentosContainer
-#     customMatrixes:
-#       Debug_Build:
-#         _configuration: Debug-netcoreapp3_1
-#         _config_short: DI
-#         _includeBenchmarkData: false
-#         _targetFramework: netcoreapp3.1
-#       Release_Build:
-#         _configuration: Release-netcoreapp3_1
-#         _config_short: RI
-#         _includeBenchmarkData: true
-#         _targetFramework: netcoreapp3.1
-#     innerLoop: true
-#     pool:
-#       name:  Hosted Ubuntu 1604
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Centos_x64_NetCoreApp31
+    buildScript: ./build.sh
+    container: CentosContainer
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_1
+        _config_short: DI
+        _includeBenchmarkData: false
+        _targetFramework: netcoreapp3.1
+      Release_Build:
+        _configuration: Release-netcoreapp3_1
+        _config_short: RI
+        _includeBenchmarkData: true
+        _targetFramework: netcoreapp3.1
+    innerLoop: true
+    pool:
+      name: Hosted Ubuntu 1604
 
-# - template: /build/ci/job-template.yml
-#   parameters:
-#     name: Ubuntu_x64_NetCoreApp21
-#     buildScript: ./build.sh
-#     container: UbuntuContainer
-#     innerLoop: true
-#     pool:
-#       name:  Hosted Ubuntu 1604
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Ubuntu_x64_NetCoreApp21
+    buildScript: ./build.sh
+    container: UbuntuContainer
+    innerLoop: true
+    pool:
+      name: Hosted Ubuntu 1604
 
-# - template: /build/ci/job-template.yml
-#   parameters:
-#     name: MacOS_x64_NetCoreApp21
-#     buildScript: ./build.sh
-#     innerLoop: true
-#     pool:
-#       name: Hosted macOS
+- template: /build/ci/job-template.yml
+  parameters:
+    name: MacOS_x64_NetCoreApp21
+    buildScript: ./build.sh
+    innerLoop: true
+    pool:
+      name: Hosted macOS
 
 - template: /build/ci/job-template.yml
   parameters:

diff --git a/Microsoft.ML.sln b/Microsoft.ML.sln
@@ -48,7 +48,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pkg", "pkg", "{D3D38B03-B55
 		pkg\Directory.Build.props = pkg\Directory.Build.props
 	EndProjectSection
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks", "test\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.PerformanceTests", "test\Microsoft.ML.PerformanceTests\Microsoft.ML.PerformanceTests.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Maml", "src\Microsoft.ML.Maml\Microsoft.ML.Maml.csproj", "{64F40A0D-D4C2-4AA7-8470-E9CC437827E4}"
 EndProject

diff --git a/build.sh b/build.sh
@@ -10,4 +10,4 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-"$DIR/eng/common/build.sh" --restore --build --pack --warnAsError false "$@"
+"$DIR/eng/common/build.sh" --restore --build --warnAsError false "$@"
diff --git a/build/ci/job-template.yml b/build/ci/job-template.yml
@@ -50,9 +50,18 @@ jobs:
       container: ${{ parameters.container }}
 
     steps:
+    # Work around MacOS Homebrew image/environment bug: https://github.com/actions/virtual-environments/issues/1811
     - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
-      - script: brew update && brew install $(Build.SourcesDirectory)/build/libomp.rb && brew unlink python@2 && brew install mono-libgdiplus gettext && brew link gettext --force && brew link libomp --force
-        displayName: Install build dependencies
+      - script: |
+          brew uninstall openssl@1.0.2t |
+          brew uninstall python@2.7.17 |
+          brew untap local/openssl |
+          brew untap local/python2
+        displayName: MacOS Homebrew bug Workaround
+    # Extra MacOS step required to install OS-specific dependencies
+    - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
+      - script: brew update && brew unlink python@3.8 && brew install mono-libgdiplus && brew install $(Build.SourcesDirectory)/build/libomp.rb && brew link libomp --force
+        displayName: Install MacOS build dependencies
     - ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
       - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
         displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
@@ -178,5 +187,5 @@ jobs:
         artifactName: ${{ parameters.name }} $(_config_short)
         artifactType: container
     - ${{ if eq(parameters.nightlyBuild, 'false') }}:
-      - script: ${{ parameters.buildScript }} -pack  -ci
+      - script: ${{ parameters.buildScript }} -pack -ci
         displayName: Build Packages
diff --git a/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Core/Properties/AssemblyInfo.cs
@@ -13,7 +13,7 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.InferenceTesting" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformerTest" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.AutoML.Tests" + PublicKey.Value)]
-[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.EntryPoints" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Maml" + PublicKey.Value)]

diff --git a/src/Microsoft.ML.CpuMath/AssemblyInfo.cs b/src/Microsoft.ML.CpuMath/AssemblyInfo.cs
@@ -15,6 +15,6 @@
 [assembly: InternalsVisibleTo("Microsoft.ML.Sweeper" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.TimeSeries" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.Transforms" + PublicKey.Value)]
-[assembly: InternalsVisibleTo("Microsoft.ML.Benchmarks.Tests" + PublicKey.Value)]
+[assembly: InternalsVisibleTo("Microsoft.ML.PerformanceTests.Tests" + PublicKey.Value)]
 
 [assembly: WantsToBeBestFriends]
diff --git a/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Data/Properties/AssemblyInfo.cs
@@ -12,10 +12,10 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformerTest" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TimeSeries.Tests" + PublicKey.Value)]
-[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.AutoML.Tests" + PublicKey.Value)]
 
-[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.EntryPoints" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Maml" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.ResultProcessor" + PublicKey.Value)]

diff --git a/src/Microsoft.ML.Maml/Properties/AssemblyInfo.cs b/src/Microsoft.ML.Maml/Properties/AssemblyInfo.cs
@@ -6,7 +6,7 @@
 using Microsoft.ML;
 
 [assembly: InternalsVisibleTo("Microsoft.ML.TestFramework" + PublicKey.Value)]
-[assembly: InternalsVisibleTo("Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo("Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.ResultProcessor" + PublicKey.Value)]
 
 [assembly: InternalsVisibleTo("RunTests" + InternalPublicKey.Value)]

diff --git a/...rosoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj b/...rosoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj
@@ -123,4 +123,6 @@
 
   <Target Name="Test" />
 
+  <Target Name="PerformanceTest"/>
+
 </Project>
diff --git a/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs b/test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs
@@ -10,13 +10,13 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Running;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.TestFramework;
 using Microsoft.ML.TestFramework.Attributes;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Microsoft.ML.Benchmarks.Tests
+namespace Microsoft.ML.PerformanceTests.Tests
 {
     public class TestConfig : RecommendedConfig
     {

diff --git a/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj b/test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj
@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <ItemGroup>
-    <ProjectReference Include="..\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.PerformanceTests\Microsoft.ML.PerformanceTests.csproj" />
 
     <NativeAssemblyReference Include="CpuMathNative" />
     <NativeAssemblyReference Include="MklImports" />

diff --git a/test/Microsoft.ML.Benchmarks/Program.cs b/test/Microsoft.ML.Benchmarks/Program.cs
diff --git a/test/Microsoft.ML.CpuMath.PerformanceTests/Microsoft.ML.CpuMath.PerformanceTests.csproj b/test/Microsoft.ML.CpuMath.PerformanceTests/Microsoft.ML.CpuMath.PerformanceTests.csproj
@@ -10,6 +10,8 @@
     -->
     <RuntimeIdentifier></RuntimeIdentifier>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <IsUnitTestProject>false</IsUnitTestProject>
+    <IsPerformanceTestProject>true</IsPerformanceTestProject>
   </PropertyGroup>
 
   <ItemGroup>

diff --git a/.../Microsoft.ML.Benchmarks/BenchmarkBase.cs → ...soft.ML.PerformanceTests/BenchmarkBase.cs b/.../Microsoft.ML.Benchmarks/BenchmarkBase.cs → ...soft.ML.PerformanceTests/BenchmarkBase.cs
@@ -8,7 +8,7 @@
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TestFrameworkCommon;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     public class BenchmarkBase
     {

diff --git a/...osoft.ML.Benchmarks/CacheDataViewBench.cs → ...ML.PerformanceTests/CacheDataViewBench.cs b/...osoft.ML.Benchmarks/CacheDataViewBench.cs → ...ML.PerformanceTests/CacheDataViewBench.cs
@@ -4,10 +4,10 @@
 
 using System;
 using BenchmarkDotNet.Attributes;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.Data;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [CIBenchmark]
     public class CacheDataViewBench : BenchmarkBase

diff --git a/...osoft.ML.Benchmarks/FeaturizeTextBench.cs → ...ML.PerformanceTests/FeaturizeTextBench.cs b/...osoft.ML.Benchmarks/FeaturizeTextBench.cs → ...ML.PerformanceTests/FeaturizeTextBench.cs
@@ -11,7 +11,7 @@
 using Microsoft.ML.Transforms.Text;
 using Xunit;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [Config(typeof(TrainConfig))]
     public class FeaturizeTextBench : BenchmarkBase

diff --git a/...icrosoft.ML.Benchmarks/Harness/Configs.cs → ...ft.ML.PerformanceTests/Harness/Configs.cs b/...icrosoft.ML.Benchmarks/Harness/Configs.cs → ...ft.ML.PerformanceTests/Harness/Configs.cs
@@ -9,9 +9,9 @@
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CsProj;
 using BenchmarkDotNet.Toolchains.DotNetCli;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     public class RecommendedConfig : ManualConfig
     {

diff --git a/...icrosoft.ML.Benchmarks/Harness/Metrics.cs → ...ft.ML.PerformanceTests/Harness/Metrics.cs b/...icrosoft.ML.Benchmarks/Harness/Metrics.cs → ...ft.ML.PerformanceTests/Harness/Metrics.cs
@@ -11,7 +11,7 @@
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     public abstract class WithExtraMetrics : BenchmarkBase
     {

diff --git a/...ML.Benchmarks/Harness/ProjectGenerator.cs → ...formanceTests/Harness/ProjectGenerator.cs b/...ML.Benchmarks/Harness/ProjectGenerator.cs → ...formanceTests/Harness/ProjectGenerator.cs
@@ -10,7 +10,7 @@
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CsProj;
 
-namespace Microsoft.ML.Benchmarks.Harness
+namespace Microsoft.ML.PerformanceTests.Harness
 {
     /// <summary>
     /// to avoid side effects of benchmarks affect each other BenchmarkDotNet runs every benchmark in a standalone, dedicated process

diff --git a/test/Microsoft.ML.Benchmarks/HashBench.cs → ...icrosoft.ML.PerformanceTests/HashBench.cs b/test/Microsoft.ML.Benchmarks/HashBench.cs → ...icrosoft.ML.PerformanceTests/HashBench.cs
@@ -5,12 +5,12 @@
 using System;
 using System.Linq;
 using BenchmarkDotNet.Attributes;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Transforms;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [CIBenchmark]
     public class HashBench : BenchmarkBase

diff --git a/...soft.ML.Benchmarks/Helpers/CIBenchmark.cs → ...L.PerformanceTests/Helpers/CIBenchmark.cs b/...soft.ML.Benchmarks/Helpers/CIBenchmark.cs → ...L.PerformanceTests/Helpers/CIBenchmark.cs
@@ -3,7 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
-namespace Microsoft.ML.Benchmarks.Harness
+namespace Microsoft.ML.PerformanceTests.Harness
 {
     /// <summary>
     /// This attribute is used to identify the benchmarks 

diff --git a/...soft.ML.Benchmarks/Helpers/EmptyWriter.cs → ...L.PerformanceTests/Helpers/EmptyWriter.cs b/...soft.ML.Benchmarks/Helpers/EmptyWriter.cs → ...L.PerformanceTests/Helpers/EmptyWriter.cs
@@ -5,7 +5,7 @@
 using System.IO;
 using System.Text;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     // Adding this class to not print anything to the console.
     // This is required for the current version of BenchmarkDotNet

diff --git a/....Benchmarks/Helpers/EnvironmentFactory.cs → ...rmanceTests/Helpers/EnvironmentFactory.cs b/....Benchmarks/Helpers/EnvironmentFactory.cs → ...rmanceTests/Helpers/EnvironmentFactory.cs
@@ -7,7 +7,7 @@
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Transforms;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     internal static class EnvironmentFactory
     {

diff --git a/...Microsoft.ML.Benchmarks/Helpers/Errors.cs → ...oft.ML.PerformanceTests/Helpers/Errors.cs b/...Microsoft.ML.Benchmarks/Helpers/Errors.cs → ...oft.ML.PerformanceTests/Helpers/Errors.cs
@@ -2,7 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     internal class Errors
     {

diff --git a/...soft.ML.Benchmarks/Helpers/ExecuteMaml.cs → ...L.PerformanceTests/Helpers/ExecuteMaml.cs b/...soft.ML.Benchmarks/Helpers/ExecuteMaml.cs → ...L.PerformanceTests/Helpers/ExecuteMaml.cs
@@ -4,7 +4,7 @@
 
 using System;
 using Microsoft.ML.Tools;
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     internal static class ExecuteMaml
     {

diff --git a/...ML.Benchmarks/ImageClassificationBench.cs → ...formanceTests/ImageClassificationBench.cs b/...ML.Benchmarks/ImageClassificationBench.cs → ...formanceTests/ImageClassificationBench.cs
@@ -15,7 +15,7 @@
 using System.Net.Http;
 using Microsoft.ML.Vision;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [Config(typeof(TrainConfig))]
     public class ImageClassificationBench : BenchmarkBase

diff --git a/...marks/KMeansAndLogisticRegressionBench.cs → ...Tests/KMeansAndLogisticRegressionBench.cs b/...marks/KMeansAndLogisticRegressionBench.cs → ...Tests/KMeansAndLogisticRegressionBench.cs
@@ -3,12 +3,12 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.Calibrators;
 using Microsoft.ML.Data;
 using Microsoft.ML.Trainers;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [CIBenchmark]
     public class KMeansAndLogisticRegressionBench : BenchmarkBase