Nightly build pipeline

Directory.Build.props

@@ -19,6 +19,7 @@
       https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json;
       https://dotnet.myget.org/F/dotnet-core/api/v3/index.json;
       https://dotnet.myget.org/F/roslyn-analyzers/api/v3/index.json;
+      https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json;
     </RestoreSources>
   </PropertyGroup>
 

Microsoft.ML.sln

@@ -285,6 +285,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Vision", "src\
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TestFrameworkCommon", "test\Microsoft.ML.TestFrameworkCommon\Microsoft.ML.TestFrameworkCommon.csproj", "{A22FAD27-77E8-4460-8B92-EC7090B7173A}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.NugetPackageVersionUpdater", "test\Microsoft.ML.NugetPackageVersionUpdater\Microsoft.ML.NugetPackageVersionUpdater.csproj", "{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}"
+EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.NightlyBuild.Tests", "test\Microsoft.ML.NightlyBuild.Tests\Microsoft.ML.NightlyBuild.Tests.csproj", "{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1716,6 +1720,42 @@ Global
 		{A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
 		{A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
 		{A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug|x64.Build.0 = Debug|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug-netcoreapp3_0|Any CPU.ActiveCfg = Debug-netcoreapp3_0|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug-netcoreapp3_0|Any CPU.Build.0 = Debug-netcoreapp3_0|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug-netcoreapp3_0|x64.ActiveCfg = Debug-netcoreapp3_0|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug-netcoreapp3_0|x64.Build.0 = Debug-netcoreapp3_0|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug-netfx|Any CPU.Build.0 = Debug-netfx|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Debug-netfx|x64.Build.0 = Debug-netfx|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release|Any CPU.Build.0 = Release|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release|x64.ActiveCfg = Release|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release|x64.Build.0 = Release|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release-netcoreapp3_0|Any CPU.ActiveCfg = Release-netcoreapp3_0|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release-netcoreapp3_0|Any CPU.Build.0 = Release-netcoreapp3_0|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release-netcoreapp3_0|x64.ActiveCfg = Release-netcoreapp3_0|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release-netcoreapp3_0|x64.Build.0 = Release-netcoreapp3_0|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Debug-netcoreapp3_0|Any CPU.ActiveCfg = Debug-netcoreapp3_0|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Debug-netcoreapp3_0|x64.ActiveCfg = Debug-netcoreapp3_0|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Release|x64.ActiveCfg = Release|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Release-netcoreapp3_0|Any CPU.ActiveCfg = Release-netcoreapp3_0|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Release-netcoreapp3_0|x64.ActiveCfg = Release-netcoreapp3_0|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1806,6 +1846,8 @@ Global
 		{3817A875-278C-4140-BF66-3C4A8CA55F0D} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
 		{419F93D5-4135-4DA0-A76E-EFC23E04093D} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{A22FAD27-77E8-4460-8B92-EC7090B7173A} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{BC091EB9-74B8-4D8C-9ADB-0FF66A601878} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{A4915012-6F2A-4CB0-BF3E-FE3CC19213E9} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

build.proj

@@ -96,6 +96,11 @@
     <MSBuild Projects="test\run-tests.proj"
              Targets="RunTests" />
   </Target>
+
+  <Target Name="RunNightlyBuildTests">
+    <MSBuild Projects="test\run-night-build-tests.proj"
+             Targets="RunNightlyBuildTests" />
+  </Target>
 
   <!-- Override CleanAllProjects from dir.traversal.targets and just remove the full BinDir -->
   <Target Name="CleanAllProjects">

build/.night-build.yml

@@ -0,0 +1,110 @@
+################################################################################
+# ML.NET's nightly build
+################################################################################
+
+# no PR builds
+pr: none
+
+# no CI builds
+trigger: none
+
+# scheduled trigger, runs at UTC 8:00 every day which is midnight of GMT-8 
+schedules:
+- cron: "0 8 * * *"
+  displayName: Nightly Build at midnight
+  branches:
+    include:
+    - master
+    - releases/1.0
+    - features/automl
+    - features/integrationPackage
+  always: true
+
+resources:
+  containers:
+  - container: CentosContainer
+    image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033
+
+  - container: UbuntuContainer
+    image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-207e097-20190312152303
+
+jobs:
+- template: /build/nightly-build/job-template.yml
+  parameters:
+    name: Centos_x64_NetCoreApp30
+    buildScript: ./build.sh
+    container: CentosContainer
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_0
+        _config_short: DI
+        _includeBenchmarkData: false
+      Release_Build:
+        _configuration: Release-netcoreapp3_0
+        _config_short: RI
+        _includeBenchmarkData: true
+    pool:
+      name: Hosted Ubuntu 1604
+
+- template: /build/nightly-build/job-template.yml
+  parameters:
+    name: Ubuntu_x64_NetCoreApp21
+    buildScript: ./build.sh
+    container: UbuntuContainer
+    pool:
+      name: Hosted Ubuntu 1604
+
+- template: /build/nightly-build/job-template.yml
+  parameters:
+    name: MacOS_x64_NetCoreApp21
+    buildScript: ./build.sh
+    pool:
+      name: Hosted macOS High Sierra
+
+- template: /build/nightly-build/job-template.yml
+  parameters:
+    name: Windows_x64_NetCoreApp30
+    buildScript: build.cmd
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_0
+        _config_short: DI
+        _includeBenchmarkData: false
+      Release_Build:
+        _configuration: Release-netcoreapp3_0
+        _config_short: RI
+        _includeBenchmarkData: true
+    pool:
+      name: NetCorePublic-Pool
+      queue: buildpool.windows.10.amd64.vs2017.open
+
+- template: /build/nightly-build/job-template.yml
+  parameters:
+    name: Windows_x64_NetCoreApp21
+    buildScript: build.cmd
+    pool:
+      name: Hosted VS2017
+
+- template: /build/nightly-build/job-template.yml
+  parameters:
+    name: Windows_x64_NetFx461
+    buildScript: build.cmd
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netfx
+        _config_short: DFX
+        _includeBenchmarkData: false
+      Release_Build:
+        _configuration: Release-netfx
+        _config_short: RFX
+        _includeBenchmarkData: false
+    pool:
+      name: Hosted VS2017
+
+- template: /build/nightly-build/job-template.yml
+  parameters:
+    name: Windows_x86_NetCoreApp21
+    architecture: x86
+    buildScript: build.cmd
+    pool:
+      name: Hosted VS2017

build/nightly-build/job-template.yml

@@ -0,0 +1,116 @@
+parameters:
+  name: ''
+  architecture: x64
+  buildScript: ''
+  pool: {}
+  customMatrixes: ''
+  container: ''
+
+jobs:
+  - job: ${{ parameters.name }}
+    timeoutInMinutes: 20
+    variables:
+      dotnetPath: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet
+      nugetFeed: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json
+      nightlyBuildProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NightlyBuild.Tests/Microsoft.ML.NightlyBuild.Tests.csproj
+      nightlyBuildRunPath: (Build.SourcesDirectory)/bin/AnyCPU.(_configuration)/Microsoft.ML.NightlyBuild.Tests/netcoreapp2.1
+      packageUpdaterProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/Microsoft.ML.NugetPackageVersionUpdater.csproj
+      versionFilePath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/latest_versions.txt
+    strategy:
+      matrix:
+        ${{ if eq(parameters.customMatrixes, '') }}:
+          Debug_Build:
+            _configuration: Debug
+            _config_short: D
+            _includeBenchmarkData: false
+          Release_Build:
+            _configuration: Release
+            _config_short: R
+            _includeBenchmarkData: true
+        ${{ if ne(parameters.customMatrixes, '') }}:
+          ${{ insert }}: ${{ parameters.customMatrixes }}
+
+    pool: ${{ parameters.pool }}
+    ${{ if ne(parameters.container, '') }}:
+      container: ${{ parameters.container }}
+
+    steps:
+    - ${{ if eq(parameters.pool.name, 'Hosted macOS High Sierra') }}:
+      - script: brew update && brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f5b1ac99a7fba27c19cee0bc4f036775c889b359/Formula/libomp.rb && brew install mono-libgdiplus gettext && brew link gettext --force && brew link libomp --force
+        displayName: Install build dependencies
+    - ${{ if eq(parameters.name, 'Ubuntu_x64_NetCoreApp21') }}:
+      - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH](nightlyBuildRunPath):LD_LIBRARY_PATH"
+        displayName: Set LD_LIBRARY_PATH for Ubuntu to locate Native shared library
+    - script: parameters.buildScript−(_configuration) -buildArch=${{ parameters.architecture }}
+      displayName: Builds
+    - script: (dotnetPath)restore(nightlyBuildProjPath)
+      displayName: Restore nightly build project
+    - script: (dotnetPath)list(nightlyBuildProjPath) package --source (nugetFeed)−−outdated>(versionFilePath)
+      displayName: List latest package versions
+    - script: (dotnetPath)run−−project(packageUpdaterProjPath)
+      displayName: Update package version
+    - script: (dotnetPath)msbuild−restore(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" /p:Configuration=(configuration)/p:TargetArchitecture={{ parameters.architecture }}
+      displayName: Build Nightly Build Project with latest package versions
+    - script: parameters.buildScript−(_configuration) -runnightlybuildtests
+      displayName: Run Nightly Build Tests
+      condition: succeeded()
+    - task: PublishTestResults@2
+      displayName: Publish Test Results
+      condition: succeededOrFailed()
+      inputs:
+        testRunner: 'vSTest'
+        searchFolder: '$(System.DefaultWorkingDirectory)/bin'
+        testResultsFiles: '**/*.trx'
+        testRunTitle: Machinelearning_Tests_Missing superscript or subscript argument(_configuration)_$(Build.BuildNumber)
+        configuration: $(_configuration)
+        mergeTestResults: true
+    - task: CopyFiles@2
+      displayName: Stage build logs
+      condition: not(succeeded())
+      inputs:
+        sourceFolder: $(Build.SourcesDirectory)
+        contents: '?(msbuild.*|binclash.log|init-tools.log)'
+        targetFolder: $(Build.ArtifactStagingDirectory)
+    - task: CopyFiles@2
+      displayName: Stage test output
+      condition: not(succeeded())
+      inputs:
+        sourceFolder: $(Build.SourcesDirectory)/bin
+        contents: |
+          **/TestOutput/**/*
+          **/*.trx
+        targetFolder: $(Build.ArtifactStagingDirectory)
+    - task: PublishBuildArtifacts@1
+      displayName: Publish build and test logs
+      condition: not(succeeded())
+      inputs:
+        pathToPublish: $(Build.ArtifactStagingDirectory)
+        artifactName: parameters.name(_config_short)
+        artifactType: container
+    - ${{ if eq(parameters.buildScript, 'build.cmd') }}:
+      - powershell: |
+                  $taskListOutput = & TaskList /FI "IMAGENAME eq dotnet.exe" | Out-String
+                  if ($taskListOutput.Contains("dotnet.exe"))
+                  {
+                      Write-Output "Killing dotnet.exe processes..."
+                      & taskkill /IM dotnet.exe /F
+                  }
+                  if (Test-Path .\packages\)
+                  {
+                      Write-Output "Removing nuget cache dir..."
+                      Remove-Item -Force -Recurse .\packages\
+                  }
+                  if (Test-Path .\test\data\external\)
+                  {
+                      Write-Output "Removing external test data..."
+                      Remove-Item -Force -Recurse .\test\data\external\
+                  }
+                  mkdir .\tmp\
+                  Move-Item .\bin\obj\packages .\tmp\
+                  Write-Output "Removing bin dir..."
+                  Remove-Item -Force -Recurse .\bin\
+                  mkdir .\bin\obj\
+                  Move-Item .\tmp\packages\ .\bin\obj\
+                  Remove-Item -Force -Recurse .\tmp\
+                  Write-Output "Done cleaning up binary folder..."
+        displayName: Clean up test data

config.json

@@ -78,6 +78,12 @@
       "values": [],
       "defaultValue": ""
     },
+	"RunNightlyBuildTests": {
+	  "description": "MsBuild target that run the nightly build tests. Call this after building.",
+      "valueType": "target",
+      "values": [],
+      "defaultValue": ""
+	},
     "Coverage": {
       "description": "Turn on code coverge.",
       "valueType": "property",
@@ -159,6 +165,12 @@
           "settings": {
             "RunTests": "default"
           }
+        },
+		    "runnightlybuildtests": {
+          "description": "Runs the nightly build tests. Call this after building.",
+          "settings": {
+            "RunNightlyBuildTests": "default"
+          }
         },
         "verbose": {
           "description": "Passes /flp:v=diag to the msbuild command or the value passed by the user.",

test/Microsoft.ML.AutoML.Tests/AutoFitTests.cs

@@ -7,7 +7,6 @@
 using System.Linq;
 using System.Runtime.InteropServices;
 using Microsoft.ML.Data;
-using Microsoft.ML.RunTests;
 using Microsoft.ML.TestFrameworkCommon;
 using Microsoft.ML.TestFramework.Attributes;
 using Xunit;

test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj

@@ -5,6 +5,7 @@
     <SignAssembly>false</SignAssembly>
     <!-- DON'T CHANGE THIS!!! We are turning off strong naming to ensure we never add `InternalsVisibleTo` for these tests -->
     <PublicSign>false</PublicSign>
+    <ReferenceTypeForTestFramework>Project</ReferenceTypeForTestFramework>
   </PropertyGroup>
 
   <ItemGroup>

test/Microsoft.ML.NightlyBuild.Tests/Microsoft.ML.NightlyBuild.Tests.csproj

@@ -0,0 +1,32 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <!-- DON'T CHANGE THIS!!! We are turning off strong naming to ensure we never add `InternalsVisibleTo` for these tests -->
+    <SignAssembly>false</SignAssembly>
+    <!-- DON'T CHANGE THIS!!! We are turning off strong naming to ensure we never add `InternalsVisibleTo` for these tests -->
+    <PublicSign>false</PublicSign>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <Import Project="..\NightlyBuildDependency.props" />
+
+  <ItemGroup>
+    <Compile Include="..\Microsoft.ML.Functional.Tests\**\*.cs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <NativeAssemblyReference Include="CpuMathNative" />
+    <NativeAssemblyReference Include="FastTreeNative" />
+    <NativeAssemblyReference Include="MatrixFactorizationNative" />
+    <NativeAssemblyReference Include="LdaNative" />
+    <NativeAssemblyReference Include="SymSgdNative" />
+    <NativeAssemblyReference Include="MklProxyNative" />
+    <NativeAssemblyReference Include="MklImports" />
+    <NativeAssemblyReference Condition="'$(OS)' == 'Windows_NT'" Include="libiomp5md" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.ML.TestFrameworkCommon\Microsoft.ML.TestFrameworkCommon.csproj" />
+  </ItemGroup>
+
+</Project>

test/Microsoft.ML.NugetPackageVersionUpdater/Microsoft.ML.NugetPackageVersionUpdater.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <StartupObject>Microsoft.ML.NugetPackageVersionUpdater.Program</StartupObject>
+    <RunWorkingDirectory>$(MSBuildProjectDirectory)</RunWorkingDirectory>
+  </PropertyGroup>
+
+</Project>