Skip to content
Permalink
Browse files

Azure Pipeline for Windows CI (#1853)

* Update Clarius.TransformOnBuild to work with VS2017

* Simplify vw_core.vcxproj

* Simplify libvw.vcxproj

* Update NuGet.exe to support SemVer

* Add explicit project dependencies to the solution

   Solution files offer a way to code-in build dependencies which do not get resolved properly by MsBuild.

   It was necessary to add the dependencies for vw_core and libvw (and vw_clr as a precaution) so that the managed projects do not kicked off until the unmanaged dependencies were done. Otherwise the unmanaged projects get kicked off immediately when the build starts and chain to a second, parallel build of vw_core. This breaks the build when the same files (e.g. vw_core.lib, or .obj files) are being written concurrently.

* Enable finding VS2017 MSBuild automatically

* Fix init.cmd to also find vstest.console when it is not provided

* Enable a way to suppres c smoke test

Currently c_test breaks CI/CD that rely on stderror spew due to vw using cerr for info

* Fix c_test.vcxproj dependency

The project was incorrectly referencing vw_core rather than libvw; unfortunately, due to explicit ordering in the soultion and an explicit reference to the lib, this was not detected until trying to fix the solution dependencies.

* Disable node reuse to avoid keeping stale MSBuild.exe processes around after build

* Fix CodeAnalysis rule files location

* Disable it for vw_clr because the rules file does not contain compatible rules for this kind of project

* Unify C# outputs

* Fix NuGet pack binary search paths

* Make package.cmd more robust

* Move output into a separate /package folder under /out
* Make the script agnostic of where we invoke it from
* Ensure output directory is generated before calling nuget pack

* Remove in-build NuGet restore

* Remove DebugLeakCheck configuration

* This was removed a long time ago, but some projects did not get cleaned up properly

* Simplify cluster.vcxproj and unify output paths

* Normalize usage of $(SolutionDir)

* Simplify cs_unittest.csproj and unify outputs

* Switch to using exit codes to perform test validation

* Create Azure Pipeline configuration for Windows CI

* Bring set closer to check for envvars
  • Loading branch information...
lokitoth committed May 2, 2019
1 parent cfef09c commit 7161aaa6f94216cf7f16015bcb9e7d0f35fa5392
@@ -0,0 +1,55 @@
resources:
- repo: self

pool:
name: Hosted VS2017
demands: Cmd

steps:
- task: VisualStudioTestPlatformInstaller@1
inputs:
packageFeedSelector: 'nugetOrg' # Options: nugetOrg, customFeed, netShare
versionSelector: 'latestStable' # Required when packageFeedSelector == NugetOrg || PackageFeedSelector == CustomFeed# Options: latestPreRelease, latestStable, specificVersion
#testPlatformVersion: # Required when versionSelector == SpecificVersion
#customFeed: # Required when packageFeedSelector == CustomFeed
#username: # Optional
#password: # Optional
#netShare: # Required when packageFeedSelector == NetShare
- script: CALL .scripts/restore.cmd
displayName: 'Restore dependencies'
env:
nugetPath: $(Build.SourcesDirectory)\vowpalwabbit\.nuget\nuget.exe
failOnStderr: true
- script: CALL .scripts/build.cmd
displayName: 'Build vw.sln'
env:
nugetPath: $(Build.SourcesDirectory)\vowpalwabbit\.nuget\nuget.exe
failOnStderr: true
- script: CALL .scripts/test.cmd
displayName: 'Run tests'
env:
nugetPath: $(Build.SourcesDirectory)\vowpalwabbit\.nuget\nuget.exe
failOnStderr: false
- task: PublishTestResults@2
displayName: 'Publish Test Results'
inputs:
testResultsFormat: VSTest
testResultsFiles: '**/vowpalwabbit/out/test/Release/x64/*.trx'
- script: CALL .scripts/package.cmd
displayName: 'Package artifacts'
env:
Tag: +$(Build.SourceVersion)
nugetPath: $(Build.SourcesDirectory)\vowpalwabbit\.nuget\nuget.exe
failOnStderr: true
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: target'
inputs:
PathtoPublish: $(Build.SourcesDirectory)\vowpalwabbit\out\target
ArtifactName: target
condition: succeededOrFailed()
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: package'
inputs:
PathtoPublish: $(Build.SourcesDirectory)\vowpalwabbit\out\package
ArtifactName: package
condition: succeededOrFailed()
@@ -10,7 +10,7 @@ PUSHD %~dp0

REM TODO: Figure out how to parametrize this script?! (is there a standard, or do we actually need parse args?)
ECHO Building "%vwRoot%\vowpalwabbit\vw.sln" for Release x64
"%msbuildPath%" /v:normal /m /p:Configuration=Release;Platform=x64 "%vwRoot%\vowpalwabbit\vw.sln"
"%msbuildPath%" /v:normal /m /nr:false /p:Configuration=Release;Platform=x64 "%vwRoot%\vowpalwabbit\vw.sln"

POPD

@@ -0,0 +1,6 @@
IF NOT DEFINED VsInstallDir (
REM Try to find VS Install
FOR /f "usebackq tokens=*" %%i IN (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -latest -products * -requires Microsoft.Component.MSBuild -property installationPath`) DO (
SET "VsInstallDir=%%i"
)
)
@@ -4,7 +4,29 @@ IF NOT DEFINED nugetPath (
)

IF NOT DEFINED msbuildPath (
SET msbuildPath=msbuild
CALL %~dp0find-vs2017.cmd
)

IF NOT DEFINED vstestPath (
CALL %~dp0find-vs2017.cmd
)

IF NOT DEFINED msbuildPath (
IF EXIST "%VsInstallDir%\MSBuild\15.0\Bin\MSBuild.exe" (
SET "msBuildPath=%VsInstallDir%\MSBuild\15.0\Bin\MSBuild.exe"
) ELSE (
ECHO ERROR: MsBuild couldn't be found
EXIT /b 1
)
)

IF NOT DEFINED vstestPath (
IF EXIST "%VsInstallDir%\Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe" (
SET "vstestPath=%VsInstallDir%\Common7\IDE\CommonExtensions\Microsoft\TestWindow\vstest.console.exe"
) ELSE (
ECHO ERROR: vstest.console couldn't be found
EXIT /b 1
)
)

REM Repo-specific paths
@@ -7,22 +7,29 @@ SETLOCAL

CALL %~dp0init.cmd

PUSHD %~dp0..

REM TODO: these should be read from the version props file, or similar
SET Version=8.6.1
SET Tag=-INTERNALONLY

SET RootRelativeOutputDirX64=%vwRoot%\vowpalwabbit\out\target\
SET RootRelativeOutputDirAnyCPU=%vwRoot%\vowpalwabbit\out\target\
SET SolutionDir=%vwRoot%\vowpalwabbit\
SET OutputDir="%SolutionDir%out\package\Release\x64"

IF NOT EXIST %OutputDir% MKDIR %OutputDir%

REM Do not add a training backslash here, as it will break the nuget xml parser for some reason
REM Do not add a trailing backslash here, as it will break the nuget xml parser for some reason
SET RepoRoot=%vwRoot%

REM TODO-pre-checkin: Figure out how to parametrize this script?! (is there a standard, or do we actually need parse args?)
nuget pack %SolutionDir%..\cs\cs\cs.nuspec -OutputDirectory "out\target\x64\Release" -Verbosity detailed -BasePath "." -Properties "RootRelativeOutputDirX64=%RootRelativeOutputDirX64%;RootRelativeOutputDirAnyCPU=%RootRelativeOutputDirAnyCPU%;Configuration=Release;Platform=X64;version=%Version%;Tag=%Tag%;RepoRoot=%RepoRoot%;SolutionDir=%SolutionDir%"
nuget pack %SolutionDir%..\cs\cs_json\cs_json.nuspec -OutputDirectory "out\target\x64\Release" -Verbosity detailed -BasePath "." -Properties "RootRelativeOutputDirX64=%RootRelativeOutputDirX64%;RootRelativeOutputDirAnyCPU=%RootRelativeOutputDirAnyCPU%;Configuration=Release;Platform=X64;version=%Version%;Tag=%Tag%;RepoRoot=%RepoRoot%;SolutionDir=%SolutionDir%"
nuget pack %SolutionDir%..\cs\cs_parallel\cs_parallel.nuspec -OutputDirectory "out\target\x64\Release" -Verbosity detailed -BasePath "." -Properties "RootRelativeOutputDirX64=%RootRelativeOutputDirX64%;RootRelativeOutputDirAnyCPU=%RootRelativeOutputDirAnyCPU%;Configuration=Release;Platform=X64;version=%Version%;Tag=%Tag%;RepoRoot=%RepoRoot%;SolutionDir=%SolutionDir%"
"%nugetPath%" pack %SolutionDir%..\cs\cs\cs.nuspec -OutputDirectory "%OutputDir%" -Verbosity detailed -BasePath "." -Properties "RootRelativeOutputDirX64=%RootRelativeOutputDirX64%;RootRelativeOutputDirAnyCPU=%RootRelativeOutputDirAnyCPU%;Configuration=Release;Platform=X64;version=%Version%;Tag=%Tag%;RepoRoot=%RepoRoot%;SolutionDir=%SolutionDir%"
"%nugetPath%" pack %SolutionDir%..\cs\cs_json\cs_json.nuspec -OutputDirectory "%OutputDir%" -Verbosity detailed -BasePath "." -Properties "RootRelativeOutputDirX64=%RootRelativeOutputDirX64%;RootRelativeOutputDirAnyCPU=%RootRelativeOutputDirAnyCPU%;Configuration=Release;Platform=X64;version=%Version%;Tag=%Tag%;RepoRoot=%RepoRoot%;SolutionDir=%SolutionDir%"
"%nugetPath%" pack %SolutionDir%..\cs\cs_parallel\cs_parallel.nuspec -OutputDirectory "%OutputDir%" -Verbosity detailed -BasePath "." -Properties "RootRelativeOutputDirX64=%RootRelativeOutputDirX64%;RootRelativeOutputDirAnyCPU=%RootRelativeOutputDirAnyCPU%;Configuration=Release;Platform=X64;version=%Version%;Tag=%Tag%;RepoRoot=%RepoRoot%;SolutionDir=%SolutionDir%"

REM TODO: Need to also include the .pdb files, once we fix all the GitLink warnings

POPD

ENDLOCAL
@@ -7,17 +7,30 @@ SETLOCAL

CALL %~dp0init.cmd

REM TODO: Determine how to pass test failure out of this script so it can be used by CI/CD setups

REM TODO: Ensure all errors caught by the smoke test will result in non-zero exit codes
ECHO Running VW C Smoke test
"%vwRoot%\vowpalwabbit\out\target\Release\x64\c_test.exe"

IF "%ERRORLEVEL%" NEQ "0" (
ENDLOCAL
EXIT /B %ERRORLEVEL%
)

ECHO Running VW Unit Tests in C++
"%vwRoot%\vowpalwabbit\out\target\Release\x64\vw_unit_test.exe"

IF "%ERRORLEVEL%" NEQ "0" (
ENDLOCAL
EXIT /B %ERRORLEVEL%
)

ECHO Running VW Unit Tests in C#
%vstestPath% /Platform:x64 /inIsolation "%vwRoot%\vowpalwabbit\x64\Release\cs_unittest.dll" /TestCaseFilter:"TestCategory!=NotOnVSO"
REM this is the same as above, just xcopied to the out folder. This is broken due to baked-in paths. The fix is to place the output straight into out.
REM %vstestPath% /Platform:x64 /inIsolation "%vwRoot%\vowpalwabbit\x64\Release\cs_unittest.dll" /TestCaseFilter:"TestCategory!=NotOnVSO"
REM TODO: Add explicit logging configuration so it can be uploaded to pipeline results.
"%vstestPath%" /Platform:x64 /inIsolation "%vwRoot%\vowpalwabbit\x64\Release\cs_unittest.dll" /TestCaseFilter:"TestCategory!=NotOnVSO" --logger:trx "--ResultsDirectory:%vwRoot%\vowpalwabbit\out\test\Release\x64"

IF "%ERRORLEVEL%" NEQ "0" (
ENDLOCAL
EXIT /B %ERRORLEVEL%
)

ENDLOCAL
@@ -23,7 +23,7 @@
<RootNamespace>c_test</RootNamespace>
<SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\vowpalwabbit\</SolutionDir>
<!-- This is the ruleset file for code analysis, you can change it in VS -->
<CodeAnalysisRuleSet>..\sdl\SDL-7.0-Recommended.ruleset</CodeAnalysisRuleSet>
<CodeAnalysisRuleSet>$(MSBuildProjectDirectory)\..\sdl\SDL-7.0-Recommended.ruleset</CodeAnalysisRuleSet>
<RunCodeAnalysis>true</RunCodeAnalysis>
</PropertyGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" />
@@ -90,19 +90,10 @@
<ClCompile Include="sample.c" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\vowpalwabbit\vw_core.vcxproj">
<Project>{1e205806-7f80-47dd-a38d-fc08083f3593}</Project>
<ProjectReference Include="..\vowpalwabbit\libvw.vcxproj">
<Project>{EA52DE0D-A5BE-4FB9-8C84-3A57BDFEBED9}</Project>
</ProjectReference>
</ItemGroup>
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
</ImportGroup>
<Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
<PropertyGroup>
<ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
</PropertyGroup>
<Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
</Target>
<Import Project="..\sdl\SDL-7.0-NativeAnalysis.targets" />
</Project>
Oops, something went wrong.

0 comments on commit 7161aaa

Please sign in to comment.
You can’t perform that action at this time.