new code coverage (#5169)

* test new code coverage * change coverage path * fix failure * try test code coverage * fix build * try new file filters * fix coverage report path * fix coverage report path * try remove GC.Collect * remove unnecessary new line
dotnet · May 30, 2020 · c8cace2 · c8cace2
1 parent 413cf82
commit c8cace2
Show file tree

Hide file tree

Showing 6 changed files with 42 additions and 24 deletions.
diff --git a/build.proj b/build.proj
@@ -38,6 +38,7 @@
       $(TraversalBuildDependsOn);
       DownloadExternalTestFiles;
       DownloadTensorflowMetaFiles;
+      DeleteTestHost;
     </TraversalBuildDependsOn>
   </PropertyGroup>
 
@@ -116,6 +117,18 @@
     </DownloadFile>
   </Target>
 
+  <!-- Delete testhost.dll and testhost.exe from output folder,
+  start test from dotnet.exe to keep consistent behavior with older version of Microsoft.NET.Test.Sdk -->
+  <Target Name="DeleteTestHost">
+    <Message Importance="High" Text="Delete testhost.dll and testhost.exe from output folder..." />
+    <ItemGroup>
+        <FilesToClean Include="$(MSBuildThisFileDirectory)\bin\**\testhost.dll" />
+        <FilesToClean Include="$(MSBuildThisFileDirectory)\bin\**\testhost.exe" />
+    </ItemGroup>
+    <Delete Files="@(FilesToClean)"/>
+    <RemoveDir Directories="@(FoldersToClean)" />
+  </Target>
+
   <Target Name="RunTests">
     <MSBuild Projects="test\run-tests.proj"
              Targets="RunTests" />

diff --git a/build/Codecoverage.proj b/build/Codecoverage.proj
@@ -16,8 +16,8 @@
       <_ReportGeneratorPath>$(PkgReportGenerator)\tools\net47\ReportGenerator.exe</_ReportGeneratorPath>
     </PropertyGroup>
 
-    <Message Importance="high" Text="&quot;$(_ReportGeneratorPath)&quot; -reports:$(BaseOutputPath)$(PlatformConfig)\coverage\*.coverage -targetdir:$(BaseOutputPath)$(PlatformConfig)\coverage -filefilters:+https*;+*.fs -reporttypes:Cobertura" />
-    <Exec Command="&quot;$(_ReportGeneratorPath)&quot; -reports:$(BaseOutputPath)$(PlatformConfig)\coverage\*.coverage -targetdir:$(BaseOutputPath)$(PlatformConfig)\coverage -filefilters:+https*;+*.fs -reporttypes:Cobertura" />
+    <Message Importance="high" Text="&quot;$(_ReportGeneratorPath)&quot; -reports:$(BaseOutputPath)$(PlatformConfig)\*\*\coverage.opencover.xml -targetdir:$(BaseOutputPath)$(PlatformConfig)\coverage -filefilters:+*.cs;+*.fs -reporttypes:Cobertura" />
+    <Exec Command="&quot;$(_ReportGeneratorPath)&quot; -reports:$(BaseOutputPath)$(PlatformConfig)\*\*\coverage.opencover.xml -targetdir:$(BaseOutputPath)$(PlatformConfig)\coverage -filefilters:+*.cs;+*.fs -reporttypes:Cobertura" />
 
     <ItemGroup>
       <_CodecovArgs Include="-f;$(BaseOutputPath)$(PlatformConfig)\coverage\Cobertura.xml" />

diff --git a/build/Dependencies.props b/build/Dependencies.props
@@ -43,7 +43,7 @@
   <PropertyGroup>
     <PublishSymbolsPackageVersion>1.0.0-beta-62824-02</PublishSymbolsPackageVersion>
     <CodecovVersion>1.9.0</CodecovVersion>
-    <CoverletVersion>2.7.0</CoverletVersion>
+    <CoverletCollectorVersion>1.2.1</CoverletCollectorVersion>
     <ReportGeneratorVersion>4.3.6</ReportGeneratorVersion>
     <MicrosoftDotNetApiCompatPackageVersion>1.0.0-beta.19225.5</MicrosoftDotNetApiCompatPackageVersion>
   </PropertyGroup>

diff --git a/test/Directory.Build.props b/test/Directory.Build.props
@@ -27,14 +27,14 @@
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.8.0" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.6.1" />
     <PackageReference Include="MSTest.TestFramework" Version="2.1.0" />
     <PackageReference Include="MSTest.TestAdapter" Version="2.1.0" />
     <PackageReference Include="xunit" Version="2.4.0" />
     <PackageReference Include="Xunit.Combinatorial" Version="$(XunitCombinatorialVersion)" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.4.0" />
     <PackageReference Include="Microsoft.DotNet.XUnitExtensions" Version="2.4.0-prerelease-63213-02" />
-    <PackageReference Include="coverlet.msbuild" Version="$(CoverletVersion)" />
+    <PackageReference Include="coverlet.collector" Version="$(CoverletCollectorVersion)" />
   </ItemGroup>
 
   <ItemGroup>
@@ -54,22 +54,4 @@
     </ProjectReference>
   </ItemGroup>
 
-  <PropertyGroup Condition="'$(Coverage)' == 'true'">
-      <!-- https://github.com/tonerdo/coverlet/issues/618 -->
-      <IncludeTestAssembly>true</IncludeTestAssembly>
-
-      <CollectCoverage>true</CollectCoverage>
-      <SingleHit>true</SingleHit>
-      <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies> <!-- https://github.com/tonerdo/coverlet/issues/72 -->
-      <CoverletOutputFormat>opencover</CoverletOutputFormat>
-      <CoverletOutput>$(BaseOutputPath)$(PlatformConfig)\coverage\$(MSBuildProjectName).coverage</CoverletOutput>
-      <Include>[Microsoft.ML.*]*</Include>
-      <!-- Excluding for perf reasons. These classes have tests that can be run conditionally
-           but they need to be migrated. Excluding these classes should have very minimal effect on code coverage.
-      -->
-      <Exclude>[*]Microsoft.ML.*Contracts*,[*]Microsoft.ML.Internal.Utilities*,[*]Microsoft.ML.Data.VBuffer*</Exclude>
-      <ExcludeByAttribute>Obsolete,ExcludeFromCodeCoverage</ExcludeByAttribute>
-      <ExcludeByFile>$(RepoRoot)src\Microsoft.ML.OnnxConverter\OnnxMl.cs,$(RepoRoot)src\Microsoft.ML.TensorFlow\TensorFlow\Buffer.cs,$(RepoRoot)src\Microsoft.ML.TensorFlow\TensorFlow\Tensor.cs,$(RepoRoot)src\Microsoft.ML.TensorFlow\TensorFlow\Tensorflow.cs</ExcludeByFile>
-  </PropertyGroup>
-
 </Project>
diff --git a/test/coverlet.runsettings b/test/coverlet.runsettings
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<RunSettings>
+  <DataCollectionRunSettings>
+    <DataCollectors>
+      <DataCollector friendlyName="XPlat code coverage">
+        <Configuration>
+          <Format>opencover</Format>          
+          <Exclude>[*]Microsoft.ML.*Contracts*,[*]Microsoft.ML.Internal.Utilities*,[*]Microsoft.ML.Data.VBuffer*</Exclude> <!-- [Assembly-Filter]Type-Filter -->
+          <Include>[Microsoft.ML.*]*</Include> <!-- [Assembly-Filter]Type-Filter -->
+          <ExcludeByAttribute>Obsolete,ExcludeFromCodeCoverage</ExcludeByAttribute>
+          <ExcludeByFile>$(RepoRoot)src\Microsoft.ML.OnnxConverter\OnnxMl.cs,$(RepoRoot)src\Microsoft.ML.TensorFlow\TensorFlow\Buffer.cs,$(RepoRoot)src\Microsoft.ML.TensorFlow\TensorFlow\Tensor.cs,$(RepoRoot)src\Microsoft.ML.TensorFlow\TensorFlow\Tensorflow.cs</ExcludeByFile> <!-- Globbing filter -->
+          <SingleHit>true</SingleHit>
+          <IncludeTestAssembly>true</IncludeTestAssembly>
+        </Configuration>
+      </DataCollector>
+    </DataCollectors>
+  </DataCollectionRunSettings>
+</RunSettings>
diff --git a/test/run-tests.proj b/test/run-tests.proj
@@ -8,7 +8,12 @@
 
   <Target Name="RunTests">
     <Message Importance="High" Text="Running tests ..." />
-    <MSBuild Targets="VSTest"
+    <MSBuild Condition="'$(Coverage)' == 'true'"
+             Targets="VSTest"
+             Projects="@(Project)"
+             Properties="VSTestNoBuild=true;VSTestBlame=true;VSTestCollect=XPlat Code Coverage;VSTestSetting=$(MSBuildThisFileDirectory)\coverlet.runsettings" />
+    <MSBuild Condition="'$(Coverage)' == 'false'"
+             Targets="VSTest"
              Projects="@(Project)"
              Properties="VSTestNoBuild=true;VSTestBlame=true" />
   </Target>