From 01908e9fe3b5a3c51935e1f6622ef4bf10b805c3 Mon Sep 17 00:00:00 2001
From: Bruce Forstall <brucefo@microsoft.com>
Date: Tue, 7 Aug 2018 14:58:12 -0700
Subject: [PATCH] Add basic check that the correct number of tests is built
 (#19290)

Fixes #19286
---
 build-test.cmd     |  9 +++++++++
 build-test.sh      | 14 ++++++++++++++
 tests/runtest.proj | 25 ++++++++++++++++++++++++-
 3 files changed, 47 insertions(+), 1 deletion(-)

diff --git a/build-test.cmd b/build-test.cmd
index 9df2033da57e..242c39e70d6d 100644
--- a/build-test.cmd
+++ b/build-test.cmd
@@ -335,6 +335,15 @@ for /l %%G in (1, 1, %__BuildLoopCount%) do (
     set __AppendToLog=true
 )
 
+REM Check that we've built about as many tests as we expect. This is primarily intended to prevent accidental changes that cause us to build
+REM drastically fewer Pri-1 tests than expected.
+echo %__MsgPrefix%Check the managed tests build
+call %__DotnetHost% msbuild %__ProjectDir%\tests\runtest.proj /t:CheckTestBuild /p:CLRTestPriorityToBuild=%__Priority% %__msbuildArgs% %__unprocessedBuildArgs%
+if errorlevel 1 (
+    echo %__MsgPrefix%Error: build failed.
+    exit /b 1
+)
+
 :SkipManagedBuild
 
 REM =========================================================================================
diff --git a/build-test.sh b/build-test.sh
index fd2d8af3d99f..b661ebaef98f 100755
--- a/build-test.sh
+++ b/build-test.sh
@@ -252,6 +252,20 @@ build_Tests()
         echo "${__MsgPrefix}Error: build failed. Refer to the build log files for details (above)"
         exit 1
     else
+        echo "Checking the Managed Tests Build..."
+
+        if [ -n __priority1 ]; then
+            __Priority=1
+        else
+            __Priority=0
+        fi
+        build_Tests_internal "Check_Test_Build" "${__ProjectDir}/tests/runtest.proj" "Check Test Build" "/t:CheckTestBuild /p:CLRTestPriorityToBuild=$__Priority"
+
+        if [ $? -ne 0 ]; then
+            echo "${__MsgPrefix}Error: Check Test Build failed."
+            exit 1
+        fi
+
         echo "Managed tests build success!"
     fi
 
diff --git a/tests/runtest.proj b/tests/runtest.proj
index 3d12bd32fb94..52eeaaf1e560 100644
--- a/tests/runtest.proj
+++ b/tests/runtest.proj
@@ -27,7 +27,7 @@
   <Target Name="FindCmdDirectories" DependsOnTargets="GetListOfTestCmds">
 
     <Error Condition="!Exists('$(XunitTestBinBase)')"
-           Text="$(XunitTestBinBase) does not exist. Please run buildtest.cmd from the (repo root)\tests at least once to get the tests built." />
+           Text="$(XunitTestBinBase) does not exist. Please run build-test.cmd from the repo root at least once to get the tests built." />
 
     <ItemGroup>
 
@@ -54,6 +54,29 @@
 
   </Target>
 
+  <!-- Target to check the test build, to see if it looks ok. We've had several cases where a change inadvertently and drastically changes
+       the set of tests that are built, and that change is unnoticed. The most common case is for a build of the Priority 1 tests
+       to only build the Priority 0 tests. This target is run after a test build to verify that the basic number of tests that were
+       built is basically what was expected. When this was written, there were about 2500 Priority 0 tests and about 12270 Priority 1
+       tests (differing slightly based on platform). We currently check that the number of Priority 0 tests is greater than 2000 and
+       less than 3000, and the number of Priority 1 tests is greater than 12000.
+  -->
+  <Target Name="CheckTestBuild" DependsOnTargets="GetListOfTestCmds">
+    <Error Condition="!Exists('$(XunitTestBinBase)')"
+        Text="$(XunitTestBinBase) does not exist. Please run build-test.cmd from the repo root at least once to get the tests built." />
+
+    <PropertyGroup>
+        <TestCount>@(AllRunnableTestPaths->Count())</TestCount>
+    </PropertyGroup>
+
+    <Message Text="Found $(TestCount) built tests"/>
+
+    <Error Condition="'$(CLRTestPriorityToBuild)' == '0' and '$(TestCount)' &lt;= 2000" Text="Unexpected test count. Expected &gt; 2000, found $(TestCount).'" />
+    <Error Condition="'$(CLRTestPriorityToBuild)' == '0' and '$(TestCount)' &gt;= 3000" Text="Unexpected test count. Expected &lt; 3000, found $(TestCount).'" />
+    <Error Condition="'$(CLRTestPriorityToBuild)' == '1' and '$(TestCount)' &lt;= 12000" Text="Unexpected test count. Expected &gt; 12000, found $(TestCount).'" />
+    <Error Condition="'$(CLRTestPriorityToBuild)' != '0' and '$(CLRTestPriorityToBuild)' != '1'" Text="Unknown priority $(CLRTestPriorityToBuild)" />
+  </Target>
+
   <Import Project="$(__Exclude)" Condition="'$(__Exclude)' != '' AND '$(XunitTestBinBase)' != ''" />
   <PropertyGroup>
     <HaveExcludes>False</HaveExcludes>