Improve perf scripts - Add MSBuild support, add static graph support,…

… add solution filter for the product code (#3621)

NuGet-Src.slnf

@@ -0,0 +1,47 @@
+{
+  "solution": {
+    "path": "NuGet.sln",
+    "projects": [
+      "src\\NuGet.Clients\\NuGet.CommandLine\\NuGet.CommandLine.csproj",
+      "src\\NuGet.Clients\\NuGet.Console\\NuGet.Console.csproj",
+      "src\\NuGet.Clients\\NuGet.Indexing\\NuGet.Indexing.csproj",
+      "src\\NuGet.Clients\\NuGet.MSSigning.Extensions\\NuGet.MSSigning.Extensions.csproj",
+      "src\\NuGet.Clients\\NuGet.PackageManagement.PowerShellCmdlets\\NuGet.PackageManagement.PowerShellCmdlets.csproj",
+      "src\\NuGet.Clients\\NuGet.PackageManagement.UI\\NuGet.PackageManagement.UI.csproj",
+      "src\\NuGet.Clients\\NuGet.PackageManagement.VisualStudio\\NuGet.PackageManagement.VisualStudio.csproj",
+      "src\\NuGet.Clients\\NuGet.SolutionRestoreManager.Interop\\NuGet.SolutionRestoreManager.Interop.csproj",
+      "src\\NuGet.Clients\\NuGet.SolutionRestoreManager\\NuGet.SolutionRestoreManager.csproj",
+      "src\\NuGet.Clients\\NuGet.Tools\\NuGet.Tools.csproj",
+      "src\\NuGet.Clients\\NuGet.VisualStudio.Client\\NuGet.VisualStudio.Client.csproj",
+      "src\\NuGet.Clients\\NuGet.VisualStudio.Common\\NuGet.VisualStudio.Common.csproj",
+      "src\\NuGet.Clients\\NuGet.VisualStudio.Contracts\\NuGet.VisualStudio.Contracts.csproj",
+      "src\\NuGet.Clients\\NuGet.VisualStudio.Implementation\\NuGet.VisualStudio.Implementation.csproj",
+      "src\\NuGet.Clients\\NuGet.VisualStudio.Internal.Contracts\\NuGet.VisualStudio.Internal.Contracts.csproj",
+      "src\\NuGet.Clients\\NuGet.VisualStudio.Interop\\NuGet.VisualStudio.Interop.csproj",
+      "src\\NuGet.Clients\\NuGet.VisualStudio.OnlineEnvironment.Client\\NuGet.VisualStudio.OnlineEnvironment.Client.csproj",
+      "src\\NuGet.Clients\\NuGet.VisualStudio\\NuGet.VisualStudio.csproj",
+      "src\\NuGet.Clients\\NuGetConsole.Host.PowerShell\\NuGetConsole.Host.PowerShell.csproj",
+      "src\\NuGet.Core\\Microsoft.Build.NuGetSdkResolver\\Microsoft.Build.NuGetSdkResolver.csproj",
+      "src\\NuGet.Core\\NuGet.Build.Tasks.Console\\NuGet.Build.Tasks.Console.csproj",
+      "src\\NuGet.Core\\NuGet.Build.Tasks.Pack\\NuGet.Build.Tasks.Pack.csproj",
+      "src\\NuGet.Core\\NuGet.Build.Tasks\\NuGet.Build.Tasks.csproj",
+      "src\\NuGet.Core\\NuGet.CommandLine.XPlat\\NuGet.CommandLine.XPlat.csproj",
+      "src\\NuGet.Core\\NuGet.Commands\\NuGet.Commands.csproj",
+      "src\\NuGet.Core\\NuGet.Common\\NuGet.Common.csproj",
+      "src\\NuGet.Core\\NuGet.Configuration\\NuGet.Configuration.csproj",
+      "src\\NuGet.Core\\NuGet.Credentials\\NuGet.Credentials.csproj",
+      "src\\NuGet.Core\\NuGet.DependencyResolver.Core\\NuGet.DependencyResolver.Core.csproj",
+      "src\\NuGet.Core\\NuGet.Frameworks\\NuGet.Frameworks.csproj",
+      "src\\NuGet.Core\\NuGet.LibraryModel\\NuGet.LibraryModel.csproj",
+      "src\\NuGet.Core\\NuGet.Localization\\NuGet.Localization.csproj",
+      "src\\NuGet.Core\\NuGet.PackageManagement\\NuGet.PackageManagement.csproj",
+      "src\\NuGet.Core\\NuGet.Packaging.Core\\NuGet.Packaging.Core.csproj",
+      "src\\NuGet.Core\\NuGet.Packaging.Extraction\\NuGet.Packaging.Extraction.csproj",
+      "src\\NuGet.Core\\NuGet.Packaging\\NuGet.Packaging.csproj",
+      "src\\NuGet.Core\\NuGet.ProjectModel\\NuGet.ProjectModel.csproj",
+      "src\\NuGet.Core\\NuGet.Protocol\\NuGet.Protocol.csproj",
+      "src\\NuGet.Core\\NuGet.Resolver\\NuGet.Resolver.csproj",
+      "src\\NuGet.Core\\NuGet.Versioning\\NuGet.Versioning.csproj"
+    ]
+  }
+}

scripts/perftests/PerformanceTestUtilities.ps1

@@ -92,6 +92,11 @@ function IsClientDotnetExe([string]$nugetClient)
     return $nugetClient.EndsWith("dotnet.exe")
 }
 
+function IsClientMSBuildExe([string]$nugetClient)
+{
+    return $nugetClient.EndsWith("MSBuild.exe", "CurrentCultureIgnoreCase")
+}
+
 # Downloads the repository at the given path.
 Function DownloadRepository([string] $repository, [string] $commitHash, [string] $sourceFolderPath)
 {
@@ -145,13 +150,18 @@ Function SetupGitRepository([string] $repository, [string] $commitHash, [string]
 }
 
 # runs locals clear all with the given client
+# If the client is msbuild.exe, the locals clear all *will* be run with dotnet.exe
 Function LocalsClearAll([string] $nugetClientFilePath)
 {
     $nugetClientFilePath = GetAbsolutePath $nugetClientFilePath
     If ($(IsClientDotnetExe $nugetClientFilePath))
     {
         . $nugetClientFilePath nuget locals -c all *>>$null
     }
+    Elseif($(IsClientMSBuildExe $nugetClientFilePath))
+    {
+        . dotnet.exe nuget locals -c all *>>$null
+    }
     Else
     {
         . $nugetClientFilePath locals -clear all -Verbosity quiet
@@ -167,6 +177,13 @@ Function GetClientVersion([string] $nugetClientFilePath)
     {
         $version = . $nugetClientFilePath --version
     }
+    ElseIf($(IsClientMSBuildExe $nugetClientFilePath))
+    {
+        $clientDir = Split-Path -Path $nugetClientFilePath
+        $nugetClientPath = Resolve-Path (Join-Path -Path $clientDir -ChildPath "../../../Common7/IDE/CommonExtensions/Microsoft/NuGet/NuGet.Build.Tasks.dll")
+        $versionInfo = Get-ChildItem $nugetClientPath | % versioninfo | Select-Object FileVersion
+        Return $(($versionInfo -split '\n')[0]).TrimStart("@{").TrimEnd('}').Substring("FileVersion=".Length)
+    }
     Else
     {
         $output = . $nugetClientFilePath
@@ -198,6 +215,7 @@ Function SetupNuGetFolders([string] $nugetClientFilePath, [string] $nugetFolders
 
     # This environment variable is not recognized by any NuGet client.
     $Env:NUGET_SOLUTION_PACKAGES_FOLDER_PATH = [System.IO.Path]::Combine($nugetFoldersPath, "sp")
+    $Env:DOTNET_MULTILEVEL_LOOKUP=0
 
     LocalsClearAll $nugetClientFilePath
 }
@@ -217,6 +235,7 @@ Function CleanNuGetFolders([string] $nugetClientFilePath, [string] $nugetFolders
     [Environment]::SetEnvironmentVariable("NUGET_PLUGINS_CACHE_PATH", $Null)
     [Environment]::SetEnvironmentVariable("NUGET_SOLUTION_PACKAGES_FOLDER_PATH", $Null)
     [Environment]::SetEnvironmentVariable("NUGET_FOLDERS_PATH", $Null)
+    [Environment]::SetEnvironmentVariable("DOTNET_MULTILEVEL_LOOKUP", $Null)
 }
 
 # Given a repository, a client and directories for the results/logs, runs the configured performance tests.
@@ -229,7 +248,8 @@ Function RunPerformanceTestsOnGitRepository(
     [string] $resultsFilePath,
     [string] $nugetFoldersPath,
     [string] $logsFolderPath,
-    [int] $iterationCount)
+    [int] $iterationCount,
+    [switch] $staticGraphRestore)
 {
     $solutionFilePath = SetupGitRepository -repository $repoUrl -commitHash $commitHash -sourceFolderPath $([System.IO.Path]::Combine($sourceRootFolderPath, $testCaseName))
     SetupNuGetFolders $nugetClientFilePath $nugetFoldersPath
@@ -239,7 +259,8 @@ Function RunPerformanceTestsOnGitRepository(
         -resultsFilePath $resultsFilePath `
         -logsFolderPath $logsFolderPath `
         -nugetFoldersPath $nugetFoldersPath `
-        -iterationCount $iterationCount
+        -iterationCount $iterationCount `
+        -staticGraphRestore:$staticGraphRestore
 }
 
 Function GetProcessorInfo()
@@ -328,9 +349,11 @@ Function RunRestore(
     [switch] $cleanHttpCache,
     [switch] $cleanPluginsCache,
     [switch] $killMsBuildAndDotnetExeProcesses,
-    [switch] $force)
+    [switch] $force,
+    [switch] $staticGraphRestore)
 {
     $isClientDotnetExe = IsClientDotnetExe $nugetClientFilePath
+    $isClientMSBuild = IsClientMSBuildExe $nugetClientFilePath
 
     If ($isClientDotnetExe -And $isPackagesConfig)
     {
@@ -371,6 +394,10 @@ Function RunRestore(
         {
             . $nugetClientFilePath nuget locals -c $localsArguments *>>$null
         }
+        ElseIf($isClientMSBuild)
+        {
+            . dotnet.exe nuget locals -c $localsArguments *>>$null
+        }
         Else
         {
             . $nugetClientFilePath locals -clear $localsArguments -Verbosity quiet
@@ -390,7 +417,14 @@ Function RunRestore(
 
     $arguments = [System.Collections.Generic.List[string]]::new()
 
-    $arguments.Add("restore")
+    If ($isClientMSBuild)
+    {
+        $arguments.Add("/t:restore")
+    }
+    Else 
+    {
+        $arguments.Add("restore")
+    }
     $arguments.Add($solutionFilePath)
 
     If ($isPackagesConfig)
@@ -413,20 +447,42 @@ Function RunRestore(
         {
             $arguments.Add("--force")
         }
+        ElseIf($isClientMSBuild)
+        {
+            $arguments.Add("/p:RestoreForce=true")
+        }
         Else
         {
             $arguments.Add("-Force")
         }
     }
 
-    If (!$isClientDotnetExe)
+    If (!$isClientDotnetExe -And !$isClientMSBuild)
     {
         $arguments.Add("-NonInteractive")
     }
+
+    If($isClientDotnetExe -Or $isClientMSBuild)
+    {   
+        If ($staticGraphRestore)
+        {
+            $staticGraphOutputValue = "true"
+            $arguments.Add("/p:RestoreUseStaticGraphEvaluation=true")
+        }
+        Else 
+        {
+            $staticGraphOutputValue = "false"
+        }
+    }
+    Else 
+    {
+        $staticGraphOutputValue = "N/A"
+    }
 
     $stopwatch = [System.Diagnostics.Stopwatch]::StartNew()
 
     $logs = . $nugetClientFilePath $arguments | Out-String
+    # TODO https://github.com/NuGet/Home/issues/9968
 
     $totalTime = $stopwatch.Elapsed.TotalSeconds
     $restoreCoreTime = ExtractRestoreElapsedTime $logs
@@ -457,15 +513,15 @@ Function RunRestore(
 
     If (!(Test-Path $resultsFilePath))
     {
-        $columnHeaders = "Client Name,Client Version,Solution Name,Test Run ID,Scenario Name,Total Time (seconds),Core Restore Time (seconds),Force," + `
+        $columnHeaders = "Client Name,Client Version,Solution Name,Test Run ID,Scenario Name,Total Time (seconds),Core Restore Time (seconds),Force,Static Graph," + `
             "Global Packages Folder .nupkg Count,Global Packages Folder .nupkg Size (MB),Global Packages Folder File Count,Global Packages Folder File Size (MB),Clean Global Packages Folder," + `
             "HTTP Cache File Count,HTTP Cache File Size (MB),Clean HTTP Cache,Plugins Cache File Count,Plugins Cache File Size (MB),Clean Plugins Cache,Kill MSBuild and dotnet Processes," + `
             "Processor Name,Processor Physical Core Count,Processor Logical Core Count"
 
         OutFileWithCreateFolders $resultsFilePath $columnHeaders
     }
 
-    $data = "$clientName,$clientVersion,$solutionName,$testRunId,$scenarioName,$totalTime,$restoreCoreTime,$force," + `
+    $data = "$clientName,$clientVersion,$solutionName,$testRunId,$scenarioName,$totalTime,$restoreCoreTime,$force,$staticGraphOutputValue," + `
         "$($globalPackagesFolderNupkgFilesInfo.Count),$($globalPackagesFolderNupkgFilesInfo.TotalSizeInMB),$($globalPackagesFolderFilesInfo.Count),$($globalPackagesFolderFilesInfo.TotalSizeInMB),$cleanGlobalPackagesFolder," + `
         "$($httpCacheFilesInfo.Count),$($httpCacheFilesInfo.TotalSizeInMB),$cleanHttpCache,$($pluginsCacheFilesInfo.Count),$($pluginsCacheFilesInfo.TotalSizeInMB),$cleanPluginsCache,$killMsBuildAndDotnetExeProcesses," + `
         "$($processorInfo.Name),$($processorInfo.NumberOfCores),$($processorInfo.NumberOfLogicalProcessors)"

scripts/perftests/README.md

@@ -22,4 +22,6 @@ The `PerformanceTestUtilities.ps1` script is a collection of utility functions t
 `RunPerformanceTests.ps1` allows you to execute the full test suite given a specific solution file, a NuGet Client and an output results file.
 `PerformanceTestRunner.ps1` will execute all the test cases added in the testCases folder with the `Test-*` pattern.
 
+Note that it's very important to initialize the script from a location that does not have a global.json in it's directory path. This can skew the results if you are dealing with SDK based projects.
+
 To run either the performance tests or the runner, run `Get-Help scriptName.ps1` and/or `Get-Help scriptName.ps1 -examples`

scripts/perftests/RunPerformanceTests.ps1

@@ -47,6 +47,9 @@ Skips force restores
 .PARAMETER skipNoOpRestores
 Skips no-op restore.
 
+.PARAMETER staticGraphRestore
+Uses static graph restore if applicable for the client.
+
 .EXAMPLE
 .\RunPerformanceTests.ps1 -nugetClientFilePath "C:\Program Files\dotnet\dotnet.exe" -solutionFilePath F:\NuGet.Client\NuGet.sln -resultsFilePath results.csv
 #>
@@ -65,7 +68,8 @@ Param(
     [switch] $skipCleanRestores,
     [switch] $skipColdRestores,
     [switch] $skipForceRestores,
-    [switch] $skipNoOpRestores
+    [switch] $skipNoOpRestores,
+    [switch] $staticGraphRestore
 )
 
 . "$PSScriptRoot\PerformanceTestUtilities.ps1"
@@ -179,14 +183,14 @@ Try
     {
         Log "Running 1x warmup restore"
         $enabledSwitches = @("cleanGlobalPackagesFolder", "cleanHttpCache", "cleanPluginsCache", "killMSBuildAndDotnetExeProcess")
-        If (!$skipForceRestores)
-        {
-            $enabledSwitches += "force"
-        }
         If ($isPackagesConfig)
         {
             $enabledSwitches += "isPackagesConfig"
         }
+        If ($staticGraphRestore)
+        {
+            $enabledSwitches += "staticGraphRestore"
+        }
         $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "warmup" -enabledSwitches $enabledSwitches
         RunRestore @arguments
     }
@@ -195,14 +199,14 @@ Try
     {
         Log "Running $($iterationCount)x clean restores"
         $enabledSwitches = @("cleanGlobalPackagesFolder", "cleanHttpCache", "cleanPluginsCache", "killMSBuildAndDotnetExeProcess")
-        If (!$skipForceRestores)
-        {
-            $enabledSwitches += "force"
-        }
         If ($isPackagesConfig)
         {
             $enabledSwitches += "isPackagesConfig"
         }
+        If ($staticGraphRestore)
+        {
+            $enabledSwitches += "staticGraphRestore"
+        }
         $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "arctic" -enabledSwitches $enabledSwitches
         1..$iterationCount | % { RunRestore @arguments }
     }
@@ -211,29 +215,41 @@ Try
     {
         Log "Running $($iterationCount)x without a global packages folder"
         $enabledSwitches = @("cleanGlobalPackagesFolder", "killMSBuildAndDotnetExeProcess")
-        If (!$skipForceRestores)
-        {
-            $enabledSwitches += "force"
-        }
         If ($isPackagesConfig)
         {
             $enabledSwitches += "isPackagesConfig"
         }
+        If ($staticGraphRestore)
+        {
+            $enabledSwitches += "staticGraphRestore"
+        }
         $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "cold" -enabledSwitches $enabledSwitches
         1..$iterationCount | % { RunRestore @arguments }
     }
 
     If (!$skipForceRestores)
     {
         Log "Running $($iterationCount)x force restores"
-        $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "force" -enabledSwitches @("force")
+        $enabledSwitches = @("force")
+        If ($staticGraphRestore)
+        {
+            $enabledSwitches += "staticGraphRestore"
+        }
+        $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "force" -enabledSwitches $enabledSwitches
         1..$iterationCount | % { RunRestore @arguments }
     }
 
     If (!$skipNoOpRestores)
     {
         Log "Running $($iterationCount)x no-op restores"
-        $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "noop"
+        If ($staticGraphRestore)
+        {
+            $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "noop" -enabledSwitches @("staticGraphRestore")
+        }
+        Else
+        {
+            $arguments = CreateNugetClientArguments $solutionFilePath $nugetClientFilePath $resultsFilePath $logsFolderPath $solutionName $testRunId "noop"
+        }
         1..$iterationCount | % { RunRestore @arguments }
     }
 

scripts/perftests/testCases/Test-NuGetClient.ps1

@@ -14,7 +14,7 @@ Param(
 . "$PSScriptRoot\..\PerformanceTestUtilities.ps1"
 
 $repoUrl = "https://github.com/NuGet/NuGet.Client.git"
-$commitHash = "203c517a85791243f53ea08d404ee5b8fae36e35"
+$commitHash = "37c31cd7c1c2429a643a881fe637dc1d718d7259"
 $repoName = GenerateNameFromGitUrl $repoUrl
 $resultsFilePath = [System.IO.Path]::Combine($resultsFolderPath, "$repoName.csv")
 $sourcePath = $([System.IO.Path]::Combine($sourceRootFolderPath, $repoName))