Refactor perf tests and create a YAML perf build (#3297)

* Fix issue with OBO and other token tests. Add Debug flag. Use explicit type in benchmarks. * Join results summary * Remove Random use and IterationSetup. * Disable GetAccounts test. Use (1, 1k) instead of (1, 10k). * Upgrade MSBuild.Sdk.Extras to 3.0.44 * Add build yaml. * PR feedback. Add comment.
AzureAD · May 12, 2022 · e68fd64 · e68fd64
1 parent 8b3b6f1
commit e68fd64
Show file tree

Hide file tree

Showing 10 changed files with 98 additions and 85 deletions.
diff --git a/LibsAndSamples.sln b/LibsAndSamples.sln
@@ -102,6 +102,7 @@ EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pipelines", "pipelines", "{EA983957-DC2F-4D9E-A2E3-79ADEB5CEB26}"
 	ProjectSection(SolutionItems) = preProject
 		build\pipeline-ci.yaml = build\pipeline-ci.yaml
+		build\pipeline-perf-tests-automation.yaml = build\pipeline-perf-tests-automation.yaml
 		build\pipeline-pullrequest.yaml = build\pipeline-pullrequest.yaml
 		build\pipeline-releasebuild.yaml = build\pipeline-releasebuild.yaml
 		build\pipeline-uiautomation.yaml = build\pipeline-uiautomation.yaml

diff --git a/build/pipeline-perf-tests-automation.yaml b/build/pipeline-perf-tests-automation.yaml
@@ -0,0 +1,48 @@
+# Build for performance tests
+trigger:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - src/client/Microsoft.Identity.Client/*
+
+pr: none
+
+pool:
+  name: MsalNetPerfTest1ESPool
+  demands: msbuild
+
+variables:
+  BuildPlatform: 'any cpu'
+  BuildConfiguration: 'release'
+  MSAL_DESKTOP_ONLY_DEV: 'false' # skips building mobile and legacy targets
+
+jobs:
+  - job: 'BuildJob'
+    displayName: 'Build and run tests'
+    steps:
+    - task: NuGetToolInstaller@1
+      displayName: 'Use NuGet 5.x'
+      inputs:
+        versionSpec: 5.x
+
+    - task: NuGetCommand@2
+      displayName: 'NuGet restore'
+      inputs:
+        restoreSolution: PerformanceTests.sln
+
+    - task: MSBuild@1
+      displayName: Build PerformanceTests.sln
+      inputs:
+        solution: PerformanceTests.sln
+        msbuildArchitecture: x64
+        platform: '$(BuildPlatform)'
+        configuration: '$(BuildConfiguration)'
+        msbuildArguments: '/p:RunCodeAnalysis=false /p:MSAL_DESKTOP_ONLY_DEV=$(MSAL_DESKTOP_ONLY_DEV)'
+        clean: true
+
+    # The parameters currently are not used, but can be uploaded into the reporting system in the future.
+    # Currently the build runs automatically but the results have to be compared manually.
+    - script: '$(Build.SourcesDirectory)\tests\Microsoft.Identity.Test.Performance\bin\$(BuildConfiguration)\$(BuildRuntime)\Microsoft.Identity.Test.Performance.exe -BranchName $(Build.SourceBranch) -BranchCommit $(Build.SourceVersion)'
+      displayName: 'Run performance tests'
diff --git a/global.json b/global.json
@@ -1,5 +1,5 @@
 {
     "msbuild-sdks": {
-        "MSBuild.Sdk.Extras": "3.0.23"
+        "MSBuild.Sdk.Extras": "3.0.44"
     }
 }
diff --git a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForClientCacheTests.cs
@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-using System;
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
@@ -19,10 +18,10 @@
 namespace Microsoft.Identity.Test.Performance
 {
     /// <summary>
-    /// Used to test the performance of token cache with large amount of items.
+    /// Used to test the performance of acquiring tokens using token cache with different number of items.
     /// </summary>
     /// <remarks>
-    /// For app cache, the number of partitions is the number of tenants
+    /// For app cache, the number of partitions is the number of tenants.
     /// </remarks>
     [MeanColumn, StdDevColumn, MedianColumn, MinColumn, MaxColumn]
     public class AcquireTokenForClientCacheTests
@@ -42,10 +41,8 @@ public class AcquireTokenForClientCacheTests
         // This is a workaround to specify the exact param combinations to be used.
         public IEnumerable<(int, int)> CacheSizeSource => new[] {
             (1, 10),
-            (1, 10000),
-            (1000, 10),
+            (1, 1000),
             (10000, 10),
-            (100000, 10),
         };
 
         [ParamsAllValues]
@@ -79,22 +76,17 @@ public async Task GlobalSetupAsync()
             }
 
             await PopulateAppCacheAsync(_cca, CacheSize.TotalTenants, CacheSize.TokensPerTenant, EnableCacheSerialization).ConfigureAwait(false);
-        }
 
-        [IterationSetup]
-        public void IterationSetup()
-        {
-            Random random = new Random();
-            _tenantId = $"{_tenantPrefix}{random.Next(0, CacheSize.TotalTenants)}";
-            _scope = $"{_scopePrefix}{random.Next(0, CacheSize.TokensPerTenant)}";
+            _tenantId = $"{_tenantPrefix}0";
+            _scope = $"{_scopePrefix}0";
         }
 
         [Benchmark(Description = "AcquireTokenForClient")]
         [BenchmarkCategory("With cache")]
         public async Task<AuthenticationResult> AcquireTokenForClient_TestAsync()
         {
             return await _cca.AcquireTokenForClient(new[] { _scope })
-              .WithAuthority($"https://login.microsoftonline.com/{_tenantId}")
+              .WithTenantId(_tenantId)
               .ExecuteAsync()
               .ConfigureAwait(false);
         }

diff --git a/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs b/tests/Microsoft.Identity.Test.Performance/AcquireTokenForOboCacheTests.cs
@@ -1,7 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-using System;
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
@@ -18,7 +17,7 @@
 namespace Microsoft.Identity.Test.Performance
 {
     /// <summary>
-    /// Used to test the performance of token cache with large amount of items.
+    /// Used to test the performance of acquiring tokens using token cache with different amount of items.
     /// </summary>
     /// <remarks>
     /// For OBO user cache, the partition key is
@@ -33,7 +32,7 @@ public class AcquireTokenForOboCacheTests
         private ConfidentialClientApplication _cca;
         private InMemoryCache _serializationCache;
         private string _scope;
-        private string _authority;
+        private string _tenantId;
         private UserAssertion _userAssertion;
 
         // i.e. (partitions, tokens per partition)
@@ -44,10 +43,8 @@ public class AcquireTokenForOboCacheTests
         // This is a workaround to specify the exact param combinations to be used.
         public IEnumerable<(int, int)> CacheSizeSource => new[] {
             (1, 10),
-            (1, 10000),
-            (1000, 10),
+            (1, 1000),
             (10000, 10),
-            (100000, 10),
         };
 
         [ParamsAllValues]
@@ -85,26 +82,18 @@ public async Task GlobalSetupAsync()
             }
 
             await PopulateUserCacheAsync(CacheSize.TotalUsers, CacheSize.TokensPerUser, EnableCacheSerialization).ConfigureAwait(false);
-        }
 
-        [IterationSetup]
-        public void IterationSetup()
-        {
-            Random random = new Random();
-            _userAssertion = new UserAssertion($"{TestConstants.DefaultAccessToken}{random.Next(0, CacheSize.TotalUsers)}");
-            string id = random.Next(0, CacheSize.TokensPerUser).ToString();
-            _scope = $"{_scopePrefix}{id}";
-            _authority = IsMultiTenant ?
-                $"https://{TestConstants.ProductionPrefNetworkEnvironment}/{_tenantPrefix}{id}" :
-                $"https://{TestConstants.ProductionPrefNetworkEnvironment}/{_tenantPrefix}";
+            _userAssertion = new UserAssertion($"{TestConstants.DefaultAccessToken}0");
+            _scope = $"{_scopePrefix}0";
+            _tenantId = IsMultiTenant ? $"{_tenantPrefix}0" : _tenantPrefix;
         }
 
         [Benchmark(Description = "AcquireTokenForOBO")]
         [BenchmarkCategory("With cache")]
         public async Task<AuthenticationResult> AcquireTokenOnBehalfOf_TestAsync()
         {
             return await _cca.AcquireTokenOnBehalfOf(new[] { _scope }, _userAssertion)
-                .WithAuthority(_authority)
+                .WithTenantId(_tenantId)
                 .ExecuteAsync()
                 .ConfigureAwait(false);
         }
@@ -138,8 +127,7 @@ private async Task PopulateUserCacheAsync(int totalUsers, int tokensPerUser, boo
                     MsalIdTokenCacheItem idtItem = TokenCacheHelper.CreateIdTokenCacheItem(
                         tenant,
                         homeAccountId,
-                        uid: user.ToString(),
-                        idToken: TestConstants.IdToken);
+                        uid: user.ToString());
                     _cca.UserTokenCacheInternal.Accessor.SaveIdToken(idtItem);
 
                     MsalAccountCacheItem accItem = TokenCacheHelper.CreateAccountItem(tenant, homeAccountId);

diff --git a/tests/Microsoft.Identity.Test.Performance/AcquireTokenNoCacheTests.cs b/tests/Microsoft.Identity.Test.Performance/AcquireTokenNoCacheTests.cs
@@ -14,15 +14,15 @@
 namespace Microsoft.Identity.Test.Performance
 {
     /// <summary>
-    /// Used to test the performance of acquire tokens without cache and mocked network calls.
+    /// Used to test the performance of acquiring tokens without cache and mocked network calls.
     /// </summary>
     [MeanColumn, StdDevColumn, MedianColumn, MinColumn, MaxColumn]
     public class AcquireTokenNoCacheTests
     {
         private ConfidentialClientApplication _cca;
         private MockHttpManager _httpManager;
-        private string[] _scope = TestConstants.s_scope.ToArray();
-        private UserAssertion _userAssertion = new UserAssertion(TestConstants.DefaultAccessToken);
+        private readonly string[] _scope = TestConstants.s_scope.ToArray();
+        private readonly UserAssertion _userAssertion = new UserAssertion(TestConstants.DefaultAccessToken);
 
         [GlobalSetup(Target = nameof(AcquireTokenForClient_TestAsync))]
         public void GlobalSetup_ForClient()

diff --git a/tests/Microsoft.Identity.Test.Performance/CryptoManagerTests.cs b/tests/Microsoft.Identity.Test.Performance/CryptoManagerTests.cs
@@ -69,8 +69,7 @@ public void IterationSetup()
         [Benchmark]
         public async Task<AuthenticationResult> BenchmarkAsync()
         {
-            var result = await _requests[_requestIdx].ExecuteAsync(System.Threading.CancellationToken.None).ConfigureAwait(true);
-            return result;
+            return await _requests[_requestIdx].ExecuteAsync(System.Threading.CancellationToken.None).ConfigureAwait(true);
         }
 
         private void AddHostToInstanceCache(IServiceBundle serviceBundle, string host)

diff --git a/tests/Microsoft.Identity.Test.Performance/LegacyCacheOperationsTests.cs b/tests/Microsoft.Identity.Test.Performance/LegacyCacheOperationsTests.cs
@@ -50,22 +50,19 @@ public void GlobalSetup()
         [Benchmark(Description = "SaveToken")]
         public async Task<string> SaveTokenResponseTestAsync()
         {
-            var result = await _cache.SaveTokenResponseAsync(_requestParams, _response).ConfigureAwait(true);
-            return result.Item1.ClientId;
+            return (await _cache.SaveTokenResponseAsync(_requestParams, _response).ConfigureAwait(true)).Item1.ClientId;
         }
 
         [Benchmark(Description = "FindToken")]
         public async Task<string> FindRefreshTokenTestAsync()
         {
-            var result = await _cache.FindRefreshTokenAsync(_requestParams).ConfigureAwait(true);
-            return result?.ClientId;
+            return (await _cache.FindRefreshTokenAsync(_requestParams).ConfigureAwait(true))?.ClientId;
         }
 
         [Benchmark(Description = "GetAllUsers")]
         public async Task GetAllAdalUsersTestAsync()
         {
-            var result = await _cache.GetAccountsAsync(_requestParams).ConfigureAwait(true);
-            result.Consume(_consumer);
+            (await _cache.GetAccountsAsync(_requestParams).ConfigureAwait(true)).Consume(_consumer);
         }
 
         [Benchmark(Description = "RemoveUser")]

diff --git a/tests/Microsoft.Identity.Test.Performance/Program.cs b/tests/Microsoft.Identity.Test.Performance/Program.cs
@@ -21,8 +21,15 @@ public static void Main(string[] args)
                     typeof(AcquireTokenForClientCacheTests),
                     typeof(AcquireTokenForOboCacheTests),
                     typeof(TokenCacheTests),
-            }).RunAll(DefaultConfig.Instance
+                    typeof(AcquireTokenNoCacheTests),
+            }).RunAll(
+#if DEBUG
+                    new DebugInProcessConfig()
+#else
+                    DefaultConfig.Instance
+#endif
                 .WithOptions(ConfigOptions.DisableLogFile)
+                .WithOptions(ConfigOptions.JoinSummary)
                 //.WithOptions(ConfigOptions.DontOverwriteResults) // Uncomment when running manually
                 .AddDiagnoser(MemoryDiagnoser.Default) // https://benchmarkdotnet.org/articles/configs/diagnosers.html
                                                        //.AddDiagnoser(new EtwProfiler()) // https://adamsitnik.com/ETW-Profiler/
@@ -32,7 +39,9 @@ public static void Main(string[] args)
             }
             catch (Exception ex)
             {
+                Logger.Log("Error running performance tests.");
                 Logger.Log(ex.ToString());
+                throw;
             }
 
             Logger.Log("Completed running performance tests.");
@@ -41,7 +50,7 @@ public static void Main(string[] args)
 
     public static class Logger
     {
-        private const string LogPrefix = "[Microsoft.Identity.Test.Performance]";
+        private const string LogPrefix = "[Test.Performance]";
         public static void Log(string message) => Console.WriteLine($"{LogPrefix} {message}");
     }
 }