-
-
Notifications
You must be signed in to change notification settings - Fork 21
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
* introduce TestUtils lib * save * build before test * fix compile * fix ci action * implement benchmark * implement benchmark action * implement benchmark * fix benchmark tool * fix merge * try new benchmark * only net6 and net7 * json report * fix path * fix ci * add benchmark ui URL * fix ci --------- Co-authored-by: Meir Blachman <meblachm@microsoft.com>
- Loading branch information
1 parent
57bb1ca
commit 3d49dce
Showing
23 changed files
with
338 additions
and
147 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,31 @@ | ||
| name: Benchmark tests | ||
| on: [pull_request] | ||
|
|
||
| permissions: | ||
| contents: write | ||
| deployments: write | ||
|
|
||
| jobs: | ||
| benchmark: | ||
| name: Performance regression check | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - uses: actions/checkout@v3 | ||
| - name: Setup .NET | ||
| uses: actions/setup-dotnet@v3 | ||
| with: | ||
| dotnet-version: '6.x' | ||
| - name: Run benchmark | ||
| run: cd src/FluentAssertions.Analyzers.BenchmarkTests && dotnet run -c Release --exporters json --filter '*' | ||
|
|
||
| - name: Store benchmark result | ||
| uses: rhysd/github-action-benchmark@v1 | ||
| with: | ||
| name: FluentAssertions.Analyzers Benchmark | ||
| tool: 'benchmarkdotnet' | ||
| output-file-path: src/FluentAssertions.Analyzers.BenchmarkTests/BenchmarkDotNet.Artifacts/results/FluentAssertions.Analyzers.BenchmarkTests.FluentAssertionsBenchmarks-report.json | ||
| github-token: ${{ secrets.GITHUB_TOKEN }} | ||
| auto-push: true | ||
| alert-threshold: '200%' | ||
| comment-on-alert: true | ||
| fail-on-alert: true |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
17 changes: 17 additions & 0 deletions
17
...luentAssertions.Analyzers.BenchmarkTests/FluentAssertions.Analyzers.BenchmarkTests.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
| <OutputType>Exe</OutputType> | ||
| <IsPackable>false</IsPackable> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="BenchmarkDotNet" Version="0.13.5" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\FluentAssertions.Analyzers.TestUtils\FluentAssertions.Analyzers.TestUtils.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
51 changes: 51 additions & 0 deletions
51
src/FluentAssertions.Analyzers.BenchmarkTests/FluentAssertionsBenchmarks.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| using BenchmarkDotNet.Attributes; | ||
| using BenchmarkDotNet.Jobs; | ||
| using Microsoft.CodeAnalysis; | ||
| using Microsoft.CodeAnalysis.Diagnostics; | ||
| using System.Collections.Immutable; | ||
| using FluentAssertions.Analyzers.TestUtils; | ||
| using System.Threading.Tasks; | ||
| using System; | ||
| using System.Collections.Generic; | ||
|
|
||
| namespace FluentAssertions.Analyzers.BenchmarkTests | ||
| { | ||
| [SimpleJob(RuntimeMoniker.Net60, baseline: true)] | ||
| [SimpleJob(RuntimeMoniker.Net70)] | ||
| [JsonExporter] | ||
| public class FluentAssertionsBenchmarks | ||
| { | ||
| private CompilationWithAnalyzers MinimalCompiliation; | ||
|
|
||
| [GlobalSetup] | ||
| public async Task GlobalSetup() | ||
| { | ||
| MinimalCompiliation = await CreateCompilationFromAsync(GenerateCode.ObjectStatement("actual.Should().Equals(expected);")); | ||
| } | ||
|
|
||
| [Benchmark] | ||
| public Task MinimalCompilation() | ||
| { | ||
| return MinimalCompiliation.GetAnalyzerDiagnosticsAsync(); | ||
| } | ||
|
|
||
| private async Task<CompilationWithAnalyzers> CreateCompilationFromAsync(params string[] sources) | ||
| { | ||
| var project = CsProjectGenerator.CreateProject(sources); | ||
| var compilation = await project.GetCompilationAsync(); | ||
|
|
||
| if (compilation is null) | ||
| { | ||
| throw new InvalidOperationException("Compilation is null"); | ||
| } | ||
|
|
||
| return compilation.WithOptions(compilation.Options.WithSpecificDiagnosticOptions(new Dictionary<string, ReportDiagnostic> | ||
| { | ||
| ["CS1701"] = ReportDiagnostic.Suppress, // Binding redirects | ||
| ["CS1702"] = ReportDiagnostic.Suppress, | ||
| ["CS1705"] = ReportDiagnostic.Suppress, | ||
| ["CS8019"] = ReportDiagnostic.Suppress // TODO: Unnecessary using directive | ||
| })).WithAnalyzers(CodeAnalyzersUtils.GetAllAnalyzers().ToImmutableArray()); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| using BenchmarkDotNet.Running; | ||
|
|
||
| namespace FluentAssertions.Analyzers.BenchmarkTests | ||
| { | ||
| public class Program | ||
| { | ||
| public static void Main() | ||
| => BenchmarkDotNet.Running.BenchmarkRunner.Run<FluentAssertionsBenchmarks>(); | ||
| } | ||
| } |
23 changes: 23 additions & 0 deletions
23
src/FluentAssertions.Analyzers.TestUtils/CodeAnalyzersUtils.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| using System; | ||
| using System.Linq; | ||
| using Microsoft.CodeAnalysis.Diagnostics; | ||
|
|
||
| namespace FluentAssertions.Analyzers.TestUtils | ||
| { | ||
| public class CodeAnalyzersUtils | ||
| { | ||
| private static readonly DiagnosticAnalyzer[] AllAnalyzers = CreateAllAnalyzers(); | ||
|
|
||
| public static DiagnosticAnalyzer[] GetAllAnalyzers() => AllAnalyzers; | ||
|
|
||
| private static DiagnosticAnalyzer[] CreateAllAnalyzers() | ||
| { | ||
| var assembly = typeof(Constants).Assembly; | ||
| var analyzersTypes = assembly.GetTypes() | ||
| .Where(type => !type.IsAbstract && typeof(DiagnosticAnalyzer).IsAssignableFrom(type)); | ||
| var analyzers = analyzersTypes.Select(type => (DiagnosticAnalyzer)Activator.CreateInstance(type)); | ||
|
|
||
| return analyzers.ToArray(); | ||
| } | ||
| } | ||
| } |
141 changes: 141 additions & 0 deletions
141
src/FluentAssertions.Analyzers.TestUtils/CsProjectGenerator.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,141 @@ | ||
| using System; | ||
| using System.Collections.Immutable; | ||
| using System.Linq; | ||
| using System.Reflection; | ||
| using FluentAssertions.Execution; | ||
| using Microsoft.CodeAnalysis; | ||
| using Microsoft.CodeAnalysis.CSharp; | ||
| using Microsoft.CodeAnalysis.Text; | ||
|
|
||
| using XunitAssert = Xunit.Assert; | ||
| using System.Net.Http; | ||
| using System.Collections.Concurrent; | ||
| using System.Collections.ObjectModel; | ||
|
|
||
| namespace FluentAssertions.Analyzers.TestUtils | ||
| { | ||
| public class CsProjectGenerator | ||
| { | ||
| static CsProjectGenerator() | ||
| { | ||
| References = new[] | ||
| { | ||
| typeof(object), // System.Private.CoreLib | ||
| typeof(Console), // System | ||
| typeof(Uri), // System.Private.Uri | ||
| typeof(Enumerable), // System.Linq | ||
| typeof(CSharpCompilation), // Microsoft.CodeAnalysis.CSharp | ||
| typeof(Compilation), // Microsoft.CodeAnalysis | ||
| typeof(AssertionScope), // FluentAssertions.Core | ||
| typeof(AssertionExtensions), // FluentAssertions | ||
| typeof(HttpRequestMessage), // System.Net.Http | ||
| typeof(ImmutableArray), // System.Collections.Immutable | ||
| typeof(ConcurrentBag<>), // System.Collections.Concurrent | ||
| typeof(ReadOnlyDictionary<,>), // System.ObjectModel | ||
| typeof(Microsoft.VisualStudio.TestTools.UnitTesting.Assert), // MsTest | ||
| typeof(XunitAssert), // Xunit | ||
| }.Select(type => type.GetTypeInfo().Assembly.Location) | ||
| .Append(GetSystemAssemblyPathByName("System.Globalization.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Text.RegularExpressions.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Runtime.Extensions.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Data.Common.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Threading.Tasks.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Runtime.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Reflection.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Xml.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Xml.XDocument.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Private.Xml.Linq.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Linq.Expressions.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Collections.dll")) | ||
| .Append(GetSystemAssemblyPathByName("netstandard.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Xml.ReaderWriter.dll")) | ||
| .Append(GetSystemAssemblyPathByName("System.Private.Xml.dll")) | ||
| .Select(location => (MetadataReference)MetadataReference.CreateFromFile(location)) | ||
| .ToImmutableArray(); | ||
|
|
||
| string GetSystemAssemblyPathByName(string assemblyName) | ||
| { | ||
| var root = System.IO.Path.GetDirectoryName(typeof(object).Assembly.Location); | ||
| return System.IO.Path.Combine(root, assemblyName); | ||
| } | ||
| } | ||
| // based on http://code.fitness/post/2017/02/using-csharpscript-with-netstandard.html | ||
| public static string GetSystemAssemblyPathByName(string assemblyName) | ||
| { | ||
| var root = System.IO.Path.GetDirectoryName(typeof(object).Assembly.Location); | ||
| return System.IO.Path.Combine(root, assemblyName); | ||
| } | ||
|
|
||
| private static readonly ImmutableArray<MetadataReference> References; | ||
|
|
||
| private static readonly string DefaultFilePathPrefix = "Test"; | ||
| private static readonly string CSharpDefaultFileExt = "cs"; | ||
| private static readonly string VisualBasicDefaultExt = "vb"; | ||
| private static readonly string TestProjectName = "TestProject"; | ||
|
|
||
| /// <summary> | ||
| /// Given an array of strings as sources and a language, turn them into a project and return the documents and spans of it. | ||
| /// </summary> | ||
| /// <param name="sources">Classes in the form of strings</param> | ||
| /// <param name="language">The language the source code is in</param> | ||
| /// <returns>A Tuple containing the Documents produced from the sources and their TextSpans if relevant</returns> | ||
| public static Document[] GetDocuments(string[] sources, string language) | ||
| { | ||
| if (language != LanguageNames.CSharp && language != LanguageNames.VisualBasic) | ||
| { | ||
| throw new ArgumentException("Unsupported Language"); | ||
| } | ||
|
|
||
| var project = CreateProject(sources, language); | ||
| var documents = project.Documents.ToArray(); | ||
|
|
||
| if (sources.Length != documents.Length) | ||
| { | ||
| throw new SystemException("Amount of sources did not match amount of Documents created"); | ||
| } | ||
|
|
||
| return documents; | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Create a Document from a string through creating a project that contains it. | ||
| /// </summary> | ||
| /// <param name="source">Classes in the form of a string</param> | ||
| /// <param name="language">The language the source code is in</param> | ||
| /// <returns>A Document created from the source string</returns> | ||
| public static Document CreateDocument(string source, string language = LanguageNames.CSharp) | ||
| { | ||
| return CreateProject(new[] { source }, language).Documents.First(); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Create a project using the inputted strings as sources. | ||
| /// </summary> | ||
| /// <param name="sources">Classes in the form of strings</param> | ||
| /// <param name="language">The language the source code is in</param> | ||
| /// <returns>A Project created out of the Documents created from the source strings</returns> | ||
| public static Project CreateProject(string[] sources, string language = LanguageNames.CSharp) | ||
| { | ||
| string fileNamePrefix = DefaultFilePathPrefix; | ||
| string fileExt = language == LanguageNames.CSharp ? CSharpDefaultFileExt : VisualBasicDefaultExt; | ||
|
|
||
| var projectId = ProjectId.CreateNewId(debugName: TestProjectName); | ||
|
|
||
| var solution = new AdhocWorkspace() | ||
| .CurrentSolution | ||
| .AddProject(projectId, TestProjectName, TestProjectName, language) | ||
| .AddMetadataReferences(projectId, References); | ||
|
|
||
| int count = 0; | ||
| foreach (var source in sources) | ||
| { | ||
| var newFileName = fileNamePrefix + count + "." + fileExt; | ||
| var documentId = DocumentId.CreateNewId(projectId, debugName: newFileName); | ||
| solution = solution.AddDocument(documentId, newFileName, SourceText.From(source)); | ||
| count++; | ||
| } | ||
| return solution.GetProject(projectId); | ||
| } | ||
| } | ||
| } | ||
|
|
21 changes: 21 additions & 0 deletions
21
src/FluentAssertions.Analyzers.TestUtils/FluentAssertions.Analyzers.TestUtils.csproj
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,21 @@ | ||
| <Project Sdk="Microsoft.NET.Sdk"> | ||
|
|
||
| <PropertyGroup> | ||
| <TargetFramework>net6.0</TargetFramework> | ||
|
|
||
| <IsPackable>false</IsPackable> | ||
| </PropertyGroup> | ||
|
|
||
| <ItemGroup> | ||
| <PackageReference Include="FluentAssertions" Version="6.1.0" /> | ||
| <PackageReference Include="Microsoft.NET.Test.Sdk" Version="16.3.0" /> | ||
| <PackageReference Include="MSTest.TestAdapter" Version="2.0.0" /> | ||
| <PackageReference Include="MSTest.TestFramework" Version="2.0.0" /> | ||
| <PackageReference Include="xunit.assert" Version="2.4.2" /> | ||
| </ItemGroup> | ||
|
|
||
| <ItemGroup> | ||
| <ProjectReference Include="..\FluentAssertions.Analyzers\FluentAssertions.Analyzers.csproj" /> | ||
| </ItemGroup> | ||
|
|
||
| </Project> |
2 changes: 1 addition & 1 deletion
2
...ssertions.Analyzers.Tests/GenerateCode.cs → ...tions.Analyzers.TestUtils/GenerateCode.cs
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.