Skip to content

Commit

Permalink
Add Benchmarks project. (#538)
Browse files Browse the repository at this point in the history
  • Loading branch information
jkoritzinsky committed Jan 11, 2021
1 parent e73c5de commit 54321e3
Show file tree
Hide file tree
Showing 6 changed files with 147 additions and 2 deletions.
2 changes: 1 addition & 1 deletion Directory.Build.props
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<TreatWarningsAsErrors>True</TreatWarningsAsErrors>
<DebugType>embedded</DebugType>
<DebugSymbols>true</DebugSymbols>
<LangVersion>Latest</LangVersion>
<LangVersion>9</LangVersion>
<IsPackable>true</IsPackable>

<!-- Set this property to false if you don't want to use the runtime
Expand Down
3 changes: 2 additions & 1 deletion DllImportGenerator/.gitignore
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
.vs/
**/bin
**/obj
*.binlog
*.binlog
BenchmarkDotNet.Artifacts
21 changes: 21 additions & 0 deletions DllImportGenerator/Benchmarks/Benchmarks.csproj
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0</TargetFramework>
<IsPackable>false</IsPackable>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="BenchmarkDotNet" Version="0.12.1" />
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\Ancillary.Interop\Ancillary.Interop.csproj" />
<ProjectReference Include="..\DllImportGenerator\DllImportGenerator.csproj" OutputItemType="Analyzer" ReferenceOutputAssembly="false" />
<ProjectReference Include="..\TestAssets\NativeExports\NativeExports.csproj" />
<ProjectReference Include="..\TestAssets\SharedTypes\SharedTypes.csproj" />
</ItemGroup>

</Project>
40 changes: 40 additions & 0 deletions DllImportGenerator/Benchmarks/BlittableTypes.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
using BenchmarkDotNet.Attributes;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Text;
using System.Threading.Tasks;

namespace Benchmarks
{
partial class NativeExportsNE
{
public const string NativeExportsNE_Binary = "Microsoft.Interop.Tests." + nameof(NativeExportsNE);

[GeneratedDllImport(NativeExportsNE_Binary, EntryPoint = "sumi")]
public static partial int Sum(int a, int b);

[GeneratedDllImport(NativeExportsNE_Binary, EntryPoint = "sumouti")]
public static partial void Sum(int a, int b, out int c);

[GeneratedDllImport(NativeExportsNE_Binary, EntryPoint = "sumrefi")]
public static partial void Sum(int a, ref int b);
}

public class BlittableTypes
{
[Benchmark]
public int IntByValue()
{
return NativeExportsNE.Sum(3, 42);
}

[Benchmark]
public int IntOutByRef()
{
NativeExportsNE.Sum(3, 42, out int result);
return result;
}
}
}
77 changes: 77 additions & 0 deletions DllImportGenerator/Benchmarks/Program.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
using BenchmarkDotNet.Configs;
using BenchmarkDotNet.Diagnosers;
using BenchmarkDotNet.Environments;
using BenchmarkDotNet.Jobs;
using BenchmarkDotNet.Loggers;
using BenchmarkDotNet.Running;
using BenchmarkDotNet.Toolchains;
using BenchmarkDotNet.Toolchains.CsProj;
using BenchmarkDotNet.Toolchains.DotNetCli;
using BenchmarkDotNet.Toolchains.Results;
using System;

namespace Benchmarks
{
class Program
{
static void Main(string[] args)
{
var config = DefaultConfig.Instance
.AddJob(GetDefaultJob())
.AddJob(GetForwarderJob())

This comment has been minimized.

Copy link
@adamsitnik

adamsitnik Apr 29, 2021

Member

@jkoritzinsky you need job.WithArguments(new[] { new MsBuildArgument("/p:SomeProperty=Value") })

This comment has been minimized.

Copy link
@jkoritzinsky

jkoritzinsky Apr 29, 2021

Author Member

That didn't fix the issue. It's still reproing locally. Is it possible that the way BenchmarkDotNet is building benchmarks in parallel is causing one file to overwrite the other? Is there a way to disable that parallel benchmark build?

This comment has been minimized.

Copy link
@adamsitnik

adamsitnik Apr 29, 2021

Member

In theory, it is possible. Does it work as expected when you define just a single job per config?

This doc explains how to do troubleshooting with BDN. To tell the long story short, you need to pass --keepFiles and see what BDN has generated.

If you confirm that this is the case, then what would work best would probably be to introduce a custom build configuration (as each is put into it's own folder) and use job.WithCustomBuildConfiguration($name)

this would work best with latest preview version of BDN (0.12.1 that you are using here is 1 year old). You can get it from:

<add key="benchmark-dotnet-prerelease" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/benchmark-dotnet-prerelease/nuget/v3/index.json" />

which is safe and MS-owned NuGet feed for BDN previews

This comment has been minimized.

Copy link
@jkoritzinsky

jkoritzinsky Apr 29, 2021

Author Member

I got it working with the separate custom build config. Thanks for the help!

.AddDiagnoser(MemoryDiagnoser.Default);

BenchmarkSwitcher.FromAssembly(typeof(Program).Assembly).Run(args, config);
}

private static Job GetDefaultJob()
{
var settings = NetCoreAppSettings.NetCoreApp50;

return Job.Default.WithToolchain(new Toolchain(".NET 6.0",
new CsProjGenerator(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.PackagesPath, "$(MicrosoftNETCoreAppVersion)"),
new DotNetCliBuilder(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.Timeout),
new DotNetCliExecutor(settings.CustomDotNetCliPath)))
.WithId("Generated");
}

private static Job GetForwarderJob()
{
var settings = NetCoreAppSettings.NetCoreApp50;
return Job.Default
.WithToolchain(new Toolchain(".NET 6.0 with Forwarders",
new CsProjGenerator(settings.TargetFrameworkMoniker, settings.CustomDotNetCliPath, settings.PackagesPath, "$(MicrosoftNETCoreAppVersion)"),
new GenerateForwarderBuilder(settings),
new DotNetCliExecutor(settings.CustomDotNetCliPath)))
.WithId("Built-in")
.AsBaseline();
}
}

internal class GenerateForwarderBuilder : IBuilder
{
private TimeSpan Timeout { get; }

private string TargetFrameworkMoniker { get; }

private string CustomDotNetCliPath { get; }

public GenerateForwarderBuilder(NetCoreAppSettings settings)
{
TargetFrameworkMoniker = settings.TargetFrameworkMoniker;
CustomDotNetCliPath = settings.CustomDotNetCliPath;
Timeout = settings.Timeout;
}

public BuildResult Build(GenerateResult generateResult, BuildPartition buildPartition, ILogger logger)
=> new DotNetCliCommand(
CustomDotNetCliPath,
"/p:DllImportGenerator_GenerateForwarders=true",
generateResult,
logger,
buildPartition,
Array.Empty<EnvironmentVariable>(),
Timeout)
.RestoreThenBuild();
}
}
6 changes: 6 additions & 0 deletions DllImportGenerator/DllImportGenerator.sln
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SharedTypes", "TestAssets\S
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "TestAssets", "TestAssets", "{C7134A89-8852-4FBE-B426-EFE007C4A819}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Benchmarks", "Benchmarks\Benchmarks.csproj", "{0914590B-C47A-4754-A7BE-E4CD843A92E4}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -67,6 +69,10 @@ Global
{D42D99E6-DA2B-48D5-BEED-F093A8E600BE}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D42D99E6-DA2B-48D5-BEED-F093A8E600BE}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D42D99E6-DA2B-48D5-BEED-F093A8E600BE}.Release|Any CPU.Build.0 = Release|Any CPU
{0914590B-C47A-4754-A7BE-E4CD843A92E4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{0914590B-C47A-4754-A7BE-E4CD843A92E4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{0914590B-C47A-4754-A7BE-E4CD843A92E4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{0914590B-C47A-4754-A7BE-E4CD843A92E4}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down

0 comments on commit 54321e3

Please sign in to comment.