ilrepack

.azuredevops/pipelines/templates/build-and-test.yml

@@ -7,20 +7,9 @@ steps:
 - task: NuGetAuthenticate@1
   displayName: NuGet Authenticate
 
-- task: DotNetCoreCLI@2
-  displayName: Restore
-  inputs:
-    command: restore
-    projects: $(Build.SourcesDirectory)/MSBuildCache.sln
-    feedsToUse: 'config'
-    nugetConfigPath: '$(Build.SourcesDirectory)/nuget.config'
-
-- task: DotNetCoreCLI@2
+- script: |
+    dotnet build $(Build.SourcesDirectory)/MSBuildCache.sln --configuration $(BuildConfiguration) -BinaryLogger:$(LogDirectory)/msbuild.binlog
   displayName: Build
-  inputs:
-    command: build
-    projects: $(Build.SourcesDirectory)/MSBuildCache.sln
-    arguments: -restore:false --configuration $(BuildConfiguration) -BinaryLogger:$(LogDirectory)/msbuild.binlog
 
 - task: DotNetCoreCLI@2
   displayName: Run Unit Tests

Directory.Packages.props

@@ -8,6 +8,7 @@
   <ItemGroup>
     <PackageVersion Include="coverlet.collector" Version="3.1.2" />
     <PackageVersion Include="DotNet.Glob" Version="2.0.3" />
+    <PackageVersion Include="ILRepack" Version="2.0.27" />
     <PackageVersion Include="Microsoft.Bcl.AsyncInterfaces" Version="7.0.0" />
     <PackageVersion Include="Microsoft.Bcl.HashCode" Version="1.1.1" />
     <PackageVersion Include="Microsoft.Build" Version="$(MSBuildPackageVersion)" />
@@ -35,8 +36,12 @@
     -->
     <PackageVersion Include="NETStandard.Library" Version="2.0.3"/>
     <PackageVersion Include="morelinq" Version="3.4.2" />
+    <PackageVersion Include="protobuf-net" Version="3.2.26" />
+    <PackageVersion Include="protobuf-net.Core" Version="3.2.26" />
+    <PackageVersion Include="protobuf-net.Grpc" Version="1.1.1" />
     <PackageVersion Include="System.Collections.Immutable" Version="7.0.0" />
     <PackageVersion Include="System.Memory" Version="4.5.5" />
+    <PackageVersion Include="System.Private.ServiceModel" Version="4.10.3" />
     <PackageVersion Include="System.Reflection.Metadata" Version="7.0.0" />
     <PackageVersion Include="System.Runtime.CompilerServices.Unsafe" Version="6.0.0" />
     <PackageVersion Include="System.Text.Json" Version="7.0.3" />

MSBuildCache.sln

@@ -32,6 +32,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Azur
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Local", "src\Local\Microsoft.MSBuildCache.Local.csproj", "{F6586428-E047-42C8-B0AC-048DF6DFAF18}"
 EndProject
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.MSBuildCache.Repack.Tests", "src\Repack.Tests\Microsoft.MSBuildCache.Repack.Tests.csproj", "{3BCB6452-B087-4A03-8418-C79F2715DDE7}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|x64 = Debug|x64
@@ -62,6 +64,10 @@ Global
 		{F6586428-E047-42C8-B0AC-048DF6DFAF18}.Debug|x64.Build.0 = Debug|x64
 		{F6586428-E047-42C8-B0AC-048DF6DFAF18}.Release|x64.ActiveCfg = Release|x64
 		{F6586428-E047-42C8-B0AC-048DF6DFAF18}.Release|x64.Build.0 = Release|x64
+		{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Debug|x64.ActiveCfg = Debug|x64
+		{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Debug|x64.Build.0 = Debug|x64
+		{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Release|x64.ActiveCfg = Release|x64
+		{3BCB6452-B087-4A03-8418-C79F2715DDE7}.Release|x64.Build.0 = Release|x64
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -73,6 +79,7 @@ Global
 		{D0195D37-E001-4283-B51A-A0B51B1D54D1} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
 		{97357681-C75E-445D-8547-46F312D01CED} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
 		{F6586428-E047-42C8-B0AC-048DF6DFAF18} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
+		{3BCB6452-B087-4A03-8418-C79F2715DDE7} = {EFFB5949-347C-4F28-8964-571D5C6B6209}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {F1CDA78F-A666-431B-BF44-56DA7DF193BA}

build/MSBuildExtensionPackage.targets

@@ -20,15 +20,73 @@
   </Target>
 
   <!-- Since this is an MSBuild extension which is loaded into the MSBuild process, much like a tool package we need to repackage our ProjectReferences and PackageReferences -->
-  <Target Name="IncludeReferenceCopyLocalPathsInBuildOutputInPackage" BeforeTargets="_GetBuildOutputFilesWithTfm" DependsOnTargets="ResolveReferences;ResolvePackageDependenciesForBuild;ResolveProjectReferences">
-    <ItemGroup>
+  <!-- When we are using ilrepack to directly merge the dependencies, we only need to copy the native files which don't get merged.-->
+  <Target Name="IncludeReferenceCopyLocalPathsInBuildOutputInPackage"
+          BeforeTargets="_GetBuildOutputFilesWithTfm"
+          DependsOnTargets="ResolveReferences;ResolvePackageDependenciesForBuild;ResolveProjectReferences"
+          >
+    <ItemGroup Condition=" '$(ILRepackEnabled)' != 'true' ">
       <BuildOutputInPackage Include="@(_ResolvedProjectReferencePaths)" TargetPath="%(RuntimeCopyLocalItems.DestinationSubDirectory)%(Filename)%(Extension)" />
-
       <BuildOutputInPackage Include="@(RuntimeCopyLocalItems-&gt;WithMetadataValue('AssetType', 'runtime'))" TargetPath="%(RuntimeCopyLocalItems.DestinationSubDirectory)%(Filename)%(Extension)" />
       <BuildOutputInPackage Include="@(RuntimeCopyLocalItems-&gt;WithMetadataValue('CopyLocal', 'true'))" TargetPath="%(RuntimeCopyLocalItems.DestinationSubDirectory)%(Filename)%(Extension)" />
-
+    </ItemGroup>
+
+    <ItemGroup>
       <!-- Somewhat hard-code this as there isn't a good way to identify it -->
       <BuildOutputInPackage Include="@(None-&gt;WithMetadataValue('Link', 'native\amd64\rocksdb.dll'))" TargetPath="%(None.Link)" />
     </ItemGroup>
   </Target>
+
+  <!-- ILRepack -->
+  <PropertyGroup Condition="'$(Configuration)' == 'Release'">
+    <ILRepackEnabled>true</ILRepackEnabled>
+  </PropertyGroup>
+
+  <ItemGroup Condition=" '$(ILRepackEnabled)' == 'true' ">
+    <!-- Make sure the types of the API between plug-in and MsBuild in in `Exclude`. CODESYNC[DO_NOT_ILMERGE_ASSEMBLIES] -->
+    <PluginAssemblies Include="System.Collections.Immutable.dll" />
+    <PluginAssemblies Include="Microsoft.Build.dll" />
+    <PluginAssemblies Include="Microsoft.Build.Framework.dll" />
+    <PluginAssemblies Include="Microsoft.Build.Utilities.Core.dll" />
+  </ItemGroup>
+
+  <ItemGroup Condition=" '$(ILRepackEnabled)' == 'true' ">
+    <PackageReference Include="ILRepack">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+    </PackageReference>
+
+    <!-- BuildXL.Cache.ContentStore.Distributed.dll depends on this but doesn't list it-->
+    <PackageReference Include="System.Private.ServiceModel"/>
+    <PackageReference Include="protobuf-net"/>
+    <PackageReference Include="protobuf-net.Core"/>
+    <PackageReference Include="protobuf-net.Grpc"/>
+  </ItemGroup>
+
+  <Target Name="ILRepacker" AfterTargets="Build" Condition="'$(ILRepack)' != ''">
+    <ItemGroup>
+      <LibraryPaths Include="$(OutputPath)"/>
+      <LibraryPaths Include="$(PkgMicrosoft_Build_Framework)\lib\net472"/>
+    </ItemGroup>
+
+    <ItemGroup>
+      <InputAssemblies 
+        Include="$(OutputPath)$(AssemblyName)$(TargetExt);$(OutputPath)*.dll" 
+        Exclude="$(OutputPath)grpc_*.dll;@(PluginAssemblies->'$(OutputPath)%(Filename)%(Extension)', ';')"
+    />
+    </ItemGroup>
+
+    <PropertyGroup>
+      <IlRepackCommand>$(ILRepack) /allowduplicateresources /union /parallel /out:$(OutputPath)$(AssemblyName)$(TargetExt) @(LibraryPaths->'/lib:%(FullPath)', ' ')</IlRepackCommand>
+      <IlRepackCommand Condition="'$(IlRepackVerbose)' == 'true'">$(IlRepackCommand) /verbose</IlRepackCommand>
+      <IlRepackCommand Condition="'$(DelaySign)' == 'true'">$(IlRepackCommand) /delaysign</IlRepackCommand>
+      <IlRepackCommand Condition="'$(AssemblyOriginatorKeyFile)' != ''">$(IlRepackCommand) /keyfile:$(AssemblyOriginatorKeyFile)</IlRepackCommand>
+      <IlRepackCommand>$(IlRepackCommand) @(InputAssemblies->'%(FullPath)', ' ')</IlRepackCommand>
+    </PropertyGroup>
+
+    <Exec
+      Command="$(IlRepackCommand)"
+      IgnoreStandardErrorWarningFormat="true"
+    />
+  </Target>
 </Project>

build/MSBuildReference.targets

@@ -10,7 +10,7 @@
   </ItemGroup>
   <ItemGroup Condition="'$(MSBuildLibraries)' == '' ">
     <PackageReference Include="Microsoft.Build" IncludeAssets="Compile" />
-    <PackageReference Include="Microsoft.Build.Framework" IncludeAssets="Compile" />
+    <PackageReference Include="Microsoft.Build.Framework" IncludeAssets="Compile" GeneratePathProperty="true"/>
     <PackageReference Include="Microsoft.Build.Utilities.Core" IncludeAssets="Compile" />
   </ItemGroup>
 </Project>

src/Common.Tests/PluginInterfaceTypeCheckTests.cs

@@ -0,0 +1,85 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Reflection;
+using Microsoft.Build.Experimental.ProjectCache;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.MSBuildCache.Tests;
+
+// Make sure that types used by the plugin interface are limited to the assemblies we expect
+[TestClass]
+public class PluginInterfaceTypeCheckTests
+{
+    public static readonly HashSet<string> PluginInterfaceNuGetAssemblies = new HashSet<string>()
+    {
+        // specific CODESYNC[DO_NOT_ILMERGE_ASSEMBLIES]
+        "Microsoft.Build.dll",
+        "Microsoft.Build.Framework.dll",
+        "Microsoft.Build.Utilities.Core.dll",
+        "System.Collections.Immutable.dll",
+    };
+
+    private static readonly HashSet<string> PluginInterfaceAssemblies = new HashSet<string>(PluginInterfaceNuGetAssemblies)
+    {
+        // general
+        "mscorlib.dll",
+        "System.Private.CoreLib.dll",
+        "System.Core.dll",
+    };
+
+    [TestMethod]
+    public void ProjectCachePluginBase()
+    {
+        CheckAssembliesForType(typeof(ProjectCachePluginBase));
+    }
+
+    private static void AssertAssembly(Type t)
+    {
+        Assert.IsTrue(PluginInterfaceAssemblies.Contains(Path.GetFileName(t.Assembly.Location)),
+            $"Type {t.FullName} is in assembly {t.Assembly.Location} which is not expected");
+    }
+
+    private static void CheckAssembliesForType(Type t)
+    {
+        var alreadyChecked = new HashSet<Type>();
+        CheckAssemblies(t, alreadyChecked, 5);
+        Assert.IsTrue(alreadyChecked.Count > 10, "Failed to find types.");
+    }
+
+    private static void CheckAssemblies(Type t, HashSet<Type> alreadyChecked, int depth)
+    {
+        if (depth <= 0 || t == null || !alreadyChecked.Add(t))
+        {
+            return;
+        }
+
+        if (t.FullName == null)
+        {
+            return;
+        }
+
+        AssertAssembly(t);
+        foreach (Type nested in t.GetNestedTypes(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance))
+        {
+            CheckAssemblies(nested, alreadyChecked, depth - 1);
+        }
+
+        foreach (PropertyInfo p in t.GetProperties(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance))
+        {
+            CheckAssemblies(p.PropertyType, alreadyChecked, depth - 1);
+        }
+
+        foreach (MethodInfo? m in t.GetMethods(BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance))
+        {
+            CheckAssemblies(m.ReturnType, alreadyChecked, depth - 1);
+            foreach (var p in m.GetParameters())
+            {
+                CheckAssemblies(p.ParameterType, alreadyChecked, depth - 1);
+            }
+        }
+    }
+}

src/Local/MSBuildCacheLocalPlugin.cs

@@ -38,7 +38,6 @@ protected override async Task<ICacheClient> CreateCacheClientAsync(PluginLoggerB
 #pragma warning restore CA2000 // Dispose objects before losing scope
         Context context = new(cacheLogger);
 
-
 #pragma warning disable CA2000 // Dispose objects before losing scope. Expected to be disposed by TwoLevelCache
         LocalCache cache = LocalCacheFactory.Create(cacheLogger, Settings.LocalCacheRootPath, Settings.LocalCacheSizeInMegabytes);
 #pragma warning restore CA2000 // Dispose objects before losing scope

src/Repack.Tests/Microsoft.MSBuildCache.Repack.Tests.csproj

@@ -0,0 +1,27 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <!-- Only supports x64 due to the RocksDB dependency -->
+    <Platform>x64</Platform>
+    <Platforms>AnyCPU;x64</Platforms>
+    <TargetFrameworks>net472;net8.0</TargetFrameworks>
+    <RootNamespace>Microsoft.MSBuildCache.Repack.Tests</RootNamespace>
+    <!-- Suppress "Avoid constant arrays as arguments". UTs have many one-off test data arrays. -->
+    <NoWarn>$(NoWarn);CA1861</NoWarn>
+    <!-- Suppress "Nested types should not be visible". UTs need to be "public" for MSTest but aren't actualy public, so nested typed might need to be exposed. -->
+    <NoWarn>$(NoWarn);CA1034</NoWarn>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="MSTest.TestAdapter" />
+    <PackageReference Include="MSTest.TestFramework" />
+    <PackageReference Include="coverlet.collector" />
+    <PackageReference Include="morelinq" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\AzureBlobStorage\Microsoft.MSBuildCache.AzureBlobStorage.csproj" />
+    <ProjectReference Include="..\AzurePipelines\Microsoft.MSBuildCache.AzurePipelines.csproj" />
+    <ProjectReference Include="..\Common.Tests\Microsoft.MSBuildCache.Common.Tests.csproj" />
+    <ProjectReference Include="..\Local\Microsoft.MSBuildCache.Local.csproj" />
+    <ProjectReference Include="..\SharedCompilation\Microsoft.MSBuildCache.SharedCompilation.csproj" />
+  </ItemGroup>
+</Project>

src/Repack.Tests/RepackTests.cs

@@ -0,0 +1,49 @@
+// Copyright (c) Microsoft. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using Microsoft.MSBuildCache.AzureBlobStorage;
+using Microsoft.MSBuildCache.AzurePipelines;
+using Microsoft.MSBuildCache.Tests;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+
+namespace Microsoft.MSBuildCache.Repack.Tests;
+
+// Check to make sure that the assemblies that defined the MSBuild/Plugin interface are not merged
+[TestClass]
+public class RepackTests
+{
+    private static readonly Type[] TypesToCheck =
+    {
+        typeof(MSBuildCacheAzureBlobStoragePlugin),
+        typeof(MSBuildCacheAzurePipelinesPlugin),
+        typeof(MSBuildCacheLocalPlugin),
+        typeof(SharedCompilation.ResolveFileAccesses),
+    };
+
+    [TestMethod]
+    public void PluginInterfaceAssembliesNotMerged()
+    {
+        foreach (Type type in TypesToCheck)
+        {
+            Dictionary<string, AssemblyName> references = type.Assembly
+                .GetReferencedAssemblies()
+                .Where(a => a.Name is not null)
+                .ToDictionary(
+                    a => a.Name!,
+                    a => a
+                );
+
+            // Check to make sure that each of the interface assemblies are still actually referenced
+            foreach (string expectedRefFileName in PluginInterfaceTypeCheckTests.PluginInterfaceNuGetAssemblies)
+            {
+                string expectedRef = Path.GetFileNameWithoutExtension(expectedRefFileName);
+                Assert.IsNotNull(references.FirstOrDefault(a => a.Value.FullName.IndexOf(expectedRef, StringComparison.Ordinal) > 0));
+            }
+        }
+    }
+}