Skip to content

Commit

Permalink
fixing race condition with FunctionAssemblyLoadContext (#6632)
Browse files Browse the repository at this point in the history
  • Loading branch information
brettsam committed Sep 11, 2020
1 parent b0eccda commit 72d04a7
Show file tree
Hide file tree
Showing 17 changed files with 347 additions and 88 deletions.
7 changes: 0 additions & 7 deletions WebJobs.Script.sln
Expand Up @@ -310,8 +310,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "HttpTrigger", "HttpTrigger"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobs.Script.Abstractions", "src\WebJobs.Script.Abstractions\WebJobs.Script.Abstractions.csproj", "{9A522D9D-2D86-4572-B7D1-ECBFBFAF312C}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobsStartupTests", "test\WebJobsStartupTests\WebJobsStartupTests.csproj", "{F5D74052-3807-410F-9A5A-B69A57127CF4}"
EndProject
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{9111825C-9831-4672-8223-82F489853F19}"
ProjectSection(SolutionItems) = preProject
.editorconfig = .editorconfig
Expand Down Expand Up @@ -368,10 +366,6 @@ Global
{9A522D9D-2D86-4572-B7D1-ECBFBFAF312C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{9A522D9D-2D86-4572-B7D1-ECBFBFAF312C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{9A522D9D-2D86-4572-B7D1-ECBFBFAF312C}.Release|Any CPU.Build.0 = Release|Any CPU
{F5D74052-3807-410F-9A5A-B69A57127CF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{F5D74052-3807-410F-9A5A-B69A57127CF4}.Debug|Any CPU.Build.0 = Debug|Any CPU
{F5D74052-3807-410F-9A5A-B69A57127CF4}.Release|Any CPU.ActiveCfg = Release|Any CPU
{F5D74052-3807-410F-9A5A-B69A57127CF4}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -431,7 +425,6 @@ Global
{0AE3CE25-4CD9-4769-AE58-399FC59CF70F} = {FF9C0818-30D3-437A-A62D-7A61CA44F459}
{BA45A727-34B7-484F-9B93-B1755AF09A2A} = {0AE3CE25-4CD9-4769-AE58-399FC59CF70F}
{9A522D9D-2D86-4572-B7D1-ECBFBFAF312C} = {16351B76-87CA-4A8C-80A1-3DD83A0C4AA6}
{F5D74052-3807-410F-9A5A-B69A57127CF4} = {AFB0F5F7-A612-4F4A-94DD-8B69CABF7970}
{A59D3F65-53E5-4666-AEFE-882987A39A49} = {FF9C0818-30D3-437A-A62D-7A61CA44F459}
{209DC34B-762E-4B1B-B094-EBD7C4B972C2} = {A59D3F65-53E5-4666-AEFE-882987A39A49}
EndGlobalSection
Expand Down
22 changes: 11 additions & 11 deletions azure-pipelines.yml
Expand Up @@ -408,7 +408,7 @@ jobs:
inputs:
command: 'test'
testRunTitle: "Node end to end tests"
arguments: '--filter "Group=NodeEndToEndTests"'
arguments: '--filter "Group=NodeEndToEndTests" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
- task: DotNetCoreCLI@2
Expand All @@ -417,7 +417,7 @@ jobs:
inputs:
command: 'test'
testRunTitle: "Direct load end to end tests"
arguments: '--filter "Group=DirectLoadEndToEndTests"'
arguments: '--filter "Group=DirectLoadEndToEndTests" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
- task: DotNetCoreCLI@2
Expand All @@ -426,7 +426,7 @@ jobs:
inputs:
command: 'test'
testRunTitle: "F# end to end tests"
arguments: '--filter "Group=FSharpEndToEndTests"'
arguments: '--filter "Group=FSharpEndToEndTests" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
- task: DotNetCoreCLI@2
Expand All @@ -435,7 +435,7 @@ jobs:
inputs:
command: 'test'
testRunTitle: "Language worker end to end tests"
arguments: '--filter "Group=LanguageWorkerSelectionEndToEndTests"'
arguments: '--filter "Group=LanguageWorkerSelectionEndToEndTests" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
- task: DotNetCoreCLI@2
Expand All @@ -444,7 +444,7 @@ jobs:
inputs:
command: 'test'
testRunTitle: "Node script host end to end tests"
arguments: '--filter "Group=NodeScriptHostTests"'
arguments: '--filter "Group=NodeScriptHostTests" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
- task: DotNetCoreCLI@2
Expand All @@ -453,16 +453,16 @@ jobs:
inputs:
command: 'test'
testRunTitle: "Raw assembly end to end tests"
arguments: '--filter "Group=RawAssemblyEndToEndTests"'
arguments: '--filter "Group=RawAssemblyEndToEndTests" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
**\WebJobs.Script.Tests.Integration.csproj
- task: DotNetCoreCLI@2
displayName: "Samples end to end tests"
condition: succeededOrFailed()
inputs:
command: 'test'
testRunTitle: "Samples end to end tests"
arguments: '--filter "Group=SamplesEndToEndTests"'
arguments: '--filter "Group=SamplesEndToEndTests" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
- task: DotNetCoreCLI@2
Expand All @@ -471,7 +471,7 @@ jobs:
inputs:
command: 'test'
testRunTitle: "Standby mode end to end tests Windows"
arguments: '--filter "Group=StandbyModeEndToEndTests_Windows"'
arguments: '--filter "Group=StandbyModeEndToEndTests_Windows" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
- task: DotNetCoreCLI@2
Expand All @@ -480,7 +480,7 @@ jobs:
inputs:
command: 'test'
testRunTitle: "Standby mode end to end tests Linux"
arguments: '--filter "Group=StandbyModeEndToEndTests_Linux"'
arguments: '--filter "Group=StandbyModeEndToEndTests_Linux" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
- task: DotNetCoreCLI@2
Expand All @@ -489,7 +489,7 @@ jobs:
inputs:
command: 'test'
testRunTitle: "Linux container end to end tests Windows"
arguments: '--filter "Group=ContainerInstanceTests"'
arguments: '--filter "Group=ContainerInstanceTests" --no-build'
projects: |
**\WebJobs.Script.Tests.Integration.csproj
- task: PowerShell@2
Expand Down
Expand Up @@ -9,7 +9,6 @@
using System.Reflection;
using System.Runtime.InteropServices;
using System.Runtime.Loader;
using System.Threading;
using Microsoft.Azure.WebJobs.Host;
using Microsoft.Azure.WebJobs.Script.Config;
using Microsoft.Extensions.DependencyModel;
Expand All @@ -25,7 +24,7 @@ public partial class FunctionAssemblyLoadContext : AssemblyLoadContext
private const string PrivateDependencyResolutionPolicy = "private";

private static readonly Lazy<Dictionary<string, ResolutionPolicyEvaluator>> _resolutionPolicyEvaluators = new Lazy<Dictionary<string, ResolutionPolicyEvaluator>>(InitializeLoadPolicyEvaluators);
private static readonly ConcurrentDictionary<string, object> _sharedContextAssembliesInFallbackLoad = new ConcurrentDictionary<string, object>();
private static readonly ConcurrentDictionary<string, int> _sharedContextAssembliesInFallbackLoad = new ConcurrentDictionary<string, int>();
private static readonly RuntimeAssembliesInfo _runtimeAssembliesInfo = new RuntimeAssembliesInfo();

private static Lazy<FunctionAssemblyLoadContext> _defaultContext = new Lazy<FunctionAssemblyLoadContext>(CreateSharedContext, true);
Expand Down Expand Up @@ -118,7 +117,8 @@ private static Assembly HandleDefaultContextFallback(AssemblyLoadContext loadCon
// If we're not currently loading this from the shared context, an attempt to load
// a user assembly from the default context might have been made (e.g. Assembly.Load or AppDomain.Load called in a
// runtime/default context loaded assembly against a function assembly)
if (!_sharedContextAssembliesInFallbackLoad.ContainsKey(assemblyName.Name))
if (!_sharedContextAssembliesInFallbackLoad.TryGetValue(assemblyName.Name, out int count)
|| count == 0)
{
if (Shared.TryLoadDepsDependency(assemblyName, out Assembly assembly))
{
Expand Down Expand Up @@ -313,15 +313,15 @@ private bool TryLoadHostEnvironmentAssembly(AssemblyName assemblyName, out Assem
assembly = null;
try
{
_sharedContextAssembliesInFallbackLoad.TryAdd(assemblyName.Name, null);
_sharedContextAssembliesInFallbackLoad.AddOrUpdate(assemblyName.Name, 1, (s, i) => i + 1);
assembly = AssemblyLoadContext.Default.LoadFromAssemblyName(assemblyName);
}
catch (FileNotFoundException)
{
}
finally
{
_sharedContextAssembliesInFallbackLoad.TryRemove(assemblyName.Name, out object _);
_sharedContextAssembliesInFallbackLoad.AddOrUpdate(assemblyName.Name, 0, (s, i) => i - 1);
}

return assembly != null;
Expand Down
@@ -0,0 +1,20 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<AzureFunctionsVersion>v3</AzureFunctionsVersion>
<_FunctionsSkipCleanOutput>true</_FunctionsSkipCleanOutput>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Net.Sdk.Functions" Version="3.0.*" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
</ItemGroup>
<ItemGroup>
<None Update="host.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Update="local.settings.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
<CopyToPublishDirectory>Never</CopyToPublishDirectory>
</None>
</ItemGroup>
</Project>
@@ -0,0 +1,55 @@
using System;
using System.Collections.Generic;
using System.Reflection;
using System.Runtime.Loader;
using System.Threading;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Azure.WebJobs;
using Microsoft.Azure.WebJobs.Extensions.Http;

namespace AssemblyLoadContextRace
{
public static class Function1
{
[FunctionName("Function1")]
public static IActionResult Run([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = null)] HttpRequest req)
{
var resetEvent = new ManualResetEvent(false);

// First, force Newtonsoft to load on many threads, which induces the race
var context = AssemblyLoadContext.GetLoadContext(typeof(Function1).Assembly);

void LoadType()
{
var assembly = context.LoadFromAssemblyName(new AssemblyName("Newtonsoft.Json, Version=12.0.3.0"));
}

List<Thread> threads = new List<Thread>();

for (int i = 0; i < 100; i++)
{
Thread t = new Thread(LoadType);
threads.Add(t);
t.Start();
}

foreach (Thread t in threads)
{
t.Join();
}

// Now, make sure the assemblies match, signifying that the race was fixed and we
// always load the host's version.
var functionAssembly = context.LoadFromAssemblyName(new AssemblyName("Newtonsoft.Json, Version=12.0.3.0"));
var defaultAssembly = AssemblyLoadContext.Default.LoadFromAssemblyName(new AssemblyName("Newtonsoft.Json, Version=12.0.3.0"));

if (!Equals(functionAssembly, defaultAssembly))
{
throw new InvalidOperationException();
}

return new OkResult();
}
}
}
@@ -0,0 +1,11 @@
{
"version": "2.0",
"logging": {
"applicationInsights": {
"samplingExcludedTypes": "Request",
"samplingSettings": {
"isEnabled": true
}
}
}
}
@@ -0,0 +1,31 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.30413.136
MinimumVisualStudioVersion = 10.0.40219.1
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WebJobsStartupTests", "WebJobsStartupTests\WebJobsStartupTests.csproj", "{EB7FC98C-1EF9-40E3-B7B2-858AA2547B1F}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "AssemblyLoadContextRace", "AssemblyLoadContextRace\AssemblyLoadContextRace.csproj", "{1E882F3E-E1D1-4D56-A575-8530596705C3}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{EB7FC98C-1EF9-40E3-B7B2-858AA2547B1F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{EB7FC98C-1EF9-40E3-B7B2-858AA2547B1F}.Debug|Any CPU.Build.0 = Debug|Any CPU
{EB7FC98C-1EF9-40E3-B7B2-858AA2547B1F}.Release|Any CPU.ActiveCfg = Release|Any CPU
{EB7FC98C-1EF9-40E3-B7B2-858AA2547B1F}.Release|Any CPU.Build.0 = Release|Any CPU
{1E882F3E-E1D1-4D56-A575-8530596705C3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{1E882F3E-E1D1-4D56-A575-8530596705C3}.Debug|Any CPU.Build.0 = Debug|Any CPU
{1E882F3E-E1D1-4D56-A575-8530596705C3}.Release|Any CPU.ActiveCfg = Release|Any CPU
{1E882F3E-E1D1-4D56-A575-8530596705C3}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {61422CE7-BD7A-46C1-9589-6ED6EE3AB8BC}
EndGlobalSection
EndGlobal
File renamed without changes.
File renamed without changes.
@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<TargetFramework>netcoreapp3.1</TargetFramework>
<AzureFunctionsVersion>V3</AzureFunctionsVersion>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Azure.WebJobs" Version="3.0.*" />
<PackageReference Include="Microsoft.Net.Sdk.Functions" Version="3.0.*" />
</ItemGroup>

<ItemGroup>
<None Update="host.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
</ItemGroup>

</Project>
File renamed without changes.
@@ -0,0 +1,32 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;

namespace Microsoft.Azure.WebJobs.Script.Tests.Integration.WebHostEndToEnd
{
public class CSharpPrecompiledEndToEndTestFixture : EndToEndTestFixture
{
private const string TestPathTemplate = "..\\..\\..\\..\\CSharpPrecompiledTestProjects\\{0}\\bin\\Debug\\netcoreapp3.1";
private readonly IDisposable _dispose;

public CSharpPrecompiledEndToEndTestFixture(string testProjectName, IDictionary<string, string> envVars = null)
: base(string.Format(TestPathTemplate, testProjectName), testProjectName, "dotnet")
{
if (envVars != null)
{
_dispose = new TestScopedEnvironmentVariable(envVars);
}
}

protected override Task CreateTestStorageEntities()
{
return Task.CompletedTask;
}

public override Task DisposeAsync()
{
_dispose?.Dispose();
return base.DisposeAsync();
}
}
}
@@ -0,0 +1,30 @@
using System;
using System.Net;
using System.Threading.Tasks;
using Xunit;

namespace Microsoft.Azure.WebJobs.Script.Tests.Integration.WebHostEndToEnd
{
public class FunctionAssemblyLoadContextEndToEndTests : IDisposable
{
HostProcessLauncher _launcher;

[Fact]
public async Task Fallback_IsThreadSafe()
{
_launcher = new HostProcessLauncher("AssemblyLoadContextRace");
await _launcher.StartHostAsync();

var client = _launcher.HttpClient;
var response = await client.GetAsync($"api/Function1");

// The function does all the validation internally.
Assert.Equal(HttpStatusCode.OK, response.StatusCode);
}

public void Dispose()
{
_launcher?.Dispose();
}
}
}

0 comments on commit 72d04a7

Please sign in to comment.