Skip to content

Commit

Permalink
fix: Eliminate intermittent deadlock on agent startup. (#2183) (#2184)
Browse files Browse the repository at this point in the history
  • Loading branch information
tippmar-nr committed Jan 11, 2024
1 parent 445f7c2 commit 11c0241
Show file tree
Hide file tree
Showing 7 changed files with 145 additions and 11 deletions.
15 changes: 5 additions & 10 deletions src/Agent/NewRelic/Agent/Core/AgentShim.cs
Expand Up @@ -20,12 +20,6 @@ static void Initialize()
AgentInitializer.InitializeAgent();
}

#if NETSTANDARD2_0
static AgentShim()
{
Initialize();
}
#else
private static bool _initialized = false;
private static object _initLock = new object();

Expand All @@ -49,6 +43,7 @@ static bool TryInitialize(string method)
private static HashSet<string> _deferInitializationOnTheseMethods = new HashSet<string>
{
"System.Net.Http.HttpClient.SendAsync",
"System.Net.Http.SocketsHttpHandler.SendAsync",
"System.Net.HttpWebRequest.SerializeHeaders",
"System.Net.HttpWebRequest.GetResponse"
};
Expand Down Expand Up @@ -93,7 +88,6 @@ static bool DeferInitialization(string method)
{
return DeferInitializationOnTheseMethods.Contains(method);
}
#endif

/// <summary>
/// Creates a tracer (if appropriate) and returns a delegate for the tracer's finish method.
Expand All @@ -112,12 +106,10 @@ static bool DeferInitialization(string method)
object[] args,
ulong functionId)
{
#if NETFRAMEWORK
if (!_initialized)
{
if (!TryInitialize($"{typeName}.{methodName}")) return NoOpFinishTracer;
}
#endif

var tracer = GetTracer(
tracerFactoryName,
Expand Down Expand Up @@ -325,7 +317,10 @@ public void FinishTracer(object returnValue, Exception exception)

/// <summary>
/// Used to stop re-entry into the agent via AgentShim entry points when the call stack contains agent code already.
/// The profiler stops reentry of GetTracer/FinishTracer twice in the same call stack, but it doesn't stop the agent from spinning up a background thread and then entering the agent from that thread. To resolve this, anytime a new thread is spun up (via any mechanism including async, Timer, Thread, ThreadPool, etc.) the work inside it needs to be wrapped in using (new IgnoreWork()) as a way of telling AgentShim to not re-enter.
/// The profiler stops reentry of GetTracer/FinishTracer twice in the same call stack, but it doesn't stop the agent
/// from spinning up a background thread and then entering the agent from that thread. To resolve this, anytime a
/// new thread is spun up (via any mechanism including async, Timer, Thread, ThreadPool, etc.) the work inside it
/// needs to be wrapped in using (new IgnoreWork()) as a way of telling AgentShim to not re-enter.
/// </summary>
public class IgnoreWork : IDisposable
{
Expand Down
@@ -0,0 +1,19 @@
<Project Sdk="Microsoft.NET.Sdk">

<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.VisualStudio.Threading.Analyzers" Version="17.8.14">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
</ItemGroup>

<ItemGroup>
<ProjectReference Include="..\..\SharedApplications\Common\SharedApplicationHelpers\SharedApplicationHelpers.csproj" />
</ItemGroup>

</Project>
@@ -0,0 +1,26 @@
// Copyright 2020 New Relic, Inc. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

using System;
using System.Diagnostics;
using NewRelic.Api.Agent;

namespace ConsoleInstrumentationStartup;

static class Program
{
[Transaction]
static void Main()
{
Console.WriteLine($"{DateTime.Now} Main enter");

var stopWatch = new Stopwatch();
stopWatch.Start();

NewRelic.Api.Agent.NewRelic.SetTransactionName("category", "name");

stopWatch.Stop();

Console.WriteLine($"{DateTime.Now} Main exit");
}
}
Expand Up @@ -64,6 +64,12 @@ public void SetHost(string host)
host);
}

public void SetHostPort(int port)
{
CommonUtils.ModifyOrCreateXmlAttributeInNewRelicConfig(_configFilePath, new[] { "configuration", "service" }, "port",
port.ToString());
}

public void SetRequestTimeout(TimeSpan duration)
{
CommonUtils.ModifyOrCreateXmlAttributeInNewRelicConfig(_configFilePath, new[] { "configuration", "service" }, "requestTimeout",
Expand Down
9 changes: 8 additions & 1 deletion tests/Agent/IntegrationTests/IntegrationTests.sln
Expand Up @@ -134,7 +134,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "AspNetCoreBasicWebApiApplic
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "SharedApplicationHelpers", "SharedApplications\Common\SharedApplicationHelpers\SharedApplicationHelpers.csproj", "{85A4B5C1-1248-4DE2-AE97-B96B6FA3AE09}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "BasicAspNetCoreRazorApplication", "Applications\BasicAspNetCoreRazorApplication\BasicAspNetCoreRazorApplication.csproj", "{8F7EBBC3-B22F-43AC-978E-2CD8AD7C02CF}"
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "BasicAspNetCoreRazorApplication", "Applications\BasicAspNetCoreRazorApplication\BasicAspNetCoreRazorApplication.csproj", "{8F7EBBC3-B22F-43AC-978E-2CD8AD7C02CF}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ConsoleInstrumentationStartup", "Applications\ConsoleInstrumentationStartup\ConsoleInstrumentationStartup.csproj", "{31DB04AF-2ED3-4379-98D7-7D02F38864F9}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Expand Down Expand Up @@ -358,6 +360,10 @@ Global
{8F7EBBC3-B22F-43AC-978E-2CD8AD7C02CF}.Debug|Any CPU.Build.0 = Debug|Any CPU
{8F7EBBC3-B22F-43AC-978E-2CD8AD7C02CF}.Release|Any CPU.ActiveCfg = Release|Any CPU
{8F7EBBC3-B22F-43AC-978E-2CD8AD7C02CF}.Release|Any CPU.Build.0 = Release|Any CPU
{31DB04AF-2ED3-4379-98D7-7D02F38864F9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{31DB04AF-2ED3-4379-98D7-7D02F38864F9}.Debug|Any CPU.Build.0 = Debug|Any CPU
{31DB04AF-2ED3-4379-98D7-7D02F38864F9}.Release|Any CPU.ActiveCfg = Release|Any CPU
{31DB04AF-2ED3-4379-98D7-7D02F38864F9}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -411,6 +417,7 @@ Global
{69F04BE2-6859-45BB-97E7-E5ACDEE53503} = {F0F6F2CE-8AE8-49E1-8EE9-A44B451EFC29}
{85A4B5C1-1248-4DE2-AE97-B96B6FA3AE09} = {30CF078E-E531-441E-83AB-24AB9B1C179F}
{8F7EBBC3-B22F-43AC-978E-2CD8AD7C02CF} = {F0F6F2CE-8AE8-49E1-8EE9-A44B451EFC29}
{31DB04AF-2ED3-4379-98D7-7D02F38864F9} = {F0F6F2CE-8AE8-49E1-8EE9-A44B451EFC29}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {3830ABDF-4AEA-4D91-83A2-13F091D1DF5F}
Expand Down
@@ -0,0 +1,56 @@
// Copyright 2020 New Relic, Inc. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

using System;
using System.Linq;
using NewRelic.Agent.IntegrationTestHelpers;
using NewRelic.Testing.Assertions;
using Xunit;
using Xunit.Abstractions;

namespace NewRelic.Agent.IntegrationTests.AgentFeatures
{
/// <summary>
/// Tests that the agent doesn't deadlock at startup
/// </summary>
[NetCoreTest]
public class InstrumentationStartupDeadlockTests : NewRelicIntegrationTest<RemoteServiceFixtures.ConsoleInstrumentationStartupFixtureCore>
{
private readonly RemoteServiceFixtures.ConsoleInstrumentationStartupFixtureCore _fixture;

public InstrumentationStartupDeadlockTests(RemoteServiceFixtures.ConsoleInstrumentationStartupFixtureCore fixture, ITestOutputHelper output) : base(fixture)
{
_fixture = fixture;
_fixture.TestLogger = output;

_fixture.Actions
(
setupConfiguration: () =>
{
var configPath = fixture.DestinationNewRelicConfigFilePath;
var configModifier = new NewRelicConfigModifier(configPath);
configModifier.SetHostPort(9999); // use a bogus port to generate an exception during HttpClient.SendAsync() on connect
configModifier.SetSendDataOnExit();
configModifier.ForceTransactionTraces();
configModifier.SetLogLevel("finest");
configModifier.DisableEventListenerSamplers(); // Required for .NET 8 to pass.
},
exerciseApplication: () =>
{
}
);

_fixture.Initialize();
}

[Fact]
public void Test()
{
var expectedLogLineRegexes = new[] { AgentLogBase.ShutdownLogLineRegex };

Assertions.LogLinesExist(expectedLogLineRegexes, _fixture.AgentLog.GetFileLines());
}
}
}
@@ -0,0 +1,25 @@
// Copyright 2020 New Relic, Inc. All rights reserved.
// SPDX-License-Identifier: Apache-2.0


using NewRelic.Agent.IntegrationTestHelpers.RemoteServiceFixtures;
using System;
using System.IO;
using System.Reflection;

namespace NewRelic.Agent.IntegrationTests.RemoteServiceFixtures
{
public class ConsoleInstrumentationStartupFixtureCore : RemoteApplicationFixture
{
private static readonly string ApplicationDirectoryName = @"ConsoleInstrumentationStartup";
private static readonly string ExecutableName = $"{ApplicationDirectoryName}.exe";

public ConsoleInstrumentationStartupFixtureCore()
: base(new RemoteConsoleApplication(ApplicationDirectoryName, ExecutableName, ApplicationType.Bounded, true, true)
.SetTimeout(TimeSpan.FromMinutes(2)))
{

}

}
}

0 comments on commit 11c0241

Please sign in to comment.