Skip to content

Commit

Permalink
feat: Enable auto-instrumentation for Confluent.Kafka versions 1.4.0 …
Browse files Browse the repository at this point in the history
…to 2.x.x. (#1990)
  • Loading branch information
tippmar-nr committed Oct 25, 2023
1 parent bf4102b commit 9f8d22f
Show file tree
Hide file tree
Showing 51 changed files with 1,474 additions and 28 deletions.
10 changes: 9 additions & 1 deletion FullAgent.sln
Expand Up @@ -167,6 +167,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Home", "src\Agent\NewRelic\
{203A8EA4-A0F2-4139-B02E-9B0F2B39C107} = {203A8EA4-A0F2-4139-B02E-9B0F2B39C107}
{22274460-3222-4474-B679-19E1F27A2CC3} = {22274460-3222-4474-B679-19E1F27A2CC3}
{230CDE78-9D21-4D10-9C4B-E00C88B8B021} = {230CDE78-9D21-4D10-9C4B-E00C88B8B021}
{270A9CC8-8031-49F4-A380-1389E7517DB7} = {270A9CC8-8031-49F4-A380-1389E7517DB7}
{279F8AD0-C959-476F-BD58-3581D9A33238} = {279F8AD0-C959-476F-BD58-3581D9A33238}
{2E6CF650-CB50-453D-830A-D00F0540FC2C} = {2E6CF650-CB50-453D-830A-D00F0540FC2C}
{2FB30555-65A4-43D7-82AA-56BF203D3A96} = {2FB30555-65A4-43D7-82AA-56BF203D3A96}
Expand Down Expand Up @@ -215,6 +216,8 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "StackExchangeRedis2Plus", "
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Elasticsearch", "src\Agent\NewRelic\Agent\Extensions\Providers\Wrapper\Elasticsearch\Elasticsearch.csproj", "{D9428449-3E4B-4723-A8AA-1191315C7AAD}"
EndProject
Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Kafka", "src\Agent\NewRelic\Agent\Extensions\Providers\Wrapper\Kafka\Kafka.csproj", "{270A9CC8-8031-49F4-A380-1389E7517DB7}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Expand Down Expand Up @@ -445,6 +448,10 @@ Global
{D9428449-3E4B-4723-A8AA-1191315C7AAD}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D9428449-3E4B-4723-A8AA-1191315C7AAD}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D9428449-3E4B-4723-A8AA-1191315C7AAD}.Release|Any CPU.Build.0 = Release|Any CPU
{270A9CC8-8031-49F4-A380-1389E7517DB7}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{270A9CC8-8031-49F4-A380-1389E7517DB7}.Debug|Any CPU.Build.0 = Debug|Any CPU
{270A9CC8-8031-49F4-A380-1389E7517DB7}.Release|Any CPU.ActiveCfg = Release|Any CPU
{270A9CC8-8031-49F4-A380-1389E7517DB7}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
Expand Down Expand Up @@ -514,10 +521,11 @@ Global
{3D69B4C9-FD16-461F-95AF-6FCA6EAA914E} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
{EC34F023-223D-432F-9401-9C3ED1B75DE4} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
{D9428449-3E4B-4723-A8AA-1191315C7AAD} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
{270A9CC8-8031-49F4-A380-1389E7517DB7} = {5E86E10A-C38F-48CB-ADE9-67B22BB2F50A}
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {D8B98070-6B8E-403C-A07F-A3F2E4A3A3D0}
EnterpriseLibraryConfigurationToolBinariesPath = packages\Unity.2.1.505.2\lib\NET35
SolutionGuid = {D8B98070-6B8E-403C-A07F-A3F2E4A3A3D0}
EndGlobalSection
GlobalSection(TestCaseManagementSettings) = postSolution
CategoryFile = FullAgent.vsmdi
Expand Down
2 changes: 2 additions & 0 deletions build/ArtifactBuilder/CoreAgentComponents.cs
Expand Up @@ -55,6 +55,7 @@ protected override void CreateAgentComponents()
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.StackExchangeRedis.dll",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.StackExchangeRedis2Plus.dll",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.NServiceBus.dll",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.dll",
};

var wrapperXmls = new[]
Expand All @@ -74,6 +75,7 @@ protected override void CreateAgentComponents()
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.StackExchangeRedis.Instrumentation.xml",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.StackExchangeRedis2Plus.Instrumentation.xml",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.NServiceBus.Instrumentation.xml",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml",
};

ExtensionXsd = $@"{SourceHomeBuilderPath}\extensions\extension.xsd";
Expand Down
4 changes: 3 additions & 1 deletion build/ArtifactBuilder/FrameworkAgentComponents.cs
Expand Up @@ -63,7 +63,8 @@ protected override void CreateAgentComponents()
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.WebOptimization.dll",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.WebServices.dll",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.AspNetCore.dll",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Owin.dll"
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Owin.dll",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.dll",
};

var wrapperXmls = new[]
Expand Down Expand Up @@ -99,6 +100,7 @@ protected override void CreateAgentComponents()
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.WebOptimization.Instrumentation.xml",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.WebServices.Instrumentation.xml",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Misc.Instrumentation.xml",
$@"{SourceHomeBuilderPath}\extensions\NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml",
};

ExtensionXsd = $@"{SourceHomeBuilderPath}\extensions\extension.xsd";
Expand Down
13 changes: 13 additions & 0 deletions src/Agent/MsiInstaller/Installer/Product.wxs
Expand Up @@ -465,6 +465,10 @@ SPDX-License-Identifier: Apache-2.0
<Component Id="ElasticsearchWrapperComponent" Guid="{9C233889-69DD-4B8B-A1A6-ABAC6E27A716}">
<File Id="ElasticsearchWrapperFile" Name="NewRelic.Providers.Wrapper.Elasticsearch.dll" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Elasticsearch.dll"/>
</Component>
<Component Id="KafkaWrapperComponent" Guid="{BE97F5F5-A392-48A0-8888-B2973EA09490}">
<File Id="KafkaWrapperFile" Name="NewRelic.Providers.Wrapper.Kafka.dll" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Kafka.dll"/>
</Component>


<!-- Reference libraries -->
<Component Id="NewRelicCoreReferenceComponent" Guid="{C196ED1E-0FBA-4D36-9C34-E969B0C9E8C9}">
Expand Down Expand Up @@ -524,6 +528,9 @@ SPDX-License-Identifier: Apache-2.0
<Component Id="CoreElasticsearchWrapperComponent" Guid="{531A713C-B46C-41C8-8B21-49CBD4C0A459}">
<File Id="CoreElasticsearchWrapperFile" Name="NewRelic.Providers.Wrapper.Elasticsearch.dll" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.Elasticsearch.dll"/>
</Component>
<Component Id="CoreKafkaWrapperComponent" Guid="{6CA6D5A4-A204-4B04-A6B5-13D8B3E736C2}">
<File Id="CoreKafkaWrapperFile" Name="NewRelic.Providers.Wrapper.Kafka.dll" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.Kafka.dll"/>
</Component>

<!-- Reference libraries -->
<Component Id="CoreNewRelicCoreReferenceComponent" Guid="{DD2BE979-7D4B-47EA-9FBE-F6B381D70E0B}">
Expand Down Expand Up @@ -629,6 +636,9 @@ SPDX-License-Identifier: Apache-2.0
<Component Id="ElasticsearchInstrumentationComponent" Guid="{42D85537-D0AB-44EB-8BE5-4B2EEB26DE91}">
<File Id="ElasticsearchInstrumentationFile" Name="NewRelic.Providers.Wrapper.Elasticsearch.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Elasticsearch.Instrumentation.xml"/>
</Component>
<Component Id="KafkaInstrumentationComponent" Guid="{9FC86A0E-CACD-4DA8-84F8-20997C904913}">
<File Id="KafkaInstrumentationFile" Name="NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)\extensions\NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml"/>
</Component>
</ComponentGroup>

<ComponentGroup Id="CoreNewRelic.Agent.Extensions.Instrumentation" Directory="CoreExtensionsFolder">
Expand Down Expand Up @@ -677,6 +687,9 @@ SPDX-License-Identifier: Apache-2.0
<Component Id="CoreElasticsearchInstrumentationComponent" Guid="{02EFBFF6-9DA8-4138-A03F-E9502B631A76}">
<File Id="CoreElasticsearchInstrumentationFile" Name="NewRelic.Providers.Wrapper.Elasticsearch.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.Elasticsearch.Instrumentation.xml"/>
</Component>
<Component Id="CoreKafkaInstrumentationComponent" Guid="{B2C4F83B-A339-4DBD-B8C4-760C8F72F9FC}">
<File Id="CoreKafkaInstrumentationFile" Name="NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml" KeyPath="yes" Source="$(var.HomeFolderPath)_coreclr\extensions\NewRelic.Providers.Wrapper.Kafka.Instrumentation.xml"/>
</Component>
</ComponentGroup>

<!-- Extensions XSD-->
Expand Down
17 changes: 16 additions & 1 deletion src/Agent/NewRelic/Agent/Core/Agent.cs
Expand Up @@ -118,6 +118,11 @@ public ITransaction CreateTransaction(MessageBrokerDestinationType destinationTy
return CreateTransaction(TransactionName.ForBrokerTransaction(destinationType, brokerVendorName, destination), true, wrapperOnCreate ?? NoOpWrapperOnCreate);
}

public ITransaction CreateKafkaTransaction(MessageBrokerDestinationType destinationType, string brokerVendorName, string destination, Action wrapperOnCreate)
{
return CreateTransaction(TransactionName.ForKafkaBrokerTransaction(destinationType, brokerVendorName, destination), true, wrapperOnCreate ?? NoOpWrapperOnCreate);
}

public ITransaction CreateTransaction(bool isWeb, string category, string transactionDisplayName, bool doNotTrackAsUnitOfWork, Action wrapperOnCreate)
{
if (transactionDisplayName == null)
Expand Down Expand Up @@ -401,11 +406,21 @@ public IStackExchangeRedisCache StackExchangeRedisCache
set { _stackExchangeRedisCache = value; }
}

public void RecordSupportabilityMetric(string metricName, int count)
public void RecordSupportabilityMetric(string metricName, long count = 1)
{
_agentHealthReporter.ReportSupportabilityCountMetric(metricName, count);
}

public void RecordCountMetric(string metricName, long count = 1)
{
_agentHealthReporter.ReportCountMetric(metricName, count);
}

public void RecordByteMetric(string metricName, long totalBytes, long? exclusiveBytes = null)
{
_agentHealthReporter.ReportByteMetric(metricName, totalBytes, exclusiveBytes);
}

public void RecordLogMessage(string frameworkName, object logEvent, Func<object, DateTime> getTimestamp, Func<object, object> getLevel, Func<object, string> getLogMessage, Func<object, Exception> getLogException, Func<object, Dictionary<string, object>> getContextData, string spanId, string traceId)
{
_agentHealthReporter.ReportLogForwardingFramework(frameworkName);
Expand Down
13 changes: 13 additions & 0 deletions src/Agent/NewRelic/Agent/Core/AgentHealth/AgentHealthReporter.cs
Expand Up @@ -107,6 +107,19 @@ private void ReportSupportabilityGaugeMetric(string metricName, float value)
TrySend(metric);
}

public void ReportCountMetric(string metricName, long count)
{
var metric = _metricBuilder.TryBuildCountMetric(metricName, count);
TrySend(metric);
}
public void ReportByteMetric(string metricName, long totalBytes, long? exclusiveBytes = null)
{
var metric = _metricBuilder.TryBuildByteMetric(metricName, totalBytes, exclusiveBytes);
TrySend(metric);
}



public void ReportDotnetVersion()
{
#if NETFRAMEWORK
Expand Down
Expand Up @@ -129,6 +129,7 @@ public interface IAgentHealthReporter : IOutOfBandMetricSource
void ReportAgentInfo();

void ReportSupportabilityCountMetric(string metricName, long count = 1);
void ReportCountMetric(string metricName, long count = 1);

void ReportInfiniteTracingSpanResponseError();
void ReportInfiniteTracingSpanEventsSeen(long count = 1);
Expand All @@ -148,6 +149,7 @@ public interface IAgentHealthReporter : IOutOfBandMetricSource
void ReportLoggingEventsDropped(int droppedCount);
void ReportLogForwardingFramework(string logFramework);
void ReportLogForwardingEnabledWithFramework(string logFramework);
void ReportByteMetric(string metricName, long totalBytes, long? exclusiveBytes = null);
void ReportLoggingEventsEmpty(int count = 1);
}
}
18 changes: 18 additions & 0 deletions src/Agent/NewRelic/Agent/Core/Metrics/MetricNames.cs
Expand Up @@ -160,6 +160,7 @@ public static class MetricNames
public const string OtherTransactionPrefix = "OtherTransaction";
public const string WebTransactionPrefix = "WebTransaction";
public const string SupportabilityPayloadsDroppedDueToMaxPayloadLimitPrefix = Supportability + PathSeparator + "DotNet/Collector" + PathSeparator + "MaxPayloadSizeLimit";
public const string KafkaMessageBrokerConsume = "Consume";

public static readonly char PathSeparatorChar = PathSeparator[0];
public static readonly char[] PathSeparatorCharArray = { PathSeparatorChar };
Expand Down Expand Up @@ -358,6 +359,7 @@ public enum MessageBrokerAction
public const string MessageBrokerNamed = "Named";
public const string MessageBrokerTemp = "Temp";
public const string Msmq = "MSMQ";
public const string Serialization = "Serialization";

public static MetricName GetMessageBroker(MessageBrokerDestinationType type, MessageBrokerAction action,
string vendor, string queueName)
Expand All @@ -368,6 +370,14 @@ public enum MessageBrokerAction
: MetricName.Create(MessageBrokerPrefix, vendor, normalizedType, action, MessageBrokerTemp);
}

public static MetricName GetMessageBrokerSerialization(MessageBrokerDestinationType type, MessageBrokerAction action,
string vendor, string queueName, string kind)
{
var normalizedType = NormalizeMessageBrokerDestinationTypeForMetricName(type);
return MetricName.Create(MessageBrokerPrefix, vendor, normalizedType, action, MessageBrokerNamed, queueName, Serialization, kind);

}

private static MessageBrokerDestinationType NormalizeMessageBrokerDestinationTypeForMetricName(
MessageBrokerDestinationType type)
{
Expand All @@ -384,6 +394,14 @@ public enum MessageBrokerAction
return type;
}

private const string KakfaTopic = "Topic";
private const string KakfaReceived = "Received";
private const string KakfaMessages = "Messages";
public static MetricName GetKafkaMessagesReceivedPerConsume(string topic)
{
return MetricName.Create(Message, "Kafka", KakfaTopic, MessageBrokerNamed, topic, KakfaReceived, KakfaMessages);
}

#endregion MessageBroker

#region Datastore
Expand Down
@@ -0,0 +1,77 @@
// Copyright 2020 New Relic, Inc. All rights reserved.
// SPDX-License-Identifier: Apache-2.0

using System;
using System.Collections.Generic;
using NewRelic.Agent.Core.Aggregators;
using NewRelic.Agent.Core.Metrics;
using NewRelic.Agent.Core.Time;
using static NewRelic.Agent.Core.WireModels.MetricWireModel;
using NewRelic.Agent.Configuration;

namespace NewRelic.Agent.Core.Segments
{
public class MessageBrokerSerializationSegmentData : AbstractSegmentData
{

private const string TransactionGuidSegmentParameterKey = "transaction_guid";

public string Vendor { get; set; }

public string Destination { get; set; }

public MetricNames.MessageBrokerDestinationType DestinationType { get; set; }

public MetricNames.MessageBrokerAction Action { get; set; }

public string Kind { get; set; }


public MessageBrokerSerializationSegmentData(string vendor, string destination, MetricNames.MessageBrokerDestinationType destinationType, MetricNames.MessageBrokerAction action, string kind)
{
Vendor = vendor;
Destination = destination;
DestinationType = destinationType;
Action = action;
Kind = kind;
}

public override bool IsCombinableWith(AbstractSegmentData otherData)
{
var otherTypedSegment = otherData as MessageBrokerSerializationSegmentData;
if (otherTypedSegment == null)
return false;

if (!Vendor.Equals(otherTypedSegment.Vendor))
return false;

if (!Destination.Equals(otherTypedSegment.Destination))
return false;

if (DestinationType != otherTypedSegment.DestinationType)
return false;

if (Action != otherTypedSegment.Action)
return false;

if (!Kind.Equals(otherTypedSegment.Kind))
return false;

return true;
}

public override string GetTransactionTraceName()
{
return MetricNames.GetMessageBrokerSerialization(DestinationType, Action, Vendor, Destination, Kind).ToString();
}

public override void AddMetricStats(Segment segment, TimeSpan durationOfChildren, TransactionMetricStatsCollection txStats, IConfigurationService configService)
{
var duration = segment.Duration.Value;
var exclusiveDuration = TimeSpanMath.Max(TimeSpan.Zero, duration - durationOfChildren);

MetricBuilder.TryBuildMessageBrokerSerializationSegmentMetric(Vendor, Destination, DestinationType, Action, Kind, duration, exclusiveDuration, txStats);

}
}
}
1 change: 1 addition & 0 deletions src/Agent/NewRelic/Agent/Core/Segments/NoOpSegment.cs
Expand Up @@ -50,6 +50,7 @@ public ISegmentExperimental MakeLeaf()
}

public void RemoveSegmentFromCallStack() { }
public void SetMessageBrokerDestination(string destination) { }

public ISegmentExperimental SetSegmentData(ISegmentData segmentData)
{
Expand Down
11 changes: 10 additions & 1 deletion src/Agent/NewRelic/Agent/Core/Segments/Segment.cs
Expand Up @@ -210,6 +210,15 @@ public void RemoveSegmentFromCallStack()
_transactionSegmentState.CallStackPop(this);
}

public void SetMessageBrokerDestination(string destination)
{
if (SegmentData is MessageBrokerSegmentData)
{
var messageBrokerSegmentData = SegmentData as MessageBrokerSegmentData;
messageBrokerSegmentData!.Destination = destination;
}
}

private const long NoEndTime = -1;
internal static NoOpSegment NoOpSegment = new NoOpSegment();
protected readonly static IEnumerable<KeyValuePair<string, object>> EmptyImmutableParameters = new KeyValuePair<string, object>[0];
Expand Down Expand Up @@ -410,7 +419,7 @@ public Segment CreateSimilar(TimeSpan newRelativeStartTime, TimeSpan newDuration

public string ToStringForFinestLogging()
{
return $"Id={UniqueId},ParentId={ParentUniqueId?.ToString() ?? "Root"},Name={Data.GetTransactionTraceName()},IsLeaf={IsLeaf},Combinable={Combinable},MethodCallData={MethodCallData}";
return $"Id={UniqueId},ParentId={ParentUniqueId?.ToString() ?? "Root"},Name={GetTransactionTraceName()},IsLeaf={IsLeaf},Combinable={Combinable},MethodCallData={MethodCallData}";
}

public ISegmentExperimental SetSegmentData(ISegmentData segmentData)
Expand Down
13 changes: 13 additions & 0 deletions src/Agent/NewRelic/Agent/Core/Transactions/NoOpTransaction.cs
Expand Up @@ -81,6 +81,14 @@ public ISegment StartMessageBrokerSegment(MethodCall methodCall, MessageBrokerDe
return Segment.NoOpSegment;
}

public ISegment StartMessageBrokerSerializationSegment(MethodCall methodCall, MessageBrokerDestinationType destinationType, MessageBrokerAction operation, string brokerVendorName, string destinationName, string kind)
{
#if DEBUG
Log.Finest("Skipping StartMessageBrokerSegment outside of a transaction");
#endif
return Segment.NoOpSegment;
}

public ISegment StartMethodSegment(MethodCall methodCall, string typeName, string methodName, bool isLeaf = false)
{
#if DEBUG
Expand Down Expand Up @@ -175,6 +183,11 @@ public void SetMessageBrokerTransactionName(MessageBrokerDestinationType destina

}

public void SetKafkaMessageBrokerTransactionName(MessageBrokerDestinationType destinationType, string brokerVendorName, string destination = null, TransactionNamePriority priority = TransactionNamePriority.Uri)
{

}

public void SetOtherTransactionName(string category, string name, TransactionNamePriority priority = TransactionNamePriority.Uri)
{

Expand Down

0 comments on commit 9f8d22f

Please sign in to comment.