-
Notifications
You must be signed in to change notification settings - Fork 4.6k
/
TraceHelper.cs
469 lines (415 loc) · 19.3 KB
/
TraceHelper.cs
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Text;
using Azure.Monitor.OpenTelemetry.Exporter.Internals.Diagnostics;
using Azure.Monitor.OpenTelemetry.Exporter.Models;
using OpenTelemetry;
namespace Azure.Monitor.OpenTelemetry.Exporter.Internals
{
internal static class TraceHelper
{
private const int Version = 2;
private const int MaxlinksAllowed = 100;
internal static List<TelemetryItem> OtelToAzureMonitorTrace(Batch<Activity> batchActivity, AzureMonitorResource? azureMonitorResource, string instrumentationKey, float sampleRate)
{
List<TelemetryItem> telemetryItems = new List<TelemetryItem>();
TelemetryItem telemetryItem;
if (batchActivity.Count > 0 && azureMonitorResource?.MonitorBaseData != null)
{
var otelResourceMetricTelemetry = new TelemetryItem(DateTime.UtcNow, azureMonitorResource, instrumentationKey!, azureMonitorResource.MonitorBaseData);
telemetryItems.Add(otelResourceMetricTelemetry);
}
foreach (var activity in batchActivity)
{
try
{
var activityTagsProcessor = EnumerateActivityTags(activity);
telemetryItem = new TelemetryItem(activity, ref activityTagsProcessor, azureMonitorResource, instrumentationKey, sampleRate);
// Check for Exceptions events
if (activity.Events.Any())
{
AddTelemetryFromActivityEvents(activity, telemetryItem, telemetryItems);
}
switch (activity.GetTelemetryType())
{
case TelemetryType.Request:
var requestData = new RequestData(Version, activity, ref activityTagsProcessor);
requestData.Name = telemetryItem.Tags.TryGetValue(ContextTagKeys.AiOperationName.ToString(), out var operationName) ? operationName.Truncate(SchemaConstants.RequestData_Name_MaxLength) : activity.DisplayName.Truncate(SchemaConstants.RequestData_Name_MaxLength);
telemetryItem.Data = new MonitorBase
{
BaseType = "RequestData",
BaseData = requestData,
};
break;
case TelemetryType.Dependency:
telemetryItem.Data = new MonitorBase
{
BaseType = "RemoteDependencyData",
BaseData = new RemoteDependencyData(Version, activity, ref activityTagsProcessor),
};
break;
}
activityTagsProcessor.Return();
telemetryItems.Add(telemetryItem);
}
catch (Exception ex)
{
AzureMonitorExporterEventSource.Log.FailedToConvertActivity(activity.Source.Name, activity.DisplayName, ex);
}
}
return telemetryItems;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void AddPropertiesToTelemetry(IDictionary<string, string> destination, ref AzMonList UnMappedTags)
{
try
{
// TODO: Iterate only interested fields. Ref: https://github.com/Azure/azure-sdk-for-net/pull/14254#discussion_r470907560
for (int i = 0; i < UnMappedTags.Length; i++)
{
var tag = UnMappedTags[i];
AddKvpToDictionary(destination, tag);
}
}
catch (Exception ex)
{
AzureMonitorExporterEventSource.Log.ErrorAddingActivityTagsAsCustomProperties(ex);
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void AddKvpToDictionary(IDictionary<string, string> destination, KeyValuePair<string, object?> keyValuePair)
{
if (keyValuePair.Key.Length <= SchemaConstants.KVP_MaxKeyLength && keyValuePair.Value != null)
{
// Note: if Key exceeds MaxLength or if Value is null, the entire KVP will be dropped.
// In case of duplicate keys, only the first occurence will be exported.
#if NET6_0_OR_GREATER
destination.TryAdd(keyValuePair.Key, Convert.ToString(keyValuePair.Value, CultureInfo.InvariantCulture).Truncate(SchemaConstants.KVP_MaxValueLength) ?? "null");
#else
if (!destination.ContainsKey(keyValuePair.Key))
{
destination.Add(keyValuePair.Key, Convert.ToString(keyValuePair.Value, CultureInfo.InvariantCulture).Truncate(SchemaConstants.KVP_MaxValueLength) ?? "null");
}
#endif
}
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
internal static void AddKvpToDictionary(IDictionary<string, string> destination, string key, string value)
{
if (key.Length <= SchemaConstants.KVP_MaxKeyLength && value != null)
{
// Note: if Key exceeds MaxLength or if Value is null, the entire KVP will be dropped.
// In case of duplicate keys, only the first occurence will be exported.
#if NET6_0_OR_GREATER
destination.TryAdd(key, value.Truncate(SchemaConstants.KVP_MaxValueLength) ?? "null");
#else
if (!destination.ContainsKey(key))
{
destination.Add(key, value.Truncate(SchemaConstants.KVP_MaxValueLength) ?? "null");
}
#endif
}
}
/// <summary>
/// Converts Activity Links to custom property with key as _MS.links.
/// Value will be a JSON string formatted as [{"operation_Id":"{TraceId}","id":"{SpanId}"}].
/// </summary>
internal static void AddActivityLinksToProperties(Activity activity, ref AzMonList UnMappedTags)
{
string msLinks = "_MS.links";
// max number of links that can fit in this json formatted string is 107. it is based on assumption that TraceId and SpanId will be of fixed length.
// Keeping max at 100 for now.
int maxLinks = MaxlinksAllowed;
if (activity.Links != null && activity.Links.Any())
{
var linksJson = new StringBuilder();
linksJson.Append('[');
foreach (ref readonly var link in activity.EnumerateLinks())
{
AddContextToMSLinks(linksJson, link);
maxLinks--;
if (maxLinks == 0)
{
if (MaxlinksAllowed < activity.Links.Count())
{
AzureMonitorExporterEventSource.Log.ActivityLinksIgnored(MaxlinksAllowed, activity.Source.Name, activity.DisplayName);
}
break;
}
}
if (linksJson.Length > 0)
{
// trim trailing comma - json does not support it
linksJson.Remove(linksJson.Length - 1, 1);
}
linksJson.Append(']');
AzMonList.Add(ref UnMappedTags, new KeyValuePair<string, object?>(msLinks, linksJson.ToString()));
}
}
internal static ActivityTagsProcessor EnumerateActivityTags(Activity activity)
{
var activityTagsProcessor = new ActivityTagsProcessor();
activityTagsProcessor.CategorizeTags(activity);
return activityTagsProcessor;
}
internal static string GetOperationName(Activity activity, ref AzMonList MappedTags)
{
var httpMethod = AzMonList.GetTagValue(ref MappedTags, SemanticConventions.AttributeHttpMethod)?.ToString();
if (!string.IsNullOrWhiteSpace(httpMethod))
{
var httpRoute = AzMonList.GetTagValue(ref MappedTags, SemanticConventions.AttributeHttpRoute)?.ToString();
// ASP.NET instrumentation assigns route as {controller}/{action}/{id} which would result in the same name for different operations.
// To work around that we will use path from httpUrl.
if (!string.IsNullOrWhiteSpace(httpRoute) && !httpRoute!.Contains("{controller}"))
{
return $"{httpMethod} {httpRoute}";
}
var httpUrl = AzMonList.GetTagValue(ref MappedTags, SemanticConventions.AttributeHttpUrl)?.ToString();
if (!string.IsNullOrWhiteSpace(httpUrl) && Uri.TryCreate(httpUrl!.ToString(), UriKind.RelativeOrAbsolute, out var uri) && uri.IsAbsoluteUri)
{
return $"{httpMethod} {uri.AbsolutePath}";
}
}
return activity.DisplayName;
}
internal static string GetOperationNameV2(Activity activity, ref AzMonList MappedTags)
{
var httpMethod = AzMonList.GetTagValue(ref MappedTags, SemanticConventions.AttributeHttpRequestMethod)?.ToString();
if (!string.IsNullOrWhiteSpace(httpMethod))
{
var httpRoute = AzMonList.GetTagValue(ref MappedTags, SemanticConventions.AttributeHttpRoute)?.ToString();
// ASP.NET instrumentation assigns route as {controller}/{action}/{id} which would result in the same name for different operations.
// To work around that we will use path from url.path.
if (!string.IsNullOrWhiteSpace(httpRoute) && !httpRoute!.Contains("{controller}"))
{
return $"{httpMethod} {httpRoute}";
}
var httpPath = AzMonList.GetTagValue(ref MappedTags, SemanticConventions.AttributeUrlPath)?.ToString();
if (!string.IsNullOrWhiteSpace(httpPath))
{
return $"{httpMethod} {httpPath}";
}
}
return activity.DisplayName;
}
private static void AddTelemetryFromActivityEvents(Activity activity, TelemetryItem telemetryItem, List<TelemetryItem> telemetryItems)
{
foreach (ref readonly var @event in activity.EnumerateEvents())
{
try
{
if (@event.Name == SemanticConventions.AttributeExceptionEventName)
{
var exceptionData = GetExceptionDataDetailsOnTelemetryItem(@event);
if (exceptionData != null)
{
var exceptionTelemetryItem = new TelemetryItem("Exception", telemetryItem, activity.SpanId, activity.Kind, @event.Timestamp);
exceptionTelemetryItem.Data = exceptionData;
telemetryItems.Add(exceptionTelemetryItem);
}
}
else
{
var messageData = GetTraceTelemetryData(@event);
if (messageData != null)
{
var traceTelemetryItem = new TelemetryItem("Message", telemetryItem, activity.SpanId, activity.Kind, @event.Timestamp);
traceTelemetryItem.Data = messageData;
telemetryItems.Add(traceTelemetryItem);
}
}
}
catch (Exception ex)
{
AzureMonitorExporterEventSource.Log.FailedToExtractActivityEvent(activity.Source.Name, activity.DisplayName, ex);
}
}
}
private static MonitorBase? GetTraceTelemetryData(ActivityEvent activityEvent)
{
if (activityEvent.Name == null)
{
return null;
}
var messageData = new MessageData(Version, activityEvent.Name);
foreach (ref readonly var tag in activityEvent.EnumerateTagObjects())
{
if (tag.Value is Array arrayValue)
{
AddKvpToDictionary(messageData.Properties, tag.Key, arrayValue.ToCommaDelimitedString());
}
else
{
AddKvpToDictionary(messageData.Properties, tag);
}
}
return new MonitorBase
{
BaseType = "MessageData",
BaseData = messageData,
};
}
private static MonitorBase? GetExceptionDataDetailsOnTelemetryItem(ActivityEvent activityEvent)
{
string? exceptionType = null;
string? exceptionStackTrace = null;
string? exceptionMessage = null;
var properties = new Dictionary<string, string>();
foreach (ref readonly var tag in activityEvent.EnumerateTagObjects())
{
// TODO: see if these can be cached
if (tag.Key == SemanticConventions.AttributeExceptionType)
{
exceptionType = tag.Value?.ToString();
}
else if (tag.Key == SemanticConventions.AttributeExceptionMessage)
{
exceptionMessage = tag.Value?.ToString();
}
else if (tag.Key == SemanticConventions.AttributeExceptionStacktrace)
{
exceptionStackTrace = tag.Value?.ToString();
}
else
{
AddKvpToDictionary(properties, tag);
}
}
if (exceptionMessage == null || exceptionType == null)
{
return null;
}
TelemetryExceptionDetails exceptionDetails = new(exceptionMessage.Truncate(SchemaConstants.ExceptionDetails_Message_MaxLength))
{
Stack = exceptionStackTrace.Truncate(SchemaConstants.ExceptionDetails_Stack_MaxLength),
HasFullStack = exceptionStackTrace != null && (exceptionStackTrace.Length <= SchemaConstants.ExceptionDetails_Stack_MaxLength),
// TODO: Update swagger schema to mandate typename.
TypeName = exceptionType.Truncate(SchemaConstants.ExceptionDetails_TypeName_MaxLength),
};
List<TelemetryExceptionDetails> exceptions = new()
{
exceptionDetails
};
TelemetryExceptionData exceptionData = new(Version, exceptions, properties);
return new MonitorBase
{
BaseType = "ExceptionData",
BaseData = exceptionData,
};
}
internal static void AddEnqueuedTimeToMeasurementsAndLinksToProperties(Activity activity, IDictionary<string, double> measurements, ref AzMonList UnMappedTags)
{
if (activity.Links != null && activity.Links.Any())
{
if (TryGetAverageQueueTimeWithLinks(activity, ref UnMappedTags, out long enqueuedTime))
{
measurements["timeSinceEnqueued"] = enqueuedTime;
}
}
}
private static bool TryGetAverageQueueTimeWithLinks(Activity activity, ref AzMonList UnMappedTags, out long avgTimeInQueue)
{
avgTimeInQueue = 0;
var linksCount = 0;
DateTimeOffset startTime = activity.StartTimeUtc;
long startEpochTime = startTime.ToUnixTimeMilliseconds();
bool isEnqueuedTimeCalculated = true;
string msLinks = "_MS.links";
var linksJson = new StringBuilder();
linksJson.Append('[');
foreach (ref readonly var link in activity.EnumerateLinks())
{
long msgEnqueuedTime = 0;
if (isEnqueuedTimeCalculated && !TryGetEnqueuedTime(link, out msgEnqueuedTime))
{
// instrumentation does not consistently report enqueued time, ignoring whole span
isEnqueuedTimeCalculated = false;
}
if (isEnqueuedTimeCalculated)
{
avgTimeInQueue += Math.Max(startEpochTime - msgEnqueuedTime, 0);
}
linksCount++;
if (linksCount <= MaxlinksAllowed)
{
AddContextToMSLinks(linksJson, link);
}
}
if (linksJson.Length > 0)
{
// trim trailing comma - json does not support it
linksJson.Remove(linksJson.Length - 1, 1);
}
linksJson.Append(']');
AzMonList.Add(ref UnMappedTags, new KeyValuePair<string, object?>(msLinks, linksJson.ToString()));
if (MaxlinksAllowed < linksCount)
{
AzureMonitorExporterEventSource.Log.ActivityLinksIgnored(MaxlinksAllowed, activity.Source.Name, activity.DisplayName);
}
if (isEnqueuedTimeCalculated)
{
avgTimeInQueue /= linksCount;
}
else
{
avgTimeInQueue = 0;
return false;
}
return true;
}
private static bool TryGetEnqueuedTime(ActivityLink link, out long enqueuedTime)
{
enqueuedTime = 0;
foreach (ref readonly var attribute in link.EnumerateTagObjects())
{
if (attribute.Key == "enqueuedTime")
{
return long.TryParse(attribute.Value?.ToString(), out enqueuedTime);
}
}
return false;
}
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private static void AddContextToMSLinks(StringBuilder linksJson, ActivityLink link)
{
linksJson
.Append('{')
.Append("\"operation_Id\":")
.Append('\"')
.Append(link.Context.TraceId.ToHexString())
.Append('\"')
.Append(',');
linksJson
.Append("\"id\":")
.Append('\"')
.Append(link.Context.SpanId.ToHexString())
.Append('\"');
linksJson
.Append("},");
}
internal static string GetAzureSDKDependencyType(ActivityKind kind, string azureNamespace)
{
// TODO: see if the values can be cached to avoid allocation.
if (kind == ActivityKind.Internal)
{
return $"InProc | {azureNamespace}";
}
else if (kind == ActivityKind.Producer)
{
return $"Queue Message | {azureNamespace}";
}
else
{
// The Azure SDK sets az.namespace with its resource provider information.
// When ActivityKind is not internal and az.namespace is present, set the value of Type to az.namespace.
return azureNamespace;
}
}
}
}