diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0ed41e624..77a119ad3 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,6 +37,7 @@ jobs: rerunMaxAttempts: 2 env: AzureWebJobsStorage: $(AzureWebJobsStorage) + APPINSIGHTS_INSTRUMENTATIONKEY: $(APPINSIGHTS_INSTRUMENTATIONKEY) - job: FunctionsV2Tests pool: @@ -78,6 +79,7 @@ jobs: rerunMaxAttempts: 2 env: AzureWebJobsStorage: $(AzureWebJobsStorage) + APPINSIGHTS_INSTRUMENTATIONKEY: $(APPINSIGHTS_INSTRUMENTATIONKEY) - job: DurableAnalyzerTests pool: @@ -116,6 +118,7 @@ jobs: rerunMaxAttempts: 2 env: AzureWebJobsStorage: $(AzureWebJobsStorage) + APPINSIGHTS_INSTRUMENTATIONKEY: $(APPINSIGHTS_INSTRUMENTATIONKEY) - job: PublishPipelineArtifact dependsOn: diff --git a/samples/correlation-csharp/FunctionAppCorrelation.sln b/samples/correlation-csharp/FunctionAppCorrelation.sln new file mode 100644 index 000000000..c1e5f9a30 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation.sln @@ -0,0 +1,25 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.29509.3 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "FunctionAppCorrelation", "FunctionAppCorrelation\FunctionAppCorrelation.csproj", "{F9428663-9742-4746-A434-D073CCD5395C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {F9428663-9742-4746-A434-D073CCD5395C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F9428663-9742-4746-A434-D073CCD5395C}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F9428663-9742-4746-A434-D073CCD5395C}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F9428663-9742-4746-A434-D073CCD5395C}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {6D2AE1B1-FA1A-4E46-B75D-B9B01E570B50} + EndGlobalSection +EndGlobal diff --git a/samples/correlation-csharp/FunctionAppCorrelation/.gitignore b/samples/correlation-csharp/FunctionAppCorrelation/.gitignore new file mode 100644 index 000000000..bd4e82a22 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/.gitignore @@ -0,0 +1,264 @@ +## Ignore Visual Studio temporary files, build results, and +## files generated by popular Visual Studio add-ons. + +# Azure Functions localsettings file +local.settings.json + +# User-specific files +*.suo +*.user +*.userosscache +*.sln.docstates + +# User-specific files (MonoDevelop/Xamarin Studio) +*.userprefs + +# Build results +[Dd]ebug/ +[Dd]ebugPublic/ +[Rr]elease/ +[Rr]eleases/ +x64/ +x86/ +bld/ +[Bb]in/ +[Oo]bj/ +[Ll]og/ + +# Visual Studio 2015 cache/options directory +.vs/ +# Uncomment if you have tasks that create the project's static files in wwwroot +#wwwroot/ + +# MSTest test Results +[Tt]est[Rr]esult*/ +[Bb]uild[Ll]og.* + +# NUNIT +*.VisualState.xml +TestResult.xml + +# Build Results of an ATL Project +[Dd]ebugPS/ +[Rr]eleasePS/ +dlldata.c + +# DNX +project.lock.json +project.fragment.lock.json +artifacts/ + +*_i.c +*_p.c +*_i.h +*.ilk +*.meta +*.obj +*.pch +*.pdb +*.pgc +*.pgd +*.rsp +*.sbr +*.tlb +*.tli +*.tlh +*.tmp +*.tmp_proj +*.log +*.vspscc +*.vssscc +.builds +*.pidb +*.svclog +*.scc + +# Chutzpah Test files +_Chutzpah* + +# Visual C++ cache files +ipch/ +*.aps +*.ncb +*.opendb +*.opensdf +*.sdf +*.cachefile +*.VC.db +*.VC.VC.opendb + +# Visual Studio profiler +*.psess +*.vsp +*.vspx +*.sap + +# TFS 2012 Local Workspace +$tf/ + +# Guidance Automation Toolkit +*.gpState + +# ReSharper is a .NET coding add-in +_ReSharper*/ +*.[Rr]e[Ss]harper +*.DotSettings.user + +# JustCode is a .NET coding add-in +.JustCode + +# TeamCity is a build add-in +_TeamCity* + +# DotCover is a Code Coverage Tool +*.dotCover + +# NCrunch +_NCrunch_* +.*crunch*.local.xml +nCrunchTemp_* + +# MightyMoose +*.mm.* +AutoTest.Net/ + +# Web workbench (sass) +.sass-cache/ + +# Installshield output folder +[Ee]xpress/ + +# DocProject is a documentation generator add-in +DocProject/buildhelp/ +DocProject/Help/*.HxT +DocProject/Help/*.HxC +DocProject/Help/*.hhc +DocProject/Help/*.hhk +DocProject/Help/*.hhp +DocProject/Help/Html2 +DocProject/Help/html + +# Click-Once directory +publish/ + +# Publish Web Output +*.[Pp]ublish.xml +*.azurePubxml +# TODO: Comment the next line if you want to checkin your web deploy settings +# but database connection strings (with potential passwords) will be unencrypted +#*.pubxml +*.publishproj + +# Microsoft Azure Web App publish settings. Comment the next line if you want to +# checkin your Azure Web App publish settings, but sensitive information contained +# in these scripts will be unencrypted +PublishScripts/ + +# NuGet Packages +*.nupkg +# The packages folder can be ignored because of Package Restore +**/packages/* +# except build/, which is used as an MSBuild target. +!**/packages/build/ +# Uncomment if necessary however generally it will be regenerated when needed +#!**/packages/repositories.config +# NuGet v3's project.json files produces more ignoreable files +*.nuget.props +*.nuget.targets + +# Microsoft Azure Build Output +csx/ +*.build.csdef + +# Microsoft Azure Emulator +ecf/ +rcf/ + +# Windows Store app package directories and files +AppPackages/ +BundleArtifacts/ +Package.StoreAssociation.xml +_pkginfo.txt + +# Visual Studio cache files +# files ending in .cache can be ignored +*.[Cc]ache +# but keep track of directories ending in .cache +!*.[Cc]ache/ + +# Others +ClientBin/ +~$* +*~ +*.dbmdl +*.dbproj.schemaview +*.jfm +*.pfx +*.publishsettings +node_modules/ +orleans.codegen.cs + +# Since there are multiple workflows, uncomment next line to ignore bower_components +# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) +#bower_components/ + +# RIA/Silverlight projects +Generated_Code/ + +# Backup & report files from converting an old project file +# to a newer Visual Studio version. Backup files are not needed, +# because we have git ;-) +_UpgradeReport_Files/ +Backup*/ +UpgradeLog*.XML +UpgradeLog*.htm + +# SQL Server files +*.mdf +*.ldf + +# Business Intelligence projects +*.rdl.data +*.bim.layout +*.bim_*.settings + +# Microsoft Fakes +FakesAssemblies/ + +# GhostDoc plugin setting file +*.GhostDoc.xml + +# Node.js Tools for Visual Studio +.ntvs_analysis.dat + +# Visual Studio 6 build log +*.plg + +# Visual Studio 6 workspace options file +*.opt + +# Visual Studio LightSwitch build output +**/*.HTMLClient/GeneratedArtifacts +**/*.DesktopClient/GeneratedArtifacts +**/*.DesktopClient/ModelManifest.xml +**/*.Server/GeneratedArtifacts +**/*.Server/ModelManifest.xml +_Pvt_Extensions + +# Paket dependency manager +.paket/paket.exe +paket-files/ + +# FAKE - F# Make +.fake/ + +# JetBrains Rider +.idea/ +*.sln.iml + +# CodeRush +.cr/ + +# Python Tools for Visual Studio (PTVS) +__pycache__/ +*.pyc diff --git a/samples/correlation-csharp/FunctionAppCorrelation/Counter.cs b/samples/correlation-csharp/FunctionAppCorrelation/Counter.cs new file mode 100644 index 000000000..ca850798d --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/Counter.cs @@ -0,0 +1,27 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System.Threading.Tasks; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Newtonsoft.Json; + +namespace FunctionAppCorrelation +{ + [JsonObject(MemberSerialization.OptIn)] + public class Counter + { + [JsonProperty("value")] + public int CurrentValue { get; set; } + + public void Add(int amount) => this.CurrentValue += amount; + + public void Reset() => this.CurrentValue = 0; + + public int Get() => this.CurrentValue; + + [FunctionName(nameof(Counter))] + public static Task Run([EntityTrigger] IDurableEntityContext ctx) + => ctx.DispatchAsync(); + } +} \ No newline at end of file diff --git a/samples/correlation-csharp/FunctionAppCorrelation/EntityOrchestration.cs b/samples/correlation-csharp/FunctionAppCorrelation/EntityOrchestration.cs new file mode 100644 index 000000000..4c6f54370 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/EntityOrchestration.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Newtonsoft.Json.Linq; + +namespace FunctionAppCorrelation +{ + /// + /// This example is for testing that Durable Entities work with the new correlation implementation. + /// + public class EntityOrchestration + { + private const string CounterName = "myCounter"; + + [FunctionName(nameof(IncrementOrchestration))] + public async Task IncrementOrchestration( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + var entityId = new EntityId(nameof(Counter), CounterName); + await context.CallEntityAsync(entityId, "Add", 1); + var counter = await context.CallEntityAsync(entityId, "Get"); + counter++; + return counter; + } + + [FunctionName(nameof(HttpStart_EntityCounter))] + public async Task HttpStart_EntityCounter( + [HttpTrigger(AuthorizationLevel.Function, "get")] HttpRequest req, + [DurableClient] IDurableEntityClient entityClient, + [DurableClient] IDurableOrchestrationClient orchestrationClient) + { + var entityId = new EntityId(nameof(Counter), CounterName); + EntityStateResponse stateResponse = await entityClient.ReadEntityStateAsync(entityId); + + await entityClient.SignalEntityAsync(entityId, "Add", 1); + var instanceId = await orchestrationClient.StartNewAsync(nameof(this.IncrementOrchestration), null); + + return orchestrationClient.CreateCheckStatusResponse(req, instanceId); + } + } +} diff --git a/samples/correlation-csharp/FunctionAppCorrelation/EternalOrchestrations.cs b/samples/correlation-csharp/FunctionAppCorrelation/EternalOrchestrations.cs new file mode 100644 index 000000000..e5e62388a --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/EternalOrchestrations.cs @@ -0,0 +1,71 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; + +namespace FunctionAppCorrelation +{ + public class EternalOrchestrations + { + private readonly TelemetryClient telemetryClient; + + public EternalOrchestrations(TelemetryConfiguration telemetryConfiguration) + { + this.telemetryClient = new TelemetryClient(telemetryConfiguration); + } + + [FunctionName(nameof(Periodic_Cleanup_Loop))] + public async Task Periodic_Cleanup_Loop( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + var expire = context.GetInput(); + await context.CallActivityAsync(nameof(this.CleanUpNotification), "CleanUp!"); + DateTime nextCleanup = context.CurrentUtcDateTime.AddSeconds(10); + if (expire.Value > nextCleanup) + { + await context.CreateTimer(nextCleanup, CancellationToken.None); + context.ContinueAsNew(expire); + } + } + + [FunctionName(nameof(CleanUpNotification))] + public void CleanUpNotification( + [ActivityTrigger] string message) + { + // You can use TrackTrace to send a custom correlated trace message to Application Insights + this.telemetryClient.TrackTrace(message); + } + + [FunctionName(nameof(HttpStart_ExternalOrchestrations))] + public async Task HttpStart_ExternalOrchestrations( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")] + HttpRequest req, + [DurableClient] IDurableOrchestrationClient starter, + ILogger log) + { + DateTime expireTime = DateTime.UtcNow.AddSeconds(60); + string instanceId = await starter.StartNewAsync(nameof(this.Periodic_Cleanup_Loop), new Expire { Value = expireTime }); + + log.LogInformation($"Started orchestration with ID = '{instanceId}'. expireTime is {expireTime}"); + return starter.CreateCheckStatusResponse(req, instanceId); + } + + public class Expire + { + public DateTime Value { get; set; } + } + } +} diff --git a/samples/correlation-csharp/FunctionAppCorrelation/ExternalEvent.cs b/samples/correlation-csharp/FunctionAppCorrelation/ExternalEvent.cs new file mode 100644 index 000000000..cd017e634 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/ExternalEvent.cs @@ -0,0 +1,110 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; +using Newtonsoft.Json; + +namespace FunctionAppCorrelation +{ + public class ExternalEvent + { + private const string ApprovalEvent1 = "Approval1"; + private const string ApprovalEvent2 = "Approval2"; + + private readonly TelemetryClient telemetryClient; + + public ExternalEvent(TelemetryConfiguration telemetryConfiguration) + { + this.telemetryClient = new TelemetryClient(telemetryConfiguration); + } + + [FunctionName(nameof(WaitForEventOrchestrator))] + public async Task WaitForEventOrchestrator( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + var event1 = context.WaitForExternalEvent(ApprovalEvent1); + var event2 = context.WaitForExternalEvent(ApprovalEvent2); + await Task.WhenAll(event1, event2); + await context.CallActivityAsync(nameof(this.CompletedAlert), $"Event1: {event1.Result} Event2: {event2.Result}"); + } + + [FunctionName(nameof(CompletedAlert))] + public void CompletedAlert( + [ActivityTrigger] string message) + { + // You can use TrackTrace to send a custom correlated trace message to Application Insights + this.telemetryClient.TrackTrace(message); + } + + [FunctionName(nameof(ApprovalOne))] + public async Task ApprovalOne( + [HttpTrigger(AuthorizationLevel.Anonymous, "post")] + HttpRequest req, + [DurableClient] IDurableOrchestrationClient client, + ILogger log) + { + var body = await req.ReadAsStringAsync(); + var payload = JsonConvert.DeserializeObject(body); + if (payload == null) + { + return new BadRequestObjectResult("Payload should be json format with message and instanceId e.g. {\"Message\": \"Approved! one\" , \"InstanceId\":\" \"} "); + } + + await client.RaiseEventAsync(payload.InstanceId, ApprovalEvent1, payload.Message); + return new OkObjectResult( + $"Approved: Number:1 InstanceId: {payload.InstanceId} Message: {payload.Message} "); + } + + [FunctionName(nameof(ApprovalTwo))] + public async Task ApprovalTwo( + [HttpTrigger(AuthorizationLevel.Anonymous, "post")] + HttpRequest req, + [DurableClient] IDurableOrchestrationClient client, + ILogger log) + { + var body = await req.ReadAsStringAsync(); + var payload = JsonConvert.DeserializeObject(body); + if (payload == null) + { + return new BadRequestObjectResult("Payload should be json format with message and instanceId e.g. {\"Message\": \"Approved! two.\" , \"InstanceId\":\" \"} "); + } + + await client.RaiseEventAsync(payload.InstanceId, ApprovalEvent2, payload.Message); + return new OkObjectResult( + $"Approved: Number:1 InstanceId: {payload.InstanceId} Message: {payload.Message} "); + } + + [FunctionName(nameof(HttpStart_ExternalEvent))] + public async Task HttpStart_ExternalEvent( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")] + HttpRequest req, + [DurableClient] IDurableOrchestrationClient starter, + ILogger log) + { + string instanceId = await starter.StartNewAsync(nameof(this.WaitForEventOrchestrator), null); + + log.LogInformation($"Started orchestration with ID = '{instanceId}'."); + return starter.CreateCheckStatusResponse(req, instanceId); + } + + public class Payload + { + public string InstanceId { get; set; } + + public string Message { get; set; } + } + } +} diff --git a/samples/correlation-csharp/FunctionAppCorrelation/FanOutFanIn.cs b/samples/correlation-csharp/FunctionAppCorrelation/FanOutFanIn.cs new file mode 100644 index 000000000..224d8b97f --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/FanOutFanIn.cs @@ -0,0 +1,52 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; + +namespace FunctionAppCorrelation +{ + public class FanOutFanIn + { + [FunctionName(nameof(FanOutFanInOrchestrator))] + public static async Task FanOutFanInOrchestrator( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + var tasks = new Task[3]; + tasks[0] = context.CallActivityAsync(nameof(FanOutFanIn_Hello), "Tokyo"); + tasks[1] = context.CallActivityAsync(nameof(FanOutFanIn_Hello), "Seattle"); + tasks[2] = context.CallActivityAsync(nameof(FanOutFanIn_Hello), "London"); + return await Task.WhenAll(tasks); + } + + [FunctionName(nameof(FanOutFanIn_Hello))] + public static string FanOutFanIn_Hello([ActivityTrigger] string name, ILogger log) + { + log.LogInformation($"Saying hello to {name} at the same time."); + return $"Hello {name}!"; + } + + [FunctionName(nameof(HttpStart_FanOutFanIn))] + public async Task HttpStart_FanOutFanIn( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")] + HttpRequest req, + [DurableClient] IDurableOrchestrationClient starter, + ILogger log) + { + string instanceId = await starter.StartNewAsync(nameof(FanOutFanInOrchestrator), null); + + log.LogInformation($"Started orchestration with ID = '{instanceId}'."); + return starter.CreateCheckStatusResponse(req, instanceId); + } + } +} diff --git a/samples/correlation-csharp/FunctionAppCorrelation/FunctionAppCorrelation.csproj b/samples/correlation-csharp/FunctionAppCorrelation/FunctionAppCorrelation.csproj new file mode 100644 index 000000000..251b7f6ac --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/FunctionAppCorrelation.csproj @@ -0,0 +1,33 @@ + + + netcoreapp3.1 + v3 + true + true + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + PreserveNewest + + + PreserveNewest + Never + + + PreserveNewest + Never + + + \ No newline at end of file diff --git a/samples/correlation-csharp/FunctionAppCorrelation/HttpEndpoints.cs b/samples/correlation-csharp/FunctionAppCorrelation/HttpEndpoints.cs new file mode 100644 index 000000000..03047878c --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/HttpEndpoints.cs @@ -0,0 +1,67 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Net.Http; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; + +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; + +namespace FunctionAppCorrelation +{ + public class HttpEndpoints + { + [FunctionName(nameof(CheckSiteAvailable))] + public async Task CheckSiteAvailable( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + Uri uri = context.GetInput(); + + // Makes an HTTP GET request to the specified endpoint + DurableHttpResponse response = + await context.CallHttpAsync(HttpMethod.Get, uri); + + if ((int)response.StatusCode >= 400) + { + throw new Exception($"HttpEndpoint can not found: {uri} statusCode: {response.StatusCode} body: {response.Content}"); + } + + return response.Content; + } + + [FunctionName(nameof(HttpEndpoint))] + public Task HttpEndpoint( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")] + HttpRequest req, + ILogger log) + { + log.LogInformation($"HttpEndpoint is called.'."); + return Task.FromResult(new OkObjectResult(new HealthCheck() { Status = "Healthy" })); + } + + [FunctionName(nameof(HttpStart_HttpEndpoints))] + public async Task HttpStart_HttpEndpoints( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")] + HttpRequest req, + [DurableClient] IDurableOrchestrationClient starter, + ILogger log) + { + var hostname = Environment.GetEnvironmentVariable("WEBSITE_HOSTNAME"); + var uri = hostname != null && !hostname.Contains("localhost") ? $"https://{hostname}" : "http://localhost:7071"; + string instanceId = + await starter.StartNewAsync(nameof(this.CheckSiteAvailable), new Uri($"{uri}/api/{nameof(this.HttpEndpoint)}")); + log.LogInformation($"Started HttpEndpoints orchestration with ID = '{instanceId}'."); + return starter.CreateCheckStatusResponse(req, instanceId); + } + + public class HealthCheck + { + public string Status { get; set; } + } + } +} diff --git a/samples/correlation-csharp/FunctionAppCorrelation/Monitoring.cs b/samples/correlation-csharp/FunctionAppCorrelation/Monitoring.cs new file mode 100644 index 000000000..248ad4736 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/Monitoring.cs @@ -0,0 +1,116 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; + +namespace FunctionAppCorrelation +{ + public class Monitoring + { + private readonly TelemetryClient telemetryClient; + + // The demo purpose implementation. For production, use Durable Entity to store the state. + private static readonly ConcurrentDictionary State = new ConcurrentDictionary(); + + public Monitoring(TelemetryConfiguration telemetryConfiguration) + { + this.telemetryClient = new TelemetryClient(telemetryConfiguration); + } + + [FunctionName(nameof(MonitorJobStatus))] + public async Task MonitorJobStatus( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + var job = context.GetInput(); + int pollingInterval = 30; // Second + while (context.CurrentUtcDateTime < job.ExpiryTime) + { + var jobStatus = await context.CallActivityAsync(nameof(this.GetJobStatus), job.JobId); + if (jobStatus == "Completed") + { + await context.CallActivityAsync(nameof(this.SendAlert), $"Job({job.JobId}) Completed."); + break; + } + + var nextCheck = context.CurrentUtcDateTime.AddSeconds(pollingInterval); + await context.CreateTimer(nextCheck, CancellationToken.None); + } + } + + [FunctionName(nameof(SendAlert))] + public void SendAlert( + [ActivityTrigger] string message) + { + this.telemetryClient.TrackTrace(message); + } + + [FunctionName(nameof(GetJobStatus))] + public Task GetJobStatus( + [ActivityTrigger] IDurableActivityContext context) + { + var jobId = context.GetInput(); + + // The demo purpose implementation. For production, use Durable Entity to store the state. + var status = State.AddOrUpdate(jobId, "Scheduled", (key, oldValue) => + { + switch (oldValue) + { + case "Scheduled": + return "Running"; + case "Running": + return "Completed"; + default: + return "Failed"; + } + }); + return Task.FromResult(status); + } + + [FunctionName(nameof(HttpStart_Monitor))] + public async Task HttpStart_Monitor( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")] + HttpRequest req, + [DurableClient] IDurableOrchestrationClient starter, + ILogger log) + { + string jobId = req.Query["JobId"]; + + if (string.IsNullOrEmpty(jobId)) + { + return new BadRequestObjectResult("Parameter JobId can not be null. Add '?JobId={someId}' on your request URI."); + } + + var expirTime = DateTime.UtcNow.AddSeconds(180); + + string instanceId = await starter.StartNewAsync(nameof(this.MonitorJobStatus), new Job() + { + JobId = jobId, + ExpiryTime = expirTime, + }); + + log.LogInformation($"Started orchestration with ID = '{instanceId}'."); + return starter.CreateCheckStatusResponse(req, instanceId); + } + + public class Job + { + public string JobId { get; set; } + + public DateTime ExpiryTime { get; set; } + } + } +} diff --git a/samples/correlation-csharp/FunctionAppCorrelation/MultiLayerOrchestrationWithRetry.cs b/samples/correlation-csharp/FunctionAppCorrelation/MultiLayerOrchestrationWithRetry.cs new file mode 100644 index 000000000..35d40c8b4 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/MultiLayerOrchestrationWithRetry.cs @@ -0,0 +1,99 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net.Http; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using DurableTask.Core; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Azure.WebJobs.Host; +using Microsoft.Extensions.Logging; +using Microsoft.Net.Http.Headers; +using RetryOptions = Microsoft.Azure.WebJobs.Extensions.DurableTask.RetryOptions; + +namespace FunctionAppCorrelation +{ + public static class MultiLayerOrchestrationWithRetry + { + private static int count1 = 0; + private static int count2 = 0; + private static object countLock = new object(); + + [FunctionName(nameof(MultiLayerOrchestrationWithRetryOrchestrator))] + public static async Task> MultiLayerOrchestrationWithRetryOrchestrator( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + var outputs = new List(); + + var child1Result = await context.CallSubOrchestratorAsync>(nameof(ChildOrchestrator), null); + var child2Result = await context.CallSubOrchestratorAsync>(nameof(ChildOrchestrator), null); + outputs.AddRange(child1Result); + outputs.AddRange(child2Result); + return outputs; + } + + [FunctionName(nameof(ChildOrchestrator))] + public static async Task> ChildOrchestrator( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + var outputs = new List(); + var tasks = new Task[3]; + var option = new RetryOptions(TimeSpan.FromSeconds(5), 3); + tasks[0] = context.CallActivityWithRetryAsync(nameof(MultiLayerOrchestrationWithRetry_Hello), option, "Osaka"); + tasks[1] = context.CallActivityWithRetryAsync(nameof(MultiLayerOrchestrationWithRetry_Hello), option, "Seattle"); + tasks[2] = context.CallActivityWithRetryAsync(nameof(MultiLayerOrchestrationWithRetry_Hello), option, "Atlanta"); + await Task.WhenAll(tasks); + return tasks.Select((i) => i.Result).ToList(); + } + + [FunctionName(nameof(MultiLayerOrchestrationWithRetry_Hello))] + public static string MultiLayerOrchestrationWithRetry_Hello([ActivityTrigger] string name, ILogger log) + { + // This is the demo purpose implementation for retrying happens for the first two execution. + // The function restart is required to start failing again. + lock (countLock) + { + if (count1 == 0) + { + count1++; + throw new Exception("Something bad happened."); + } + + if (count2 == 3) + { + count2++; + throw new Exception("Something wrong happened."); + } + + count1++; + count2++; + } + + log.LogInformation($"Saying hello to {name}."); + return $"Hello {name}!"; + } + + [FunctionName(nameof(HttpStart_MultiLayerOrchestrationWithRetry))] + public static async Task HttpStart_MultiLayerOrchestrationWithRetry( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")]HttpRequest req, + [DurableClient]IDurableOrchestrationClient starter, + ILogger log) + { + string instanceId = await starter.StartNewAsync(nameof(MultiLayerOrchestrationWithRetryOrchestrator), null); + log.LogInformation($"Started orchestration with ID = '{instanceId}'."); + return starter.CreateCheckStatusResponse(req, instanceId); + } + } +} \ No newline at end of file diff --git a/samples/correlation-csharp/FunctionAppCorrelation/SimpleCorrelationDemo.cs b/samples/correlation-csharp/FunctionAppCorrelation/SimpleCorrelationDemo.cs new file mode 100644 index 000000000..c48f6c260 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/SimpleCorrelationDemo.cs @@ -0,0 +1,109 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using DurableTask.Core; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; + +namespace FunctionAppCorrelation +{ + public class SimpleCorrelationDemo + { + private readonly TelemetryClient telemetryClient; + private readonly HttpClient httpClient; + + public SimpleCorrelationDemo(TelemetryConfiguration telemetryConfiguration, HttpClient client) + { + this.telemetryClient = new TelemetryClient(telemetryConfiguration); + this.httpClient = client; + } + + [FunctionName(nameof(Orchestration_W3C))] + public async Task> Orchestration_W3C( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + if (!CorrelationTraceContext.Current is W3CTraceContext correlationContext) + { + throw new InvalidOperationException($"This sample expects a correlation trace context of {nameof(W3CTraceContext)}, but the context isof type {CorrelationTraceContext.Current.GetType()}"); + } + + var trace = new TraceTelemetry( + $"Activity Id: {correlationContext.TraceParent} ParentSpanId: {correlationContext.ParentSpanId}"); + trace.Context.Operation.Id = correlationContext.TelemetryContextOperationId; + trace.Context.Operation.ParentId = correlationContext.TelemetryContextOperationParentId; + this.telemetryClient.Track(trace); + + var outputs = new List + { + await context.CallActivityAsync(nameof(this.Hello_W3C), "Tokyo"), + await context.CallActivityAsync(nameof(this.Hello_W3C), "Seattle"), + await context.CallActivityAsync(nameof(this.Hello_W3C), "London"), + }; + + // returns ["Hello Tokyo!", "Hello Seattle!", "Hello London!"] + return outputs; + } + + [FunctionName(nameof(Hello_W3C))] + public string Hello_W3C([ActivityTrigger] string name, ILogger log) + { + // Send Custom Telemetry + var currentActivity = Activity.Current; + this.telemetryClient.TrackTrace($"Message from Activity: {name}."); + + log.LogInformation($"Saying hello to {name}."); + return $"Hello {name}!"; + } + + [FunctionName(nameof(HttpStart_With_W3C))] + public async Task HttpStart_With_W3C( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")]HttpRequestMessage req, + [DurableClient]IDurableOrchestrationClient starter, + ILogger log) + { + var traceparent = req.Headers.GetValues("traceparent").FirstOrDefault(); // HttpCorrelationProtocol uses Request-Id + var currentActivity = Activity.Current; + + if (string.IsNullOrEmpty(traceparent)) + { + log.LogInformation("Traceparent can not be empty."); + return req.CreateErrorResponse(HttpStatusCode.BadRequest, "Traceparent header is required."); + } + + string instanceId = await starter.StartNewAsync(nameof(this.Orchestration_W3C), null); + + log.LogInformation($"Started orchestration with ID = '{instanceId}'."); + + return starter.CreateCheckStatusResponse(req, instanceId); + } + + [FunctionName(nameof(HttpStart_AnExternalSystem))] + public async Task HttpStart_AnExternalSystem( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")] + HttpRequest req, + ILogger log) + { + log.LogInformation($"An external system send a request. "); + var currentActivity = Activity.Current; + var hostname = Environment.GetEnvironmentVariable("WEBSITE_HOSTNAME"); + var uri = $"https://{hostname}"; // Change from https to http with port number for Local Debugging. + await this.httpClient.GetAsync($"{uri}/api/{nameof(this.HttpStart_With_W3C)}"); + return new OkObjectResult("Telemetry Sent."); + } + } +} diff --git a/samples/correlation-csharp/FunctionAppCorrelation/SimpleOrchestration.cs b/samples/correlation-csharp/FunctionAppCorrelation/SimpleOrchestration.cs new file mode 100644 index 000000000..9a57ef857 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/SimpleOrchestration.cs @@ -0,0 +1,55 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Net.Http; +using System.Text; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Azure.WebJobs; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; +using Microsoft.Azure.WebJobs.Extensions.Http; +using Microsoft.Extensions.Logging; + +namespace FunctionAppCorrelation +{ + public class SimpleOrchestration + { + [FunctionName(nameof(SimpleOrchestrator))] + public static async Task> SimpleOrchestrator( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + var outputs = new List(); + + // Replace "hello" with the name of your Durable Activity Function. + outputs.Add(await context.CallActivityAsync(nameof(SayHello), "Tokyo")); + outputs.Add(await context.CallActivityAsync(nameof(SayHello), "Seattle")); + outputs.Add(await context.CallActivityAsync(nameof(SayHello), "London")); + + return outputs; + } + + [FunctionName(nameof(SayHello))] + public static string SayHello([ActivityTrigger] string name, ILogger log) + { + var currentActivity = Activity.Current; + log.LogInformation($"Saying hello to {name}."); + return $"Hello {name}!"; + } + + [FunctionName(nameof(HttpStart_SimpleOrchestration))] + public static async Task HttpStart_SimpleOrchestration( + [HttpTrigger(AuthorizationLevel.Anonymous, "get")]HttpRequest req, + [DurableClient]IDurableOrchestrationClient starter, + ILogger log) + { + string instanceId = await starter.StartNewAsync(nameof(SimpleOrchestrator), null); + log.LogInformation($"Started orchestration with ID = '{instanceId}'."); + return starter.CreateCheckStatusResponse(req, instanceId); + } + } +} diff --git a/samples/correlation-csharp/FunctionAppCorrelation/host.json b/samples/correlation-csharp/FunctionAppCorrelation/host.json new file mode 100644 index 000000000..37e6030d7 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/host.json @@ -0,0 +1,17 @@ +{ + "extensions": { + "durableTask": { + "tracing": { + "DistributedTracingProtocol": "W3CTraceContext" + } + } + }, + "logging": { + "applicationInsights": { + "httpAutoCollectionOptions": { + "enableW3CDistributedTracing": true + } + } + }, + "version": "2.0" +} \ No newline at end of file diff --git a/samples/correlation-csharp/FunctionAppCorrelation/local.settings.json.example b/samples/correlation-csharp/FunctionAppCorrelation/local.settings.json.example new file mode 100644 index 000000000..fc45fa6e0 --- /dev/null +++ b/samples/correlation-csharp/FunctionAppCorrelation/local.settings.json.example @@ -0,0 +1,8 @@ +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet", + "APPINSIGHTS_INSTRUMENTATIONKEY": "" + } +} diff --git a/samples/correlation-csharp/Readme.md b/samples/correlation-csharp/Readme.md new file mode 100644 index 000000000..8e9d58f20 --- /dev/null +++ b/samples/correlation-csharp/Readme.md @@ -0,0 +1,64 @@ +# Distributed Tracing for Durable Functions + +It does not only demonstrate how to use the Distributed Tracing but also used for testing, including non-supported orchestrations by the Distributed Tracing. + +![Overview](images/overview.png) + +## Video: Distributed Tracing for Durable Functions +Overview of the Distributed Tracing + +[![Distributed Tracing for Durable Functions](https://img.youtube.com/vi/lzvgPps37G4/0.jpg)](https://www.youtube.com/watch?v=lzvgPps37G4) + +## Video External Correlation +How to correlate with external systems + +[![External Correlation](https://img.youtube.com/vi/fv9D2pVE4eY/0.jpg)](https://www.youtube.com/watch?v=fv9D2pVE4eY) + + +# Getting Started + +* [Getting Started](getting-started.md) + +# Configuration + +* [Configuration](configuration.md) + +# Reference + +* [Reference](reference.md) + +# Architecture + +Durable Functions Distributed Tracing is mainly implemented DurableTask library. For more details of the architecture, you can refer [durabletask: Distributed Tracing](https://github.com/Azure/durabletask/tree/correlation/samples/Correlation.Samples). + +# Sample Scenario + +Refer to the scenario that includes the samples. Some scenarios do not support distributed tracing. We'll support it in the future. For executing samples, call `HttpStart_*` functions as the endpoints. + +| Scenario | Description | Endpoint | Supported | comment | +| ------- | ------------------- | -------- | --------- | ------- | +| Simple Orchestration | A simple Orchestration with an Activity | /api/HttpStart_SimpleOrchestration | yes | | +| Multiple LayerOrchestration With Retry | A complex orchestration that includes sub orchestration, errors, and retry | /api/HttpStart_MultiLayerOrchestrationWithRetry | yes | | +| Correlation integration | Getting a tracing Id from outside of the function, also getting tracing Id in the orchestrator and activity | /api/HttpStart_AnExternalSystem | yes | The endpoint represents an external sytem, it call a function that start an orchestration | +| Fan Out Fan In | Simple [Fan Out/Fan In](https://docs.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-overview?tabs=csharp#fan-in-out) Pattern implementation | /api/HttpStart_FanOutFanIn | +| Durable Entity Orchestration | [Stateful Entities Pattern](https://docs.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-orchestrations?tabs=csharp) implementation | /api/HttpStart_EntityCounter |no | Entity has not supported yet, however, it correlate for the other parts | +| HTTP endpoints | [Calling HTTP endpoints](https://docs.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-orchestrations?tabs=csharp) implementation | /api/HttpStart_HttpEndpoints | no | | +| Human Interaction | [Human Interaction](https://docs.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-overview?tabs=csharp#human) pattern implementation | /api/HttpStart_ExternalEvent | no | After staring the orchestration, call /api/ApprovalOne and /api/ApprovalTwo with `{"Message": "Approved!" , "InstanceId":""}` with POST | +| ContinueAsNew | [Eternal Orchestrator](https://docs.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-eternal-orchestrations?tabs=csharp) implementation| /api/HttpStart_ExternalOrchestrations | yes | | +| Monitoring | [Monitor ](https://docs.microsoft.com/en-us/azure/azure-functions/durable/durable-functions-overview?tabs=csharp#monitoring) Pattern implementation | /api/HttpStart_Monitor?JobId={JobId} | no | You can see all the telemetries on the End to End transaction as the same Operation ID telemetry. However, the parent-children relationship sometimes missing. | + +# Limitation + +Current version doesn't include the custom properties of a telemetry of the Application Insights. + +![Custom Properties](images/custom-properties.png) + +The custom property is available for ActivityTrigger once this pull request is accepted and a change added to the Durable Functions side. +[Track original activity at the Application Insights Logger](https://github.com/Azure/azure-webjobs-sdk/pull/2474) + +# FAQ + +### 1. In some supported Scenario, the correlation relationship looks broken + +The Distributed Tracing system track the Orchestration Request Telemetry after the completion of the orchestrator. Wait for a while, then look at the telemetry again. + diff --git a/samples/correlation-csharp/configuration.md b/samples/correlation-csharp/configuration.md new file mode 100644 index 000000000..91ff374fa --- /dev/null +++ b/samples/correlation-csharp/configuration.md @@ -0,0 +1,58 @@ +# Configuration + +## host.json + +### durableTask + +#### tracing + +| property | Default | Description | +| -------- | ------- | ----------- | +| DistributedTracingEnabled | true | Make it true if you don't need Distributed Tracing for Durable Functions | +| DistributedTracingProtocol | HttpCorrelationProtocol | Set Protocol of Distributed Tracing. Possible values are HttpCorrelationProtocol and W3CTraceContext | + +**NOTE:** You need to specify the same protocol as `logging.applicationInsights.httpAutoCollectionOptions.enableW3CDistributedTracing` + +### Sample + +Enable Distributed Tracing with W3C Trace Context. + +_host.json_ + +```json + + "extensions": { + "durableTask": { + "tracing": { + "DistributedTracingProtocol": "W3CTraceContext" + } + } + }, + "logging": { + "applicationInsights": { + "httpAutoCollectionOptions": { + "enableW3CDistributedTracing": true + } + } + }, + "version": "2.0" +} +``` + +## AppSettings + +You need to specify the Application Insights Instrumentation key on AppSettings or local.settings.json or Environment Variables. + + +_local.settings.json_ + +```json +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet", + "APPINSIGHTS_INSTRUMENTATIONKEY": "" + } +} +``` diff --git a/samples/correlation-csharp/getting-started.md b/samples/correlation-csharp/getting-started.md new file mode 100644 index 000000000..2f1145258 --- /dev/null +++ b/samples/correlation-csharp/getting-started.md @@ -0,0 +1,164 @@ +# Getting Started for Distributed Tracing for Durable Functions + +In this article, you use command-line tools to create a function app, then publish the sample applications using Visual Studio then learn how to diagnose the telemetry. In this tutorial, we use Azure CLI on the Cloud Shell to create an environment. + +* An Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?ref=microsoft.com&utm_source=microsoft.com&utm_medium=docs&utm_campaign=visualstudio) +* Visual Studio 2019. Ensure you select the Azure Development workload during installation. For more details [Quickstart: Create your first function in Azure using Visual Studio](https://docs.microsoft.com/en-us/azure/azure-functions/functions-create-your-first-function-visual-studio) + + +# Create a function app + +Go to Azure Portal, Click the cloud shell on the right top of the Azure Portal. Open Bash. + +![Cloud Shell](images/cloud-shell.png) + +## Create a Resource Group + +Create a resource group. + +```bash +ResourceGroup=DurableFunctionsQucikstart-rg +Location=westus +az group create --name $ResourceGroup --location $Location +``` +## Create a Storage Account + +Create a storage account. The storage account name should be globally unique. + +```bash +StorageAccountName= +az storage account create --name $StorageAccountName --location $Location --resource-group $ResourceGroup --sku Standard_LRS +``` + +## Create an Application Insights +FunctionAppName should be globally unique. + +```bash +az extension add -n application-insights +FunctionAppName= +InstrumentationKey=`az monitor app-insights component create --app $FunctionAppName --location $Location --kind web -g $ResourceGroup --application-type web | jq .instrumentationKey | xargs` +``` + +You can see your InstrumentationKey of the Application Insights. We'll use this key at the configuration part of the sample applications. + +```bash +echo $InstrumentationKey +``` + +## Create a Function App + + +```bash +az functionapp create --resource-group $ResourceGroup --consumption-plan-location $Location --runtime dotnet --functions-version 3 --name $FunctionAppName --storage-account $StorageAccountName --app-insights $FunctionAppName --app-insights-key $InstrumentationKey +``` + +**NOTE:** The samples targeting functions version 3. Distributed Tracing works for version 2 but not for version 1. + +# Publish samples + +```bash +git clone git@github.com:Azure/azure-functions-durable-extension.git +cd azure-functions-durable-extension +git switch -c correlation origin/correlation +``` + +## Open the .sln file + +Go to `samples/correlation-csharp/` then open FunctionAppCorrelation.sln with Visual Studio 2019. + +## Create a local.settings.json + +Create your `local.settings.json` for local execution. +You can copy and modify `local.settings.json.example.` Set the Application Insights Key. If you are not familiar with the Application Insights key, refer to [Create an Application Insights resource](https://docs.microsoft.com/en-us/azure/azure-monitor/app/create-new-resource). + +```json +{ + "IsEncrypted": false, + "Values": { + "AzureWebJobsStorage": "UseDevelopmentStorage=true", + "FUNCTIONS_WORKER_RUNTIME": "dotnet", + "APPINSIGHTS_INSTRUMENTATIONKEY": "" + } +} +``` + +## host.json + +Configure host.json. This JSON is the sample of the Distributed Tracing with the W3C trace context. Distributed Tracing is enabled by default. You need to configure the telemetry protocol. For more details, refer [reference](reference.md). + +```json +{ + "extensions": { + "durableTask": { + "tracing": { + "DistributedTracingProtocol": "W3CTraceContext" + } + } + }, + "logging": { + "applicationInsights": { + "httpAutoCollectionOptions": { + "enableW3CDistributedTracing": true + } + } + }, + "version": "2.0" +} +``` + +## Storage Emulator +For the local execution, you need the [Strorage Emulator](https://docs.microsoft.com/en-us/azure/storage/common/storage-use-emulator). Download it from the link and execute it before you run the functions. + +## Configure NuGet + +Goto the Package source on Visual Studio. Right-click FunctionAppCorrelation project, select `Manage NuGet packages...`, then click the icon. + +![NuGet Manager](images/nuget-manager.png) + +Add Available package sources. +NOTE: Distributed Tracing is pre-release. We currently use myget.org until it is going to GA. + +Name: azure-appservice-staging +Source: https://www.myget.org/F/azure-appservice-staging/api/v3/index.json + +During pre-release, we use `Microsoft.Azure.WebJobs.Extensions.DurableTask.Telemetry` NuGet package start with Version `2.2.0-alpha`. + +## Push the samples to the Function App + +Right-click the `FunctionAppCorrelation` project, then select `Publish.` + +Pick a publish target will be `Azure Functions Consumption Plan` with Select Existing with Run from package file. Click `Create Profile` + +![Pick a publish target](images/pick-a-publish-target.png) + +Then select the target function app. Then click `OK.` + +![App Service](images/app-service.png) + +## Run the samples + +Refer to [the scenario](Readme.md#sample-scenario) that includes the samples. Some scenarios do not support distributed tracing. We'll support it in the future. For executing samples, call `HttpStart_*` functions as the endpoints. + +If you can't pick one, you can try `/api/HttpStart_sampleOrchestration` on your FunctionApp. For the complex orchestration, try `/api/HttpStart_MultiLayerOrchestrationWithRetry` + +Refer the [Sample scenario](Readme.md#sample-scenario). + +## Diagnose the Telemetry + +Go to your Azure Portal, then go to your Application Insights resource. +Click `Search` on your left list. Filter it with `Last 30 minutes` and `Event types: Request.` You can see the `Start Orchestration` request. Click it. + +![Search](images/search.png) + +Then You can see end-to-end tracing. Click and see how it correlates with each other. + +![End To End Tracing](images/end-to-end.png) + +**NOTE:** When you see the correlation breaks, you might wait for a while. The request telemetry of the first orchestrator comes last. + +# Next Steps + +* [Configuration](configuration.md) +* [Reference](reference.md) + + diff --git a/samples/correlation-csharp/images/app-service.png b/samples/correlation-csharp/images/app-service.png new file mode 100644 index 000000000..4f6b40adc Binary files /dev/null and b/samples/correlation-csharp/images/app-service.png differ diff --git a/samples/correlation-csharp/images/cloud-shell.png b/samples/correlation-csharp/images/cloud-shell.png new file mode 100644 index 000000000..2571dd471 Binary files /dev/null and b/samples/correlation-csharp/images/cloud-shell.png differ diff --git a/samples/correlation-csharp/images/custom-properties.png b/samples/correlation-csharp/images/custom-properties.png new file mode 100644 index 000000000..8fbed2b86 Binary files /dev/null and b/samples/correlation-csharp/images/custom-properties.png differ diff --git a/samples/correlation-csharp/images/end-to-end.png b/samples/correlation-csharp/images/end-to-end.png new file mode 100644 index 000000000..83dd9ce82 Binary files /dev/null and b/samples/correlation-csharp/images/end-to-end.png differ diff --git a/samples/correlation-csharp/images/nuget-manager.png b/samples/correlation-csharp/images/nuget-manager.png new file mode 100644 index 000000000..3af9b56e0 Binary files /dev/null and b/samples/correlation-csharp/images/nuget-manager.png differ diff --git a/samples/correlation-csharp/images/overview.png b/samples/correlation-csharp/images/overview.png new file mode 100644 index 000000000..cbe252caf Binary files /dev/null and b/samples/correlation-csharp/images/overview.png differ diff --git a/samples/correlation-csharp/images/pick-a-publish-target.png b/samples/correlation-csharp/images/pick-a-publish-target.png new file mode 100644 index 000000000..1b167e757 Binary files /dev/null and b/samples/correlation-csharp/images/pick-a-publish-target.png differ diff --git a/samples/correlation-csharp/images/search.png b/samples/correlation-csharp/images/search.png new file mode 100644 index 000000000..9534b86c2 Binary files /dev/null and b/samples/correlation-csharp/images/search.png differ diff --git a/samples/correlation-csharp/reference.md b/samples/correlation-csharp/reference.md new file mode 100644 index 000000000..70c34c5d7 --- /dev/null +++ b/samples/correlation-csharp/reference.md @@ -0,0 +1,113 @@ +# Reference + +You can get correlation information from these classes. + +## CorrelationTraceContext + +CorrelationTraceContext share the current TraceContext. + +### Properties + +| Property | Type | Description | +| -------- | -----|------ | +| Current | TraceContextBase |Get or Set the current TraceContext with AsyncLocal | + +## TraceContextBase + +Represents TraceContext Base class, that preserve the Correlation information. + +### Properties + +| Property | Type | Description | +| -------- | -----|------ | +| StartTime | DateTimeOffset |Get or Set the start time of this telemetry | +| TelemetryType | TlemetryType | Get or Set the telemetry type. Possible values are RequestTelemetry or DepdendencyTelemetry | +| OrchestrationTraceContexts | `Stack` | Get or Set the TraceContextBase object with Stack structure. This object represent Orchestration TraceContexts. It includes both Request and Dependnecy telemetries. | +| OperationName | string | Get or Set Operation name | + +### Methods + +| Method | Type | Description | +| -------- | -----|------ | +| GetCurrentOrchestrationRequestTraceContext() | TraceContextBase | GetRequestTraceContext of CurrentOrchestration | + + +## W3CTraceContext + +W3CTraceContext represent TraceContext for [W3CTraceContext](https://www.w3.org/TR/trace-context/) protocol. Implementation of the TraceContextBase. + +### Properties + +| Property | Type | Description | +| -------- | -----|------ | +| TraceParent | string |Get or Set the traceparent | +| TraceState | string | Get or Set the tracestate | +| ParentSpanId | string | Get or Set the ParentSpanId | +| Duration | TimeSpan | Get the duration of this extecution | +| TelemetryId | string | Get the telemetryId. This value is sent to the Application Insights. | +| TelemetryContextOperationId | string | Get the TelemetryContextOperationId. This value is sent to the Application Insights. | +| TelemetryContextOperationParentId | string | Get the TelemetryContextOperationParentId. This value is sent to the Application Insights. | + + +## HttpCorrelationTraceContext +HttpCorrelationTraceContext represent TraceContext for [HttpCorrelationProtocol](https://github.com/dotnet/runtime/blob/4f9ae42d861fcb4be2fcd5d3d55d5f227d30e723/src/libraries/System.Diagnostics.DiagnosticSource/src/HttpCorrelationProtocol.md) protocol. Implementation of the TraceContextBase. + +### Properties + +| Property | Type | Description | +| -------- | -----|------ | +| ParentId | string |Get or Set the ParentId | +| ParentParentId | string | Get or Set the ParentId of the parent | +| Duration | TimeSpan | Get the duration of this extecution | +| TelemetryId | string | Get the telemetryId. This value is sent to the Application Insights. | +| TelemetryContextOperationId | string | Get the TelemetryContextOperationId. This value is sent to the Application Insights. | +| TelemetryContextOperationParentId | string | Get the TelemetryContextOperationParentId. This value is sent to the Application Insights. | + +## Activity + +[Activity Class](https://docs.microsoft.com/en-us/dotnet/api/system.diagnostics.activity?view=netcore-3.1) represents an operation with context to be used for logging. If you want to get correlation information on Activity Functions, please refer to this object with `Activity.Current` property. For more details, please refer to [Activity User Guide](https://github.com/dotnet/runtime/blob/4f9ae42d861fcb4be2fcd5d3d55d5f227d30e723/src/libraries/System.Diagnostics.DiagnosticSource/src/ActivityUserGuide.md1). + +## Samples + +### Get TraceContextBase object on an orchestrator function + +```csharp + [FunctionName(nameof(Orchestration_W3C))] + public async Task> Orchestration_W3C( + [OrchestrationTrigger] IDurableOrchestrationContext context) + { + + var correlationContext = CorrelationTraceContext.Current as W3CTraceContext; + var trace = new TraceTelemetry( + $"Activity Id: {correlationContext?.TraceParent} ParentSpanId: {correlationContext?.ParentSpanId}"); + trace.Context.Operation.Id = correlationContext?.TelemetryContextOperationId; + trace.Context.Operation.ParentId = correlationContext?.TelemetryContextOperationParentId; + _telemetryClient.Track(trace); + + var outputs = new List(); + + // Replace "hello" with the name of your Durable Activity Function. + outputs.Add(await context.CallActivityAsync(nameof(Hello_W3C), "Tokyo")); + outputs.Add(await context.CallActivityAsync(nameof(Hello_W3C), "Seattle")); + outputs.Add(await context.CallActivityAsync(nameof(Hello_W3C), "London")); + + // returns ["Hello Tokyo!", "Hello Seattle!", "Hello London!"] + return outputs; + } +``` + +### Get Activity on Activity functions + +```csharp + [FunctionName(nameof(Hello_W3C))] + public string Hello_W3C([ActivityTrigger] string name, ILogger log) + { + // Send Custom Telemetry + var currentActivity = Activity.Current; + _telemetryClient.TrackTrace($"Message from Activity: {name}."); + + log.LogInformation($"Saying hello to {name}."); + return $"Hello {name}!"; + } +``` + diff --git a/src/WebJobs.Extensions.DurableTask/AssemblyInfo.cs b/src/WebJobs.Extensions.DurableTask/AssemblyInfo.cs index a1bec28d6..b7082cbda 100644 --- a/src/WebJobs.Extensions.DurableTask/AssemblyInfo.cs +++ b/src/WebJobs.Extensions.DurableTask/AssemblyInfo.cs @@ -5,4 +5,4 @@ [assembly: InternalsVisibleTo("WebJobs.Extensions.DurableTask.Tests.V1, PublicKey=0024000004800000940000000602000000240000525341310004000001000100cd1dabd5a893b40e75dc901fe7293db4a3caf9cd4d3e3ed6178d49cd476969abe74a9e0b7f4a0bb15edca48758155d35a4f05e6e852fff1b319d103b39ba04acbadd278c2753627c95e1f6f6582425374b92f51cca3deb0d2aab9de3ecda7753900a31f70a236f163006beefffe282888f85e3c76d1205ec7dfef7fa472a17b1")] [assembly: InternalsVisibleTo("WebJobs.Extensions.DurableTask.Tests.V2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100cd1dabd5a893b40e75dc901fe7293db4a3caf9cd4d3e3ed6178d49cd476969abe74a9e0b7f4a0bb15edca48758155d35a4f05e6e852fff1b319d103b39ba04acbadd278c2753627c95e1f6f6582425374b92f51cca3deb0d2aab9de3ecda7753900a31f70a236f163006beefffe282888f85e3c76d1205ec7dfef7fa472a17b1")] -[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2,PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] +[assembly: InternalsVisibleTo("DynamicProxyGenAssembly2, PublicKey=0024000004800000940000000602000000240000525341310004000001000100c547cac37abd99c8db225ef2f6c8a3602f3b3606cc9891605d02baa56104f4cfc0734aa39b93bf7852f7d9266654753cc297e7d2edfe0bac1cdcf9f717241550e0a7b191195b7667bb4f64bcb8e2121380fd1d9d46ad2d92d2d15605093924cceaf74c4861eff62abf69b9291ed0a340e113be11e6a7d3113e92484cf7045cc7")] diff --git a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableOrchestrationContext.cs b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableOrchestrationContext.cs index f920fa07c..4aff2242a 100644 --- a/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableOrchestrationContext.cs +++ b/src/WebJobs.Extensions.DurableTask/ContextImplementations/DurableOrchestrationContext.cs @@ -481,6 +481,10 @@ void IDurableOrchestrationContext.SignalEntity(EntityId entity, DateTime startTi /// string IDurableOrchestrationContext.StartNewOrchestration(string functionName, object input, string instanceId) { + // correlation +#if NETSTANDARD2_0 + var context = CorrelationTraceContext.Current; +#endif this.ThrowIfInvalidAccess(); var actualInstanceId = string.IsNullOrEmpty(instanceId) ? this.NewGuid().ToString() : instanceId; var alreadyCompletedTask = this.CallDurableTaskFunctionAsync(functionName, FunctionType.Orchestrator, true, actualInstanceId, null, null, input, null); diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/DurableTaskCorrelationTelemetryInitializer.cs b/src/WebJobs.Extensions.DurableTask/Correlation/DurableTaskCorrelationTelemetryInitializer.cs new file mode 100644 index 000000000..e8d8e8a30 --- /dev/null +++ b/src/WebJobs.Extensions.DurableTask/Correlation/DurableTaskCorrelationTelemetryInitializer.cs @@ -0,0 +1,319 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; +using DurableTask.Core; +using DurableTask.Core.Settings; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.ApplicationInsights.Extensibility.Implementation; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation +{ + /// + /// Telemetry Initializer that sets correlation ids for W3C. + /// This source is based on W3COperationCorrelationTelemetryInitializer.cs + /// 1. Modified with CorrelationTraceContext.Current + /// 2. Avoid to be overriden when it is RequestTelemetry + /// Original Source is here https://github.com/microsoft/ApplicationInsights-dotnet-server/blob/2.8.0/Src/Common/W3C/W3COperationCorrelationTelemetryInitializer.cs. + /// + internal + class DurableTaskCorrelationTelemetryInitializer : ITelemetryInitializer + { + private const string RddDiagnosticSourcePrefix = "rdddsc"; + private const string SqlRemoteDependencyType = "SQL"; + + /// These internal property is copied from W3CConstants + /// Trace-Id tag name. + internal const string TraceIdTag = "w3c_traceId"; + + /// Span-Id tag name. + internal const string SpanIdTag = "w3c_spanId"; + + /// Parent span-Id tag name. + internal const string ParentSpanIdTag = "w3c_parentSpanId"; + + /// Version tag name. + internal const string VersionTag = "w3c_version"; + + /// Sampled tag name. + internal const string SampledTag = "w3c_sampled"; + + /// Tracestate tag name. + internal const string TracestateTag = "w3c_tracestate"; + + /// Default version value. + internal const string DefaultVersion = "00"; + + /// + /// Default sampled flag value: may be recorded, not requested. + /// + internal const string TraceFlagRecordedAndNotRequested = "02"; + + /// Recorded and requested sampled flag value. + internal const string TraceFlagRecordedAndRequested = "03"; + + /// Requested trace flag. + internal const byte RequestedTraceFlag = 1; + + /// Legacy root Id tag name. + internal const string LegacyRootIdProperty = "ai_legacyRootId"; + + /// Legacy root Id tag name. + internal const string LegacyRequestIdProperty = "ai_legacyRequestId"; + + /// + /// Constructor. + /// + public DurableTaskCorrelationTelemetryInitializer() + { + this.ExcludeComponentCorrelationHttpHeadersOnDomains = new HashSet(); + } + + /// + /// Set of suppress telemetry tracking if you add Host name on this. + /// + public HashSet ExcludeComponentCorrelationHttpHeadersOnDomains { get; set; } + + /// + /// Initializes telemetry item. + /// + /// Telemetry item. + public void Initialize(ITelemetry telemetry) + { + if (this.IsSuppressedTelemetry(telemetry)) + { + this.SuppressTelemetry(telemetry); + return; + } + + if (!(telemetry is RequestTelemetry)) + { + Activity currentActivity = Activity.Current; + if (telemetry is ExceptionTelemetry) + { + Console.WriteLine("exception!"); + } + + if (currentActivity == null) + { + if (CorrelationTraceContext.Current != null) + { + UpdateTelemetry(telemetry, CorrelationTraceContext.Current); + } + } + else + { + if (CorrelationTraceContext.Current != null) + { + UpdateTelemetry(telemetry, CorrelationTraceContext.Current); + } + else if (CorrelationSettings.Current.Protocol == Protocol.W3CTraceContext) + { + UpdateTelemetry(telemetry, currentActivity, false); + } + else if (CorrelationSettings.Current.Protocol == Protocol.HttpCorrelationProtocol + && telemetry is ExceptionTelemetry) + { + UpdateTelemetryExceptionForHTTPCorrelationProtocol((ExceptionTelemetry)telemetry, currentActivity); + } + } + } + } + + internal static void UpdateTelemetry(ITelemetry telemetry, TraceContextBase contextBase) + { + switch (contextBase) + { + case NullObjectTraceContext nullObjectContext: + return; + case W3CTraceContext w3cContext: + UpdateTelemetryW3C(telemetry, w3cContext); + break; + case HttpCorrelationProtocolTraceContext httpCorrelationProtocolTraceContext: + UpdateTelemetryHttpCorrelationProtocol(telemetry, httpCorrelationProtocolTraceContext); + break; + default: + return; + } + } + + internal static void UpdateTelemetryHttpCorrelationProtocol(ITelemetry telemetry, HttpCorrelationProtocolTraceContext context) + { + OperationTelemetry opTelemetry = telemetry as OperationTelemetry; + + bool initializeFromCurrent = opTelemetry != null; + + if (initializeFromCurrent) + { + initializeFromCurrent &= !(opTelemetry is DependencyTelemetry dependency && + dependency.Type == SqlRemoteDependencyType && + dependency.Context.GetInternalContext().SdkVersion + .StartsWith(RddDiagnosticSourcePrefix, StringComparison.Ordinal)); + } + + if (initializeFromCurrent) + { + opTelemetry.Id = !string.IsNullOrEmpty(opTelemetry.Id) ? opTelemetry.Id : context.TelemetryId; + telemetry.Context.Operation.ParentId = !string.IsNullOrEmpty(telemetry.Context.Operation.ParentId) ? telemetry.Context.Operation.ParentId : context.TelemetryContextOperationParentId; + } + else + { + telemetry.Context.Operation.Id = !string.IsNullOrEmpty(telemetry.Context.Operation.Id) ? telemetry.Context.Operation.Id : context.TelemetryContextOperationId; + if (telemetry is ExceptionTelemetry) + { + telemetry.Context.Operation.ParentId = context.TelemetryId; + } + else + { + telemetry.Context.Operation.ParentId = !string.IsNullOrEmpty(telemetry.Context.Operation.ParentId) ? telemetry.Context.Operation.ParentId : context.TelemetryContextOperationParentId; + } + } + } + + internal static void UpdateTelemetryW3C(ITelemetry telemetry, W3CTraceContext context) + { + OperationTelemetry opTelemetry = telemetry as OperationTelemetry; + + bool initializeFromCurrent = opTelemetry != null; + + if (initializeFromCurrent) + { + initializeFromCurrent &= !(opTelemetry is DependencyTelemetry dependency && + dependency.Type == SqlRemoteDependencyType && + dependency.Context.GetInternalContext().SdkVersion + .StartsWith(RddDiagnosticSourcePrefix, StringComparison.Ordinal)); + } + + if (!string.IsNullOrEmpty(context.TraceState)) + { + opTelemetry.Properties["w3c_tracestate"] = context.TraceState; + } + + TraceParent traceParent = TraceParent.FromString(context.TraceParent); + + if (initializeFromCurrent) + { + if (string.IsNullOrEmpty(opTelemetry.Id)) + { + opTelemetry.Id = traceParent.SpanId; + } + + if (string.IsNullOrEmpty(context.ParentSpanId)) + { + telemetry.Context.Operation.ParentId = telemetry.Context.Operation.Id; + } + } + else + { + if (telemetry.Context.Operation.Id == null) + { + telemetry.Context.Operation.Id = traceParent.TraceId; + } + + if (telemetry.Context.Operation.ParentId == null) + { + telemetry.Context.Operation.ParentId = traceParent.SpanId; + } + } + } + + internal void SuppressTelemetry(ITelemetry telemetry) + { + // TODO For suppressing Dependency, I make the Id as suppressed. This strategy increases the number of telemetery. + // However, new implementation is already supressed. Once we've fully tested this logic, remove the suppression logic on this class. + telemetry.Context.Operation.Id = "suppressed"; + telemetry.Context.Operation.ParentId = "suppressed"; +#pragma warning disable 618 + + // Context. Properties. ai_legacyRequestId , ai_legacyRequestId + foreach (var key in telemetry.Context.Properties.Keys) + { + if (key == "ai_legacyRootId" || + key == "ai_legacyRequestId") + { + telemetry.Context.Properties[key] = "suppressed"; + } + } +#pragma warning restore 618 + + ((OperationTelemetry)telemetry).Id = "suppressed"; + } + + internal bool IsSuppressedTelemetry(ITelemetry telemetry) + { + OperationTelemetry opTelemetry = telemetry as OperationTelemetry; + if (telemetry is DependencyTelemetry) + { + DependencyTelemetry dTelemetry = telemetry as DependencyTelemetry; +#pragma warning disable 618 + if (!string.IsNullOrEmpty(dTelemetry.CommandName)) + { + var host = new Uri(dTelemetry.CommandName).Host; +#pragma warning restore 618 + if (this.ExcludeComponentCorrelationHttpHeadersOnDomains.Contains(host)) + { + return true; + } + } + } + + return false; + } + + internal static void UpdateTelemetryExceptionForHTTPCorrelationProtocol(ExceptionTelemetry telemetry, Activity activity) + { + telemetry.Context.Operation.ParentId = activity.Id; + telemetry.Context.Operation.Id = activity.RootId; + } + + [SuppressMessage("Microsoft.Usage", "CA1801:ReviewUnusedParameters", Justification = "This method has different code for Net45/NetCore")] + internal static void UpdateTelemetry(ITelemetry telemetry, Activity activity, bool forceUpdate) + { + if (activity == null) + { + return; + } + + // Requests and dependencies are initialized from the Activity.Current. + // (i.e. telemetry.Id = current.Id). Activity is created for such requests specifically + // Traces, exceptions, events on the other side are children of current activity + // There is one exception - SQL DiagnosticSource where current Activity is a parent + // for dependency calls. + + OperationTelemetry opTelemetry = telemetry as OperationTelemetry; + bool initializeFromCurrent = opTelemetry != null; + + if (initializeFromCurrent) + { + initializeFromCurrent &= !(opTelemetry is DependencyTelemetry dependency && + dependency.Type == SqlRemoteDependencyType && + dependency.Context.GetInternalContext().SdkVersion + .StartsWith(RddDiagnosticSourcePrefix, StringComparison.Ordinal)); + } + + if (telemetry is OperationTelemetry operation) + { + operation.Properties[TracestateTag] = activity.TraceStateString; + } + + if (initializeFromCurrent) + { + opTelemetry.Id = activity.SpanId.ToHexString(); + if (activity.ParentSpanId != null) + { + opTelemetry.Context.Operation.ParentId = activity.ParentSpanId.ToHexString(); + } + } + else + { + telemetry.Context.Operation.ParentId = activity.SpanId.ToHexString(); + } + } + } +} diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/ITelemetryActivator.cs b/src/WebJobs.Extensions.DurableTask/Correlation/ITelemetryActivator.cs new file mode 100644 index 000000000..1e2071da4 --- /dev/null +++ b/src/WebJobs.Extensions.DurableTask/Correlation/ITelemetryActivator.cs @@ -0,0 +1,16 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation +{ + /// + /// ITelemetryActivator is an interface. + /// + public interface ITelemetryActivator + { + /// + /// Initialize is initialize the telemetry client. + /// + void Initialize(); + } +} diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/NoOpTelemetryChannel.cs b/src/WebJobs.Extensions.DurableTask/Correlation/NoOpTelemetryChannel.cs new file mode 100644 index 000000000..c7dee9c9c --- /dev/null +++ b/src/WebJobs.Extensions.DurableTask/Correlation/NoOpTelemetryChannel.cs @@ -0,0 +1,83 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using Microsoft.ApplicationInsights.Channel; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation +{ + /// + /// A stub of . + /// + public sealed class NoOpTelemetryChannel : ITelemetryChannel + { + /// + /// Initializes a new instance of the class. + /// + public NoOpTelemetryChannel() + { + this.OnSend = telemetry => { }; + this.OnFlush = () => { }; + this.OnDispose = () => { }; + } + + /// + /// Gets or sets a value indicating whether this channel is in developer mode. + /// + public bool? DeveloperMode { get; set; } + + /// + /// Gets or sets a value indicating the channel's URI. To this URI the telemetry is expected to be sent. + /// + public string EndpointAddress { get; set; } + + /// + /// Gets or sets a value indicating whether to throw an error. + /// + public bool ThrowError { get; set; } + + /// + /// Gets or sets the callback invoked by the method. + /// + public Action OnSend { get; set; } + + /// + /// Gets or sets the callback invoked by the method. + /// + public Action OnFlush { get; set; } + + /// + /// Gets or sets the callback invoked by the method. + /// + public Action OnDispose { get; set; } + + /// + /// Implements the method by invoking the callback. + /// + public void Send(ITelemetry item) + { + if (this.ThrowError) + { + throw new Exception("test error"); + } + + this.OnSend(item); + } + + /// + /// Implements the method. + /// + public void Dispose() + { + this.OnDispose(); + } + + /// + /// Implements the method. + /// + public void Flush() + { + this.OnFlush(); + } + } +} diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/TelemetryActivator.cs b/src/WebJobs.Extensions.DurableTask/Correlation/TelemetryActivator.cs new file mode 100644 index 000000000..a391e6a7e --- /dev/null +++ b/src/WebJobs.Extensions.DurableTask/Correlation/TelemetryActivator.cs @@ -0,0 +1,101 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using DurableTask.Core; +using DurableTask.Core.Settings; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.Extensibility; +using Microsoft.Extensions.Options; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation +{ + /// + /// TelemetryActivator initializes Distributed Tracing. This class only works for netstandard2.0. + /// + public class TelemetryActivator : ITelemetryActivator + { + private TelemetryClient telemetryClient; + private IOptions options; + + /// + /// Constructor for initializing Distributed Tracing. + /// + /// DurableTask options. + public TelemetryActivator(IOptions options) + { + this.options = options; + } + + /// + /// OnSend is an action that enable to hook of sending telemetry. + /// You can use this property for testing. + /// + public Action OnSend { get; set; } = null; + + /// + /// Initialize is initialize the telemetry client. + /// + public void Initialize() + { + this.SetUpDistributedTracing(); + if (CorrelationSettings.Current.EnableDistributedTracing) + { + this.SetUpTelemetryClient(); + this.SetUpTelemetryCallbacks(); + } + } + + private void SetUpDistributedTracing() + { + DurableTaskOptions durableTaskOptions = this.options.Value; + CorrelationSettings.Current.EnableDistributedTracing = + durableTaskOptions.Tracing.DistributedTracingEnabled; + CorrelationSettings.Current.Protocol = + durableTaskOptions.Tracing.DistributedTracingProtocol == Protocol.W3CTraceContext.ToString() + ? Protocol.W3CTraceContext + : Protocol.HttpCorrelationProtocol; + } + + private void SetUpTelemetryCallbacks() + { + CorrelationTraceClient.SetUp( + (TraceContextBase requestTraceContext) => + { + requestTraceContext.Stop(); + + var requestTelemetry = requestTraceContext.CreateRequestTelemetry(); + this.telemetryClient.TrackRequest(requestTelemetry); + }, + (TraceContextBase dependencyTraceContext) => + { + dependencyTraceContext.Stop(); + var dependencyTelemetry = dependencyTraceContext.CreateDependencyTelemetry(); + this.telemetryClient.TrackDependency(dependencyTelemetry); + }, + (Exception e) => + { + this.telemetryClient.TrackException(e); + }); + } + + private void SetUpTelemetryClient() + { + TelemetryConfiguration config = TelemetryConfiguration.CreateDefault(); + if (this.OnSend != null) + { + config.TelemetryChannel = new NoOpTelemetryChannel { OnSend = this.OnSend }; + } + + var telemetryInitializer = new DurableTaskCorrelationTelemetryInitializer(); + + telemetryInitializer.ExcludeComponentCorrelationHttpHeadersOnDomains.Add("127.0.0.1"); + config.TelemetryInitializers.Add(telemetryInitializer); + + config.InstrumentationKey = Environment.GetEnvironmentVariable("APPINSIGHTS_INSTRUMENTATIONKEY"); + + this.telemetryClient = new TelemetryClient(config); + } + } +} \ No newline at end of file diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/TraceContextBaseExtensions.cs b/src/WebJobs.Extensions.DurableTask/Correlation/TraceContextBaseExtensions.cs new file mode 100644 index 000000000..d0562d5b9 --- /dev/null +++ b/src/WebJobs.Extensions.DurableTask/Correlation/TraceContextBaseExtensions.cs @@ -0,0 +1,50 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using DurableTask.Core; +using Microsoft.ApplicationInsights; +using Microsoft.ApplicationInsights.DataContracts; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation +{ + /// + /// TraceContextBase extension methods. + /// + public static class TraceContextBaseExtensions + { + /// + /// Create RequestTelemetry from the TraceContext. + /// + /// TraceContext. + /// RequestTelemetry. + public static RequestTelemetry CreateRequestTelemetry(this TraceContextBase context) + { + var telemetry = new RequestTelemetry { Name = context.OperationName }; + telemetry.Duration = context.Duration; + telemetry.Timestamp = context.StartTime; + telemetry.Id = context.TelemetryId; + telemetry.Context.Operation.Id = context.TelemetryContextOperationId; + telemetry.Context.Operation.ParentId = context.TelemetryContextOperationParentId; + + return telemetry; + } + + /// + /// Create DependencyTelemetry from the Activity. + /// + /// TraceContext. + /// DependencyTelemetry. + public static DependencyTelemetry CreateDependencyTelemetry(this TraceContextBase context) + { + var telemetry = new DependencyTelemetry { Name = context.OperationName }; + telemetry.Start(); + telemetry.Duration = context.Duration; + telemetry.Timestamp = context.StartTime; // TimeStamp is the time of ending the Activity. + telemetry.Id = context.TelemetryId; + telemetry.Context.Operation.Id = context.TelemetryContextOperationId; + telemetry.Context.Operation.ParentId = context.TelemetryContextOperationParentId; + + return telemetry; + } + } +} diff --git a/src/WebJobs.Extensions.DurableTask/Correlation/TraceParent.cs b/src/WebJobs.Extensions.DurableTask/Correlation/TraceParent.cs new file mode 100644 index 000000000..51cdbb387 --- /dev/null +++ b/src/WebJobs.Extensions.DurableTask/Correlation/TraceParent.cs @@ -0,0 +1,62 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation +{ + /// + /// Represents a traceParent that is defined W3C TraceContext. + /// + public class TraceParent + { + /// + /// Gets or sets the Version of the traceParent. + /// + public string Version { get; set; } + + /// + /// Gets or sets the TraceId of the traceParent. + /// + public string TraceId { get; set; } + + /// + /// Gets or sets the SpanId of the traceParent. + /// + public string SpanId { get; set; } + + /// + /// Gets or sets the TraceFlags of the traceParent. + /// + public string TraceFlags { get; set; } + + /// + /// Convert a traceParent string to TraceParent object. + /// + /// string representations of traceParent. + /// TraceParent object. + public static TraceParent FromString(string traceparent) + { + var exceptionString = + $"Traceparent doesn't respect the spec. spec: {{version}}-{{traceId}}-{{spanId}}-{{traceFlags}} actual: {traceparent}"; + if (!string.IsNullOrEmpty(traceparent)) + { + var substrings = traceparent.Split('-'); + if (substrings.Length != 4) + { + throw new ArgumentException(exceptionString); + } + + return new TraceParent + { + Version = substrings[0], + TraceId = substrings[1], + SpanId = substrings[2], + TraceFlags = substrings[3], + }; + } + + throw new ArgumentException(exceptionString); + } + } +} diff --git a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs index e85857857..a49c594b3 100644 --- a/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs +++ b/src/WebJobs.Extensions.DurableTask/DurableTaskExtension.cs @@ -18,6 +18,9 @@ using DurableTask.Core.History; using DurableTask.Core.Middleware; using Microsoft.Azure.WebJobs.Description; +#if !FUNCTIONS_V1 +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; +#endif using Microsoft.Azure.WebJobs.Extensions.DurableTask.Listener; using Microsoft.Azure.WebJobs.Host; using Microsoft.Azure.WebJobs.Host.Config; @@ -60,7 +63,11 @@ public class DurableTaskExtension : new ConcurrentDictionary(); private readonly AsyncLock taskHubLock = new AsyncLock(); - +#if !FUNCTIONS_V1 +#pragma warning disable CS0169 + private readonly ITelemetryActivator telemetryActivator; +#pragma warning restore CS0169 +#endif private readonly bool isOptionsConfigured; private readonly IApplicationLifetimeWrapper hostLifetimeService = HostLifecycleService.NoOp; @@ -87,7 +94,7 @@ public DurableTaskExtension() this.isOptionsConfigured = false; } #endif - +#pragma warning disable CS1572 /// /// Initializes a new instance of the . /// @@ -100,6 +107,8 @@ public DurableTaskExtension() /// The lifecycle notification helper used for custom orchestration tracking. /// The factory used to create for message settings. /// The factory used to create for error settings. + /// The activator of DistributedTracing. .netstandard2.0 only. +#pragma warning restore CS1572 public DurableTaskExtension( IOptions options, ILoggerFactory loggerFactory, @@ -109,7 +118,14 @@ public DurableTaskExtension() IDurableHttpMessageHandlerFactory durableHttpMessageHandlerFactory = null, ILifeCycleNotificationHelper lifeCycleNotificationHelper = null, IMessageSerializerSettingsFactory messageSerializerSettingsFactory = null, +#if !FUNCTIONS_V1 + IErrorSerializerSettingsFactory errorSerializerSettingsFactory = null, +#pragma warning disable SA1113, SA1001, SA1115 + ITelemetryActivator telemetryActivator = null) +#pragma warning restore SA1113, SA1001, SA1115 +#else IErrorSerializerSettingsFactory errorSerializerSettingsFactory = null) +#endif { // Options will be null in Functions v1 runtime - populated later. this.Options = options?.Value ?? new DurableTaskOptions(); @@ -144,6 +160,8 @@ public DurableTaskExtension() // The RPC server is started when the extension is initialized. // The RPC server is stopped when the host has finished shutting down. this.hostLifetimeService.OnStopped.Register(this.StopLocalRcpServer); + this.telemetryActivator = telemetryActivator; + this.telemetryActivator?.Initialize(); #endif } diff --git a/src/WebJobs.Extensions.DurableTask/DurableTaskJobHostConfigurationExtensions.cs b/src/WebJobs.Extensions.DurableTask/DurableTaskJobHostConfigurationExtensions.cs index 9e763c58a..27d8e3676 100644 --- a/src/WebJobs.Extensions.DurableTask/DurableTaskJobHostConfigurationExtensions.cs +++ b/src/WebJobs.Extensions.DurableTask/DurableTaskJobHostConfigurationExtensions.cs @@ -6,6 +6,7 @@ using System.Threading; #if !FUNCTIONS_V1 using Microsoft.Azure.WebJobs.Extensions.DurableTask.ContextImplementations; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; using Microsoft.Azure.WebJobs.Extensions.DurableTask.Options; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection.Extensions; @@ -45,6 +46,9 @@ public static IWebJobsBuilder AddDurableTask(this IWebJobsBuilder builder) serviceCollection.TryAddSingleton(); serviceCollection.TryAddSingleton(); serviceCollection.TryAddSingleton(); +#if !FUNCTIONS_V1 + serviceCollection.AddSingleton(); +#endif serviceCollection.TryAddSingleton(); return builder; diff --git a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask-net461.xml b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask-net461.xml index 89ea235dd..883148ee9 100644 --- a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask-net461.xml +++ b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask-net461.xml @@ -2090,6 +2090,7 @@ The lifecycle notification helper used for custom orchestration tracking. The factory used to create for message settings. The factory used to create for error settings. + The activator of DistributedTracing. .netstandard2.0 only. diff --git a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml index 0e72f3670..2113d6c89 100644 --- a/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml +++ b/src/WebJobs.Extensions.DurableTask/Microsoft.Azure.WebJobs.Extensions.DurableTask.xml @@ -1548,6 +1548,208 @@ Now obsolete: use instead. + + + Telemetry Initializer that sets correlation ids for W3C. + This source is based on W3COperationCorrelationTelemetryInitializer.cs + 1. Modified with CorrelationTraceContext.Current + 2. Avoid to be overriden when it is RequestTelemetry + Original Source is here https://github.com/microsoft/ApplicationInsights-dotnet-server/blob/2.8.0/Src/Common/W3C/W3COperationCorrelationTelemetryInitializer.cs. + + + + These internal property is copied from W3CConstants + Trace-Id tag name. + + + Span-Id tag name. + + + Parent span-Id tag name. + + + Version tag name. + + + Sampled tag name. + + + Tracestate tag name. + + + Default version value. + + + + Default sampled flag value: may be recorded, not requested. + + + + Recorded and requested sampled flag value. + + + Requested trace flag. + + + Legacy root Id tag name. + + + Legacy root Id tag name. + + + + Constructor. + + + + + Set of suppress telemetry tracking if you add Host name on this. + + + + + Initializes telemetry item. + + Telemetry item. + + + + ITelemetryActivator is an interface. + + + + + Initialize is initialize the telemetry client. + + + + + A stub of . + + + + + Initializes a new instance of the class. + + + + + Gets or sets a value indicating whether this channel is in developer mode. + + + + + Gets or sets a value indicating the channel's URI. To this URI the telemetry is expected to be sent. + + + + + Gets or sets a value indicating whether to throw an error. + + + + + Gets or sets the callback invoked by the method. + + + + + Gets or sets the callback invoked by the method. + + + + + Gets or sets the callback invoked by the method. + + + + + Implements the method by invoking the callback. + + + + + Implements the method. + + + + + Implements the method. + + + + + TelemetryActivator initializes Distributed Tracing. This class only works for netstandard2.0. + + + + + Constructor for initializing Distributed Tracing. + + DurableTask options. + + + + OnSend is an action that enable to hook of sending telemetry. + You can use this property for testing. + + + + + Initialize is initialize the telemetry client. + + + + + TraceContextBase extension methods. + + + + + Create RequestTelemetry from the TraceContext. + + TraceContext. + RequestTelemetry. + + + + Create DependencyTelemetry from the Activity. + + TraceContext. + DependencyTelemetry. + + + + Represents a traceParent that is defined W3C TraceContext. + + + + + Gets or sets the Version of the traceParent. + + + + + Gets or sets the TraceId of the traceParent. + + + + + Gets or sets the SpanId of the traceParent. + + + + + Gets or sets the TraceFlags of the traceParent. + + + + + Convert a traceParent string to TraceParent object. + + string representations of traceParent. + TraceParent object. + Attribute used with the Durable Functions Analyzer to label a method as Deterministic. This allows the method to be called in an Orchestration function without causing a compiler warning. @@ -2077,7 +2279,7 @@ Configuration for the Durable Functions extension. - + Initializes a new instance of the . @@ -2090,6 +2292,7 @@ The lifecycle notification helper used for custom orchestration tracking. The factory used to create for message settings. The factory used to create for error settings. + The activator of DistributedTracing. .netstandard2.0 only. @@ -3735,6 +3938,19 @@ Boolean value specifying if the replay events should be logged. + + + Gets or sets a flag indicating whether to enable distributed tracing. + The default value is false. + + + + + Gets or sets a protocol for distributed Tracing. + Possible values are "HttpCorrelationProtocol" and "W3CTraceContext". + The default value is "HttpCorrelationProtocol". + + Deprecated attribute to bind a function parameter to a . diff --git a/src/WebJobs.Extensions.DurableTask/Options/TraceOptions.cs b/src/WebJobs.Extensions.DurableTask/Options/TraceOptions.cs index f5ebdb14d..590bc32e4 100644 --- a/src/WebJobs.Extensions.DurableTask/Options/TraceOptions.cs +++ b/src/WebJobs.Extensions.DurableTask/Options/TraceOptions.cs @@ -50,6 +50,21 @@ public class TraceOptions /// public bool TraceReplayEvents { get; set; } +#if !FUNCTIONS_V1 + /// + /// Gets or sets a flag indicating whether to enable distributed tracing. + /// The default value is false. + /// + public bool DistributedTracingEnabled { get; set; } = false; + + /// + /// Gets or sets a protocol for distributed Tracing. + /// Possible values are "HttpCorrelationProtocol" and "W3CTraceContext". + /// The default value is "HttpCorrelationProtocol". + /// + public string DistributedTracingProtocol { get; set; } = "HttpCorrelationProtocol"; + +#endif internal void AddToDebugString(StringBuilder builder) { builder.Append(nameof(this.TraceReplayEvents)).Append(": ").Append(this.TraceReplayEvents).Append(", "); diff --git a/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj b/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj index 79a3d318f..b63864f0d 100644 --- a/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj +++ b/src/WebJobs.Extensions.DurableTask/WebJobs.Extensions.DurableTask.csproj @@ -43,8 +43,14 @@ FUNCTIONS_V1 + false + + + + + @@ -62,6 +68,8 @@ + + diff --git a/test/Common/DurableClientBaseTests.cs b/test/Common/DurableClientBaseTests.cs index 0190a09de..088a6e75f 100644 --- a/test/Common/DurableClientBaseTests.cs +++ b/test/Common/DurableClientBaseTests.cs @@ -166,7 +166,7 @@ public async void DurableClient_ExternalApp_GetStatusAsync_ReturnsStatus() orchestrationServiceClientMock.Setup(x => x.GetOrchestrationStateAsync(It.IsAny(), It.IsAny())) .ReturnsAsync(GetInstanceState(OrchestrationStatus.Running)); - var durableOrchestrationClient = this.GetDurableClient(orchestrationServiceClientMock.Object); + var durableOrchestrationClient = this.GetDurableClient(orchestrationServiceClientMock.Object); var status = await durableOrchestrationClient.GetStatusAsync("testInstanceId"); Assert.Equal(OrchestrationRuntimeStatus.Running, status.RuntimeStatus); } diff --git a/test/Common/DurableTaskEndToEndTests.cs b/test/Common/DurableTaskEndToEndTests.cs index 9454a3641..e7455fa28 100644 --- a/test/Common/DurableTaskEndToEndTests.cs +++ b/test/Common/DurableTaskEndToEndTests.cs @@ -2676,7 +2676,7 @@ public async Task SetStatusOrchestration(bool extendedSessions, string storagePr // Test clearing an existing custom status await client.RaiseEventAsync("UpdateStatus", null, this.output); - await client.WaitForCustomStatusAsync(TimeSpan.FromSeconds(10), this.output, JValue.CreateNull()); + await client.WaitForCustomStatusAsync(TimeSpan.FromSeconds(30), this.output, JValue.CreateNull()); // Test setting the custom status to a complex object. var newCustomStatus = new { Foo = "Bar", Count = 2, }; diff --git a/test/Common/TestHelpers.cs b/test/Common/TestHelpers.cs index be71e54d9..948925839 100644 --- a/test/Common/TestHelpers.cs +++ b/test/Common/TestHelpers.cs @@ -7,6 +7,10 @@ using System.Net.Http; using System.Threading.Tasks; using DurableTask.AzureStorage; +using Microsoft.ApplicationInsights.Channel; +#if !FUNCTIONS_V1 +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; +#endif using Microsoft.Azure.WebJobs.Host.TestCommon; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -57,6 +61,7 @@ internal static class TestHelpers IMessageSerializerSettingsFactory serializerSettings = null, bool? localRpcEndpointEnabled = false, DurableTaskOptions options = null, + Action onSend = null, bool rollbackEntityOperationsOnExceptions = true, int entityMessageReorderWindowInMinutes = 30, string exactTaskHubName = null) @@ -82,12 +87,11 @@ internal static class TestHelpers // task hub name and require the usage of that task hub. Otherwise, generate a partially random task hub from the // test name and properties of the test. options.HubName = exactTaskHubName ?? GetTaskHubNameFromTestName(testName, enableExtendedSessions); - options.Tracing = new TraceOptions() - { - TraceInputsAndOutputs = true, - TraceReplayEvents = traceReplayEvents, - AllowVerboseLinuxTelemetry = allowVerboseLinuxTelemetry, - }; + + options.Tracing.TraceInputsAndOutputs = true; + options.Tracing.TraceReplayEvents = traceReplayEvents; + options.Tracing.AllowVerboseLinuxTelemetry = allowVerboseLinuxTelemetry; + options.Notifications = new NotificationOptions() { EventGrid = new EventGridNotificationOptions() @@ -149,6 +153,7 @@ internal static class TestHelpers durableHttpMessageHandler, lifeCycleNotificationHelper, serializerSettings, + onSend, durabilityProviderFactoryType); } @@ -160,6 +165,7 @@ internal static class TestHelpers IDurableHttpMessageHandlerFactory durableHttpMessageHandler = null, ILifeCycleNotificationHelper lifeCycleNotificationHelper = null, IMessageSerializerSettingsFactory serializerSettings = null, + Action onSend = null, Type durabilityProviderFactoryType = null) { if (serializerSettings == null) @@ -175,16 +181,17 @@ internal static class TestHelpers } return PlatformSpecificHelpers.CreateJobHost( - optionsWrapper, - storageProviderType, + options: optionsWrapper, + storageProvider: storageProviderType, #if !FUNCTIONS_V1 - durabilityProviderFactoryType, + durabilityProviderFactoryType: durabilityProviderFactoryType, #endif - loggerProvider, - testNameResolver, - durableHttpMessageHandler, - lifeCycleNotificationHelper, - serializerSettings); + loggerProvider: loggerProvider, + nameResolver: testNameResolver, + durableHttpMessageHandler: durableHttpMessageHandler, + lifeCycleNotificationHelper: lifeCycleNotificationHelper, + serializerSettingsFactory: serializerSettings, + onSend: onSend); } public static DurableTaskOptions GetDurableTaskOptionsForStorageProvider(string storageProvider) diff --git a/test/FunctionsV1/PlatformSpecificHelpers.FunctionsV1.cs b/test/FunctionsV1/PlatformSpecificHelpers.FunctionsV1.cs index 8eb673aa6..8e0045822 100644 --- a/test/FunctionsV1/PlatformSpecificHelpers.FunctionsV1.cs +++ b/test/FunctionsV1/PlatformSpecificHelpers.FunctionsV1.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Reflection; using System.Threading.Tasks; +using Microsoft.ApplicationInsights.Channel; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; @@ -27,8 +28,9 @@ public static class PlatformSpecificHelpers INameResolver nameResolver, IDurableHttpMessageHandlerFactory durableHttpMessageHandler, ILifeCycleNotificationHelper lifeCycleNotificationHelper, - IMessageSerializerSettingsFactory serializerSettings, - IApplicationLifetimeWrapper shutdownNotificationService = null) + IMessageSerializerSettingsFactory serializerSettingsFactory, + IApplicationLifetimeWrapper shutdownNotificationService = null, + Action onSend = null) { var config = new JobHostConfiguration { HostId = "durable-task-host" }; config.TypeLocator = TestHelpers.GetTypeLocator(); @@ -52,7 +54,7 @@ public static class PlatformSpecificHelpers shutdownNotificationService ?? new TestHostShutdownNotificationService(), durableHttpMessageHandler, lifeCycleNotificationHelper, - serializerSettings); + serializerSettingsFactory); config.UseDurableTask(extension); // Mock INameResolver for not setting EnvironmentVariables. diff --git a/test/FunctionsV2/CorrelationEndToEndTests.cs b/test/FunctionsV2/CorrelationEndToEndTests.cs new file mode 100644 index 000000000..1b8bd5831 --- /dev/null +++ b/test/FunctionsV2/CorrelationEndToEndTests.cs @@ -0,0 +1,257 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using DurableTask.AzureStorage; +using DurableTask.Core; +using DurableTask.Core.Settings; +using FluentAssertions.Collections; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.ApplicationInsights.DataContracts; +using Microsoft.ApplicationInsights.Extensibility.Implementation; +using Microsoft.Azure.WebJobs.Extensions.DurableTask; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests; +using Microsoft.Azure.WebJobs.Host.TestCommon; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests +{ + [Collection("Non-Parallel Collection")] + public class CorrelationEndToEndTests + { + private readonly ITestOutputHelper output; + private readonly TestLoggerProvider loggerProvider; + + public CorrelationEndToEndTests(ITestOutputHelper output) + { + this.output = output; + this.loggerProvider = new TestLoggerProvider(output); + } + + [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + [InlineData(false, "W3CTraceContext")] + [InlineData(true, "HttpCorrelationProtocol")] + [InlineData(true, "W3CTraceContext")] + [InlineData(false, "HttpCorrelationProtocol")] + public async Task SingleOrchestration_With_Activity(bool extendedSessions, string protocol) + { + string[] orchestrationFunctionNames = + { + nameof(TestOrchestrations.SayHelloWithActivity), + }; + + var result = await + this.ExecuteOrchestrationWithExceptionAsync( + orchestrationFunctionNames, + "SingleOrchestration", + "world", + extendedSessions, + protocol); + var actual = result.Item1; + Assert.Equal(5, actual.Count); + Assert.Empty(result.Item2); + Assert.Equal( + new (Type, string)[] + { + (typeof(RequestTelemetry), $"{TraceConstants.Client}: "), + (typeof(DependencyTelemetry), TraceConstants.Client), + (typeof(RequestTelemetry), $"{TraceConstants.Orchestrator} SayHelloWithActivity"), + (typeof(DependencyTelemetry), $"{TraceConstants.Orchestrator} Hello"), + (typeof(RequestTelemetry), $"{TraceConstants.Activity} Hello"), + }.ToList(), actual.Select(x => (x.GetType(), x.Name)).ToList()); + } + + /* + [Theory] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + [InlineData(false, "W3CTraceContext")] + [InlineData(true, "HttpCorrelationProtocol")] + [InlineData(true, "W3CTraceContext")] + [InlineData(false, "HttpCorrelationProtocol")] + public async Task AllOrchestrationActivityActions(bool extendedSessions, string protocol) + { + string[] orchestrationFunctionNames = + { + nameof(TestOrchestrations.AllOrchestratorActivityActions), + }; + + var counterEntityId = new EntityId("Counter", Guid.NewGuid().ToString()); + + var result = await + this.ExecuteOrchestrationWithExceptionAsync( + orchestrationFunctionNames, + nameof(this.AllOrchestrationActivityActions), + counterEntityId, + extendedSessions, + protocol); + var actual = result.Item1; + Assert.Equal(15, actual.Count); + // TODO: This part of the test is failing. This is commented out temporarily since Correlation Tracing is a WIP. + // Assert.Single(result.Item2); // Error inside of HttpActivity since the request set to null. + Assert.Equal( + new (Type, string)[] + { + (typeof(RequestTelemetry), $"{TraceConstants.Client}: "), // start orchestration + (typeof(DependencyTelemetry), TraceConstants.Client), + (typeof(RequestTelemetry), $"{TraceConstants.Orchestrator} AllOrchestratorActivityActions"), // Orchestrator started + (typeof(DependencyTelemetry), $"{TraceConstants.Orchestrator} Hello"), + (typeof(RequestTelemetry), $"{TraceConstants.Activity} Hello"), // Activity Hello Started + (typeof(DependencyTelemetry), $"{TraceConstants.Orchestrator} Hello"), + (typeof(RequestTelemetry), $"{TraceConstants.Activity} Hello"), // Activity Hello Started + (typeof(DependencyTelemetry), $"{TraceConstants.Orchestrator} SayHelloInline"), + (typeof(RequestTelemetry), $"{TraceConstants.Orchestrator} SayHelloInline"), // SubOrchestrator SayHelloInline Started + (typeof(DependencyTelemetry), $"{TraceConstants.Orchestrator} SayHelloWithActivity"), + (typeof(RequestTelemetry), $"{TraceConstants.Orchestrator} SayHelloWithActivity"), // SubOrchestrator SayHelloWithActivity Started + (typeof(DependencyTelemetry), $"{TraceConstants.Orchestrator} Hello"), + (typeof(RequestTelemetry), $"{TraceConstants.Activity} Hello"), // Activity Hello Started by SubOrchestrator SayHelloWithActivity + (typeof(DependencyTelemetry), $"{TraceConstants.Orchestrator} BuiltIn::HttpActivity"), + (typeof(RequestTelemetry), $"{TraceConstants.Activity} BuiltIn::HttpActivity"), // HttpActivity Started + }.ToList(), actual.Select(x => (x.GetType(), x.Name)).ToList()); + } + */ + + internal async Task, List>> + ExecuteOrchestrationWithExceptionAsync( + string[] orchestratorFunctionNames, + string testName, + object input, + bool extendedSessions, + string protocol) + { + ConcurrentQueue sendItems = new ConcurrentQueue(); + TraceOptions traceOptions = new TraceOptions() + { + DistributedTracingEnabled = true, + DistributedTracingProtocol = protocol, + }; + DurableTaskOptions options = new DurableTaskOptions(); + options.Tracing = traceOptions; + var sendAction = new Action( + delegate(ITelemetry telemetry) { sendItems.Enqueue(telemetry); }); + using (var host = TestHelpers.GetJobHost( + this.loggerProvider, + testName, + extendedSessions, + options: options, + onSend: sendAction)) + { + await host.StartAsync(); + var client = await host.StartOrchestratorAsync(orchestratorFunctionNames[0], input, this.output); + var status = await client.WaitForCompletionAsync(this.output, timeout: TimeSpan.FromSeconds(90)); + await host.StopAsync(); + } + + var sendItemList = this.ConvertTo(sendItems); + var operationTelemetryList = sendItemList.OfType(); + var exceptionTelemetryList = sendItemList.OfType().ToList(); + var result = this.FilterOperationTelemetry(operationTelemetryList).ToList(); + return new Tuple, List>(result.CorrelationSort(), exceptionTelemetryList); + } + + private IEnumerable FilterOperationTelemetry(IEnumerable operationTelemetries) + { + return operationTelemetries.Where( + p => p.Name.Contains(TraceConstants.Activity) || p.Name.Contains(TraceConstants.Orchestrator) || p.Name.Contains(TraceConstants.Client) || p.Name.Contains("Operation")); + } + + private List ConvertTo(ConcurrentQueue queue) + { + var converted = new List(); + while (!queue.IsEmpty) + { + ITelemetry x; + if (queue.TryDequeue(out x)) + { + converted.Add(x); + } + } + + return converted; + } + } + +#pragma warning disable SA1402 + public static class ListExtensions + { + public static List CorrelationSort(this List telemetries) + { + var result = new List(); + if (telemetries.Count == 0) + { + return result; + } + + // Sort by the timestamp + var sortedTelemetries = telemetries.OrderBy(p => p.Timestamp.Ticks).ToList(); + + // pick the first one as the parent. remove it from the list. + var parent = sortedTelemetries.First(); + result.Add(parent); + sortedTelemetries.RemoveOperationTelemetry(parent); + + // find the child recursively and remove the child and pass it as a parameter + var sortedList = GetCorrelationSortedList(parent, sortedTelemetries); + result.AddRange(sortedList); + return result; + } + + public static bool RemoveOperationTelemetry(this List telemetries, OperationTelemetry telemetry) + { + int index = -1; + for (var i = 0; i < telemetries.Count; i++) + { + if (telemetries[i].Id == telemetry.Id) + { + index = i; + } + } + + if (index == -1) + { + return false; + } + + telemetries.RemoveAt(index); + return true; + } + + private static List GetCorrelationSortedList(OperationTelemetry parent, List current) + { + var result = new List(); + if (current.Count != 0) + { + foreach (var some in current) + { + if (parent.Id == some.Context.Operation.ParentId) + { + Console.WriteLine("match"); + } + } + + IOrderedEnumerable nexts = current.Where(p => p.Context.Operation.ParentId == parent.Id).OrderBy(p => p.Timestamp.Ticks); + foreach (OperationTelemetry next in nexts) + { + current.RemoveOperationTelemetry(next); + result.Add(next); + var childResult = GetCorrelationSortedList(next, current); + result.AddRange(childResult); + } + } + + return result; + } + } + + [CollectionDefinition("Non-Parallel Collection", DisableParallelization = true)] + public class NonParallelCollectionDefinitionClass + { + } +#pragma warning restore SA1402 +} diff --git a/test/FunctionsV2/PlatformSpecificHelpers.FunctionsV2.cs b/test/FunctionsV2/PlatformSpecificHelpers.FunctionsV2.cs index a117cc11b..bd223d96e 100644 --- a/test/FunctionsV2/PlatformSpecificHelpers.FunctionsV2.cs +++ b/test/FunctionsV2/PlatformSpecificHelpers.FunctionsV2.cs @@ -5,6 +5,8 @@ using System.Collections.Generic; using System.Reflection; using System.Threading.Tasks; +using Microsoft.ApplicationInsights.Channel; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; @@ -29,7 +31,8 @@ public static class PlatformSpecificHelpers INameResolver nameResolver, IDurableHttpMessageHandlerFactory durableHttpMessageHandler, ILifeCycleNotificationHelper lifeCycleNotificationHelper, - IMessageSerializerSettingsFactory serializerSettingsFactory) + IMessageSerializerSettingsFactory serializerSettingsFactory, + Action onSend) { IHost host = new HostBuilder() .ConfigureLogging( @@ -60,6 +63,19 @@ public static class PlatformSpecificHelpers { serviceCollection.AddSingleton(serializerSettingsFactory); } + + if (onSend != null) + { + serviceCollection.AddSingleton(serviceProvider => + { + var durableTaskOptions = serviceProvider.GetService>(); + var telemetryActivator = new TelemetryActivator(durableTaskOptions) + { + OnSend = onSend, + }; + return telemetryActivator; + }); + } }) .Build(); diff --git a/test/FunctionsV2/TraceParentTest.cs b/test/FunctionsV2/TraceParentTest.cs new file mode 100644 index 000000000..d898456c2 --- /dev/null +++ b/test/FunctionsV2/TraceParentTest.cs @@ -0,0 +1,41 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the MIT License. See LICENSE in the project root for license information. + +using System; +using System.Collections.Generic; +using System.Text; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Correlation; +using Microsoft.Azure.WebJobs.Extensions.DurableTask.Tests; +using Xunit; +using Xunit.Extensions; + +namespace WebJobs.Extensions.DurableTask.Tests.V2 +{ + public class TraceParentTest + { + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void FromString_NormalCase() + { + var expectedVersion = "00"; + var expectedTraceId = "0af7651916cd43dd8448eb211c80319c"; + var expectedSpanId = "00f067aa0ba902b7"; + var expectedTraceFlags = "01"; + var actual = TraceParent.FromString($"{expectedVersion}-{expectedTraceId}-{expectedSpanId}-{expectedTraceFlags}"); + Assert.Equal(expectedVersion, actual.Version); + Assert.Equal(expectedTraceId, actual.TraceId); + Assert.Equal(expectedSpanId, actual.SpanId); + Assert.Equal(expectedTraceFlags, actual.TraceFlags); + } + + [Fact] + [Trait("Category", PlatformSpecificHelpers.TestCategory)] + public void FromString_Exception() + { + var ex = Assert.Throws(() => { TraceParent.FromString("foobar"); }); + Assert.Contains("foobar", ex.Message); + ex = Assert.Throws(() => { TraceParent.FromString(""); }); + Assert.Contains("Traceparent", ex.Message); + } + } +} diff --git a/test/FunctionsV2/WebJobs.Extensions.DurableTask.Tests.V2.csproj b/test/FunctionsV2/WebJobs.Extensions.DurableTask.Tests.V2.csproj index a8de23e7f..7c0662c26 100644 --- a/test/FunctionsV2/WebJobs.Extensions.DurableTask.Tests.V2.csproj +++ b/test/FunctionsV2/WebJobs.Extensions.DurableTask.Tests.V2.csproj @@ -18,7 +18,7 @@ - +